(BREAKING) refactor: rename dropWorst option to warmupRuns

.changeset/thirty-roses-arrive.md

@@ -0,0 +1,6 @@
+---
+'@callstack/reassure-measure': minor
+'reassure': minor
+---
+
+(BREAKING) refactor: renamed `dropWorst` option to `warmupRuns` with slight change of logic behind it.

README.md

@@ -384,14 +384,14 @@ async function measurePerformance(ui: React.ReactElement, options?: MeasureOptio
 ```ts
 interface MeasureOptions {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   wrapper?: React.ComponentType<{ children: ReactElement }>;
   scenario?: (view?: RenderResult) => Promise<any>;
 }
 ```
 
 - **`runs`**: number of runs per series for the particular test
-- **`dropWorst`**: number of worst (highest) runs dropped from a test series
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1).
 - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured.
 - **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions
 
@@ -405,7 +405,7 @@ of the `configure` function.
 ```ts
 type Config = {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   outputFile?: string;
   verbose?: boolean;
   testingLibrary?:
@@ -418,15 +418,15 @@ type Config = {
 ```ts
 const defaultConfig: Config = {
   runs: 10,
-  dropWorst: 1,
+  warmupRuns: 1,
   outputFile: '.reassure/current.perf',
   verbose: false,
   testingLibrary: undefined, // Will try auto-detect first RNTL, then RTL
 };
 ```
 
 **`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care.
-**`dropWorst`**: number of worst dropped results from the series per test (used to remove test run outliers)
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs.
 **`outputFile`**: name of the file the records will be saved to
 **`verbose`**: make Reassure log more, e.g. for debugging purposes
 **`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions

docusaurus/docs/api.md

@@ -40,14 +40,14 @@ test('Test with scenario', async () => {
 ```ts
 interface MeasureOptions {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   wrapper?: React.ComponentType<{ children: ReactElement }>;
   scenario?: (view?: RenderResult) => Promise<any>;
 }
 ```
 
 - **`runs`**: number of runs per series for the particular test
-- **`dropWorst`**: number of worst (highest) runs dropped from a test series
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs.
 - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured.
 - **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions
 
@@ -61,7 +61,7 @@ of the `configure` function.
 ```ts
 type Config = {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   outputFile?: string;
   verbose?: boolean;
   testingLibrary?:
@@ -74,15 +74,15 @@ type Config = {
 ```ts
 const defaultConfig: Config = {
   runs: 10,
-  dropWorst: 1,
+  warmupRuns: 1,
   outputFile: '.reassure/current.perf',
   verbose: false,
   testingLibrary: undefined, // Will try auto-detect first RNTL, then RTL
 };
 ```
 
 **`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care.
-**`dropWorst`**: number of worst dropped results from the series per test (used to remove test run outliers)
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs.
 **`outputFile`**: name of the file the records will be saved to
 **`verbose`**: make Reassure log more, e.g. for debugging purposes
 **`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions

packages/reassure-measure/src/__tests__/measure.test.tsx

@@ -108,12 +108,12 @@ test('processRunResults calculates correct means and stdevs', () => {
   });
 });
 
-test('processRunResults applies dropWorst option', () => {
+test('processRunResults applies warmupRuns option', () => {
   const input = [
+    { duration: 23, count: 1 },
     { duration: 20, count: 5 },
     { duration: 24, count: 5 },
     { duration: 22, count: 5 },
-    { duration: 1000, count: 1 },
   ];
 
   expect(processRunResults(input, 1)).toEqual({

packages/reassure-measure/src/config.ts

@@ -5,14 +5,14 @@ export type Cleanup = () => void;
 
 type Config = {
   runs: number;
-  dropWorst: number;
+  warmupRuns: number;
   outputFile: string;
   testingLibrary?: TestingLibrary;
 };
 
 const defaultConfig: Config = {
   runs: 10,
-  dropWorst: 1,
+  warmupRuns: 1,
   outputFile: process.env.REASSURE_OUTPUT_FILE ?? '.reassure/current.perf',
   testingLibrary: undefined,
 };

packages/reassure-measure/src/measure.tsx

@@ -13,7 +13,7 @@ logger.configure({
 
 export interface MeasureOptions {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   wrapper?: React.ComponentType<{ children: React.ReactElement }>;
   scenario?: (screen: any) => Promise<any>;
 }
@@ -31,15 +31,15 @@ export async function measurePerformance(
 export async function measureRender(ui: React.ReactElement, options?: MeasureOptions): Promise<MeasureRenderResult> {
   const runs = options?.runs ?? config.runs;
   const scenario = options?.scenario;
-  const dropWorst = options?.dropWorst ?? config.dropWorst;
+  const warmupRuns = options?.warmupRuns ?? config.warmupRuns;
 
   const { render, cleanup } = resolveTestingLibrary();
 
   showFlagsOuputIfNeeded();
 
   const runResults: RunResult[] = [];
   let hasTooLateRender = false;
-  for (let i = 0; i < runs + dropWorst; i += 1) {
+  for (let i = 0; i < runs + warmupRuns; i += 1) {
     let duration = 0;
     let count = 0;
     let isFinished = false;
@@ -75,7 +75,7 @@ export async function measureRender(ui: React.ReactElement, options?: MeasureOpt
     );
   }
 
-  return processRunResults(runResults, dropWorst);
+  return processRunResults(runResults, warmupRuns);
 }
 
 export function buildUiToRender(
@@ -97,9 +97,9 @@ interface RunResult {
   count: number;
 }
 
-export function processRunResults(results: RunResult[], dropWorst: number) {
+export function processRunResults(results: RunResult[], warmupRuns: number) {
+  results = results.slice(warmupRuns);
   results.sort((first, second) => second.duration - first.duration); // duration DESC
-  results = results.slice(dropWorst);
 
   const durations = results.map((result) => result.duration);
   const meanDuration = math.mean(durations) as number;

packages/reassure/README.md

@@ -384,14 +384,14 @@ async function measurePerformance(ui: React.ReactElement, options?: MeasureOptio
 ```ts
 interface MeasureOptions {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   wrapper?: React.ComponentType<{ children: ReactElement }>;
   scenario?: (view?: RenderResult) => Promise<any>;
 }
 ```
 
 - **`runs`**: number of runs per series for the particular test
-- **`dropWorst`**: number of worst (highest) runs dropped from a test series
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs.
 - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured.
 - **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions
 
@@ -405,7 +405,7 @@ of the `configure` function.
 ```ts
 type Config = {
   runs?: number;
-  dropWorst?: number;
+  warmupRuns?: number;
   outputFile?: string;
   verbose?: boolean;
   testingLibrary?:
@@ -418,15 +418,15 @@ type Config = {
 ```ts
 const defaultConfig: Config = {
   runs: 10,
-  dropWorst: 1,
+  warmupRuns: 1,
   outputFile: '.reassure/current.perf',
   verbose: false,
   testingLibrary: undefined, // Will try auto-detect first RNTL, then RTL
 };
 ```
 
 **`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care.
-**`dropWorst`**: number of worst dropped results from the series per test (used to remove test run outliers)
+- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs.
 **`outputFile`**: name of the file the records will be saved to
 **`verbose`**: make Reassure log more, e.g. for debugging purposes
 **`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions