From d0d175f9acb2566c598532b2a991372312ac79cc Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Fri, 25 Nov 2022 00:18:23 +0100 Subject: [PATCH 01/24] feat: base measureExecution API fix: lint refactor: rename `measureExecution` to `UNSTABLE_measurePerformance` feat: measurePerformanceInternal executs code number of times feat: allow setting custom measure code refactor: re-structure measure functions refactor: cleanup tests refactor: rename measure results type --- README.md | 2 +- docusaurus/docs/api.md | 2 +- .../src/__tests__/measure-function.test.tsx | 43 +++++++++++++++++++ .../src/__tests__/measure-helpers.test.ts | 38 ++++++++++++++++ ...asure.test.tsx => measure-render.test.tsx} | 41 +----------------- packages/reassure-measure/src/index.ts | 5 ++- .../reassure-measure/src/measure-function.tsx | 38 ++++++++++++++++ .../reassure-measure/src/measure-helpers.tsx | 30 +++++++++++++ .../src/{measure.tsx => measure-render.tsx} | 39 ++--------------- packages/reassure-measure/src/output.ts | 9 ++-- packages/reassure-measure/src/types.ts | 26 +++++------ packages/reassure/README.md | 2 +- test-apps/native/.eslintrc.js | 8 +++- test-apps/native/src/fib.perf-test.tsx | 23 ++++++++++ 14 files changed, 207 insertions(+), 99 deletions(-) create mode 100644 packages/reassure-measure/src/__tests__/measure-function.test.tsx create mode 100644 packages/reassure-measure/src/__tests__/measure-helpers.test.ts rename packages/reassure-measure/src/__tests__/{measure.test.tsx => measure-render.test.tsx} (74%) create mode 100644 packages/reassure-measure/src/measure-function.tsx create mode 100644 packages/reassure-measure/src/measure-helpers.tsx rename packages/reassure-measure/src/{measure.tsx => measure-render.tsx} (70%) create mode 100644 test-apps/native/src/fib.perf-test.tsx diff --git a/README.md b/README.md index 6b18d5a19..e6457ecd5 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,7 @@ measuring its performance and writing results to the output file. You can use th of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { ``` #### `MeasureOptions` type diff --git a/docusaurus/docs/api.md b/docusaurus/docs/api.md index 495482dbe..2c26a4939 100644 --- a/docusaurus/docs/api.md +++ b/docusaurus/docs/api.md @@ -14,7 +14,7 @@ measuring its performance and writing results to the output file. You can use op of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { ``` #### Example diff --git a/packages/reassure-measure/src/__tests__/measure-function.test.tsx b/packages/reassure-measure/src/__tests__/measure-function.test.tsx new file mode 100644 index 000000000..e3e6f29ef --- /dev/null +++ b/packages/reassure-measure/src/__tests__/measure-function.test.tsx @@ -0,0 +1,43 @@ +import { measureFunctionInternal } from '../measure-function'; + +// Exponentially slow function +function fib(n: number): number { + if (n <= 1) { + return n; + } + + return fib(n - 1) + fib(n - 2); +} + +test('measureFunctionInternal captures results', () => { + const fn = jest.fn(() => fib(5)); + const results = measureFunctionInternal(fn, { runs: 1, dropWorst: 0 }); + + expect(fn).toHaveBeenCalledTimes(1); + expect(results.runs).toBe(1); + expect(results.counts).toEqual([1]); +}); + +test('measureFunctionInternal runs specified number of times', () => { + const fn = jest.fn(() => fib(5)); + const results = measureFunctionInternal(fn, { runs: 20, dropWorst: 0 }); + + expect(fn).toHaveBeenCalledTimes(20); + expect(results.runs).toBe(20); + expect(results.durations).toHaveLength(20); + expect(results.counts).toHaveLength(20); + expect(results.meanCount).toBe(1); + expect(results.stdevCount).toBe(0); +}); + +test('measureFunctionInternal applies dropsWorst option', () => { + const fn = jest.fn(() => fib(5)); + const results = measureFunctionInternal(fn, { runs: 10, dropWorst: 1 }); + + expect(fn).toHaveBeenCalledTimes(11); + expect(results.runs).toBe(10); + expect(results.durations).toHaveLength(10); + expect(results.counts).toHaveLength(10); + expect(results.meanCount).toBe(1); + expect(results.stdevCount).toBe(0); +}); diff --git a/packages/reassure-measure/src/__tests__/measure-helpers.test.ts b/packages/reassure-measure/src/__tests__/measure-helpers.test.ts new file mode 100644 index 000000000..714e5ac52 --- /dev/null +++ b/packages/reassure-measure/src/__tests__/measure-helpers.test.ts @@ -0,0 +1,38 @@ +import { processRunResults } from '../measure-helpers'; + +test('processRunResults calculates correct means and stdevs', () => { + const input = [ + { duration: 10, count: 2 }, + { duration: 12, count: 2 }, + { duration: 14, count: 2 }, + ]; + + expect(processRunResults(input, 0)).toEqual({ + runs: 3, + meanDuration: 12, + stdevDuration: 2, + durations: [14, 12, 10], + meanCount: 2, + stdevCount: 0, + counts: [2, 2, 2], + }); +}); + +test('processRunResults applies warmupRuns option', () => { + const input = [ + { duration: 23, count: 1 }, + { duration: 20, count: 5 }, + { duration: 24, count: 5 }, + { duration: 22, count: 5 }, + ]; + + expect(processRunResults(input, 1)).toEqual({ + runs: 3, + meanDuration: 22, + stdevDuration: 2, + durations: [24, 22, 20], + meanCount: 5, + stdevCount: 0, + counts: [5, 5, 5], + }); +}); diff --git a/packages/reassure-measure/src/__tests__/measure.test.tsx b/packages/reassure-measure/src/__tests__/measure-render.test.tsx similarity index 74% rename from packages/reassure-measure/src/__tests__/measure.test.tsx rename to packages/reassure-measure/src/__tests__/measure-render.test.tsx index 68f4f2d17..a51c68b33 100644 --- a/packages/reassure-measure/src/__tests__/measure.test.tsx +++ b/packages/reassure-measure/src/__tests__/measure-render.test.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { View } from 'react-native'; -import { buildUiToRender, measureRender, processRunResults } from '../measure'; +import { buildUiToRender, measureRender } from '../measure-render'; import { resetHasShownFlagsOutput } from '../output'; const errorsToIgnore = ['❌ Measure code is running under incorrect Node.js configuration.']; @@ -23,7 +23,7 @@ test('measureRender run test given number of times', async () => { expect(result.meanCount).toBe(1); expect(result.stdevCount).toBe(0); - // Test is actually run 21 times = 20 runs + 1 drop worst + // Test is actually run 21 times = 20 runs + 1 warmup runs expect(scenario).toHaveBeenCalledTimes(21); }); @@ -89,40 +89,3 @@ test('buildUiToRender does not wrap when no wrapper is passed', () => { `); }); - -test('processRunResults calculates correct means and stdevs', () => { - const input = [ - { duration: 10, count: 2 }, - { duration: 12, count: 2 }, - { duration: 14, count: 2 }, - ]; - - expect(processRunResults(input, 0)).toEqual({ - runs: 3, - meanDuration: 12, - stdevDuration: 2, - durations: [14, 12, 10], - meanCount: 2, - stdevCount: 0, - counts: [2, 2, 2], - }); -}); - -test('processRunResults applies warmupRuns option', () => { - const input = [ - { duration: 23, count: 1 }, - { duration: 20, count: 5 }, - { duration: 24, count: 5 }, - { duration: 22, count: 5 }, - ]; - - expect(processRunResults(input, 1)).toEqual({ - runs: 3, - meanDuration: 22, - stdevDuration: 2, - durations: [24, 22, 20], - meanCount: 5, - stdevCount: 0, - counts: [5, 5, 5], - }); -}); diff --git a/packages/reassure-measure/src/index.ts b/packages/reassure-measure/src/index.ts index d36a0d27e..1960bd41a 100644 --- a/packages/reassure-measure/src/index.ts +++ b/packages/reassure-measure/src/index.ts @@ -1,3 +1,4 @@ export { configure, resetToDefault } from './config'; -export { measurePerformance } from './measure'; -export type { MeasureOptions } from './measure'; +export { measurePerformance } from './measure-render'; +export { measureFunction } from './measure-function'; +export type { MeasureOptions } from './measure-render'; diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx new file mode 100644 index 000000000..f8f0d83a4 --- /dev/null +++ b/packages/reassure-measure/src/measure-function.tsx @@ -0,0 +1,38 @@ +import { config } from './config'; +import { RunResult, MeasureResults, processRunResults } from './measure-helpers'; +import { showFlagsOuputIfNeeded, writeTestStats } from './output'; + +interface MeasureFunctionOptions { + runs?: number; + dropWorst?: number; +} + +export async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { + const stats = await measureFunctionInternal(fn, options); + await writeTestStats(stats); + + return stats; +} + +export function measureFunctionInternal(fn: () => void, options?: MeasureFunctionOptions): MeasureResults { + const runs = options?.runs ?? config.runs; + const dropWorst = options?.dropWorst ?? config.dropWorst; + + showFlagsOuputIfNeeded(); + + const runResults: RunResult[] = []; + for (let i = 0; i < runs + dropWorst; i += 1) { + const timeStart = getCurrentTime(); + fn(); + const timeEnd = getCurrentTime(); + + const duration = Number((timeEnd - timeStart) / 1_000_000n); + runResults.push({ duration, count: 1 }); + } + + return processRunResults(runResults, dropWorst); +} + +function getCurrentTime() { + return process.hrtime.bigint(); +} diff --git a/packages/reassure-measure/src/measure-helpers.tsx b/packages/reassure-measure/src/measure-helpers.tsx new file mode 100644 index 000000000..8713cf607 --- /dev/null +++ b/packages/reassure-measure/src/measure-helpers.tsx @@ -0,0 +1,30 @@ +import * as math from 'mathjs'; +import type { MeasureResults } from './types'; + +export interface RunResult { + duration: number; + count: number; +} + +export function processRunResults(results: RunResult[], warmupRuns: number): MeasureResults { + results = results.slice(warmupRuns); + results.sort((first, second) => second.duration - first.duration); // duration DESC + + const durations = results.map((result) => result.duration); + const meanDuration = math.mean(durations) as number; + const stdevDuration = math.std(...durations); + + const counts = results.map((result) => result.count); + const meanCount = math.mean(counts) as number; + const stdevCount = math.std(...counts); + + return { + runs: results.length, + meanDuration, + stdevDuration, + durations, + meanCount, + stdevCount, + counts, + }; +} diff --git a/packages/reassure-measure/src/measure.tsx b/packages/reassure-measure/src/measure-render.tsx similarity index 70% rename from packages/reassure-measure/src/measure.tsx rename to packages/reassure-measure/src/measure-render.tsx index 58130194f..9911899a1 100644 --- a/packages/reassure-measure/src/measure.tsx +++ b/packages/reassure-measure/src/measure-render.tsx @@ -1,10 +1,10 @@ import * as React from 'react'; -import * as math from 'mathjs'; import { logger } from '@callstack/reassure-logger'; import { config } from './config'; +import { RunResult, processRunResults } from './measure-helpers'; import { showFlagsOuputIfNeeded, writeTestStats } from './output'; import { resolveTestingLibrary } from './testingLibrary'; -import type { MeasureRenderResult } from './types'; +import type { MeasureResults } from './types'; logger.configure({ verbose: process.env.REASSURE_VERBOSE === 'true' || process.env.REASSURE_VERBOSE === '1', @@ -18,17 +18,14 @@ export interface MeasureOptions { scenario?: (screen: any) => Promise; } -export async function measurePerformance( - ui: React.ReactElement, - options?: MeasureOptions -): Promise { +export async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { const stats = await measureRender(ui, options); await writeTestStats(stats); return stats; } -export async function measureRender(ui: React.ReactElement, options?: MeasureOptions): Promise { +export async function measureRender(ui: React.ReactElement, options?: MeasureOptions): Promise { const runs = options?.runs ?? config.runs; const scenario = options?.scenario; const warmupRuns = options?.warmupRuns ?? config.warmupRuns; @@ -91,31 +88,3 @@ export function buildUiToRender( return Wrapper ? {uiWithProfiler} : uiWithProfiler; } - -interface RunResult { - duration: number; - count: number; -} - -export function processRunResults(results: RunResult[], warmupRuns: number) { - results = results.slice(warmupRuns); - results.sort((first, second) => second.duration - first.duration); // duration DESC - - const durations = results.map((result) => result.duration); - const meanDuration = math.mean(durations) as number; - const stdevDuration = math.std(...durations); - - const counts = results.map((result) => result.count); - const meanCount = math.mean(counts) as number; - const stdevCount = math.std(...counts); - - return { - runs: results.length, - meanDuration, - stdevDuration, - durations, - meanCount, - stdevCount, - counts, - }; -} diff --git a/packages/reassure-measure/src/output.ts b/packages/reassure-measure/src/output.ts index 09d6dac1b..3cef54a43 100644 --- a/packages/reassure-measure/src/output.ts +++ b/packages/reassure-measure/src/output.ts @@ -1,14 +1,11 @@ import * as fs from 'fs/promises'; import { logger } from '@callstack/reassure-logger'; import { config } from './config'; -import type { MeasureRenderResult } from './types'; +import type { MeasureResults } from './measure-helpers'; -export async function writeTestStats( - result: MeasureRenderResult, - outputFilePath: string = config.outputFile -): Promise { +export async function writeTestStats(stats: MeasureResults, outputFilePath: string = config.outputFile): Promise { const name = expect.getState().currentTestName; - const line = JSON.stringify({ name, ...result }) + '\n'; + const line = JSON.stringify({ name, ...stats }) + '\n'; try { await fs.appendFile(outputFilePath, line); diff --git a/packages/reassure-measure/src/types.ts b/packages/reassure-measure/src/types.ts index 7650b6056..9bfe28869 100644 --- a/packages/reassure-measure/src/types.ts +++ b/packages/reassure-measure/src/types.ts @@ -1,32 +1,32 @@ /** - * Type representing output from `measureRender` function. + * Type representing output of measure functions, e.g. `measurePerformance`, `measureFunction`. */ -export interface MeasureRenderResult { - /** Number of times the measurment test was run */ +export interface MeasureResults { + /** Number of times the test subject was run */ runs: number; - /** Array of measured render durations for each run */ - durations: number[]; - - /** Arithmetic average of measured render durations for each run */ + /** Arithmetic average of measured execution durations for each run */ meanDuration: number; - /* Standard deviation of measured render durations for each run */ + /** Standard deviation of measured execution durations for each run */ stdevDuration: number; - /** Array of measured render counts for each run */ - counts: number[]; + /** Array of measured execution durations for each run */ + durations: number[]; - /** Arithmetic average of measured render counts for each run */ + /** Arithmetic average of measured execution count for each run */ meanCount: number; - /** Standard deviation of measured render counts for each run */ + /** Standard deviation of measured execution count for each run */ stdevCount: number; + + /** Array of measured execution count for each run */ + counts: number[]; } /** * Output of specific test scenarion as written to perf results file. */ -export interface PerformanceEntry extends MeasureRenderResult { +export interface PerformanceEntry extends MeasureResults { name: string; } diff --git a/packages/reassure/README.md b/packages/reassure/README.md index 8c0b4c6fe..acaf4cb6f 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -376,7 +376,7 @@ measuring its performance and writing results to the output file. You can use op of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { ``` #### `MeasureOptions` type diff --git a/test-apps/native/.eslintrc.js b/test-apps/native/.eslintrc.js index 2d59e2eb3..273dfba6e 100644 --- a/test-apps/native/.eslintrc.js +++ b/test-apps/native/.eslintrc.js @@ -7,7 +7,13 @@ module.exports = { 'import/no-unresolved': 'off', 'jest/expect-expect': [ 'error', - { assertFunctionNames: ['expect', 'measurePerformance'] }, + { + assertFunctionNames: [ + 'expect', + 'measurePerformance', + 'measureFunction', + ], + }, ], }, }; diff --git a/test-apps/native/src/fib.perf-test.tsx b/test-apps/native/src/fib.perf-test.tsx new file mode 100644 index 000000000..42480e0c0 --- /dev/null +++ b/test-apps/native/src/fib.perf-test.tsx @@ -0,0 +1,23 @@ +import { measureFunction } from '@callstack/reassure-measure'; + +function fib(n: number): number { + if (n <= 1) { + return n; + } + + return fib(n - 1) + fib(n - 2); +} + +jest.setTimeout(60_000); + +test('fib 30', async () => { + await measureFunction(() => fib(30)); +}); + +test('fib 31', async () => { + await measureFunction(() => fib(31)); +}); + +test('fib 32', async () => { + await measureFunction(() => fib(32)); +}); From d19738b9b041091e42143c66ed404f1894d4d18c Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Fri, 29 Sep 2023 13:31:25 +0200 Subject: [PATCH 02/24] fix: build errors due to invalid imports --- packages/reassure-measure/src/measure-function.tsx | 5 +++-- packages/reassure-measure/src/output.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx index f8f0d83a4..1e3a1b64e 100644 --- a/packages/reassure-measure/src/measure-function.tsx +++ b/packages/reassure-measure/src/measure-function.tsx @@ -1,5 +1,6 @@ import { config } from './config'; -import { RunResult, MeasureResults, processRunResults } from './measure-helpers'; +import type { MeasureResults } from './types'; +import { type RunResult, processRunResults } from './measure-helpers'; import { showFlagsOuputIfNeeded, writeTestStats } from './output'; interface MeasureFunctionOptions { @@ -16,7 +17,7 @@ export async function measureFunction(fn: () => void, options?: MeasureFunctionO export function measureFunctionInternal(fn: () => void, options?: MeasureFunctionOptions): MeasureResults { const runs = options?.runs ?? config.runs; - const dropWorst = options?.dropWorst ?? config.dropWorst; + const dropWorst = options?.dropWorst ?? config.warmupRuns; showFlagsOuputIfNeeded(); diff --git a/packages/reassure-measure/src/output.ts b/packages/reassure-measure/src/output.ts index 3cef54a43..c20e348af 100644 --- a/packages/reassure-measure/src/output.ts +++ b/packages/reassure-measure/src/output.ts @@ -1,7 +1,7 @@ import * as fs from 'fs/promises'; import { logger } from '@callstack/reassure-logger'; import { config } from './config'; -import type { MeasureResults } from './measure-helpers'; +import type { MeasureResults } from './types'; export async function writeTestStats(stats: MeasureResults, outputFilePath: string = config.outputFile): Promise { const name = expect.getState().currentTestName; From 9e1919749f57a5cfaa30d2ca4d3ca97ecfa1c592 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Fri, 29 Sep 2023 13:32:44 +0200 Subject: [PATCH 03/24] chore: rename dropWorst to warmupRuns for consistency --- .../src/__tests__/measure-function.test.tsx | 6 +++--- packages/reassure-measure/src/measure-function.tsx | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/reassure-measure/src/__tests__/measure-function.test.tsx b/packages/reassure-measure/src/__tests__/measure-function.test.tsx index e3e6f29ef..ade058e30 100644 --- a/packages/reassure-measure/src/__tests__/measure-function.test.tsx +++ b/packages/reassure-measure/src/__tests__/measure-function.test.tsx @@ -11,7 +11,7 @@ function fib(n: number): number { test('measureFunctionInternal captures results', () => { const fn = jest.fn(() => fib(5)); - const results = measureFunctionInternal(fn, { runs: 1, dropWorst: 0 }); + const results = measureFunctionInternal(fn, { runs: 1, warmupRuns: 0 }); expect(fn).toHaveBeenCalledTimes(1); expect(results.runs).toBe(1); @@ -20,7 +20,7 @@ test('measureFunctionInternal captures results', () => { test('measureFunctionInternal runs specified number of times', () => { const fn = jest.fn(() => fib(5)); - const results = measureFunctionInternal(fn, { runs: 20, dropWorst: 0 }); + const results = measureFunctionInternal(fn, { runs: 20, warmupRuns: 0 }); expect(fn).toHaveBeenCalledTimes(20); expect(results.runs).toBe(20); @@ -32,7 +32,7 @@ test('measureFunctionInternal runs specified number of times', () => { test('measureFunctionInternal applies dropsWorst option', () => { const fn = jest.fn(() => fib(5)); - const results = measureFunctionInternal(fn, { runs: 10, dropWorst: 1 }); + const results = measureFunctionInternal(fn, { runs: 10, warmupRuns: 1 }); expect(fn).toHaveBeenCalledTimes(11); expect(results.runs).toBe(10); diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx index 1e3a1b64e..b2ea6b346 100644 --- a/packages/reassure-measure/src/measure-function.tsx +++ b/packages/reassure-measure/src/measure-function.tsx @@ -5,7 +5,7 @@ import { showFlagsOuputIfNeeded, writeTestStats } from './output'; interface MeasureFunctionOptions { runs?: number; - dropWorst?: number; + warmupRuns?: number; } export async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { @@ -17,12 +17,12 @@ export async function measureFunction(fn: () => void, options?: MeasureFunctionO export function measureFunctionInternal(fn: () => void, options?: MeasureFunctionOptions): MeasureResults { const runs = options?.runs ?? config.runs; - const dropWorst = options?.dropWorst ?? config.warmupRuns; + const warmupRuns = options?.warmupRuns ?? config.warmupRuns; showFlagsOuputIfNeeded(); const runResults: RunResult[] = []; - for (let i = 0; i < runs + dropWorst; i += 1) { + for (let i = 0; i < runs + warmupRuns; i += 1) { const timeStart = getCurrentTime(); fn(); const timeEnd = getCurrentTime(); @@ -31,7 +31,7 @@ export function measureFunctionInternal(fn: () => void, options?: MeasureFunctio runResults.push({ duration, count: 1 }); } - return processRunResults(runResults, dropWorst); + return processRunResults(runResults, warmupRuns); } function getCurrentTime() { From c6963498fdca1c09de831624fffa542d7f587ad4 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Fri, 29 Sep 2023 15:24:38 +0200 Subject: [PATCH 04/24] chore: adjust naming convention to drop render prefix where possible --- README.md | 6 +++--- packages/reassure-compare/src/compare.ts | 8 ++++---- packages/reassure-compare/src/output/console.ts | 12 ++++++------ packages/reassure-compare/src/output/markdown.ts | 16 ++++++++-------- packages/reassure-compare/src/utils/format.ts | 12 ++++++------ packages/reassure-compare/src/utils/validate.ts | 12 ++++++------ packages/reassure/README.md | 6 +++--- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index e6457ecd5..eb655a74a 100644 --- a/README.md +++ b/README.md @@ -340,9 +340,9 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example, you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Render Duration** shows test scenarios where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Render Duration** shows test scenarios where the change is not statistically significant -- **Changes To Render Count** shows test scenarios where the render count did change +- **Significant Changes To Duration** shows test scenarios where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the change is not statistically significant +- **Changes To Count** shows test scenarios where the count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements diff --git a/packages/reassure-compare/src/compare.ts b/packages/reassure-compare/src/compare.ts index 185518d79..7ceea3721 100644 --- a/packages/reassure-compare/src/compare.ts +++ b/packages/reassure-compare/src/compare.ts @@ -21,7 +21,7 @@ import { parseHeader, parsePerformanceEntries } from './utils/validate'; const PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02; /** - * Render duration threshold (in ms) for treating given difference as significant. + * Duration threshold (in ms) for treating given difference as significant. * * This is additional filter, in addition to probability threshold above. * Too small duration difference might be result of measurement grain of 1 ms. @@ -29,8 +29,8 @@ const PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02; const DURATION_DIFF_THRESHOLD_SIGNIFICANT = 4; /** - * Threshold for considering render count change as significant. This implies inclusion - * of scenario results in Render Count Changed output section. + * Threshold for considering count change as significant. This implies inclusion + * of scenario results in Count Changed output section. */ const COUNT_DIFF_THRESHOLD = 0.5; @@ -176,7 +176,7 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult } /** - * Establish statisticial significance of render duration difference build compare entry. + * Establish statisticial significance of duration difference build compare entry. */ function buildCompareEntry(name: string, current: PerformanceEntry, baseline: PerformanceEntry): CompareEntry { const durationDiff = current.meanDuration - baseline.meanDuration; diff --git a/packages/reassure-compare/src/output/console.ts b/packages/reassure-compare/src/output/console.ts index 0e063bacb..70a1b66b8 100644 --- a/packages/reassure-compare/src/output/console.ts +++ b/packages/reassure-compare/src/output/console.ts @@ -4,8 +4,8 @@ import { formatCount, formatDuration, formatMetadata, - formatRenderCountChange, - formatRenderDurationChange, + formatOutputCountChange, + formatOutputDurationChange, } from '../utils/format'; import type { PerformanceMetadata } from '../types'; @@ -16,13 +16,13 @@ export function printToConsole(data: CompareResult) { printMetadata('Current', data.metadata.current); printMetadata('Baseline', data.metadata.baseline); - logger.log('\n➡️ Significant changes to render duration'); + logger.log('\n➡️ Significant changes to duration'); data.significant.forEach(printRegularLine); - logger.log('\n➡️ Meaningless changes to render duration'); + logger.log('\n➡️ Meaningless changes to duration'); data.meaningless.forEach(printRegularLine); - logger.log('\n➡️ Render count changes'); + logger.log('\n➡️ Count changes'); data.countChanged.forEach(printRegularLine); logger.log('\n➡️ Added scenarios'); @@ -39,7 +39,7 @@ function printMetadata(name: string, metadata?: PerformanceMetadata) { } function printRegularLine(entry: CompareEntry) { - logger.log(` - ${entry.name}: ${formatRenderDurationChange(entry)} | ${formatRenderCountChange(entry)}`); + logger.log(` - ${entry.name}: ${formatOutputDurationChange(entry)} | ${formatOutputCountChange(entry)}`); } function printAddedLine(entry: AddedEntry) { diff --git a/packages/reassure-compare/src/output/markdown.ts b/packages/reassure-compare/src/output/markdown.ts index 39145a0f9..4d1eaf4ae 100644 --- a/packages/reassure-compare/src/output/markdown.ts +++ b/packages/reassure-compare/src/output/markdown.ts @@ -10,8 +10,8 @@ import { formatDuration, formatMetadata, formatPercent, - formatRenderCountChange, - formatRenderDurationChange, + formatOutputCountChange, + formatOutputDurationChange, } from '../utils/format'; import type { AddedEntry, @@ -22,7 +22,7 @@ import type { PerformanceMetadata, } from '../types'; -const tableHeader = ['Name', 'Render Duration', 'Render Count'] as const; +const tableHeader = ['Name', 'Duration', 'Count'] as const; export const writeToMarkdown = async (filePath: string, data: CompareResult) => { try { @@ -68,13 +68,13 @@ function buildMarkdown(data: CompareResult) { }); } - result += `\n\n${headers.h3('Significant Changes To Render Duration')}`; + result += `\n\n${headers.h3('Significant Changes To Duration')}`; result += `\n${buildSummaryTable(data.significant)}`; result += `\n${buildDetailsTable(data.significant)}`; - result += `\n\n${headers.h3('Meaningless Changes To Render Duration')}`; + result += `\n\n${headers.h3('Meaningless Changes To Duration')}`; result += `\n${buildSummaryTable(data.meaningless, true)}`; result += `\n${buildDetailsTable(data.meaningless)}`; - result += `\n\n${headers.h3('Changes To Render Count')}`; + result += `\n\n${headers.h3('Changes To Count')}`; result += `\n${buildSummaryTable(data.countChanged)}`; result += `\n${buildDetailsTable(data.countChanged)}`; result += `\n\n${headers.h3('Added Scenarios')}`; @@ -111,14 +111,14 @@ function buildDetailsTable(entries: Array 0.15) return '🔴'; if (entry.relativeDurationDiff < -0.15) return '🟢'; @@ -75,7 +75,7 @@ function getRenderDurationSymbols(entry: CompareEntry) { return ''; } -export function formatRenderCountChange(entry: CompareEntry) { +export function formatOutputCountChange(entry: CompareEntry) { const { baseline, current } = entry; let output = `${formatCount(baseline.meanCount)} → ${formatCount(current.meanCount)}`; @@ -84,12 +84,12 @@ export function formatRenderCountChange(entry: CompareEntry) { output += ` (${formatCountChange(entry.countDiff)}, ${formatPercentChange(entry.relativeCountDiff)})`; } - output += ` ${getRenderCountSymbols(entry)}`; + output += ` ${getOutputCountSymbols(entry)}`; return output; } -function getRenderCountSymbols(entry: CompareEntry) { +function getOutputCountSymbols(entry: CompareEntry) { if (entry.countDiff > 1.5) return '🔴🔴'; if (entry.countDiff > 0.5) return '🔴'; if (entry.countDiff < -1.5) return '🟢🟢'; diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index 28fef9dfe..71538430a 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -22,22 +22,22 @@ export const performanceEntrySchema = z.object({ /** Number of times the measurment test was run */ runs: z.number(), - /** Arithmetic average of measured render durations for each run */ + /** Arithmetic average of measured durations for each run */ meanDuration: z.number(), - /** Standard deviation of measured render durations for each run */ + /** Standard deviation of measured durations for each run */ stdevDuration: z.number(), - /** Array of measured render durations for each run */ + /** Array of measured durations for each run */ durations: z.array(z.number()), - /** Arithmetic average of measured render counts for each run */ + /** Arithmetic average of measured counts for each run */ meanCount: z.number(), - /** Standard deviation of measured render counts for each run */ + /** Standard deviation of measured counts for each run */ stdevCount: z.number(), - /** Array of measured render counts for each run */ + /** Array of measured counts for each run */ counts: z.array(z.number()), }); diff --git a/packages/reassure/README.md b/packages/reassure/README.md index acaf4cb6f..2738761ef 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -359,9 +359,9 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Render Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Render Duration** shows test scenarios where the change is not stastatistically significant -- **Changes To Render Count** shows test scenarios where render count did change +- **Significant Changes To Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the change is not stastatistically significant +- **Changes To Count** shows test scenarios where count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements From e5975f8bd0e122d4863afd3b9117cece5d8a7dc4 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Fri, 29 Sep 2023 16:23:00 +0200 Subject: [PATCH 05/24] refactor: collect current time using performance api --- packages/reassure-measure/src/measure-function.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx index b2ea6b346..35866b6ec 100644 --- a/packages/reassure-measure/src/measure-function.tsx +++ b/packages/reassure-measure/src/measure-function.tsx @@ -1,3 +1,4 @@ +import { performance } from 'perf_hooks'; import { config } from './config'; import type { MeasureResults } from './types'; import { type RunResult, processRunResults } from './measure-helpers'; @@ -27,7 +28,7 @@ export function measureFunctionInternal(fn: () => void, options?: MeasureFunctio fn(); const timeEnd = getCurrentTime(); - const duration = Number((timeEnd - timeStart) / 1_000_000n); + const duration = timeEnd - timeStart; runResults.push({ duration, count: 1 }); } @@ -35,5 +36,5 @@ export function measureFunctionInternal(fn: () => void, options?: MeasureFunctio } function getCurrentTime() { - return process.hrtime.bigint(); + return performance.now(); } From 5cad008c894561745ba859b1ad45016b78d1e4a7 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 09:41:00 +0200 Subject: [PATCH 06/24] feat: add type property to each measurement --- packages/reassure-compare/src/compare.ts | 15 +++++++++++---- packages/reassure-compare/src/types.ts | 18 ++++++++++++------ .../reassure-compare/src/utils/validate.ts | 3 +++ .../reassure-measure/src/measure-function.tsx | 2 +- .../reassure-measure/src/measure-render.tsx | 2 +- packages/reassure-measure/src/output.ts | 10 +++++++--- packages/reassure-measure/src/types.ts | 7 +------ 7 files changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/reassure-compare/src/compare.ts b/packages/reassure-compare/src/compare.ts index 7ceea3721..fecd19f86 100644 --- a/packages/reassure-compare/src/compare.ts +++ b/packages/reassure-compare/src/compare.ts @@ -8,6 +8,7 @@ import type { PerformanceResults, PerformanceEntry, PerformanceHeader, + MeasureType, } from './types'; import { printToConsole } from './output/console'; import { writeToJson } from './output/json'; @@ -143,11 +144,11 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult const baselineEntry = baseline?.entries[name]; if (currentEntry && baselineEntry) { - compared.push(buildCompareEntry(name, currentEntry, baselineEntry)); + compared.push(buildCompareEntry(name, currentEntry.type, currentEntry, baselineEntry)); } else if (currentEntry) { - added.push({ name, current: currentEntry }); + added.push({ name, type: currentEntry.type, current: currentEntry }); } else if (baselineEntry) { - removed.push({ name, baseline: baselineEntry }); + removed.push({ name, type: baselineEntry.type, baseline: baselineEntry }); } }); @@ -178,7 +179,12 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult /** * Establish statisticial significance of duration difference build compare entry. */ -function buildCompareEntry(name: string, current: PerformanceEntry, baseline: PerformanceEntry): CompareEntry { +function buildCompareEntry( + name: string, + type: MeasureType, + current: PerformanceEntry, + baseline: PerformanceEntry +): CompareEntry { const durationDiff = current.meanDuration - baseline.meanDuration; const relativeDurationDiff = durationDiff / baseline.meanDuration; const countDiff = current.meanCount - baseline.meanCount; @@ -192,6 +198,7 @@ function buildCompareEntry(name: string, current: PerformanceEntry, baseline: Pe return { name, + type, baseline, current, durationDiff, diff --git a/packages/reassure-compare/src/types.ts b/packages/reassure-compare/src/types.ts index 1ce299102..bfa682976 100644 --- a/packages/reassure-compare/src/types.ts +++ b/packages/reassure-compare/src/types.ts @@ -5,6 +5,7 @@ import type { performanceEntrySchema, performanceHeaderSchema, performanceMetada export type PerformanceHeader = z.infer; export type PerformanceMetadata = z.infer; export type PerformanceEntry = z.infer; +export type MeasureType = PerformanceEntry['type']; export interface PerformanceResults { metadata?: PerformanceMetadata; @@ -12,10 +13,17 @@ export interface PerformanceResults { } /** - * Compare entry for tests that have both baseline and current entry + * Common interface for all compare entries */ -export interface CompareEntry { +interface BaseEntry { name: string; + type: MeasureType; +} + +/** + * Compare entry for tests that have both baseline and current entry + */ +export interface CompareEntry extends BaseEntry { current: PerformanceEntry; baseline: PerformanceEntry; durationDiff: number; @@ -28,16 +36,14 @@ export interface CompareEntry { /** * Compare entry for tests that have only current entry */ -export interface AddedEntry { - name: string; +export interface AddedEntry extends BaseEntry { current: PerformanceEntry; } /** * Compare entry for tests that have only baseline entry */ -export interface RemovedEntry { - name: string; +export interface RemovedEntry extends BaseEntry { baseline: PerformanceEntry; } diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index 71538430a..c3483d02d 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -19,6 +19,9 @@ export const performanceEntrySchema = z.object({ /** Name of the test scenario */ name: z.string(), + /** Type of the test scenario */ + type: z.enum(['render', 'syncFunction']), + /** Number of times the measurment test was run */ runs: z.number(), diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx index 35866b6ec..771981fc3 100644 --- a/packages/reassure-measure/src/measure-function.tsx +++ b/packages/reassure-measure/src/measure-function.tsx @@ -11,7 +11,7 @@ interface MeasureFunctionOptions { export async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { const stats = await measureFunctionInternal(fn, options); - await writeTestStats(stats); + await writeTestStats('syncFunction', stats); return stats; } diff --git a/packages/reassure-measure/src/measure-render.tsx b/packages/reassure-measure/src/measure-render.tsx index 9911899a1..ed9e91deb 100644 --- a/packages/reassure-measure/src/measure-render.tsx +++ b/packages/reassure-measure/src/measure-render.tsx @@ -20,7 +20,7 @@ export interface MeasureOptions { export async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { const stats = await measureRender(ui, options); - await writeTestStats(stats); + await writeTestStats('render', stats); return stats; } diff --git a/packages/reassure-measure/src/output.ts b/packages/reassure-measure/src/output.ts index c20e348af..676ce7cb3 100644 --- a/packages/reassure-measure/src/output.ts +++ b/packages/reassure-measure/src/output.ts @@ -1,11 +1,15 @@ import * as fs from 'fs/promises'; import { logger } from '@callstack/reassure-logger'; import { config } from './config'; -import type { MeasureResults } from './types'; +import type { MeasureResults, MeasureType } from './types'; -export async function writeTestStats(stats: MeasureResults, outputFilePath: string = config.outputFile): Promise { +export async function writeTestStats( + type: MeasureType, + stats: MeasureResults, + outputFilePath: string = config.outputFile +): Promise { const name = expect.getState().currentTestName; - const line = JSON.stringify({ name, ...stats }) + '\n'; + const line = JSON.stringify({ name, type, ...stats }) + '\n'; try { await fs.appendFile(outputFilePath, line); diff --git a/packages/reassure-measure/src/types.ts b/packages/reassure-measure/src/types.ts index 9bfe28869..512b96943 100644 --- a/packages/reassure-measure/src/types.ts +++ b/packages/reassure-measure/src/types.ts @@ -24,9 +24,4 @@ export interface MeasureResults { counts: number[]; } -/** - * Output of specific test scenarion as written to perf results file. - */ -export interface PerformanceEntry extends MeasureResults { - name: string; -} +export type MeasureType = 'render' | 'syncFunction'; From a5b94d5817ef76433d1c9eb22b12f1655ba86bc3 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 09:41:25 +0200 Subject: [PATCH 07/24] feat: render test type within the output --- packages/reassure-compare/src/output/console.ts | 12 +++++++++--- packages/reassure-compare/src/output/markdown.ts | 11 ++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/reassure-compare/src/output/console.ts b/packages/reassure-compare/src/output/console.ts index 70a1b66b8..0ec9d149e 100644 --- a/packages/reassure-compare/src/output/console.ts +++ b/packages/reassure-compare/src/output/console.ts @@ -39,15 +39,21 @@ function printMetadata(name: string, metadata?: PerformanceMetadata) { } function printRegularLine(entry: CompareEntry) { - logger.log(` - ${entry.name}: ${formatOutputDurationChange(entry)} | ${formatOutputCountChange(entry)}`); + logger.log( + ` - [${entry.type}] ${entry.name}: ${formatOutputDurationChange(entry)} | ${formatOutputCountChange(entry)}` + ); } function printAddedLine(entry: AddedEntry) { const { current } = entry; - logger.log(` - ${entry.name}: ${formatDuration(current.meanDuration)} | ${formatCount(current.meanCount)}`); + logger.log( + ` - [${entry.type}] ${entry.name}: ${formatDuration(current.meanDuration)} | ${formatCount(current.meanCount)}` + ); } function printRemovedLine(entry: RemovedEntry) { const { baseline } = entry; - logger.log(` - ${entry.name}: ${formatDuration(baseline.meanDuration)} | ${formatCount(baseline.meanCount)}`); + logger.log( + ` - [${entry.type}] ${entry.name}: ${formatDuration(baseline.meanDuration)} | ${formatCount(baseline.meanCount)}` + ); } diff --git a/packages/reassure-compare/src/output/markdown.ts b/packages/reassure-compare/src/output/markdown.ts index 4d1eaf4ae..4e973a7dc 100644 --- a/packages/reassure-compare/src/output/markdown.ts +++ b/packages/reassure-compare/src/output/markdown.ts @@ -22,7 +22,7 @@ import type { PerformanceMetadata, } from '../types'; -const tableHeader = ['Name', 'Duration', 'Count'] as const; +const tableHeader = ['Name', 'Type', 'Duration', 'Count'] as const; export const writeToMarkdown = async (filePath: string, data: CompareResult) => { try { @@ -95,7 +95,7 @@ function buildMetadataMarkdown(name: string, metadata: PerformanceMetadata | und function buildSummaryTable(entries: Array, collapse: boolean = false) { if (!entries.length) return emphasis.i('There are no entries'); - const rows = entries.map((entry) => [entry.name, formatEntryDuration(entry), formatEntryCount(entry)]); + const rows = entries.map((entry) => [entry.name, entry.type, formatEntryDuration(entry), formatEntryCount(entry)]); const content = markdownTable([tableHeader, ...rows]); return collapse ? collapsibleSection('Show entries', content) : content; @@ -104,7 +104,12 @@ function buildSummaryTable(entries: Array) { if (!entries.length) return ''; - const rows = entries.map((entry) => [entry.name, buildDurationDetailsEntry(entry), buildCountDetailsEntry(entry)]); + const rows = entries.map((entry) => [ + entry.name, + entry.type, + buildDurationDetailsEntry(entry), + buildCountDetailsEntry(entry), + ]); const content = markdownTable([tableHeader, ...rows]); return collapsibleSection('Show details', content); From c74fdcdb14147787656f279280ab1bd076c85642 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 11:29:11 +0200 Subject: [PATCH 08/24] docs: align methodology categories with the new output --- docusaurus/docs/methodology.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docusaurus/docs/methodology.md b/docusaurus/docs/methodology.md index 7239b9f5b..9b9968096 100644 --- a/docusaurus/docs/methodology.md +++ b/docusaurus/docs/methodology.md @@ -34,8 +34,8 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Render Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Render Duration** shows test scenarios where the change is not stastatistically significant -- **Changes To Render Count** shows test scenarios where render count did change +- **Significant Changes To Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the change is not stastatistically significant +- **Changes To Count** shows test scenarios where count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements -- **Removed Scenarios** shows test scenarios which do not exist in the current measurements \ No newline at end of file +- **Removed Scenarios** shows test scenarios which do not exist in the current measurements From 994cb0daad85c0c0fcb39970ca6b37f7f3332adb Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 11:30:03 +0200 Subject: [PATCH 09/24] docs: add measureFunction api, fix formatting --- docusaurus/docs/api.md | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/docusaurus/docs/api.md b/docusaurus/docs/api.md index 2c26a4939..9eb6712e5 100644 --- a/docusaurus/docs/api.md +++ b/docusaurus/docs/api.md @@ -35,6 +35,26 @@ test('Test with scenario', async () => { }); ``` +### `measureFunction` function + +Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. + +```ts +async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { +``` + +#### Example + +```ts +// sample.perf-test.tsx +import { measureFunction } from '@callstack/reassure-measure'; +import { fib } from './fib'; + +test('fib 30', async () => { + await measureFunction(() => fib(30)); +}); +``` + ### `MeasureOptions` type ```ts @@ -51,6 +71,18 @@ interface MeasureOptions { - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured. - **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions +### `MeasureFunctionOptions` type + +```ts +interface MeasureFunctionOptions { + runs?: number; + warmupRuns?: number; +} +``` + +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. + ## Configuration ### Default configuration @@ -81,11 +113,11 @@ const defaultConfig: Config = { }; ``` -**`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care. +- **`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care. - **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. -**`outputFile`**: name of the file the records will be saved to -**`verbose`**: make Reassure log more, e.g. for debugging purposes -**`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions +- **`outputFile`**: name of the file the records will be saved to +- **`verbose`**: make Reassure log more, e.g. for debugging purposes +- **`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions ### `configure` function From 0a60da05dd4a1d196746c4a1a79d79921725d061 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 13:45:34 +0200 Subject: [PATCH 10/24] fix: unit tests for compare module --- .../test/__snapshots__/compare.test.js.snap | 76 ++++++++++++++++++- .../reassure-compare/src/test/compare.test.js | 6 +- .../src/test/invalid-entry.perf | 9 ++- .../src/test/invalid-json.perf | 9 ++- .../src/test/valid-header.perf | 9 ++- .../src/test/valid-no-header.perf | 9 ++- 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap index e81655b7c..68d5734e6 100644 --- a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap +++ b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`loadFile should fail for file with invalid JSON structure 1`] = `"Unexpected token : in JSON at position 6"`; +exports[`loadFile should fail for file with invalid JSON structure 1`] = `"Unexpected non-whitespace character after JSON at position 6"`; exports[`loadFile should fail for file with invalid entry 1`] = ` "[ @@ -61,6 +61,7 @@ exports[`loadFile should load results file with header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 7.554248252914823, + "type": "render", }, "Other Component 10": { "counts": [ @@ -93,6 +94,7 @@ exports[`loadFile should load results file with header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 4.831608887776871, + "type": "render", }, "Other Component 10 legacy scenario": { "counts": [ @@ -125,6 +127,7 @@ exports[`loadFile should load results file with header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 4.552166761249221, + "type": "render", }, "Other Component 20": { "counts": [ @@ -177,6 +180,40 @@ exports[`loadFile should load results file with header 1`] = ` "runs": 20, "stdevCount": 0, "stdevDuration": 4.595191995301633, + "type": "render", + }, + "fib 30": { + "counts": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + ], + "durations": [ + 80, + 80, + 80, + 80, + 80, + 79, + 79, + 79, + 79, + 79, + ], + "meanCount": 1, + "meanDuration": 79.9, + "name": "fib 30", + "runs": 10, + "stdevCount": 0, + "stdevDuration": 0.4532780026900862, + "type": "syncFunction", }, }, "metadata": { @@ -220,6 +257,7 @@ exports[`loadFile should load results file without header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 7.554248252914823, + "type": "render", }, "Other Component 10": { "counts": [ @@ -252,6 +290,7 @@ exports[`loadFile should load results file without header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 4.831608887776871, + "type": "render", }, "Other Component 10 legacy scenario": { "counts": [ @@ -284,6 +323,7 @@ exports[`loadFile should load results file without header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 4.552166761249221, + "type": "render", }, "Other Component 20": { "counts": [ @@ -336,6 +376,40 @@ exports[`loadFile should load results file without header 1`] = ` "runs": 20, "stdevCount": 0, "stdevDuration": 4.595191995301633, + "type": "render", + }, + "fib 30": { + "counts": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + ], + "durations": [ + 80, + 80, + 80, + 80, + 80, + 79, + 79, + 79, + 79, + 79, + ], + "meanCount": 1, + "meanDuration": 79.9, + "name": "fib 30", + "runs": 10, + "stdevCount": 0, + "stdevDuration": 0.4532780026900862, + "type": "syncFunction", }, }, "metadata": undefined, diff --git a/packages/reassure-compare/src/test/compare.test.js b/packages/reassure-compare/src/test/compare.test.js index 4aa828398..9f34c1a30 100644 --- a/packages/reassure-compare/src/test/compare.test.js +++ b/packages/reassure-compare/src/test/compare.test.js @@ -10,12 +10,13 @@ describe('loadFile', () => { }); const entries = Object.keys(results.entries); - expect(entries).toHaveLength(4); + expect(entries).toHaveLength(5); expect(entries).toEqual([ 'Other Component 10', 'Other Component 10 legacy scenario', 'Other Component 20', 'Async Component', + 'fib 30', ]); expect(results).toMatchSnapshot(); }); @@ -26,12 +27,13 @@ describe('loadFile', () => { expect(results.metadata).toBeUndefined(); const entries = Object.keys(results.entries); - expect(entries).toHaveLength(4); + expect(entries).toHaveLength(5); expect(entries).toEqual([ 'Other Component 10', 'Other Component 10 legacy scenario', 'Other Component 20', 'Async Component', + 'fib 30', ]); expect(results).toMatchSnapshot(); }); diff --git a/packages/reassure-compare/src/test/invalid-entry.perf b/packages/reassure-compare/src/test/invalid-entry.perf index 63f831581..61e63df19 100644 --- a/packages/reassure-compare/src/test/invalid-entry.perf +++ b/packages/reassure-compare/src/test/invalid-entry.perf @@ -1,5 +1,6 @@ {"metadata":{"branch":"feat/perf-file-validation","commitHash":"991427a413b1ff05497a881287c9ddcba7b8de54"}} -{"name":"Other Component 10","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 10 legacy scenario","runs":"10","stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 20","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} -{"name":"Async Component","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"Other Component 10","type":"render","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 10 legacy scenario","type":"render","runs":"10","stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} +{"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file diff --git a/packages/reassure-compare/src/test/invalid-json.perf b/packages/reassure-compare/src/test/invalid-json.perf index f15b3f4b3..9522426c5 100644 --- a/packages/reassure-compare/src/test/invalid-json.perf +++ b/packages/reassure-compare/src/test/invalid-json.perf @@ -1,4 +1,5 @@ -{"name":"Other Component 10","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 10 legacy scenario","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -"name":"Other Component 20","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} -{"name":"Async Component","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"Other Component 10","type":"render","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} +{"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} diff --git a/packages/reassure-compare/src/test/valid-header.perf b/packages/reassure-compare/src/test/valid-header.perf index 2ec700314..e090fb414 100644 --- a/packages/reassure-compare/src/test/valid-header.perf +++ b/packages/reassure-compare/src/test/valid-header.perf @@ -1,5 +1,6 @@ {"metadata":{"branch":"feat/perf-file-validation","commitHash":"991427a413b1ff05497a881287c9ddcba7b8de54"}} -{"name":"Other Component 10","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 10 legacy scenario","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 20","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} -{"name":"Async Component","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"Other Component 10","type":"render","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} +{"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file diff --git a/packages/reassure-compare/src/test/valid-no-header.perf b/packages/reassure-compare/src/test/valid-no-header.perf index d42f11022..2a20ca99e 100644 --- a/packages/reassure-compare/src/test/valid-no-header.perf +++ b/packages/reassure-compare/src/test/valid-no-header.perf @@ -1,4 +1,5 @@ -{"name":"Other Component 10","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 10 legacy scenario","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} -{"name":"Other Component 20","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} -{"name":"Async Component","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"Other Component 10","type":"render","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} +{"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} +{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file From e6335dd213d8eb841ff3b4f5aaba270570414a69 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 16:16:46 +0200 Subject: [PATCH 11/24] fix: save compare test snapshot with a matching node version --- .../src/test/__snapshots__/compare.test.js.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap index 68d5734e6..ad1f9d697 100644 --- a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap +++ b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`loadFile should fail for file with invalid JSON structure 1`] = `"Unexpected non-whitespace character after JSON at position 6"`; +exports[`loadFile should fail for file with invalid JSON structure 1`] = `"Unexpected token : in JSON at position 6"`; exports[`loadFile should fail for file with invalid entry 1`] = ` "[ From f9009b4cd461751a329efa3c2162727e8c550249 Mon Sep 17 00:00:00 2001 From: Adam Horodyski Date: Tue, 10 Oct 2023 17:26:03 +0200 Subject: [PATCH 12/24] fix: default to render when validating performance entries --- packages/reassure-compare/src/utils/validate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index c3483d02d..220670934 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -20,7 +20,7 @@ export const performanceEntrySchema = z.object({ name: z.string(), /** Type of the test scenario */ - type: z.enum(['render', 'syncFunction']), + type: z.enum(['render', 'syncFunction']).default('render'), /** Number of times the measurment test was run */ runs: z.number(), From 9ed67908514e0db8d6b0605211d112755549bd7f Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 11:56:35 +0200 Subject: [PATCH 13/24] docs: tweaks --- README.md | 56 +++++++++++- docusaurus/docs/api.md | 34 ++++---- packages/reassure/README.md | 169 ++++++++++++++++++++++-------------- 3 files changed, 174 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index eb655a74a..c791137ad 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ You can think about it as a React performance testing library. In fact, Reassure Reassure works by measuring render characteristics – duration and count – of the testing scenario you provide and comparing that to the stable version. It repeats the scenario multiple times to reduce the impact of random variations in render times caused by the runtime environment. Then, it applies statistical analysis to determine whether the code changes are statistically significant. As a result, it generates a human-readable report summarizing the results and displays it on the CI or as a comment to your pull request. +In addition to measuring component render times it can also measure execution of regular JavaScript functions. + ## Installation and setup To install Reassure, run the following command in your app folder: @@ -158,7 +160,7 @@ To measure your first test performance, you need to run the following command in yarn reassure ``` -This command will run your tests multiple times using Jest, gathering render statistics and will write them to `.reassure/current.perf` file. To check your setup, check if the output file exists after running the command for the first time. +This command will run your tests multiple times using Jest, gathering performance statistics and will write them to `.reassure/current.perf` file. To check your setup, check if the output file exists after running the command for the first time. > **Note:** You can add `.reassure/` folder to your `.gitignore` file to avoid accidentally committing your results. @@ -374,8 +376,60 @@ interface MeasureOptions { - **`runs`**: number of runs per series for the particular test - **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1). - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. +- **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL or RTL functions + +#### `measurePerformance` function + +Custom wrapper for the RNTL `render` function responsible for rendering the passed screen inside a `React.Profiler` component, +measuring its performance and writing results to the output file. You can use the optional `options` object that allows customizing aspects +of the testing + +```ts +async function measurePerformance( + ui: React.ReactElement, + options?: MeasureOptions +): Promise { +``` + +#### `MeasureOptions` type + +```ts +interface MeasureOptions { + runs?: number; + warmupRuns?: number; + wrapper?: React.ComponentType<{ children: ReactElement }>; + scenario?: (view?: RenderResult) => Promise; +} +``` + +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1) +- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. - **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL functions +#### `measureFunction` function + +Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. + +```ts +async function measureFunction( + fn: () => void, + options?: MeasureFunctionOptions +): Promise { +``` + +#### `MeasureFunctionOptions` type + +```ts +interface MeasureFunctionOptions { + runs?: number; + warmupRuns?: number; +} +``` + +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. + ### Configuration #### Default configuration diff --git a/docusaurus/docs/api.md b/docusaurus/docs/api.md index 9eb6712e5..bd5b97d99 100644 --- a/docusaurus/docs/api.md +++ b/docusaurus/docs/api.md @@ -35,6 +35,22 @@ test('Test with scenario', async () => { }); ``` +### `MeasureOptions` type + +```ts +interface MeasureOptions { + runs?: number; + warmupRuns?: number; + wrapper?: React.ComponentType<{ children: ReactElement }>; + scenario?: (view?: RenderResult) => Promise; +} +``` + +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. +- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured. +- **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions + ### `measureFunction` function Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. @@ -47,7 +63,7 @@ async function measureFunction(fn: () => void, options?: MeasureFunctionOptions) ```ts // sample.perf-test.tsx -import { measureFunction } from '@callstack/reassure-measure'; +import { measureFunction } from 'reassure'; import { fib } from './fib'; test('fib 30', async () => { @@ -55,22 +71,6 @@ test('fib 30', async () => { }); ``` -### `MeasureOptions` type - -```ts -interface MeasureOptions { - runs?: number; - warmupRuns?: number; - wrapper?: React.ComponentType<{ children: ReactElement }>; - scenario?: (view?: RenderResult) => Promise; -} -``` - -- **`runs`**: number of runs per series for the particular test -- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. -- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured. -- **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions - ### `MeasureFunctionOptions` type ```ts diff --git a/packages/reassure/README.md b/packages/reassure/README.md index 2738761ef..c791137ad 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -45,17 +45,17 @@ You want your React Native app to perform well and fast at all times. As a part ## This solution -Reassure allows you to automate React Native app performance regression testing on CI or a local machine. The same way you write your -integration and unit tests that automatically verify that your app is still _working correctly_, you can write -performance tests that verify that your app still _working performantly_. +Reassure allows you to automate React Native app performance regression testing on CI or a local machine. In the same way, you write your integration and unit tests that automatically verify that your app is still _working correctly_, you can write performance tests that verify that your app is still _working performantly_. You can think about it as a React performance testing library. In fact, Reassure is designed to reuse as much of your [React Native Testing Library](https://github.com/callstack/react-native-testing-library) tests and setup as possible. -Reassure works by measuring render characteristics – duration and count – of the testing scenario you provide and comparing that to the stable version. It repeates the scenario multiple times to reduce impact of random variations in render times caused by the runtime environment. Then it applies statistical analysis to figure out whether the code changes are statistically significant or not. As a result, it generates a human-readable report summarizing the results and displays it on the CI or as a comment to your pull request. +Reassure works by measuring render characteristics – duration and count – of the testing scenario you provide and comparing that to the stable version. It repeats the scenario multiple times to reduce the impact of random variations in render times caused by the runtime environment. Then, it applies statistical analysis to determine whether the code changes are statistically significant. As a result, it generates a human-readable report summarizing the results and displays it on the CI or as a comment to your pull request. + +In addition to measuring component render times it can also measure execution of regular JavaScript functions. ## Installation and setup -In order to install Reassure run following command in your app folder: +To install Reassure, run the following command in your app folder: Using yarn @@ -78,8 +78,7 @@ You can check our example projects: - [React (Next.js)](https://github.com/callstack/reassure-examples/tree/main/examples/web-nextjs) - [React (Vite)](https://github.com/callstack/reassure-examples/tree/main/examples/native-expo) -Reassure will try to detect which Testing Library you have installed. In case both React Native Testing Library and React Testing Library are present it will -warn you about that and give a precedence to React Native Testing Library. You can explicitly specify Testing Library to by used by using [`configure`](#configure-function) option: +Reassure will try to detect which Testing Library you have installed. If both React Native Testing Library and React Testing Library are present, it will warn you about that and give precedence to React Native Testing Library. You can explicitly specify Testing Library to be used by using [`configure`](#configure-function) option: ``` configure({ testingLibrary: 'react-native' }) @@ -87,11 +86,11 @@ configure({ testingLibrary: 'react-native' }) configure({ testingLibrary: 'react' }) ``` -You should set it in your Jest setup file and you can override it in particular test files if needed. +You should set it in your Jest setup file, and you can override it in particular test files if needed. ### Writing your first test -Now that the library is installed, you can write you first test scenario in a file with `.perf-test.js`/`.perf-test.tsx` extension: +Now that the library is installed, you can write your first test scenario in a file with `.perf-test.js`/`.perf-test.tsx` extension: ```ts // ComponentUnderTest.perf-test.tsx @@ -105,11 +104,11 @@ test('Simple test', async () => { This test will measure render times of `ComponentUnderTest` during mounting and resulting sync effects. -> **Note**: Reassure will automatically match test filenames using Jest's `--testMatch` option with value `"/**/*.perf-test.[jt]s?(x)"`. However, if you would like to pass a custom `--testMatch` option, you may add it to the `reassure measure` script in order to pass your own glob. More about `--testMatch` in [Jest docs](https://jestjs.io/docs/configuration#testmatch-arraystring) +> **Note**: Reassure will automatically match test filenames using Jest's `--testMatch` option with value `"/**/*.perf-test.[jt]s?(x)"`. However, if you want to pass a custom `--testMatch` option, you may add it to the `reassure measure` script to pass your own glob. More about `--testMatch` in [Jest docs](https://jestjs.io/docs/configuration#testmatch-arraystring) #### Writing async tests -If your component contains any async logic or you want to test some interaction you should pass the `scenario` option: +If your component contains any async logic or you want to test some interaction, you should pass the `scenario` option: ```ts import { measurePerformance } from 'reassure'; @@ -144,10 +143,9 @@ test('Test with scenario', async () => { }); ``` -If your test contains any async changes, you will need to make sure that the scenario waits for these changes to settle, e.g. using -`findBy` queries, `waitFor` or `waitForElementToBeRemoved` functions from RNTL. +If your test contains any async changes, you will need to make sure that the scenario waits for these changes to settle, e.g. using `findBy` queries, `waitFor` or `waitForElementToBeRemoved` functions from RNTL. -For more examples look into our example apps: +For more examples, look into our example apps: - [React Native (CLI)](https://github.com/callstack/reassure-examples/tree/main/examples/native) - [React Native (Expo)](https://github.com/callstack/reassure-examples/tree/main/examples/native-expo) @@ -156,19 +154,17 @@ For more examples look into our example apps: ### Measuring test performance -In order to measure your first test performance you need to run following command in terminal: +To measure your first test performance, you need to run the following command in the terminal: ```sh yarn reassure ``` -This command will run your tests multiple times using Jest, gathering render statistics, and will write them to -`.reassure/current.perf` file. In order to check your setup, check if the output file exists after running the -command for the first time. +This command will run your tests multiple times using Jest, gathering performance statistics and will write them to `.reassure/current.perf` file. To check your setup, check if the output file exists after running the command for the first time. > **Note:** You can add `.reassure/` folder to your `.gitignore` file to avoid accidentally committing your results. -Reassure CLI will automatically try to detect your source code branch name and commit hash when you are using Git. You can override these options, e.g. if you are using different version control system: +Reassure CLI will automatically try to detect your source code branch name and commit hash when you are using Git. You can override these options, e.g. if you are using a different version control system: ```sh yarn reassure --branch [branch name] --commit-hash [commit hash] @@ -176,14 +172,11 @@ yarn reassure --branch [branch name] --commit-hash [commit hash] ### Write performance testing script -In order to detect performance changes, you need to measure the performance of two versions of your code -current (your modified code), and baseline (your reference point, e.g. `main` branch). In order to measure performance -on two different branches you need to either switch branches in git or clone two copies of your repository. +To detect performance changes, you must measure the performance of two versions of your code current (your modified code) and baseline (your reference point, e.g. `main` branch). To measure performance on two branches, you must switch branches in Git or clone two copies of your repository. -We want to automate this task, so it can run on the CI. In order to do that you will need to create a -performance testing script. You should save it in your repository, e.g. as `reassure-tests.sh`. +We want to automate this task to run on the CI. To do that, you will need to create a performance-testing script. You should save it in your repository, e.g. as `reassure-tests.sh`. -A simple version of such script, using branch changing approach is as follows: +A simple version of such script, using a branch-changing approach, is as follows: ```sh #!/usr/bin/env bash @@ -207,7 +200,7 @@ yarn reassure ## CI setup -To make setting up the CI integration and all prerequisites more convenient, we have prepared a CLI command which will generate all necessary templates for you to get started with. +To make setting up the CI integration and all prerequisites more convenient, we have prepared a CLI command to generate all necessary templates for you to start with. Simply run: @@ -226,7 +219,7 @@ This will generate the following file structure ### Options -You can also use the following options in order to further adjust the script +You can also use the following options to adjust the script further #### `--verbose` (optional) @@ -244,22 +237,19 @@ Basic script allowing you to run Reassure on CI. More on the importance and stru #### Dangerfile -If your project already contains a `dangerfile.ts/js`, the CLI will not override it in any way. Instead, it will generate a `dangerfile.reassure.ts/js` file which will allow you to compare and update your own at your own convenience. +If your project already contains a `dangerfile.ts/js`, the CLI will not override it in any way. Instead, it will generate a `dangerfile.reassure.ts/js` file, allowing you to compare and update your own at your convenience. #### `.gitignore` -If .gitignore file is present and no mentions of `reassure` appear within it, the script will append the `.reassure/` directory to its end. +If the `.gitignore` file is present and no mentions of `reassure` appear, the script will append the `.reassure/` directory to its end. ### CI script (`reassure-tests.sh`) -In order to detect performance changes, you need to measure the performance of two versions of your code -current (your modified code), and baseline (your reference point, e.g. `main` branch). In order to measure performance -on two different branches you need to either switch branches in git or clone two copies of your repository. +To detect performance changes, you must measure the performance of two versions of your code current (your modified code) and baseline (your reference point, e.g. `main` branch). To measure performance on two branches, you must switch branches in Git or clone two copies of your repository. -We want to automate this task, so it can run on the CI. In order to do that you will need to create a -performance testing script. You should save it in your repository, e.g. as `reassure-tests.sh`. +We want to automate this task to run on the CI. To do that, you will need to create a performance-testing script. You should save it in your repository, e.g. as `reassure-tests.sh`. -A simple version of such script, using branch changing approach is as follows: +A simple version of such script, using a branch-changing approach, is as follows: ```sh #!/usr/bin/env bash @@ -283,14 +273,14 @@ yarn reassure ### Integration -As a final setup step you need to configure your CI to run the performance testing script and output the result. -For presenting output at the moment we integrate with Danger JS, which supports all major CI tools. +As a final setup step, you must configure your CI to run the performance testing script and output the result. +For presenting output at the moment, we integrate with Danger JS, which supports all major CI tools. #### Updating existing Dangerfile You will need a working [Danger JS setup](https://danger.systems/js/guides/getting_started.html). -Then add Reassure Danger JS plugin to your dangerfile : +Then add Reassure Danger JS plugin to your dangerfile: ```ts // //dangerfile.reassure.ts (generated by the init script) @@ -307,11 +297,11 @@ dangerReassure({ If you do not have a Dangerfile (`dangerfile.js` or `dangerfile.ts`) yet, you can use the one generated by the `reassure init` script without making any additional changes. -You can also find it in our example file [Dangerfile](https://github.com/callstack/reassure/blob/main/dangerfile.ts). +It is also in our example file [Dangerfile](https://github.com/callstack/reassure/blob/main/dangerfile.ts). #### Updating the CI configuration file -Finally run both performance testing script & danger in your CI config: +Finally, run both the performance testing script & danger in your CI config: ```yaml - name: Run performance testing script @@ -327,27 +317,20 @@ You can also check our example [GitHub workflow](https://github.com/callstack/re The above example is based on GitHub Actions, but it should be similar to other CI config files and should only serve as a reference in such cases. -> **Note**: Your performance test will run much longer than regular integration tests. It's because we run each test scenario multiple times (by default 10), and we repeat that for two branches of your code. Hence, each test will run 20 times by default. That's unless you increase that number even higher. +> **Note**: Your performance test will run much longer than regular integration tests. It's because we run each test scenario multiple times (by default, 10) and repeat that for two branches of your code. Hence, each test will run 20 times by default. That's unless you increase that number even higher. ## Assessing CI stability -During performance measurements we measure React component render times with microsecond precision using `React.Profiler`. This means -that the same code will run faster or slower depending on the machine. For this reason, -baseline & current measurements need to be run on the same machine. Optimally, they should be run one after another. +We measure React component render times with microsecond precision during performance measurements using `React.Profiler`. This means the same code will run faster or slower, depending on the machine. For this reason, baseline & current measurements need to be run on the same machine. Optimally, they should be run one after another. -Moreover, in order to achieve meaningful results your CI agent needs to have stable performance. It does not matter -really if your agent is fast or slow as long as it is consistent in its performance. That's why during the performance -tests the agent should not be used for any other work that might impact measuring render times. +Moreover, your CI agent needs to have stable performance to achieve meaningful results. It does not matter if your agent is fast or slow as long as it is consistent in its performance. That's why the agent should not be used during the performance tests for any other work that might impact measuring render times. -In order to help you assess your machine stability, you can use `reassure check-stability` command. It runs performance -measurements twice for the current code, so baseline and current measurements refer to the same code. In such case the -expected changes are 0% (no change). The degree of random performance changes will reflect the stability of your machine. +To help you assess your machine's stability, you can use the `reassure check-stability` command. It runs performance measurements twice for the current code, so baseline and current measurements refer to the same code. In such a case, the expected changes are 0% (no change). The degree of random performance changes will reflect the stability of your machine. This command can be run both on CI and local machines. -Normally, the random changes should be below 5%. Results of 10% and more considered too high and mean that you should -work on tweaking your machine stability. +Normally, the random changes should be below 5%. Results of 10% and more are considered too high, meaning you should work on tweaking your machine's stability. -> **Note**: As a trick of last resort you can increase the `run` option, from the default value of 10 to 20, 50 or even 100, for all or some of your tests, based on the assumption that more test runs will even out measurement fluctuations. That will however make your tests run even longer. +> **Note**: As a trick of last resort, you can increase the `run` option from the default value of 10 to 20, 50 or even 100 for all or some of your tests, based on the assumption that more test runs will even out measurement fluctuations. That will, however, make your tests run even longer. You can refer to our example [GitHub workflow](https://github.com/callstack/reassure/blob/main/.github/workflows/stability.yml). @@ -357,11 +340,11 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Markdown report

-Looking at the example you can notice that test scenarios can be assigned to certain categories: +Looking at the example, you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Duration** shows test scenarios where the change is not stastatistically significant -- **Changes To Count** shows test scenarios where count did change +- **Significant Changes To Duration** shows test scenarios where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the change is not statistically significant +- **Changes To Count** shows test scenarios where the count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements @@ -372,7 +355,7 @@ Looking at the example you can notice that test scenarios can be assigned to cer #### `measurePerformance` function Custom wrapper for the RNTL `render` function responsible for rendering the passed screen inside a `React.Profiler` component, -measuring its performance and writing results to the output file. You can use optional `options` object allows customizing aspects +measuring its performance and writing results to the output file. You can use the optional `options` object that allows customizing aspects of the testing ```ts @@ -390,10 +373,62 @@ interface MeasureOptions { } ``` +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1). +- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. +- **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL or RTL functions + +#### `measurePerformance` function + +Custom wrapper for the RNTL `render` function responsible for rendering the passed screen inside a `React.Profiler` component, +measuring its performance and writing results to the output file. You can use the optional `options` object that allows customizing aspects +of the testing + +```ts +async function measurePerformance( + ui: React.ReactElement, + options?: MeasureOptions +): Promise { +``` + +#### `MeasureOptions` type + +```ts +interface MeasureOptions { + runs?: number; + warmupRuns?: number; + wrapper?: React.ComponentType<{ children: ReactElement }>; + scenario?: (view?: RenderResult) => Promise; +} +``` + +- **`runs`**: number of runs per series for the particular test +- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1) +- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. +- **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL functions + +#### `measureFunction` function + +Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. + +```ts +async function measureFunction( + fn: () => void, + options?: MeasureFunctionOptions +): Promise { +``` + +#### `MeasureFunctionOptions` type + +```ts +interface MeasureFunctionOptions { + runs?: number; + warmupRuns?: number; +} +``` + - **`runs`**: number of runs per series for the particular test - **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. -- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results, only the wrapped component is measured. -- **`scenario`**: a custom async function, which defines user interaction within the ui by utilized RNTL functions ### Configuration @@ -425,9 +460,9 @@ const defaultConfig: Config = { }; ``` -**`runs`**: number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care. -- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs. -**`outputFile`**: name of the file the records will be saved to +**`runs`**: the number of repeated runs in a series per test (allows for higher accuracy by aggregating more data). Should be handled with care. +- **`warmupRuns`**: the number of additional warmup runs that will be done and discarded before the actual runs. +**`outputFile`**: the name of the file the records will be saved to **`verbose`**: make Reassure log more, e.g. for debugging purposes **`testingLibrary`**: where to look for `render` and `cleanup` functions, supported values `'react-native'`, `'react'` or object providing custom `render` and `cleanup` functions @@ -437,7 +472,7 @@ const defaultConfig: Config = { function configure(customConfig: Partial): void; ``` -You can use the `configure` function to override the default config parameters. +The `configure` function can override the default config parameters. #### `resetToDefault` function @@ -445,11 +480,11 @@ You can use the `configure` function to override the default config parameters. resetToDefault(): void ``` -Reset current config to the original `defaultConfig` object +Reset the current config to the original `defaultConfig` object #### Environmental variables -You can use available environmental variables in order to alter your test runner settings. +You can use available environmental variables to alter your test runner settings. - `TEST_RUNNER_PATH`: an alternative path for your test runner. Defaults to `'node_modules/.bin/jest'` or on Windows `'node_modules/jest/bin/jest'` - `TEST_RUNNER_ARGS`: a set of arguments fed to the runner. Defaults to `'--runInBand --testMatch "/**/*.perf-test.[jt]s?(x)"'` @@ -478,6 +513,6 @@ Reassure is an Open Source project and will always remain free to use. The proje partnership with [Entain](https://entaingroup.com/) and was originally their in-house project. Thanks to their willingness to develop the React & React Native ecosystem, we decided to make it Open Source. If you think it's cool, please star it 🌟 -Callstack is a group of React and React Native experts. If you need any help with these or just want to say hi, contact us at hello@callstack.com! +Callstack is a group of React and React Native experts. If you need help with these or want to say hi, contact us at hello@callstack.com! Like the project? ⚛️ [Join the Callstack team](https://callstack.com/careers/?utm_campaign=Senior_RN&utm_source=github&utm_medium=readme) who does amazing stuff for clients and drives React Native Open Source! 🔥 From 46444767f493c0b8aedf7eb316e0c015da5c708e Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 12:00:03 +0200 Subject: [PATCH 14/24] docs: more tweaks --- README.md | 35 +++-------------------------------- packages/reassure/README.md | 35 +++-------------------------------- 2 files changed, 6 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index c791137ad..13e88a140 100644 --- a/README.md +++ b/README.md @@ -342,9 +342,9 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example, you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Duration** shows test scenarios where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Duration** shows test scenarios where the change is not statistically significant -- **Changes To Count** shows test scenarios where the count did change +- **Significant Changes To Duration** shows test scenarios where the performance change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the performance change is not statistically significant +- **Changes To Count** shows test scenarios where the render or execution count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements @@ -378,35 +378,6 @@ interface MeasureOptions { - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. - **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL or RTL functions -#### `measurePerformance` function - -Custom wrapper for the RNTL `render` function responsible for rendering the passed screen inside a `React.Profiler` component, -measuring its performance and writing results to the output file. You can use the optional `options` object that allows customizing aspects -of the testing - -```ts -async function measurePerformance( - ui: React.ReactElement, - options?: MeasureOptions -): Promise { -``` - -#### `MeasureOptions` type - -```ts -interface MeasureOptions { - runs?: number; - warmupRuns?: number; - wrapper?: React.ComponentType<{ children: ReactElement }>; - scenario?: (view?: RenderResult) => Promise; -} -``` - -- **`runs`**: number of runs per series for the particular test -- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1) -- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. -- **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL functions - #### `measureFunction` function Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. diff --git a/packages/reassure/README.md b/packages/reassure/README.md index c791137ad..13e88a140 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -342,9 +342,9 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example, you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Duration** shows test scenarios where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Duration** shows test scenarios where the change is not statistically significant -- **Changes To Count** shows test scenarios where the count did change +- **Significant Changes To Duration** shows test scenarios where the performance change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the performance change is not statistically significant +- **Changes To Count** shows test scenarios where the render or execution count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements @@ -378,35 +378,6 @@ interface MeasureOptions { - **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. - **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL or RTL functions -#### `measurePerformance` function - -Custom wrapper for the RNTL `render` function responsible for rendering the passed screen inside a `React.Profiler` component, -measuring its performance and writing results to the output file. You can use the optional `options` object that allows customizing aspects -of the testing - -```ts -async function measurePerformance( - ui: React.ReactElement, - options?: MeasureOptions -): Promise { -``` - -#### `MeasureOptions` type - -```ts -interface MeasureOptions { - runs?: number; - warmupRuns?: number; - wrapper?: React.ComponentType<{ children: ReactElement }>; - scenario?: (view?: RenderResult) => Promise; -} -``` - -- **`runs`**: number of runs per series for the particular test -- **`warmupRuns`**: number of additional warmup runs that will be done and discarded before the actual runs (default 1) -- **`wrapper`**: React component, such as a `Provider`, which the `ui` will be wrapped with. Note: the render duration of the `wrapper` itself is excluded from the results; only the wrapped component is measured. -- **`scenario`**: a custom async function, which defines user interaction within the UI by utilising RNTL functions - #### `measureFunction` function Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. From 327d0e261f2afe96b74f0d4eec39a0f20bd37fc3 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 12:08:04 +0200 Subject: [PATCH 15/24] refactor: code review change --- docusaurus/docs/methodology.md | 6 +++--- packages/reassure-compare/src/compare.ts | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docusaurus/docs/methodology.md b/docusaurus/docs/methodology.md index 9b9968096..f071d22ea 100644 --- a/docusaurus/docs/methodology.md +++ b/docusaurus/docs/methodology.md @@ -34,8 +34,8 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas Looking at the example you can notice that test scenarios can be assigned to certain categories: -- **Significant Changes To Duration** shows test scenario where the change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement -- **Meaningless Changes To Duration** shows test scenarios where the change is not stastatistically significant -- **Changes To Count** shows test scenarios where count did change +- **Significant Changes To Duration** shows test scenario where the performance change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement +- **Meaningless Changes To Duration** shows test scenarios where the performance change is not stastatistically significant +- **Changes To Count** shows test scenarios where the render or execution count did change - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements - **Removed Scenarios** shows test scenarios which do not exist in the current measurements diff --git a/packages/reassure-compare/src/compare.ts b/packages/reassure-compare/src/compare.ts index fecd19f86..4d793687d 100644 --- a/packages/reassure-compare/src/compare.ts +++ b/packages/reassure-compare/src/compare.ts @@ -30,7 +30,7 @@ const PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02; const DURATION_DIFF_THRESHOLD_SIGNIFICANT = 4; /** - * Threshold for considering count change as significant. This implies inclusion + * Threshold for considering render or execution count change as significant. This implies inclusion * of scenario results in Count Changed output section. */ const COUNT_DIFF_THRESHOLD = 0.5; @@ -144,7 +144,7 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult const baselineEntry = baseline?.entries[name]; if (currentEntry && baselineEntry) { - compared.push(buildCompareEntry(name, currentEntry.type, currentEntry, baselineEntry)); + compared.push(buildCompareEntry(name, currentEntry, baselineEntry)); } else if (currentEntry) { added.push({ name, type: currentEntry.type, current: currentEntry }); } else if (baselineEntry) { @@ -181,7 +181,6 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult */ function buildCompareEntry( name: string, - type: MeasureType, current: PerformanceEntry, baseline: PerformanceEntry ): CompareEntry { @@ -198,7 +197,7 @@ function buildCompareEntry( return { name, - type, + type: current.type, baseline, current, durationDiff, From 5f4cd7e07c41b6b0add3376afbdabba9cc91d975 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 12:13:59 +0200 Subject: [PATCH 16/24] refactor: cleanup --- packages/reassure-compare/src/compare.ts | 9 ++------- .../src/test/__snapshots__/compare.test.js.snap | 4 ++-- packages/reassure-compare/src/test/invalid-entry.perf | 2 +- packages/reassure-compare/src/test/invalid-json.perf | 2 +- packages/reassure-compare/src/test/valid-header.perf | 2 +- packages/reassure-compare/src/test/valid-no-header.perf | 2 +- packages/reassure-compare/src/utils/validate.ts | 2 +- packages/reassure-measure/src/measure-function.tsx | 2 +- packages/reassure-measure/src/measure-render.tsx | 2 +- packages/reassure-measure/src/output.ts | 2 +- packages/reassure-measure/src/types.ts | 5 +++-- 11 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/reassure-compare/src/compare.ts b/packages/reassure-compare/src/compare.ts index 4d793687d..3ce07ab13 100644 --- a/packages/reassure-compare/src/compare.ts +++ b/packages/reassure-compare/src/compare.ts @@ -8,7 +8,6 @@ import type { PerformanceResults, PerformanceEntry, PerformanceHeader, - MeasureType, } from './types'; import { printToConsole } from './output/console'; import { writeToJson } from './output/json'; @@ -177,13 +176,9 @@ function compareResults(current: PerformanceResults, baseline: PerformanceResult } /** - * Establish statisticial significance of duration difference build compare entry. + * Establish statisticial significance of render/execution duration difference build compare entry. */ -function buildCompareEntry( - name: string, - current: PerformanceEntry, - baseline: PerformanceEntry -): CompareEntry { +function buildCompareEntry(name: string, current: PerformanceEntry, baseline: PerformanceEntry): CompareEntry { const durationDiff = current.meanDuration - baseline.meanDuration; const relativeDurationDiff = durationDiff / baseline.meanDuration; const countDiff = current.meanCount - baseline.meanCount; diff --git a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap index ad1f9d697..2584c38d3 100644 --- a/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap +++ b/packages/reassure-compare/src/test/__snapshots__/compare.test.js.snap @@ -213,7 +213,7 @@ exports[`loadFile should load results file with header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 0.4532780026900862, - "type": "syncFunction", + "type": "function", }, }, "metadata": { @@ -409,7 +409,7 @@ exports[`loadFile should load results file without header 1`] = ` "runs": 10, "stdevCount": 0, "stdevDuration": 0.4532780026900862, - "type": "syncFunction", + "type": "function", }, }, "metadata": undefined, diff --git a/packages/reassure-compare/src/test/invalid-entry.perf b/packages/reassure-compare/src/test/invalid-entry.perf index 61e63df19..06789aeae 100644 --- a/packages/reassure-compare/src/test/invalid-entry.perf +++ b/packages/reassure-compare/src/test/invalid-entry.perf @@ -3,4 +3,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":"10","stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file diff --git a/packages/reassure-compare/src/test/invalid-json.perf b/packages/reassure-compare/src/test/invalid-json.perf index 9522426c5..8f93440cb 100644 --- a/packages/reassure-compare/src/test/invalid-json.perf +++ b/packages/reassure-compare/src/test/invalid-json.perf @@ -2,4 +2,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} "name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} diff --git a/packages/reassure-compare/src/test/valid-header.perf b/packages/reassure-compare/src/test/valid-header.perf index e090fb414..c7c31e5f4 100644 --- a/packages/reassure-compare/src/test/valid-header.perf +++ b/packages/reassure-compare/src/test/valid-header.perf @@ -3,4 +3,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file diff --git a/packages/reassure-compare/src/test/valid-no-header.perf b/packages/reassure-compare/src/test/valid-no-header.perf index 2a20ca99e..895d1b899 100644 --- a/packages/reassure-compare/src/test/valid-no-header.perf +++ b/packages/reassure-compare/src/test/valid-no-header.perf @@ -2,4 +2,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"syncFunction","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index 220670934..a6abda907 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -20,7 +20,7 @@ export const performanceEntrySchema = z.object({ name: z.string(), /** Type of the test scenario */ - type: z.enum(['render', 'syncFunction']).default('render'), + type: z.enum(['render', 'function']).default('render'), /** Number of times the measurment test was run */ runs: z.number(), diff --git a/packages/reassure-measure/src/measure-function.tsx b/packages/reassure-measure/src/measure-function.tsx index 771981fc3..15a59b7a7 100644 --- a/packages/reassure-measure/src/measure-function.tsx +++ b/packages/reassure-measure/src/measure-function.tsx @@ -11,7 +11,7 @@ interface MeasureFunctionOptions { export async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { const stats = await measureFunctionInternal(fn, options); - await writeTestStats('syncFunction', stats); + await writeTestStats(stats, 'function'); return stats; } diff --git a/packages/reassure-measure/src/measure-render.tsx b/packages/reassure-measure/src/measure-render.tsx index ed9e91deb..e9b56c474 100644 --- a/packages/reassure-measure/src/measure-render.tsx +++ b/packages/reassure-measure/src/measure-render.tsx @@ -20,7 +20,7 @@ export interface MeasureOptions { export async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { const stats = await measureRender(ui, options); - await writeTestStats('render', stats); + await writeTestStats(stats, 'render'); return stats; } diff --git a/packages/reassure-measure/src/output.ts b/packages/reassure-measure/src/output.ts index 676ce7cb3..42f3f1278 100644 --- a/packages/reassure-measure/src/output.ts +++ b/packages/reassure-measure/src/output.ts @@ -4,8 +4,8 @@ import { config } from './config'; import type { MeasureResults, MeasureType } from './types'; export async function writeTestStats( - type: MeasureType, stats: MeasureResults, + type: MeasureType, outputFilePath: string = config.outputFile ): Promise { const name = expect.getState().currentTestName; diff --git a/packages/reassure-measure/src/types.ts b/packages/reassure-measure/src/types.ts index 512b96943..6a79248aa 100644 --- a/packages/reassure-measure/src/types.ts +++ b/packages/reassure-measure/src/types.ts @@ -1,3 +1,6 @@ +/** Type (natute) of measured characteristic. */ +export type MeasureType = 'render' | 'function'; + /** * Type representing output of measure functions, e.g. `measurePerformance`, `measureFunction`. */ @@ -23,5 +26,3 @@ export interface MeasureResults { /** Array of measured execution count for each run */ counts: number[]; } - -export type MeasureType = 'render' | 'syncFunction'; From 9c97a8a2b57d197a3d55a316fd0005ffa634b088 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 12:27:59 +0200 Subject: [PATCH 17/24] refactor: code review changes --- .../reassure-compare/src/utils/validate.ts | 18 +++++++++--------- packages/reassure-measure/src/types.ts | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index a6abda907..a912c8ceb 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -16,31 +16,31 @@ export const performanceHeaderSchema = z.object({ /** Entry in the performance results file. */ export const performanceEntrySchema = z.object({ - /** Name of the test scenario */ + /** Name of the test scenario. */ name: z.string(), - /** Type of the test scenario */ + /** Type of the measured characteristic (render, function execution). */ type: z.enum(['render', 'function']).default('render'), - /** Number of times the measurment test was run */ + /** Number of times the measurement test was run. */ runs: z.number(), - /** Arithmetic average of measured durations for each run */ + /** Arithmetic average of measured render/execution durations for each run. */ meanDuration: z.number(), - /** Standard deviation of measured durations for each run */ + /** Standard deviation of measured render/execution durations for each run. */ stdevDuration: z.number(), - /** Array of measured durations for each run */ + /** Array of measured render/execution durations for each run. */ durations: z.array(z.number()), - /** Arithmetic average of measured counts for each run */ + /** Arithmetic average of measured render/execution counts for each run. */ meanCount: z.number(), - /** Standard deviation of measured counts for each run */ + /** Standard deviation of measured render/execution counts for each run. */ stdevCount: z.number(), - /** Array of measured counts for each run */ + /** Array of measured render/execution counts for each run. */ counts: z.array(z.number()), }); diff --git a/packages/reassure-measure/src/types.ts b/packages/reassure-measure/src/types.ts index 6a79248aa..7f766ad44 100644 --- a/packages/reassure-measure/src/types.ts +++ b/packages/reassure-measure/src/types.ts @@ -1,28 +1,28 @@ -/** Type (natute) of measured characteristic. */ +/** Type of measured performance characteristic. */ export type MeasureType = 'render' | 'function'; /** - * Type representing output of measure functions, e.g. `measurePerformance`, `measureFunction`. + * Type representing the result of `measure*` functions. */ export interface MeasureResults { /** Number of times the test subject was run */ runs: number; - /** Arithmetic average of measured execution durations for each run */ + /** Arithmetic average of measured render/execution durations for each run */ meanDuration: number; - /** Standard deviation of measured execution durations for each run */ + /** Standard deviation of measured render/execution durations for each run */ stdevDuration: number; - /** Array of measured execution durations for each run */ + /** Array of measured render/execution durations for each run */ durations: number[]; - /** Arithmetic average of measured execution count for each run */ + /** Arithmetic average of measured render/execution count for each run */ meanCount: number; - /** Standard deviation of measured execution count for each run */ + /** Standard deviation of measured render/execution count for each run */ stdevCount: number; - /** Array of measured execution count for each run */ + /** Array of measured render/execution count for each run */ counts: number[]; } From 64b89aa7351c42e1515e4c7c6c94123b77d7723f Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 12:47:24 +0200 Subject: [PATCH 18/24] refactor: code review changes --- .../reassure-compare/src/output/console.ts | 12 ++--------- .../reassure-compare/src/output/markdown.ts | 8 ++++---- packages/reassure-compare/src/types.ts | 18 ++++++++--------- packages/reassure-compare/src/utils/format.ts | 20 +++++++++---------- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/packages/reassure-compare/src/output/console.ts b/packages/reassure-compare/src/output/console.ts index 0ec9d149e..44097d9d1 100644 --- a/packages/reassure-compare/src/output/console.ts +++ b/packages/reassure-compare/src/output/console.ts @@ -1,12 +1,6 @@ import { logger } from '@callstack/reassure-logger'; import type { AddedEntry, CompareResult, CompareEntry, RemovedEntry } from '../types'; -import { - formatCount, - formatDuration, - formatMetadata, - formatOutputCountChange, - formatOutputDurationChange, -} from '../utils/format'; +import { formatCount, formatDuration, formatMetadata, formatCountChange, formatDurationChange } from '../utils/format'; import type { PerformanceMetadata } from '../types'; export function printToConsole(data: CompareResult) { @@ -39,9 +33,7 @@ function printMetadata(name: string, metadata?: PerformanceMetadata) { } function printRegularLine(entry: CompareEntry) { - logger.log( - ` - [${entry.type}] ${entry.name}: ${formatOutputDurationChange(entry)} | ${formatOutputCountChange(entry)}` - ); + logger.log(` - [${entry.type}] ${entry.name}: ${formatDurationChange(entry)} | ${formatCountChange(entry)}`); } function printAddedLine(entry: AddedEntry) { diff --git a/packages/reassure-compare/src/output/markdown.ts b/packages/reassure-compare/src/output/markdown.ts index 4e973a7dc..d2a86e8d7 100644 --- a/packages/reassure-compare/src/output/markdown.ts +++ b/packages/reassure-compare/src/output/markdown.ts @@ -10,8 +10,8 @@ import { formatDuration, formatMetadata, formatPercent, - formatOutputCountChange, - formatOutputDurationChange, + formatCountDiff, + formatDurationChange, } from '../utils/format'; import type { AddedEntry, @@ -116,14 +116,14 @@ function buildDetailsTable(entries: Array 0) { return `+${formatDuration(value)}`; } @@ -34,7 +34,7 @@ export function formatDurationChange(value: number): string { export function formatCount(value: number) { return Number.isInteger(value) ? `${value}` : `${value.toFixed(2)}`; } -export function formatCountChange(value: number): string { +export function formatCountDiff(value: number): string { if (value > 0) return `+${value}`; if (value < 0) return `${value}`; return '±0'; @@ -46,21 +46,21 @@ export function formatChange(value: number): string { return '0'; } -export function formatOutputDurationChange(entry: CompareEntry) { +export function formatDurationChange(entry: CompareEntry) { const { baseline, current } = entry; let output = `${formatDuration(baseline.meanDuration)} → ${formatDuration(current.meanDuration)}`; if (baseline.meanDuration != current.meanDuration) { - output += ` (${formatDurationChange(entry.durationDiff)}, ${formatPercentChange(entry.relativeDurationDiff)})`; + output += ` (${formatDurationDiff(entry.durationDiff)}, ${formatPercentChange(entry.relativeDurationDiff)})`; } - output += ` ${getDurationSymbols(entry)}`; + output += ` ${getDurationChangeSymbols(entry)}`; return output; } -function getDurationSymbols(entry: CompareEntry) { +function getDurationChangeSymbols(entry: CompareEntry) { if (!entry.isDurationDiffSignificant) { if (entry.relativeDurationDiff > 0.15) return '🔴'; if (entry.relativeDurationDiff < -0.15) return '🟢'; @@ -75,21 +75,21 @@ function getDurationSymbols(entry: CompareEntry) { return ''; } -export function formatOutputCountChange(entry: CompareEntry) { +export function formatCountChange(entry: CompareEntry) { const { baseline, current } = entry; let output = `${formatCount(baseline.meanCount)} → ${formatCount(current.meanCount)}`; if (baseline.meanCount != current.meanCount) { - output += ` (${formatCountChange(entry.countDiff)}, ${formatPercentChange(entry.relativeCountDiff)})`; + output += ` (${formatCountDiff(entry.countDiff)}, ${formatPercentChange(entry.relativeCountDiff)})`; } - output += ` ${getOutputCountSymbols(entry)}`; + output += ` ${getCountChangeSymbols(entry)}`; return output; } -function getOutputCountSymbols(entry: CompareEntry) { +function getCountChangeSymbols(entry: CompareEntry) { if (entry.countDiff > 1.5) return '🔴🔴'; if (entry.countDiff > 0.5) return '🔴'; if (entry.countDiff < -1.5) return '🟢🟢'; From 936541968a3a79867abc195b44748ed906add807 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 13:08:04 +0200 Subject: [PATCH 19/24] chore: add tests --- packages/reassure-compare/src/test/compare.test.js | 7 +++++++ packages/reassure-compare/src/test/default-type.perf | 4 ++++ packages/reassure-compare/src/test/invalid-entry.perf | 2 +- packages/reassure-compare/src/test/valid-header.perf | 2 +- packages/reassure-compare/src/test/valid-no-header.perf | 2 +- 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 packages/reassure-compare/src/test/default-type.perf diff --git a/packages/reassure-compare/src/test/compare.test.js b/packages/reassure-compare/src/test/compare.test.js index 9f34c1a30..00b76b2e4 100644 --- a/packages/reassure-compare/src/test/compare.test.js +++ b/packages/reassure-compare/src/test/compare.test.js @@ -45,4 +45,11 @@ describe('loadFile', () => { it('should fail for file with invalid entry', async () => { expect(() => loadFile(`${__dirname}/invalid-entry.perf`)).toThrowErrorMatchingSnapshot(); }); + + it('should support entries without type', async () => { + const results = loadFile(`${__dirname}/default-type.perf`); + + const types = Object.entries(results.entries).map(([key, value]) => value.type); + expect(types).toEqual(['render', 'render', 'function']); + }); }); diff --git a/packages/reassure-compare/src/test/default-type.perf b/packages/reassure-compare/src/test/default-type.perf new file mode 100644 index 000000000..4e55e3d37 --- /dev/null +++ b/packages/reassure-compare/src/test/default-type.perf @@ -0,0 +1,4 @@ +{"metadata":{"branch":"feat/perf-file-validation","commitHash":"991427a413b1ff05497a881287c9ddcba7b8de54"}} +{"name":"Other Component 10 with type","type":"render","runs":10,"meanDuration":92.7,"stdevDuration":4.831608887776871,"durations":[100,97,95,94,94,94,93,90,86,84],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"Other Component 10 without type","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} diff --git a/packages/reassure-compare/src/test/invalid-entry.perf b/packages/reassure-compare/src/test/invalid-entry.perf index 06789aeae..1380a6f12 100644 --- a/packages/reassure-compare/src/test/invalid-entry.perf +++ b/packages/reassure-compare/src/test/invalid-entry.perf @@ -3,4 +3,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":"10","stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} diff --git a/packages/reassure-compare/src/test/valid-header.perf b/packages/reassure-compare/src/test/valid-header.perf index c7c31e5f4..0117cf3a8 100644 --- a/packages/reassure-compare/src/test/valid-header.perf +++ b/packages/reassure-compare/src/test/valid-header.perf @@ -3,4 +3,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} diff --git a/packages/reassure-compare/src/test/valid-no-header.perf b/packages/reassure-compare/src/test/valid-no-header.perf index 895d1b899..a384fe862 100644 --- a/packages/reassure-compare/src/test/valid-no-header.perf +++ b/packages/reassure-compare/src/test/valid-no-header.perf @@ -2,4 +2,4 @@ {"name":"Other Component 10 legacy scenario","type":"render","runs":10,"meanDuration":90.5,"stdevDuration":4.552166761249221,"durations":[97,96,94,92,91,91,88,88,85,83],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4]} {"name":"Other Component 20","type":"render","runs":20,"meanDuration":92.2,"stdevDuration":4.595191995301633,"durations":[99,99,98,98,96,95,94,93,93,93,93,92,90,90,89,88,88,87,87,82],"meanCount":4,"stdevCount":0,"counts":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]} {"name":"Async Component","type":"render","runs":10,"meanDuration":150.8,"stdevDuration":7.554248252914823,"durations":[160,158,158,155,153,150,149,145,144,136],"meanCount":7,"stdevCount":0,"counts":[7,7,7,7,7,7,7,7,7,7]} -{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} \ No newline at end of file +{"name":"fib 30","type":"function","runs":10,"meanDuration":79.9,"stdevDuration":0.4532780026900862,"durations":[80,80,80,80,80,79,79,79,79,79],"meanCount":1,"stdevCount":0,"counts":[1,1,1,1,1,1,1,1,1,1]} From d4bb6661a6221857b54d0857262c0dc2bf0c1097 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 13:23:30 +0200 Subject: [PATCH 20/24] docs: tweak --- README.md | 2 +- packages/reassure/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 13e88a140..bd227430a 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,7 @@ interface MeasureOptions { #### `measureFunction` function -Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. +Allows you to wrap any synchronous function, measure its execution times and write results to the output file. You can use optional `options` to customize aspects of the testing. Note: the execution count will always be one. ```ts async function measureFunction( diff --git a/packages/reassure/README.md b/packages/reassure/README.md index 13e88a140..bd227430a 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -380,7 +380,7 @@ interface MeasureOptions { #### `measureFunction` function -Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. +Allows you to wrap any synchronous function, measure its execution times and write results to the output file. You can use optional `options` to customize aspects of the testing. Note: the execution count will always be one. ```ts async function measureFunction( From 8e081258edbb9aea31177f97118edec1555bd8aa Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 13:56:11 +0200 Subject: [PATCH 21/24] chore: fix build --- packages/reassure-compare/src/output/console.ts | 6 +++--- packages/reassure-compare/src/output/markdown.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/reassure-compare/src/output/console.ts b/packages/reassure-compare/src/output/console.ts index 44097d9d1..cc6d62e98 100644 --- a/packages/reassure-compare/src/output/console.ts +++ b/packages/reassure-compare/src/output/console.ts @@ -33,19 +33,19 @@ function printMetadata(name: string, metadata?: PerformanceMetadata) { } function printRegularLine(entry: CompareEntry) { - logger.log(` - [${entry.type}] ${entry.name}: ${formatDurationChange(entry)} | ${formatCountChange(entry)}`); + logger.log(` - ${entry.name} [${entry.type}]: ${formatDurationChange(entry)} | ${formatCountChange(entry)}`); } function printAddedLine(entry: AddedEntry) { const { current } = entry; logger.log( - ` - [${entry.type}] ${entry.name}: ${formatDuration(current.meanDuration)} | ${formatCount(current.meanCount)}` + ` - ${entry.name} [${entry.type}]: ${formatDuration(current.meanDuration)} | ${formatCount(current.meanCount)}` ); } function printRemovedLine(entry: RemovedEntry) { const { baseline } = entry; logger.log( - ` - [${entry.type}] ${entry.name}: ${formatDuration(baseline.meanDuration)} | ${formatCount(baseline.meanCount)}` + ` - ${entry.name} [${entry.type}]: ${formatDuration(baseline.meanDuration)} | ${formatCount(baseline.meanCount)}` ); } diff --git a/packages/reassure-compare/src/output/markdown.ts b/packages/reassure-compare/src/output/markdown.ts index d2a86e8d7..e55a8f009 100644 --- a/packages/reassure-compare/src/output/markdown.ts +++ b/packages/reassure-compare/src/output/markdown.ts @@ -10,7 +10,7 @@ import { formatDuration, formatMetadata, formatPercent, - formatCountDiff, + formatCountChange, formatDurationChange, } from '../utils/format'; import type { @@ -123,7 +123,7 @@ function formatEntryDuration(entry: CompareEntry | AddedEntry | RemovedEntry) { } function formatEntryCount(entry: CompareEntry | AddedEntry | RemovedEntry) { - if ('baseline' in entry && 'current' in entry) return formatCountDiff(entry); + if ('baseline' in entry && 'current' in entry) return formatCountChange(entry); if ('baseline' in entry) return formatCount(entry.baseline.meanCount); if ('current' in entry) return formatCount(entry.current.meanCount); return ''; From ccca9771a504216fc3cd8a237c84c0717047c8fb Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 11 Oct 2023 14:41:54 +0200 Subject: [PATCH 22/24] refactor: cleanup --- packages/reassure-compare/src/type-schemas.ts | 43 +++++++++++++++++++ packages/reassure-compare/src/types.ts | 2 +- .../reassure-compare/src/utils/validate.ts | 43 +------------------ 3 files changed, 45 insertions(+), 43 deletions(-) create mode 100644 packages/reassure-compare/src/type-schemas.ts diff --git a/packages/reassure-compare/src/type-schemas.ts b/packages/reassure-compare/src/type-schemas.ts new file mode 100644 index 000000000..b0eebd031 --- /dev/null +++ b/packages/reassure-compare/src/type-schemas.ts @@ -0,0 +1,43 @@ +import { z } from 'zod'; + +/** Metadata information for performance results. */ +export const performanceMetadataSchema = z.object({ + branch: z.string().optional(), + commitHash: z.string().optional(), + creationDate: z.string().datetime().optional(), +}); + +/** Header of performance results file. */ +export const performanceHeaderSchema = z.object({ + metadata: performanceMetadataSchema, +}); + +/** Entry in the performance results file. */ +export const performanceEntrySchema = z.object({ + /** Name of the test scenario. */ + name: z.string(), + + /** Type of the measured characteristic (render, function execution). */ + type: z.enum(['render', 'function']).default('render'), + + /** Number of times the measurement test was run. */ + runs: z.number(), + + /** Arithmetic average of measured render/execution durations for each run. */ + meanDuration: z.number(), + + /** Standard deviation of measured render/execution durations for each run. */ + stdevDuration: z.number(), + + /** Array of measured render/execution durations for each run. */ + durations: z.array(z.number()), + + /** Arithmetic average of measured render/execution counts for each run. */ + meanCount: z.number(), + + /** Standard deviation of measured render/execution counts for each run. */ + stdevCount: z.number(), + + /** Array of measured render/execution counts for each run. */ + counts: z.array(z.number()), +}); diff --git a/packages/reassure-compare/src/types.ts b/packages/reassure-compare/src/types.ts index 20b52e107..a3bdc80fa 100644 --- a/packages/reassure-compare/src/types.ts +++ b/packages/reassure-compare/src/types.ts @@ -1,6 +1,6 @@ /** Parsed performance results file. */ import type { z } from 'zod'; -import type { performanceEntrySchema, performanceHeaderSchema, performanceMetadataSchema } from './utils/validate'; +import type { performanceEntrySchema, performanceHeaderSchema, performanceMetadataSchema } from './type-schemas'; export type PerformanceHeader = z.infer; export type PerformanceMetadata = z.infer; diff --git a/packages/reassure-compare/src/utils/validate.ts b/packages/reassure-compare/src/utils/validate.ts index a912c8ceb..a71a4e0cd 100644 --- a/packages/reassure-compare/src/utils/validate.ts +++ b/packages/reassure-compare/src/utils/validate.ts @@ -1,49 +1,8 @@ import type { PerformanceEntry, PerformanceHeader } from 'src/types'; import { z } from 'zod'; +import { performanceHeaderSchema, performanceEntrySchema } from '../type-schemas'; import { hasDuplicateValues } from './array'; -/** Metadata information for performance results. */ -export const performanceMetadataSchema = z.object({ - branch: z.string().optional(), - commitHash: z.string().optional(), - creationDate: z.string().datetime().optional(), -}); - -/** Header of performance results file. */ -export const performanceHeaderSchema = z.object({ - metadata: performanceMetadataSchema, -}); - -/** Entry in the performance results file. */ -export const performanceEntrySchema = z.object({ - /** Name of the test scenario. */ - name: z.string(), - - /** Type of the measured characteristic (render, function execution). */ - type: z.enum(['render', 'function']).default('render'), - - /** Number of times the measurement test was run. */ - runs: z.number(), - - /** Arithmetic average of measured render/execution durations for each run. */ - meanDuration: z.number(), - - /** Standard deviation of measured render/execution durations for each run. */ - stdevDuration: z.number(), - - /** Array of measured render/execution durations for each run. */ - durations: z.array(z.number()), - - /** Arithmetic average of measured render/execution counts for each run. */ - meanCount: z.number(), - - /** Standard deviation of measured render/execution counts for each run. */ - stdevCount: z.number(), - - /** Array of measured render/execution counts for each run. */ - counts: z.array(z.number()), -}); - const performanceEntriesSchema = z .array(performanceEntrySchema) .refine((val) => !hasDuplicateValues(val.map((val) => val.name)), { From 0febcd6fdb9677b272b2066bbca7648b5b5e7f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Jastrze=CC=A8bski?= Date: Wed, 11 Oct 2023 21:48:16 +0200 Subject: [PATCH 23/24] docs: tweaks --- README.md | 5 ++++- docusaurus/docs/api.md | 10 ++++++++-- packages/reassure/README.md | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bd227430a..1f894bb01 100644 --- a/README.md +++ b/README.md @@ -359,7 +359,10 @@ measuring its performance and writing results to the output file. You can use th of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance( + ui: React.ReactElement, + options?: MeasureOptions, +): Promise { ``` #### `MeasureOptions` type diff --git a/docusaurus/docs/api.md b/docusaurus/docs/api.md index bd5b97d99..0e13c0a8d 100644 --- a/docusaurus/docs/api.md +++ b/docusaurus/docs/api.md @@ -14,7 +14,10 @@ measuring its performance and writing results to the output file. You can use op of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance( + ui: React.ReactElement, + options?: MeasureOptions, +): Promise { ``` #### Example @@ -56,7 +59,10 @@ interface MeasureOptions { Allows you to wrap any synchronous function, measure its performance and write results to the output file. You can use optional `options` to customize aspects of the testing. ```ts -async function measureFunction(fn: () => void, options?: MeasureFunctionOptions): Promise { +async function measureFunction( + fn: () => void, + options?: MeasureFunctionOptions, +): Promise { ``` #### Example diff --git a/packages/reassure/README.md b/packages/reassure/README.md index bd227430a..1f894bb01 100644 --- a/packages/reassure/README.md +++ b/packages/reassure/README.md @@ -359,7 +359,10 @@ measuring its performance and writing results to the output file. You can use th of the testing ```ts -async function measurePerformance(ui: React.ReactElement, options?: MeasureOptions): Promise { +async function measurePerformance( + ui: React.ReactElement, + options?: MeasureOptions, +): Promise { ``` #### `MeasureOptions` type From 30c07675efff685fe211b50cf1af417643f0fc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Jastrze=CC=A8bski?= Date: Wed, 11 Oct 2023 22:01:15 +0200 Subject: [PATCH 24/24] refactor: tweaks --- packages/reassure-compare/src/output/markdown.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/reassure-compare/src/output/markdown.ts b/packages/reassure-compare/src/output/markdown.ts index e55a8f009..bf74de0f4 100644 --- a/packages/reassure-compare/src/output/markdown.ts +++ b/packages/reassure-compare/src/output/markdown.ts @@ -154,7 +154,7 @@ function buildDurationDetails(title: string, entry: PerformanceEntry) { emphasis.b(title), `Mean: ${formatDuration(entry.meanDuration)}`, `Stdev: ${formatDuration(entry.stdevDuration)} (${formatPercent(relativeStdev)})`, - entry.durations ? `Runs: ${entry.durations.join(' ')}` : '', + entry.durations ? `Runs: ${formatRunDurations(entry.durations)}` : '', ] .filter(Boolean) .join(`
`); @@ -176,3 +176,7 @@ function buildCountDetails(title: string, entry: PerformanceEntry) { export function collapsibleSection(title: string, content: string) { return `
\n${title}\n\n${content}\n
\n\n`; } + +export function formatRunDurations(values: number[]) { + return values.map((v) => (Number.isInteger(v) ? `${v}` : `${v.toFixed(1)}`)).join(' '); +}