feat: detect unnecessary render and warn user (#468)

* feat: detect unnecessary render and warn user

feat: capture json representations in an array to compare after for unnecessary rendering

fix: get current testingLibrary for comparasion only react-native for now

fix: save json representation instead of string for rendered component state and compare changes between states using dfs

fix: test name

fix: update comparasion function and save compare results into output.json and show comparation results in the end of the test

fix: update interface and variable names based on pr recommendatations

chore: rebase to v1

* refactor: improve output formatting

* refactor: fix typo

* refactor: remove unnecessary warning

* refactor: improve testing

* refactor: improve report criteria

* refactor: improve naming

* refactor: improve code structure & tests

* refactor: tweaks

* feat: improve markdown output

* refactor: custom tree comparer

* refactor: clean up code

* refactor: update JSON structure

* refactor: use "initial update count" naming

* chore: fix lint

* chore: improve tests

* refactor: tweaks

* docs: update

* refactor: self code review

* docs: tweaks

* chore: add changeset

* refactor: final tweaks

---------

Co-authored-by: Guven Karanfil <guven.karanfil@smartface.io>
Co-authored-by: Maciej Jastrzebski <mdjastrzebski@gmail.com>

.changeset/hot-ties-double.md

@@ -0,0 +1,8 @@
+---
+'reassure': minor
+'@callstack/reassure-compare': minor
+'@callstack/reassure-measure': minor
+'@callstack/reassure-cli': minor
+---
+
+Detect render issues (initial render updates, redundant renders)

docusaurus/docs/methodology.md

@@ -32,10 +32,31 @@ You can refer to our example [GitHub workflow](https://github.com/callstack/reas
 <img src="https://github.com/callstack/reassure/raw/main/packages/reassure/docs/report-markdown.png" width="920px" alt="Markdown report" />
 </p>
 
+### Results categorization
+
 Looking at the example you can notice that test scenarios can be assigned to certain categories:
 
 - **Significant Changes To Duration** shows test scenario where the performance change is statistically significant and **should** be looked into as it marks a potential performance loss/improvement
-- **Meaningless Changes To Duration** shows test scenarios where the performance change is not stastatistically significant
+- **Meaningless Changes To Duration** shows test scenarios where the performance change is not statistically significant
 - **Changes To Count** shows test scenarios where the render or execution count did change
 - **Added Scenarios** shows test scenarios which do not exist in the baseline measurements
 - **Removed Scenarios** shows test scenarios which do not exist in the current measurements
+
+### Render issues (experimental)
+
+:::note
+
+This feature is experimental, and its behavior might change without increasing the major version of the package.
+
+:::
+
+Reassure analyses your components' render patterns during the initial test run (usually the warm-up run) to spot signs of potential issues.
+
+Currently, it's able to inform you about the following types of issues:
+
+- **Initial updates** informs about the number of updates (= re-renders) that happened immediately (synchronously) after the mount (= initial render). This is most likely caused by `useEffect` hook triggering immediate re-renders using set state. In the optimal case, the initial render should not cause immediate re-renders by itself. Next, renders should be caused by some external source: user action, system event, API call response, timers, etc.
+
+- **Redundant updates** inform about renders that resulted in the same host element tree as the previous render. After each update, this check inspects the host element structure and compares it to the previous structure. If they are the same, the subsequent render could be avoided as it resulted in no visible change to the user.
+  - This feature is available only on React Native at this time
+  - The host element tree comparison ignores references to event handlers. This means that differences in function props (e.g. event handlers) are ignored and only non-function props (e.g. strings, numbers, objects, arrays, etc.) are considered
+  - The report includes the indices of redundant renders for easier diagnose, 0th render is the mount (initial render), renders 1 and later are updates (re-renders)

packages/compare/src/compare.ts

@@ -151,6 +151,8 @@ function compareResults(current: MeasureResults, baseline: MeasureResults | null
     }
   });
 
+  const withCurrent = [...compared, ...added];
+
   const significant = compared
     .filter((item) => item.isDurationDiffSignificant)
     .sort((a, b) => b.durationDiff - a.durationDiff);
@@ -160,6 +162,9 @@ function compareResults(current: MeasureResults, baseline: MeasureResults | null
   const countChanged = compared
     .filter((item) => Math.abs(item.countDiff) > COUNT_DIFF_THRESHOLD)
     .sort((a, b) => b.countDiff - a.countDiff);
+  const renderIssues = withCurrent.filter(
+    (item) => item.current.issues?.initialUpdateCount || item.current.issues?.redundantUpdates?.length
+  );
   added.sort((a, b) => a.name.localeCompare(b.name));
   removed.sort((a, b) => a.name.localeCompare(b.name));
 
@@ -170,13 +175,14 @@ function compareResults(current: MeasureResults, baseline: MeasureResults | null
     significant,
     meaningless,
     countChanged,
+    renderIssues,
     added,
     removed,
   };
 }
 
 /**
- * Establish statisticial significance of render/execution duration difference build compare entry.
+ * Establish statistical significance of render/execution duration difference build compare entry.
  */
 function buildCompareEntry(name: string, current: MeasureEntry, baseline: MeasureEntry): CompareEntry {
   const durationDiff = current.meanDuration - baseline.meanDuration;

packages/compare/src/output/console.ts

@@ -12,18 +12,39 @@ export function printToConsole(data: CompareResult) {
 
   logger.log('\n➡️  Significant changes to duration');
   data.significant.forEach(printRegularLine);
+  if (data.significant.length === 0) {
+    logger.log(' - (none)');
+  }
 
   logger.log('\n➡️  Meaningless changes to duration');
   data.meaningless.forEach(printRegularLine);
+  if (data.meaningless.length === 0) {
+    logger.log(' - (none)');
+  }
 
-  logger.log('\n➡️  Count changes');
+  logger.log('\n➡️  Render count changes');
   data.countChanged.forEach(printRegularLine);
+  if (data.countChanged.length === 0) {
+    logger.log(' - (none)');
+  }
+
+  logger.log('\n➡️  Render issues');
+  data.renderIssues.forEach(printRenderIssuesLine);
+  if (data.renderIssues.length === 0) {
+    logger.log(' - (none)');
+  }
 
   logger.log('\n➡️  Added scenarios');
   data.added.forEach(printAddedLine);
+  if (data.added.length === 0) {
+    logger.log(' - (none)');
+  }
 
   logger.log('\n➡️  Removed scenarios');
   data.removed.forEach(printRemovedLine);
+  if (data.removed.length === 0) {
+    logger.log(' - (none)');
+  }
 
   logger.newLine();
 }
@@ -33,7 +54,28 @@ function printMetadata(name: string, metadata?: MeasureMetadata) {
 }
 
 function printRegularLine(entry: CompareEntry) {
-  logger.log(` - ${entry.name} [${entry.type}]: ${formatDurationChange(entry)} | ${formatCountChange(entry)}`);
+  logger.log(
+    ` - ${entry.name} [${entry.type}]: ${formatDurationChange(entry)} | ${formatCountChange(
+      entry.current.meanCount,
+      entry.baseline.meanCount
+    )}`
+  );
+}
+
+function printRenderIssuesLine(entry: CompareEntry | AddedEntry) {
+  const issues = [];
+
+  const initialUpdateCount = entry.current.issues?.initialUpdateCount;
+  if (initialUpdateCount) {
+    issues.push(formatInitialUpdates(initialUpdateCount));
+  }
+
+  const redundantUpdates = entry.current.issues?.redundantUpdates;
+  if (redundantUpdates?.length) {
+    issues.push(formatRedundantUpdates(redundantUpdates));
+  }
+
+  logger.log(` - ${entry.name}: ${issues.join(' | ')}`);
 }
 
 function printAddedLine(entry: AddedEntry) {
@@ -49,3 +91,17 @@ function printRemovedLine(entry: RemovedEntry) {
     ` - ${entry.name} [${entry.type}]: ${formatDuration(baseline.meanDuration)} | ${formatCount(baseline.meanCount)}`
   );
 }
+
+export function formatInitialUpdates(count: number) {
+  if (count === 0) return '-';
+  if (count === 1) return '1 initial update 🔴';
+
+  return `${count} initial updates 🔴`;
+}
+
+export function formatRedundantUpdates(redundantUpdates: number[]) {
+  if (redundantUpdates.length === 0) return '-';
+  if (redundantUpdates.length === 1) return `1 redundant update (${redundantUpdates.join(', ')}) 🔴`;
+
+  return `${redundantUpdates.length} redundant updates (${redundantUpdates.join(', ')}) 🔴`;
+}

packages/compare/src/output/markdown.ts

@@ -13,6 +13,7 @@ import {
 } from '../utils/format';
 import * as md from '../utils/markdown';
 import type { AddedEntry, CompareEntry, CompareResult, RemovedEntry, MeasureEntry, MeasureMetadata } from '../types';
+import { collapsibleSection } from '../utils/markdown';
 
 const tableHeader = ['Name', 'Type', 'Duration', 'Count'] as const;
 
@@ -66,9 +67,18 @@ function buildMarkdown(data: CompareResult) {
   result += `\n\n${md.heading3('Meaningless Changes To Duration')}`;
   result += `\n${buildSummaryTable(data.meaningless, true)}`;
   result += `\n${buildDetailsTable(data.meaningless)}`;
-  result += `\n\n${md.heading3('Changes To Count')}`;
-  result += `\n${buildSummaryTable(data.countChanged)}`;
-  result += `\n${buildDetailsTable(data.countChanged)}`;
+
+  // Skip renders counts if user only has function measurements
+  const allEntries = [...data.significant, ...data.meaningless, ...data.added, ...data.removed];
+  const hasRenderEntries = allEntries.some((e) => e.type === 'render');
+  if (hasRenderEntries) {
+    result += `\n\n${md.heading3('Render Count Changes')}`;
+    result += `\n${buildSummaryTable(data.countChanged)}`;
+    result += `\n${buildDetailsTable(data.countChanged)}`;
+    result += `\n\n${md.heading3('Render Issues')}`;
+    result += `\n${buildRedundantRendersTable(data.renderIssues)}`;
+  }
+
   result += `\n\n${md.heading3('Added Scenarios')}`;
   result += `\n${buildSummaryTable(data.added)}`;
   result += `\n${buildDetailsTable(data.added)}`;
@@ -108,22 +118,23 @@ function buildDetailsTable(entries: Array<CompareEntry | AddedEntry | RemovedEnt
 }
 
 function formatEntryDuration(entry: CompareEntry | AddedEntry | RemovedEntry) {
-  if ('baseline' in entry && 'current' in entry) return formatDurationChange(entry);
-  if ('baseline' in entry) return formatDuration(entry.baseline.meanDuration);
+  if (entry.baseline != null && 'current' in entry) return formatDurationChange(entry);
+  if (entry.baseline != null) return formatDuration(entry.baseline.meanDuration);
   if ('current' in entry) return formatDuration(entry.current.meanDuration);
   return '';
 }
 
 function formatEntryCount(entry: CompareEntry | AddedEntry | RemovedEntry) {
-  if ('baseline' in entry && 'current' in entry) return formatCountChange(entry);
-  if ('baseline' in entry) return formatCount(entry.baseline.meanCount);
+  if (entry.baseline != null && 'current' in entry)
+    return formatCountChange(entry.current.meanCount, entry.baseline.meanCount);
+  if (entry.baseline != null) return formatCount(entry.baseline.meanCount);
   if ('current' in entry) return formatCount(entry.current.meanCount);
   return '';
 }
 
 function buildDurationDetailsEntry(entry: CompareEntry | AddedEntry | RemovedEntry) {
   return [
-    'baseline' in entry ? buildDurationDetails('Baseline', entry.baseline) : '',
+    entry.baseline != null ? buildDurationDetails('Baseline', entry.baseline) : '',
     'current' in entry ? buildDurationDetails('Current', entry.current) : '',
   ]
     .filter(Boolean)
@@ -132,7 +143,7 @@ function buildDurationDetailsEntry(entry: CompareEntry | AddedEntry | RemovedEnt
 
 function buildCountDetailsEntry(entry: CompareEntry | AddedEntry | RemovedEntry) {
   return [
-    'baseline' in entry ? buildCountDetails('Baseline', entry.baseline) : '',
+    entry.baseline != null ? buildCountDetails('Baseline', entry.baseline) : '',
     'current' in entry ? buildCountDetails('Current', entry.current) : '',
   ]
     .filter(Boolean)
@@ -165,10 +176,33 @@ function buildCountDetails(title: string, entry: MeasureEntry) {
     .join(`<br/>`);
 }
 
-export function collapsibleSection(title: string, content: string) {
-  return `<details>\n<summary>${title}</summary>\n\n${content}\n</details>\n\n`;
+function formatRunDurations(values: number[]) {
+  return values.map((v) => (Number.isInteger(v) ? `${v}` : `${v.toFixed(1)}`)).join(' ');
 }
 
-export function formatRunDurations(values: number[]) {
-  return values.map((v) => (Number.isInteger(v) ? `${v}` : `${v.toFixed(1)}`)).join(' ');
+function buildRedundantRendersTable(entries: Array<CompareEntry | AddedEntry>) {
+  if (!entries.length) return md.italic('There are no entries');
+
+  const tableHeader = ['Name', 'Initial Updates', 'Redundant Updates'] as const;
+  const rows = entries.map((entry) => [
+    entry.name,
+    formatInitialUpdates(entry.current.issues?.initialUpdateCount),
+    formatRedundantUpdates(entry.current.issues?.redundantUpdates),
+  ]);
+
+  return markdownTable([tableHeader, ...rows]);
+}
+
+function formatInitialUpdates(count: number | undefined) {
+  if (count == null) return '?';
+  if (count === 0) return '-';
+
+  return `${count} 🔴`;
+}
+
+function formatRedundantUpdates(redundantUpdates: number[] | undefined) {
+  if (redundantUpdates == null) return '?';
+  if (redundantUpdates.length === 0) return '-';
+
+  return `${redundantUpdates.length} (${redundantUpdates.join(', ')}) 🔴`;
 }

packages/compare/src/type-schemas.ts

@@ -43,4 +43,11 @@ export const MeasureEntryScheme = z.object({
 
   /** Array of measured render/execution counts for each run. */
   counts: z.array(z.number()),
+
+  issues: z.optional(
+    z.object({
+      initialUpdateCount: z.number().optional(),
+      redundantUpdates: z.array(z.number()).optional(),
+    })
+  ),
 });

packages/compare/src/types.ts

@@ -34,6 +34,7 @@ export interface AddedEntry {
   name: string;
   type: MeasureType;
   current: MeasureEntry;
+  baseline?: undefined;
 }
 
 /**
@@ -56,6 +57,7 @@ export interface CompareResult {
   significant: CompareEntry[];
   meaningless: CompareEntry[];
   countChanged: CompareEntry[];
+  renderIssues: Array<CompareEntry | AddedEntry>;
   added: AddedEntry[];
   removed: RemovedEntry[];
   errors: string[];

packages/compare/src/utils/format.ts

@@ -32,15 +32,39 @@ export function formatDurationDiff(value: number): string {
   return '0 ms';
 }
 
-export function formatCount(value: number) {
+export function formatCount(value?: number) {
+  if (value == null) {
+    return '?';
+  }
+
   return Number.isInteger(value) ? `${value}` : `${value.toFixed(2)}`;
 }
-export function formatCountDiff(value: number): string {
-  if (value > 0) return `+${value}`;
-  if (value < 0) return `${value}`;
+
+export function formatCountDiff(current: number, baseline: number): string {
+  const diff = current - baseline;
+  if (diff > 0) return `+${diff}`;
+  if (diff < 0) return `${diff}`;
   return '±0';
 }
 
+export function formatCountChange(current?: number, baseline?: number): string {
+  let output = `${formatCount(baseline)} → ${formatCount(current)}`;
+
+  if (baseline != null && current != null && baseline !== current) {
+    const parts = [formatCountDiff(current, baseline)];
+
+    if (baseline > 0) {
+      const relativeDiff = (current - baseline) / baseline;
+      parts.push(formatPercentChange(relativeDiff));
+    }
+
+    output += ` (${parts.join(', ')})`;
+  }
+
+  output += ` ${getCountChangeSymbols(current, baseline)}`;
+  return output;
+}
+
 export function formatChange(value: number): string {
   if (value > 0) return `+${value}`;
   if (value < 0) return `${value}`;
@@ -76,25 +100,16 @@ function getDurationChangeSymbols(entry: CompareEntry) {
   return '';
 }
 
-export function formatCountChange(entry: CompareEntry) {
-  const { baseline, current } = entry;
-
-  let output = `${formatCount(baseline.meanCount)} → ${formatCount(current.meanCount)}`;
-
-  if (baseline.meanCount != current.meanCount) {
-    output += ` (${formatCountDiff(entry.countDiff)}, ${formatPercentChange(entry.relativeCountDiff)})`;
+function getCountChangeSymbols(current?: number, baseline?: number) {
+  if (current == null || baseline == null) {
+    return '';
   }
 
-  output += ` ${getCountChangeSymbols(entry)}`;
-
-  return output;
-}
-
-function getCountChangeSymbols(entry: CompareEntry) {
-  if (entry.countDiff > 1.5) return '🔴🔴';
-  if (entry.countDiff > 0.5) return '🔴';
-  if (entry.countDiff < -1.5) return '🟢🟢';
-  if (entry.countDiff < -0.5) return '🟢';
+  const diff = current - baseline;
+  if (diff > 1.5) return '🔴🔴';
+  if (diff > 0.5) return '🔴';
+  if (diff < -1.5) return '🟢🟢';
+  if (diff < -0.5) return '🟢';
 
   return '';
 }

packages/compare/src/utils/markdown.ts

@@ -17,3 +17,7 @@ export function bold(text: string) {
 export function italic(text: string) {
   return `*${text}*`;
 }
+
+export function collapsibleSection(title: string, content: string) {
+  return `<details>\n<summary>${title}</summary>\n\n${content}\n</details>\n\n`;
+}

packages/measure/package.json

@@ -36,7 +36,8 @@
   "homepage": "https://github.com/callstack/reassure#readme",
   "dependencies": {
     "@callstack/reassure-logger": "1.0.0-rc.4",
-    "mathjs": "^12.4.2"
+    "mathjs": "^12.4.2",
+    "pretty-format": "^29.7.0"
   },
   "devDependencies": {
     "@babel/core": "^7.24.5",