Merge pull request #134 from argos-ci/feat-threshold

feat: allow to specify a threshold to control diff sensitivity

__fixtures__/screenshots/penelope.png.argos.json

@@ -5,5 +5,6 @@
   "mediaType": "screen",
   "browser": { "name": "chromium", "version": "119.0.6045.9" },
   "automationLibrary": { "name": "playwright", "version": "1.39.0" },
-  "sdk": { "name": "@argos-ci/playwright", "version": "0.0.7" }
+  "sdk": { "name": "@argos-ci/playwright", "version": "0.0.7" },
+  "threshold": 0.2
 }

packages/cli/src/index.ts

@@ -84,6 +84,12 @@ program
       "Commit used as baseline for screenshot comparison",
     ).env("ARGOS_REFERENCE_COMMIT"),
   )
+  .addOption(
+    new Option(
+      "--threshold <number>",
+      "Sensitivity threshold between 0 and 1. The higher the threshold, the less sensitive the diff will be. Default to 0.5",
+    ).env("ARGOS_THRESHOLD"),
+  )
   .action(async (directory, options) => {
     const spinner = ora("Uploading screenshots").start();
     try {
@@ -104,6 +110,7 @@ program
         referenceBranch: options.referenceBranch,
         referenceCommit: options.referenceCommit,
         mode: options.mode,
+        threshold: options.threshold,
       });
       spinner.succeed(`Build created: ${result.build.url}`);
     } catch (error: any) {

packages/core/src/api-client.test.ts

@@ -55,12 +55,14 @@ describe("#createArgosApiClient", () => {
             name: "screenshot 1",
             metadata: null,
             pwTraceKey: null,
+            threshold: null,
           },
           {
             key: "456",
             name: "screenshot 2",
             metadata: null,
             pwTraceKey: null,
+            threshold: null,
           },
         ],
       });

packages/core/src/api-client.ts

@@ -48,6 +48,7 @@ export interface UpdateBuildInput {
     name: string;
     metadata: ScreenshotMetadata | null;
     pwTraceKey: string | null;
+    threshold: number | null;
   }[];
   parallel?: boolean | null;
   parallelTotal?: number | null;

packages/core/src/config.ts

@@ -28,6 +28,18 @@ const mustBeArgosToken = (value: any) => {
   }
 };
 
+convict.addFormat({
+  name: "float-percent",
+  validate: function (val) {
+    if (val !== 0 && (!val || val > 1 || val < 0)) {
+      throw new Error("Must be a float between 0 and 1, inclusive.");
+    }
+  },
+  coerce: function (val) {
+    return parseFloat(val);
+  },
+});
+
 const schema = {
   apiBaseUrl: {
     env: "ARGOS_API_BASE_URL",
@@ -138,6 +150,12 @@ const schema = {
     default: null,
     nullable: true,
   },
+  threshold: {
+    env: "ARGOS_THRESHOLD",
+    format: "float-percent",
+    default: null,
+    nullable: true,
+  },
 };
 
 export interface Config {
@@ -161,6 +179,7 @@ export interface Config {
   prHeadCommit: string | null;
   mode: "ci" | "monitoring" | null;
   ciProvider: string | null;
+  threshold: number | null;
 }
 
 const createConfig = () => {

packages/core/src/upload.test.ts

@@ -26,6 +26,7 @@ describe("#upload", () => {
           optimizedPath: expect.any(String),
           hash: expect.stringMatching(/^[A-Fa-f0-9]{64}$/),
           metadata: null,
+          threshold: null,
         },
         {
           name: "penelope.png",
@@ -56,6 +57,7 @@ describe("#upload", () => {
               width: 1024,
             },
           },
+          threshold: 0.2,
         },
         {
           name: "nested/alicia.jpg",
@@ -66,6 +68,7 @@ describe("#upload", () => {
           optimizedPath: expect.any(String),
           hash: expect.stringMatching(/^[A-Fa-f0-9]{64}$/),
           metadata: null,
+          threshold: null,
         },
       ],
     });

packages/core/src/upload.ts

@@ -50,6 +50,12 @@ export interface UploadParameters {
   referenceBranch?: string;
   /** Commit used as baseline for screenshot comparison */
   referenceCommit?: string;
+  /**
+   * Sensitivity threshold between 0 and 1.
+   * The higher the threshold, the less sensitive the diff will be.
+   * @default 0.5
+   */
+  threshold?: number;
 }
 
 async function getConfigFromOptions({
@@ -130,15 +136,23 @@ export async function upload(params: UploadParameters) {
         getPlaywrightTracePath(screenshot.path),
         optimizeScreenshot(screenshot.path),
       ]);
+
       const [hash, pwTraceHash] = await Promise.all([
         hashFile(optimizedPath),
         pwTracePath ? hashFile(pwTracePath) : null,
       ]);
+
+      const threshold = metadata?.threshold ?? null;
+      if (metadata) {
+        delete metadata.threshold;
+      }
+
       return {
         ...screenshot,
         hash,
         optimizedPath,
         metadata,
+        threshold,
         pwTrace:
           pwTracePath && pwTraceHash
             ? { path: pwTracePath, hash: pwTraceHash }
@@ -216,13 +230,15 @@ export async function upload(params: UploadParameters) {
 
   // Update build
   debug("Updating build");
+
   await apiClient.updateBuild({
     buildId: result.build.id,
     screenshots: screenshots.map((screenshot) => ({
       key: screenshot.hash,
       name: screenshot.name,
       metadata: screenshot.metadata,
       pwTraceKey: screenshot.pwTrace?.hash ?? null,
+      threshold: screenshot.threshold ?? config?.threshold ?? null,
     })),
     parallel: config.parallel,
     parallelTotal: config.parallelTotal,

packages/cypress/cypress/e2e/argosScreenshot.cy.ts

@@ -32,4 +32,11 @@ describe("argosScreenshot", () => {
       argosCSS: "body { background: blue; }",
     });
   });
+
+  it("supports threshold option", () => {
+    cy.visit("cypress/pages/index.html");
+    cy.argosScreenshot("threshold-option", {
+      threshold: 0.2,
+    });
+  });
 });

packages/cypress/cypress/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../tsconfig.json",
   "compilerOptions": {
-    "types": ["cypress", "@argos-ci/cypress/support"],
+    "types": ["cypress", "../dist/support"],
     "paths": {}
   }
 }

packages/cypress/src/support.ts

@@ -9,6 +9,7 @@ import {
   getMetadataPath,
   getScreenshotName,
   ScreenshotMetadata,
+  validateThreshold,
 } from "@argos-ci/util/browser";
 // @ts-ignore
 import { version } from "../package.json";
@@ -24,6 +25,12 @@ type ArgosScreenshotOptions = Partial<
    * Custom CSS evaluated during the screenshot process.
    */
   argosCSS?: string;
+  /**
+   * Sensitivity threshold between 0 and 1.
+   * The higher the threshold, the less sensitive the diff will be.
+   * @default 0.5
+   */
+  threshold?: number;
 };
 
 declare global {
@@ -33,13 +40,19 @@ declare global {
       /**
        * Stabilize the UI and takes a screenshot of the application under test.
        *
-       * @see https://on.cypress.io/screenshot
+       * @see https://argos-ci.com/docs/cypress#api-overview
        * @example
        *    cy.argosScreenshot("my-screenshot")
        *    cy.get(".post").argosScreenshot()
        */
       argosScreenshot: (
+        /**
+         * Name of the screenshot. Must be unique.
+         */
         name: string,
+        /**
+         * Options for the screenshot.
+         */
         options?: ArgosScreenshotOptions,
       ) => Chainable<null>;
     }
@@ -148,6 +161,11 @@ Cypress.Commands.add(
           },
         };
 
+        if (options.threshold !== undefined) {
+          validateThreshold(options.threshold);
+          metadata.threshold = options.threshold;
+        }
+
         cy.writeFile(getMetadataPath(ref.props.path), JSON.stringify(metadata));
       });
     }

packages/playwright/e2e.spec.ts

@@ -126,6 +126,14 @@ test.describe("#argosScreenshot", () => {
     });
   });
 
+  test.describe("with custom threshold", () => {
+    test("works", async ({ page }) => {
+      await argosScreenshot(page, "threshold-option", {
+        threshold: 0.2,
+      });
+    });
+  });
+
   test.describe("with cjs version", () => {
     test("works", async ({ page }) => {
       await argosScreenshotCjs(page, "full-page-cjs");

packages/playwright/src/index.ts

@@ -17,6 +17,7 @@ import {
   getMetadataPath,
   getScreenshotName,
   ScreenshotMetadata,
+  validateThreshold,
   writeMetadata,
 } from "@argos-ci/util";
 import { getAttachmentName } from "./attachment";
@@ -49,6 +50,12 @@ export type ArgosScreenshotOptions = {
    * @default true
    */
   disableHover?: boolean;
+  /**
+   * Sensitivity threshold between 0 and 1.
+   * The higher the threshold, the less sensitive the diff will be.
+   * @default 0.5
+   */
+  threshold?: number;
 } & LocatorOptions &
   ScreenshotOptions<LocatorScreenshotOptions> &
   ScreenshotOptions<PageScreenshotOptions>;
@@ -108,9 +115,25 @@ async function setup(page: Page, options: ArgosScreenshotOptions) {
   };
 }
 
+/**
+ * Stabilize the UI and takes a screenshot of the application under test.
+ *
+ * @example
+ *    argosScreenshot(page, "my-screenshot")
+ * @see https://argos-ci.com/docs/playwright#api-overview
+ */
 export async function argosScreenshot(
+  /**
+   * Playwright `page` object.
+   */
   page: Page,
+  /**
+   * Name of the screenshot. Must be unique.
+   */
   name: string,
+  /**
+   * Options for the screenshot.
+   */
   options: ArgosScreenshotOptions = {},
 ) {
   const { element, has, hasText, viewports, argosCSS, ...playwrightOptions } =
@@ -193,6 +216,10 @@ export async function argosScreenshot(
     );
 
     const metadata = await collectMetadata(testInfo);
+    if (options.threshold !== undefined) {
+      validateThreshold(options.threshold);
+      metadata.threshold = options.threshold;
+    }
     const nameInProject = testInfo?.project.name
       ? `${testInfo.project.name}/${name}`
       : name;

packages/puppeteer/src/index.ts

@@ -12,6 +12,7 @@ import {
   ScreenshotMetadata,
   getScreenshotName,
   readVersionFromPackage,
+  validateThreshold,
   writeMetadata,
 } from "@argos-ci/util";
 
@@ -28,6 +29,9 @@ async function injectArgos(page: Page) {
   await page.addScriptTag({ content: getGlobalScript() });
 }
 
+/**
+ * Accepts all Puppeteer screenshot options and adds Argos-specific options.
+ */
 export type ArgosScreenshotOptions = Omit<
   ScreenshotOptions,
   "encoding" | "type" | "omitBackground" | "path"
@@ -49,6 +53,12 @@ export type ArgosScreenshotOptions = Omit<
    * @default true
    */
   disableHover?: boolean;
+  /**
+   * Sensitivity threshold between 0 and 1.
+   * The higher the threshold, the less sensitive the diff will be.
+   * @default 0.5
+   */
+  threshold?: number;
 };
 
 async function getPuppeteerVersion(): Promise<string> {
@@ -121,16 +131,24 @@ async function setup(page: Page, options: ArgosScreenshotOptions) {
 }
 
 /**
- * @param page Puppeteer `page` object.
- * @param name The name of the screenshot or the full path to the screenshot.
- * @param options In addition to Puppeteer's `ScreenshotOptions`, you can pass:
- * @param options.element ElementHandle or string selector of the element to take a screenshot of.
- * @param options.viewports Viewports to take screenshots of.
- * @param options.argosCSS Custom CSS evaluated during the screenshot process.
+ * Stabilize the UI and takes a screenshot of the application under test.
+ *
+ * @example
+ *    argosScreenshot(page, "my-screenshot")
+ * @see https://argos-ci.com/docs/puppeteer#api-overview
  */
 export async function argosScreenshot(
+  /**
+   * Puppeteer `page` object.
+   */
   page: Page,
+  /**
+   * Name of the screenshot. Must be unique.
+   */
   name: string,
+  /**
+   * Options for the screenshot.
+   */
   options: ArgosScreenshotOptions = {},
 ) {
   const { element, viewports, argosCSS, ...puppeteerOptions } = options;
@@ -194,6 +212,11 @@ export async function argosScreenshot(
       },
     };
 
+    if (options?.threshold !== undefined) {
+      validateThreshold(options.threshold);
+      metadata.threshold = options.threshold;
+    }
+
     return metadata;
   }