core(user-flow): dry run flag

docs/user-flows.md

@@ -221,6 +221,7 @@ As this flow has multiple steps, the flow report summarizes everything and allow
 - Keep timespan recordings _short_ and focused on a single interaction sequence or page transition.
 - Use snapshot recordings when a substantial portion of the page content has changed.
 - Always wait for transitions and interactions to finish before ending a timespan. The puppeteer APIs `page.waitForSelector`/`page.waitForFunction`/`page.waitForResponse`/`page.waitForTimeout` are your friends here.
+- Quickly validate your user flow with: `api.startFlow(page, {dryRun: true})`.
 
 ## Related Reading
 

lighthouse-core/fraggle-rock/gather/dry-run.js

@@ -0,0 +1,59 @@
+/**
+ * @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Driver = require('./driver.js');
+const emulation = require('../../lib/emulation.js');
+const {initializeConfig} = require('../config/config.js');
+const {gotoURL} = require('../../gather/driver/navigation.js');
+
+/**
+ * @param {LH.Gatherer.GatherMode} gatherMode
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRunSetup(gatherMode, options) {
+  const {page, config: configJson, configContext} = options;
+  const {config} = await initializeConfig(configJson, {...configContext, gatherMode});
+  const driver = new Driver(page);
+  await driver.connect();
+  await emulation.emulate(driver.defaultSession, config.settings);
+  return {driver, config};
+}
+
+/**
+ * @param {Exclude<LH.Gatherer.GatherMode, 'navigation'>} gatherMode
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRun(gatherMode, options) {
+  const {driver} = await dryRunSetup(gatherMode, options);
+  await driver.disconnect();
+}
+
+/**
+ * @param {LH.NavigationRequestor} requestor
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRunNavigation(requestor, options) {
+  const {driver, config} = await dryRunSetup('navigation', options);
+  if (!config.navigations || !config.navigations.length) {
+    throw new Error('No navigations configured');
+  }
+
+  const navigation = config.navigations[0];
+  await gotoURL(driver, requestor, {
+    ...navigation,
+    maxWaitForFcp: config.settings.maxWaitForFcp,
+    maxWaitForLoad: config.settings.maxWaitForLoad,
+    waitUntil: navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
+  });
+
+  await driver.disconnect();
+}
+
+module.exports = {
+  dryRun,
+  dryRunNavigation,
+};

lighthouse-core/fraggle-rock/user-flow.js

@@ -9,11 +9,12 @@
 const {snapshotGather} = require('./gather/snapshot-runner.js');
 const {startTimespanGather} = require('./gather/timespan-runner.js');
 const {navigationGather} = require('./gather/navigation-runner.js');
+const {dryRun, dryRunNavigation} = require('./gather/dry-run.js');
 const Runner = require('../runner.js');
 const {initializeConfig} = require('./config/config.js');
 
 /** @typedef {Parameters<snapshotGather>[0]} FrOptions */
-/** @typedef {Omit<FrOptions, 'page'> & {name?: string}} UserFlowOptions */
+/** @typedef {Omit<FrOptions, 'page'> & {name?: string, dryRun?: boolean}} UserFlowOptions */
 /** @typedef {Omit<FrOptions, 'page'> & {stepName?: string}} StepOptions */
 /** @typedef {WeakMap<LH.UserFlow.GatherStep, LH.Gatherer.FRGatherResult['runnerOptions']>} GatherStepRunnerOptions */
 
@@ -24,9 +25,11 @@
    */
   constructor(page, options) {
     /** @type {FrOptions} */
-    this.options = {page, ...options};
+    this._options = {page, ...options};
     /** @type {string|undefined} */
-    this.name = options?.name;
+    this._name = options?.name;
+    /** @type {boolean} */
+    this._dryRun = options?.dryRun === true;
     /** @type {LH.UserFlow.GatherStep[]} */
     this._gatherSteps = [];
     /** @type {GatherStepRunnerOptions} */
@@ -62,7 +65,7 @@
    * @param {StepOptions=} stepOptions
    */
   _getNextNavigationOptions(stepOptions) {
-    const options = {...this.options, ...stepOptions};
+    const options = {...this._options, ...stepOptions};
     const configContext = {...options.configContext};
     const settingsOverrides = {...configContext.settingsOverrides};
 
@@ -111,8 +114,13 @@
     if (this.currentNavigation) throw new Error('Navigation already in progress');
 
     const options = this._getNextNavigationOptions(stepOptions);
-    const gatherResult = await navigationGather(requestor, options);
 
+    if (this._dryRun) {
+      await dryRunNavigation(requestor, options);
+      return;
+    }
+
+    const gatherResult = await navigationGather(requestor, options);
     this._addGatherStep(gatherResult, options);
   }
 
@@ -174,13 +182,19 @@
   async startTimespan(stepOptions) {
     if (this.currentTimespan) throw new Error('Timespan already in progress');
     if (this.currentNavigation) throw new Error('Navigation already in progress');
+    const options = {...this._options, ...stepOptions};
+
+    if (this._dryRun) {
+      await dryRun('timespan', options);
+      return;
+    }
 
-    const options = {...this.options, ...stepOptions};
     const timespan = await startTimespanGather(options);
     this.currentTimespan = {timespan, options};
   }
 
   async endTimespan() {
+    if (this._dryRun) return;
     if (!this.currentTimespan) throw new Error('No timespan in progress');
     if (this.currentNavigation) throw new Error('Navigation already in progress');
 
@@ -197,20 +211,30 @@
   async snapshot(stepOptions) {
     if (this.currentTimespan) throw new Error('Timespan already in progress');
     if (this.currentNavigation) throw new Error('Navigation already in progress');
+    const options = {...this._options, ...stepOptions};
 
-    const options = {...this.options, ...stepOptions};
-    const gatherResult = await snapshotGather(options);
+    if (this._dryRun) {
+      await dryRun('snapshot', options);
+      return;
+    }
 
+    const gatherResult = await snapshotGather(options);
     this._addGatherStep(gatherResult, options);
   }
 
   /**
    * @returns {Promise<LH.FlowResult>}
    */
   async createFlowResult() {
+    if (this._dryRun) {
+      return {
+        name: 'Dry run',
+        steps: [],
+      };
+    }
     return auditGatherSteps(this._gatherSteps, {
-      name: this.name,
-      config: this.options.config,
+      name: this._name,
+      config: this._options.config,
       gatherStepRunnerOptions: this._gatherStepRunnerOptions,
     });
   }
@@ -219,6 +243,7 @@
    * @return {Promise<string>}
    */
   async generateReport() {
+    if (this._dryRun) return '<h1>Cannot generate a flow report from a dry run</h1>';
     const flowResult = await this.createFlowResult();
     return generateFlowReportHtml(flowResult);
   }
@@ -229,7 +254,7 @@
   createArtifactsJson() {
     return {
       gatherSteps: this._gatherSteps,
-      name: this.name,
+      name: this._name,
     };
   }
 }

lighthouse-core/scripts/update-flow-fixtures.js

@@ -89,7 +89,7 @@ async function rebaselineArtifacts(artifactKeys) {
   });
 
   const page = await browser.newPage();
-  const flow = await api.startFlow(page, {config});
+  const flow = await api.startFlow(page, {config, dryRun: true});
 
   await flow.navigate('https://www.mikescerealshack.co');
 

lighthouse-core/test/fraggle-rock/scenarios/dry-run-test-pptr.js

@@ -0,0 +1,72 @@
+/**
+ * @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+import {jest} from '@jest/globals';
+
+import * as api from '../../../fraggle-rock/api.js';
+import {createTestState} from './pptr-test-utils.js';
+import {LH_ROOT} from '../../../../root.js';
+
+/* eslint-env jest */
+/* eslint-env browser */
+
+jest.setTimeout(90_000);
+
+describe('Dry Run', () => {
+  const state = createTestState();
+
+  state.installSetupAndTeardownHooks();
+
+  beforeAll(() => {
+    state.server.baseDir = `${LH_ROOT}/lighthouse-core/test/fixtures/fraggle-rock/snapshot-basic`;
+  });
+
+  it('should setup environment and perform flow actions', async () => {
+    const pageUrl = `${state.serverBaseUrl}/onclick.html`;
+
+    const flow = await api.startFlow(state.page, {dryRun: true});
+    await flow.navigate(pageUrl);
+
+    await flow.startNavigation();
+    await state.page.goto(pageUrl);
+    await flow.endNavigation();
+
+    await flow.startTimespan();
+    await state.page.click('button');
+    await flow.endTimespan();
+
+    await flow.snapshot();
+
+    // Ensure page navigated occurred and button was clicked.
+    const finalUrl = await state.page.url();
+    expect(finalUrl).toEqual(`${pageUrl}#done`);
+
+    // Ensure Lighthouse emulated a mobile device.
+    // Device scale factor override is not shared between sessions
+    // https://bugs.chromium.org/p/chromium/issues/detail?id=1337089
+    const deviceMetrics = await state.page.evaluate(() => ({
+      width: document.documentElement.clientWidth,
+      height: document.documentElement.clientHeight,
+      // deviceScaleFactor: window.devicePixelRatio,
+    }));
+    expect(deviceMetrics).toEqual({
+      height: 640,
+      width: 360,
+      // deviceScaleFactor: 2,
+    });
+
+    expect(flow.createArtifactsJson()).toEqual({
+      gatherSteps: [],
+    });
+    expect(await flow.createFlowResult()).toEqual({
+      name: 'Dry run',
+      steps: [],
+    });
+    expect(await flow.generateReport()).toEqual(
+      '<h1>Cannot generate a flow report from a dry run</h1>'
+    );
+  });
+});

lighthouse-core/test/fraggle-rock/user-flow-test.js

@@ -27,6 +27,8 @@ const navigationModule = {navigationGather: jest.fn()};
 jest.mock('../../fraggle-rock/gather/navigation-runner.js', () => navigationModule);
 const timespanModule = {startTimespanGather: jest.fn()};
 jest.mock('../../fraggle-rock/gather/timespan-runner.js', () => timespanModule);
+const dryRunModule = {dryRun: jest.fn(), dryRunNavigation: jest.fn()};
+jest.mock('../../fraggle-rock/gather/dry-run.js', () => dryRunModule);
 
 const mockRunner = mockRunnerModule();
 
@@ -38,6 +40,9 @@ describe('UserFlow', () => {
 
     mockRunner.reset();
 
+    dryRunModule.dryRun.mockReset();
+    dryRunModule.dryRunNavigation.mockReset();
+
     snapshotModule.snapshotGather.mockReset();
     snapshotModule.snapshotGather.mockResolvedValue({
       artifacts: {
@@ -162,6 +167,14 @@ describe('UserFlow', () => {
       expect(configContext).toEqual({settingsOverrides: {maxWaitForLoad: 1000}});
       expect(configContextExplicit).toEqual({skipAboutBlank: false});
     });
+
+    it('should perform a minimal navigation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.navigate('https://example.com/1');
+
+      expect(navigationModule.navigationGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRunNavigation).toHaveBeenCalled();
+    });
   });
 
   describe('.startNavigation()', () => {
@@ -238,13 +251,29 @@ describe('UserFlow', () => {
         {name: 'Timespan report (www.example.com/)'},
       ]);
     });
+
+    it('should setup emulation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.startTimespan();
+
+      expect(timespanModule.startTimespanGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).toHaveBeenCalled();
+    });
   });
 
   describe('.endTimespan()', () => {
     it('should throw if a timespan has not started', async () => {
       const flow = new UserFlow(mockPage.asPage());
       await expect(flow.endTimespan()).rejects.toBeTruthy();
     });
+
+    it('should do nothing in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.endTimespan();
+
+      expect(timespanModule.startTimespanGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).not.toHaveBeenCalled();
+    });
   });
 
   describe('.snapshot()', () => {
@@ -266,6 +295,14 @@ describe('UserFlow', () => {
         {name: 'Snapshot report (www.example.com/)'},
       ]);
     });
+
+    it('should setup emulation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.snapshot();
+
+      expect(snapshotModule.snapshotGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).toHaveBeenCalled();
+    });
   });
 
   describe('.getFlowResult', () => {
@@ -275,6 +312,16 @@ describe('UserFlow', () => {
       await expect(flowResultPromise).rejects.toThrow(/Need at least one step/);
     });
 
+    it('should return shell object after a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.snapshot();
+      const flowResult = await flow.createFlowResult();
+      await expect(flowResult).toEqual({
+        name: 'Dry run',
+        steps: [],
+      });
+    });
+
     it('should audit active gather steps', async () => {
       mockRunner.audit.mockImplementation(artifacts => ({
         lhr: {