tests(smoke): convert to single LH run per test

lighthouse-cli/test/smokehouse/frontends/back-compat-util.js

@@ -0,0 +1,42 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * COMPAT: update from the old TestDefn format (array of `expectations` per
+ * definition) to the new format (single `expectations` per def), doing our best
+ * generating some unique IDs.
+ * TODO: remove in Lighthouse 9+ once PubAds (and others?) are updated.
+ * @see https://github.com/GoogleChrome/lighthouse/issues/11950
+ * @param {ReadonlyArray<Smokehouse.BackCompatTestDefn>} allTestDefns
+ * @return {Array<Smokehouse.TestDfn>}
+ */
+function updateTestDefnFormat(allTestDefns) {
+  const expandedTestDefns = allTestDefns.map(testDefn => {
+    if (!Array.isArray(testDefn.expectations)) {
+      // New object to make tsc happy.
+      return {
+        ...testDefn,
+        expectations: testDefn.expectations,
+      };
+    } else {
+      // Create a testDefn per expectation.
+      return testDefn.expectations.map((expectations, index) => {
+        return {
+          ...testDefn,
+          id: `${testDefn.id}-${index}`,
+          expectations,
+        };
+      });
+    }
+  });
+
+  return expandedTestDefns.flat();
+}
+
+module.exports = {
+  updateTestDefnFormat,
+};

lighthouse-cli/test/smokehouse/frontends/lib.js

@@ -16,36 +16,31 @@
 const cloneDeep = require('lodash.clonedeep');
 const smokeTests = require('../test-definitions/core-tests.js');
 const {runSmokehouse} = require('../smokehouse.js');
+const {updateTestDefnFormat} = require('./back-compat-util.js');
 
 /**
  * @param {Smokehouse.SmokehouseLibOptions} options
  */
 async function smokehouse(options) {
   const {urlFilterRegex, skip, modify, ...smokehouseOptions} = options;
 
-  const clonedTests = cloneDeep(smokeTests);
-  const modifiedTests = clonedTests.map(test => {
-    const modifiedExpectations = [];
-    for (const expected of test.expectations) {
-      if (urlFilterRegex && !expected.lhr.requestedUrl.match(urlFilterRegex)) {
-        continue;
-      }
-
-      const reasonToSkip = skip && skip(test, expected);
-      if (reasonToSkip) {
-        console.log(`skipping ${expected.lhr.requestedUrl}: ${reasonToSkip}`);
-        continue;
-      }
-
-      modify && modify(test, expected);
-      modifiedExpectations.push(expected);
+  const updatedCoreTests = updateTestDefnFormat(smokeTests);
+  const clonedTests = cloneDeep(updatedCoreTests);
+  const modifiedTests = [];
+  for (const test of clonedTests) {
+    if (urlFilterRegex && !test.expectations.lhr.requestedUrl.match(urlFilterRegex)) {
+      continue;
     }
 
-    return {
-      ...test,
-      expectations: modifiedExpectations,
-    };
-  }).filter(test => test.expectations.length > 0);
+    const reasonToSkip = skip && skip(test, test.expectations);
+    if (reasonToSkip) {
+      console.log(`skipping ${test.expectations.lhr.requestedUrl}: ${reasonToSkip}`);
+      continue;
+    }
+
+    modify && modify(test, test.expectations);
+    modifiedTests.push(test);
+  }
 
   return runSmokehouse(modifiedTests, smokehouseOptions);
 }

lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -18,6 +18,7 @@ const cloneDeep = require('lodash.clonedeep');
 const yargs = require('yargs');
 const log = require('lighthouse-logger');
 const {runSmokehouse} = require('../smokehouse.js');
+const {updateTestDefnFormat} = require('./back-compat-util.js');
 
 const coreTestDefnsPath = path.join(__dirname, '../test-definitions/core-tests.js');
 
@@ -46,12 +47,17 @@ function getDefinitionsToRun(allTestDefns, requestedIds, {invertMatch}) {
     console.log('Running ALL smoketests. Equivalent to:');
     console.log(usage);
   } else {
-    smokes = allTestDefns.filter(test => invertMatch !== requestedIds.includes(test.id));
+    smokes = allTestDefns.filter(test => {
+      // Include all tests that *include* requested id.
+      // e.g. a requested 'pwa' will match 'pwa-airhorner', 'pwa-caltrain', etc
+      const isRequested = requestedIds.some(requestedId => test.id.includes(requestedId));
+      return invertMatch !== isRequested;
+    });
     console.log(`Running ONLY smoketests for: ${smokes.map(t => t.id).join(' ')}\n`);
   }
 
   const unmatchedIds = requestedIds.filter(requestedId => {
-    return !allTestDefns.map(t => t.id).includes(requestedId);
+    return !allTestDefns.map(t => t.id).some(id => id.includes(requestedId));
   });
   if (unmatchedIds.length) {
     console.log(log.redify(`Smoketests not found for: ${unmatchedIds.join(' ')}`));
@@ -80,23 +86,21 @@ function pruneExpectedNetworkRequests(testDefns, takeNetworkRequestUrls) {
 
   const clonedDefns = cloneDeep(testDefns);
   for (const {id, expectations, runSerially} of clonedDefns) {
-    for (const expectation of expectations) {
-      if (!runSerially && expectation.networkRequests) {
-        throw new Error(`'${id}' must be set to 'runSerially: true' to assert 'networkRequests'`);
-      }
+    if (!runSerially && expectations.networkRequests) {
+      throw new Error(`'${id}' must be set to 'runSerially: true' to assert 'networkRequests'`);
+    }
 
-      if (pruneNetworkRequests && expectation.networkRequests) {
-        // eslint-disable-next-line max-len
-        const msg = `'networkRequests' cannot be asserted in test '${id}'. They should only be asserted on tests from an in-process server`;
-        if (process.env.CI) {
-          // If we're in CI, we require any networkRequests expectations to be asserted.
-          throw new Error(msg);
-        }
-
-        console.warn(log.redify('Warning:'),
-            `${msg}. Pruning expectation: ${JSON.stringify(expectation.networkRequests)}`);
-        expectation.networkRequests = undefined;
+    if (pruneNetworkRequests && expectations.networkRequests) {
+      // eslint-disable-next-line max-len
+      const msg = `'networkRequests' cannot be asserted in test '${id}'. They should only be asserted on tests from an in-process server`;
+      if (process.env.CI) {
+        // If we're in CI, we require any networkRequests expectations to be asserted.
+        throw new Error(msg);
       }
+
+      console.warn(log.redify('Warning:'),
+          `${msg}. Pruning expectation: ${JSON.stringify(expectations.networkRequests)}`);
+      expectations.networkRequests = undefined;
     }
   }
 
@@ -166,7 +170,8 @@ async function begin() {
   let testDefnPath = argv.testsPath || coreTestDefnsPath;
   testDefnPath = path.resolve(process.cwd(), testDefnPath);
   const requestedTestIds = argv._;
-  const allTestDefns = require(testDefnPath);
+  const rawTestDefns = require(testDefnPath);
+  const allTestDefns = updateTestDefnFormat(rawTestDefns);
   const invertMatch = argv.invertMatch;
   const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds, {invertMatch});
 

lighthouse-cli/test/smokehouse/lib/concurrent-mapper.js

@@ -106,6 +106,21 @@ class ConcurrentMapper {
 
     return Promise.all(result);
   }
+
+  /**
+   * Runs `fn` concurrent to other operations in the pool, at a max of
+   * `concurrency` at a time across all callers on this instance. Default
+   * `concurrency` limit is `Infinity`.
+   * @template U
+   * @param {() => Promise<U>} fn
+   * @param {{concurrency: number}} [options]
+   * @return {Promise<U>}
+   */
+  async runInPool(fn, options = {concurrency: Infinity}) {
+    // Let pooledMap handle the pool management for the cost of boxing a fake `value`.
+    const result = await this.pooledMap([''], fn, options);
+    return result[0];
+  }
 }
 
 module.exports = ConcurrentMapper;

lighthouse-cli/test/smokehouse/report-assert.js

@@ -333,15 +333,6 @@ function reportAssertion(localConsole, assertion) {
   RegExp.prototype.toJSON = _toJSON;
 }
 
-/**
- * @param {number} count
- * @return {string}
- */
-function assertLogString(count) {
-  const plural = count === 1 ? '' : 's';
-  return `${count} assertion${plural}`;
-}
-
 /**
  * Log all the comparisons between actual and expected test results, then print
  * summary. Returns count of passed and failed tests.
@@ -371,17 +362,6 @@ function report(actual, expected, reportOptions = {}) {
     }
   });
 
-  const correctStr = assertLogString(correctCount);
-  const colorFn = correctCount === 0 ? log.redify : log.greenify;
-  localConsole.log(`  Correctly passed ${colorFn(correctStr)}`);
-
-  if (failedCount) {
-    const failedString = assertLogString(failedCount);
-    const failedColorFn = failedCount === 0 ? log.greenify : log.redify;
-    localConsole.log(`  Failed ${failedColorFn(failedString)}`);
-  }
-  localConsole.write('\n');
-
   return {
     passed: correctCount,
     failed: failedCount,