core(gather): new computed artifact, js-bundles

.eslintignore

@@ -4,6 +4,8 @@
 
 **/node_modules/**
 **/third_party/**
+**/generated/**
+**/source-maps/**
 
 /dist/**
 

.travis.yml

@@ -29,6 +29,8 @@ before_script:
   - google-chrome-stable --version
   - yarn build-all
 script:
+  # Fail if any changes were written to source files (ex, from: build/build-cdt-lib.js).
+  - git diff --exit-code
   - yarn bundlesize
   - yarn diff:sample-json
   - yarn lint

build/build-cdt-lib.js

@@ -0,0 +1,137 @@
+/**
+ * @license Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const ts = require('typescript');
+
+const outDir = `${__dirname}/../lighthouse-core/lib/cdt/generated`;
+const files = {
+  'node_modules/chrome-devtools-frontend/front_end/sdk/SourceMap.js': 'SourceMap.js',
+};
+
+console.log('making modifications ...');
+
+for (const [inFilename, outFilename] of Object.entries(files)) {
+  const code = fs.readFileSync(inFilename, 'utf-8');
+  const codeWithoutLegacyExports = code.substr(0, code.indexOf('/* Legacy exported object'));
+  const codeTranspiledToCommonJS = ts.transpileModule(codeWithoutLegacyExports, {
+    compilerOptions: {module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2019},
+  }).outputText;
+
+  const sourceFile = ts.createSourceFile('', codeTranspiledToCommonJS,
+    ts.ScriptTarget.ES2019, true, ts.ScriptKind.JS);
+  const simplePrinter = ts.createPrinter({newLine: ts.NewLineKind.LineFeed});
+
+  const classesToRemove = [
+    // Not needed.
+    'EditResult',
+    // Not needed.
+    'WasmSourceMap',
+  ];
+  const methodsToRemove = [
+    // Not needed.
+    'load',
+    // Not needed.
+    'sourceContentProvider',
+  ];
+  const expressionsToRemove = [
+    /* Original:
+
+    let url = Common.ParsedURL.completeURL(this._baseURL, href) || href;
+    const source = sourceMap.sourcesContent && sourceMap.sourcesContent[i];
+    if (url === this._compiledURL && source) {
+      url += Common.UIString('? [sm]')
+    }
+    this._sourceInfos.set(url, new TextSourceMap.SourceInfo(source, null));
+    sourcesList.push(url);
+    ----
+    If a source file is the same as the compiled url and there is a sourcesContent,
+    then `entry.sourceURL` (what is returned from .mappings) will have `? [sm]` appended.
+    This is useful in DevTools - to show that a sources panel tab not a real network resource -
+    but for us it is not wanted. The sizing function uses `entry.sourceURL` to index the byte
+    counts, and is further used in the details to specify a file within a source map.
+    */
+    `url += Common.UIString('? [sm]')`,
+    // Delete all the export statements.
+    'exports.',
+  ];
+  // Complicated expressions are hard detect with the TS lib, so instead work with the raw code.
+  const rawCodeToReplace = {
+    // Similar to the reason for removing `url += Common.UIString('? [sm]')`.
+    // The entries in `.mappings` should not have their url property modified.
+    'Common.ParsedURL.completeURL(this._baseURL, href)': `''`,
+  };
+
+  // Verify that all the above code is present.
+  const codeFragments = [
+    ...classesToRemove,
+    ...methodsToRemove,
+    ...expressionsToRemove,
+    ...Object.values(rawCodeToReplace),
+  ];
+  for (const codeFragment of codeFragments) {
+    if (!codeTranspiledToCommonJS.includes(codeFragment)) {
+      throw new Error(`did not find expected code fragment: ${codeFragment}`);
+    }
+  }
+
+  const printer = ts.createPrinter({newLine: ts.NewLineKind.LineFeed}, {
+    substituteNode(hint, node) {
+      let removeNode = false;
+
+      if (ts.isMethodDeclaration(node) &&
+          // Make typescript happy.
+          !ts.isComputedPropertyName(node.name) &&
+          methodsToRemove.includes(node.name.text)) {
+        removeNode = true;
+      }
+
+      if (ts.isClassDeclaration(node) && node.name && classesToRemove.includes(node.name.text)) {
+        removeNode = true;
+      }
+
+      if (ts.isExpressionStatement(node)) {
+        const asString = simplePrinter.printNode(ts.EmitHint.Unspecified, node, sourceFile);
+        if (classesToRemove.some(className => asString.includes(className))) {
+          removeNode = true;
+        }
+        if (expressionsToRemove.some(className => asString.includes(className))) {
+          removeNode = true;
+        }
+      }
+
+      if (removeNode) {
+        return ts.createNode(ts.SyntaxKind.Unknown);
+      }
+
+      return node;
+    },
+  });
+
+  let sourceFilePrinted = '';
+  sourceFile.forEachChild(node => {
+    sourceFilePrinted += printer.printNode(ts.EmitHint.Unspecified, node, sourceFile) + '\n';
+  });
+
+  for (const [code, replacement] of Object.entries(rawCodeToReplace)) {
+    sourceFilePrinted = sourceFilePrinted.replace(code, replacement);
+    if (sourceFilePrinted.includes(code)) {
+      throw new Error(`expected only one occurrence for: ${code}`);
+    }
+  }
+
+  const modifiedFile = [
+    '// @ts-nocheck\n',
+    '// generated by yarn build-cdt-lib\n',
+    'const Common = require(\'../Common.js\')\n',
+    sourceFilePrinted,
+    'module.exports = TextSourceMap',
+  ].join('');
+  fs.writeFileSync(`${outDir}/${outFilename}`, modifiedFile);
+}

lighthouse-core/computed/js-bundles.js

@@ -0,0 +1,122 @@
+/**
+ * @license Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const log = require('lighthouse-logger');
+const makeComputedArtifact = require('./computed-artifact.js');
+const SDK = require('../lib/cdt/SDK.js');
+
+/**
+ * Calculate the number of bytes contributed by each source file
+ * @param {LH.Artifacts.Bundle['map']} map
+ * @param {string} content
+ * @return {LH.Artifacts.Bundle['sizes']}
+ */
+function computeGeneratedFileSizes(map, content) {
+  const lines = content.split('\n');
+  /** @type {Record<string, number>} */
+  const files = {};
+  const totalBytes = content.length;
+  let unmappedBytes = totalBytes;
+
+  // If the map + contents don't line up, return a result that
+  // denotes nothing is mapped.
+  const failureResult = {files: {}, unmappedBytes, totalBytes};
+
+  // @ts-ignore: This function is added in SDK.js. This will eventually be added to CDT.
+  map.computeLastGeneratedColumns();
+
+  for (const mapping of map.mappings()) {
+    const source = mapping.sourceURL;
+    const lineNum = mapping.lineNumber;
+    const colNum = mapping.columnNumber;
+    // @ts-ignore: `lastColumnNumber` is not on types yet. This will eventually be added to CDT.
+    const lastColNum = /** @type {number=} */ (mapping.lastColumnNumber);
+
+    // Webpack sometimes emits null mappings.
+    // https://github.com/mozilla/source-map/pull/303
+    if (!source) continue;
+
+    // Lines and columns are zero-based indices. Visually, lines are shown as a 1-based index.
+
+    const line = lines[lineNum];
+    if (line === null) {
+      log.error('JSBundles', `${map.url()} mapping for line out of bounds: ${lineNum + 1}`);
+      return failureResult;
+    }
+
+    if (colNum > line.length) {
+      // eslint-disable-next-line max-len
+      log.error('JSBundles', `${map.url()} mapping for column out of bounds: ${lineNum + 1}:${colNum}`);
+      return failureResult;
+    }
+
+    let mappingLength = 0;
+    if (lastColNum !== undefined) {
+      if (lastColNum > line.length) {
+        // eslint-disable-next-line max-len
+        log.error('JSBundles', `${map.url()} mapping for last column out of bounds: ${lineNum + 1}:${lastColNum}`);
+        return failureResult;
+      }
+      mappingLength = lastColNum - colNum;
+    } else {
+      // Add +1 to account for the newline.
+      mappingLength = line.length - colNum + 1;
+    }
+    files[source] = (files[source] || 0) + mappingLength;
+    unmappedBytes -= mappingLength;
+  }
+
+  return {
+    files,
+    unmappedBytes,
+    totalBytes,
+  };
+}
+
+class JSBundles {
+  /**
+   * @param {Pick<LH.Artifacts, 'SourceMaps'|'ScriptElements'>} artifacts
+   */
+  static async compute_(artifacts) {
+    const {SourceMaps, ScriptElements} = artifacts;
+
+    /** @type {LH.Artifacts.Bundle[]} */
+    const bundles = [];
+
+    // Collate map and script, compute file sizes.
+    for (const SourceMap of SourceMaps) {
+      if (!SourceMap.map) continue;
+      const {scriptUrl, map: rawMap} = SourceMap;
+
+      if (!rawMap.mappings) continue;
+
+      const scriptElement = ScriptElements.find(s => s.src === scriptUrl);
+      if (!scriptElement) continue;
+
+      const compiledUrl = SourceMap.scriptUrl || 'compiled.js';
+      const mapUrl = SourceMap.sourceMapUrl || 'compiled.js.map';
+      // @ts-ignore: CDT expects undefined properties to be explicit.
+      const rawMapForCdt = /** @type {any} */ (rawMap);
+      const map = new SDK.TextSourceMap(compiledUrl, mapUrl, rawMapForCdt);
+
+      const content = scriptElement && scriptElement.content ? scriptElement.content : '';
+      const sizes = computeGeneratedFileSizes(map, content);
+
+      const bundle = {
+        rawMap,
+        script: scriptElement,
+        map,
+        sizes,
+      };
+      bundles.push(bundle);
+    }
+
+    return bundles;
+  }
+}
+
+module.exports = makeComputedArtifact(JSBundles);

lighthouse-core/gather/gatherers/source-maps.js

@@ -127,6 +127,9 @@ class SourceMaps extends Gatherer {
       const map = isSourceMapADataUri ?
           this.parseSourceMapFromDataUrl(rawSourceMapUrl) :
           await this.fetchSourceMapInPage(driver, rawSourceMapUrl);
+      if (map.sections) {
+        map.sections = map.sections.filter(section => section.map);
+      }
       return {
         scriptUrl,
         sourceMapUrl,

lighthouse-core/lib/cdt/Common.js

@@ -0,0 +1,14 @@
+/**
+ * @license Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/** @fileoverview Needed for the generated/SourceMap.js file */
+
+const Common = {
+  console,
+};
+
+module.exports = Common;