diff --git a/lighthouse-cli/test/fixtures/accessibility/accessibility_tester.html b/lighthouse-cli/test/fixtures/a11y/a11y_tester.html similarity index 92% rename from lighthouse-cli/test/fixtures/accessibility/accessibility_tester.html rename to lighthouse-cli/test/fixtures/a11y/a11y_tester.html index 8dea189a14a5..afa2429441cb 100644 --- a/lighthouse-cli/test/fixtures/accessibility/accessibility_tester.html +++ b/lighthouse-cli/test/fixtures/a11y/a11y_tester.html @@ -120,7 +120,7 @@
@@ -197,5 +197,13 @@
+ diff --git a/lighthouse-cli/test/smokehouse/a11y/a11y-config.js b/lighthouse-cli/test/smokehouse/a11y/a11y-config.js new file mode 100644 index 000000000000..363d13f5cdfc --- /dev/null +++ b/lighthouse-cli/test/smokehouse/a11y/a11y-config.js @@ -0,0 +1,18 @@ +/** + * @license Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Config file for running PWA smokehouse audits for axe. + */ +module.exports = { + extends: 'lighthouse:default', + settings: { + onlyCategories: [ + 'accessibility', + ], + }, +}; diff --git a/lighthouse-cli/test/smokehouse/a11y/expectations.js b/lighthouse-cli/test/smokehouse/a11y/expectations.js new file mode 100644 index 000000000000..81ecde776b57 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/a11y/expectations.js @@ -0,0 +1,242 @@ +/** + * @license Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Expected Lighthouse audit values for byte efficiency tests + */ +module.exports = [ + { + initialUrl: 'http://localhost:10200/a11y/a11y_tester.html', + url: 'http://localhost:10200/a11y/a11y_tester.html', + audits: { + 'accesskeys': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-allowed-attr': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-required-children': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-required-parent': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-roles': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-valid-attr-value': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'aria-valid-attr': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'button-name': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'bypass': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'color-contrast': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'definition-list': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'dlitem': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'document-title': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'duplicate-id': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'frame-title': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'html-has-lang': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'image-alt': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'input-image-alt': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'label': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'layout-table': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'link-name': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'list': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'listitem': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'meta-viewport': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'object-alt': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'tabindex': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'td-headers-attr': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + 'valid-lang': { + score: false, + details: { + items: { + length: 1, + }, + }, + }, + }, + }, +]; diff --git a/lighthouse-cli/test/smokehouse/a11y/run-tests.sh b/lighthouse-cli/test/smokehouse/a11y/run-tests.sh new file mode 100755 index 000000000000..f51c4c704949 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/a11y/run-tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +node lighthouse-cli/test/fixtures/static-server.js & + +sleep 0.5s + +config="lighthouse-cli/test/smokehouse/a11y/a11y-config.js" +expectations="lighthouse-cli/test/smokehouse/a11y/expectations.js" + +yarn smokehouse -- --config-path=$config --expectations-path=$expectations +exit_code=$? + +# kill test servers +kill $(jobs -p) + +exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/run-all-tests.sh b/lighthouse-cli/test/smokehouse/run-all-tests.sh new file mode 100644 index 000000000000..c4ed77028f5a --- /dev/null +++ b/lighthouse-cli/test/smokehouse/run-all-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +for d in "$DIR"/*/ ; do + bash "${d}run-tests.sh" + if [ $? -ne 0 ] + then + echo "Error: $d smoke test failed" + exit 1 + fi +done diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index 4c81f2a3679b..76431926fdcf 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -41,6 +41,7 @@ class Driver { this._devtoolsLog = new DevtoolsLog(/^(Page|Network)\./); this.online = true; this._domainEnabledCounts = new Map(); + this._isolatedExecutionContextId = undefined; /** * Used for monitoring network status events during gotoURL. @@ -216,12 +217,29 @@ class Driver { } /** - * Evaluate an expression in the context of the current page. + * Evaluate an expression in the context of the current page. If useIsolation is true, the expression + * will be evaluated in a content script that has access to the page's DOM but whose JavaScript state + * is completely separate. * Returns a promise that resolves on the expression's value. * @param {string} expression + * @param {{useIsolation: boolean}=} options * @return {!Promise<*>} */ - evaluateAsync(expression) { + evaluateAsync(expression, options = {}) { + const contextIdPromise = options.useIsolation ? + this._getOrCreateIsolatedContextId() : + Promise.resolve(undefined); + return contextIdPromise.then(contextId => this._evaluateInContext(expression, contextId)); + } + + /** + * Evaluate an expression in the given execution context; an undefined contextId implies the main + * page without isolation. + * @param {string} expression + * @param {number|undefined} contextId + * @return {!Promise<*>} + */ + _evaluateInContext(expression, contextId) { return new Promise((resolve, reject) => { // If this gets to 60s and it hasn't been resolved, reject the Promise. const asyncTimeout = setTimeout( @@ -229,7 +247,7 @@ class Driver { 60000 ); - this.sendCommand('Runtime.evaluate', { + const evaluationParams = { // We need to explicitly wrap the raw expression for several purposes: // 1. Ensure that the expression will be a native Promise and not a polyfill/non-Promise. // 2. Ensure that errors in the expression are captured by the Promise. @@ -247,7 +265,10 @@ class Driver { includeCommandLineAPI: true, awaitPromise: true, returnByValue: true, - }).then(result => { + contextId, + }; + + this.sendCommand('Runtime.evaluate', evaluationParams).then(result => { clearTimeout(asyncTimeout); const value = result.result.value; @@ -431,10 +452,12 @@ class Driver { let lastTimeout; let cancelled = false; + + const checkForQuietExpression = `(${checkTimeSinceLastLongTask.toString()})()`; function checkForQuiet(driver, resolve) { if (cancelled) return; - return driver.evaluateAsync(`(${checkTimeSinceLastLongTask.toString()})()`) + return driver.evaluateAsync(checkForQuietExpression) .then(timeSinceLongTask => { if (cancelled) return; @@ -595,6 +618,32 @@ class Driver { return finalUrl; } + /** + * Returns the cached isolated execution context ID or creates a new execution context for the main + * frame. The cached execution context is cleared on every gotoURL invocation, so a new one will + * always be created on the first call on a new page. + * @return {!Promise} + */ + _getOrCreateIsolatedContextId() { + if (typeof this._isolatedExecutionContextId === 'number') { + return Promise.resolve(this._isolatedExecutionContextId); + } + + return this.sendCommand('Page.getResourceTree') + .then(data => { + const mainFrameId = data.frameTree.frame.id; + return this.sendCommand('Page.createIsolatedWorld', { + frameId: mainFrameId, + worldName: 'lighthouse_isolated_context', + }); + }) + .then(data => this._isolatedExecutionContextId = data.executionContextId); + } + + _clearIsolatedContextId() { + this._isolatedExecutionContextId = undefined; + } + /** * Navigate to the given URL. Direct use of this method isn't advised: if * the current page is already at the given URL, navigation will not occur and @@ -623,6 +672,7 @@ class Driver { /* eslint-enable max-len */ return this._beginNetworkStatusMonitoring(url) + .then(_ => this._clearIsolatedContextId()) .then(_ => { // These can 'race' and that's OK. // We don't want to wait for Page.navigate's resolution, as it can now @@ -949,7 +999,7 @@ class Driver { const globalVarToPopulate = `window['__${funcName}StackTraces']`; const collectUsage = () => { return this.evaluateAsync( - `Promise.resolve(Array.from(${globalVarToPopulate}).map(item => JSON.parse(item)))`) + `Array.from(${globalVarToPopulate}).map(item => JSON.parse(item))`) .then(result => { if (!Array.isArray(result)) { throw new Error( diff --git a/lighthouse-core/gather/gatherers/accessibility.js b/lighthouse-core/gather/gatherers/accessibility.js index bbc09e7be9c4..913cc43ed2f8 100644 --- a/lighthouse-core/gather/gatherers/accessibility.js +++ b/lighthouse-core/gather/gatherers/accessibility.js @@ -96,7 +96,7 @@ class Accessibility extends Gatherer { return (${runA11yChecks.toString()}()); })()`; - return driver.evaluateAsync(expression).then(returnedValue => { + return driver.evaluateAsync(expression, {useIsolation: true}).then(returnedValue => { if (!returnedValue) { throw new Error('No axe-core results returned'); } diff --git a/lighthouse-core/test/gather/driver-test.js b/lighthouse-core/test/gather/driver-test.js index a5424f54390f..08a047b18d1c 100644 --- a/lighthouse-core/test/gather/driver-test.js +++ b/lighthouse-core/test/gather/driver-test.js @@ -58,6 +58,8 @@ connection.sendCommand = function(command, params) { return Promise.resolve({ nodeIds: params.selector === 'invalid' ? [] : [231], }); + case 'Runtime.evaluate': + return Promise.resolve({result: {value: 123}}); case 'Runtime.getProperties': return Promise.resolve({ result: params.objectId === 'invalid' ? [] : [{ @@ -69,6 +71,10 @@ connection.sendCommand = function(command, params) { name: 'novalue', }], }); + case 'Page.getResourceTree': + return Promise.resolve({frameTree: {frame: {id: 1}}}); + case 'Page.createIsolatedWorld': + return Promise.resolve({executionContextId: 1}); case 'Page.enable': case 'Tracing.start': case 'ServiceWorker.enable': @@ -131,6 +137,30 @@ describe('Browser Driver', () => { }); }); + it('evaluates an expression', () => { + return driverStub.evaluateAsync('120 + 3').then(value => { + assert.deepEqual(value, 123); + assert.equal(sendCommandParams[0].command, 'Runtime.evaluate'); + }); + }); + + it('evaluates an expression in isolation', () => { + return driverStub.evaluateAsync('120 + 3', {useIsolation: true}).then(value => { + assert.deepEqual(value, 123); + + assert.ok(sendCommandParams.length > 1, 'did not create execution context'); + const evaluateCommand = sendCommandParams.find(data => data.command === 'Runtime.evaluate'); + assert.equal(evaluateCommand.params.contextId, 1); + + // test repeat isolation evaluations + sendCommandParams = []; + return driverStub.evaluateAsync('120 + 3', {useIsolation: true}); + }).then(value => { + assert.deepEqual(value, 123); + assert.ok(sendCommandParams.length === 1, 'created unnecessary 2nd execution context'); + }); + }); + it('will track redirects through gotoURL load', () => { const delay = _ => new Promise(resolve => setTimeout(resolve)); diff --git a/package.json b/package.json index 468e782592fc..e2cb79576f09 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "build-viewer": "cd ./lighthouse-viewer && yarn build", "clean": "rimraf *.report.html *.report.dom.html *.report.json *.screenshots.html *.devtoolslog.json *.trace.json || true", "lint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .", - "smoke": "bash lighthouse-cli/test/smokehouse/offline-local/run-tests.sh && bash lighthouse-cli/test/smokehouse/perf/run-tests.sh && bash lighthouse-cli/test/smokehouse/dobetterweb/run-tests.sh && bash lighthouse-cli/test/smokehouse/byte-efficiency/run-tests.sh && bash lighthouse-cli/test/smokehouse/tricky-ttci/run-tests.sh && bash lighthouse-cli/test/smokehouse/seo/run-tests.sh && bash lighthouse-cli/test/smokehouse/redirects/run-tests.sh", + "smoke": "bash lighthouse-cli/test/smokehouse/run-all-tests.sh", "coverage": "istanbul cover -x \"**/third_party/**\" _mocha -- $(find */test -name '*-test.js') --timeout 10000 --reporter progress --report lcovonly", "start": "node ./lighthouse-cli/index.js", "test": "yarn lint --quiet && yarn unit && yarn closure && yarn test-cli-formatting",