fail tests on CSP violation

README.md

@@ -49,6 +49,10 @@ interface EmberCLIContentSecurityPolicyConfig {
   // Controls if addon is enabled at all.
   enabled?: boolean,
 
+  // Controls if addon causes tests to fail if they violate configured CSP
+  // policy.
+  failTests: true,
+
   // A hash of options representing a Content Security Policy. The key must be
   // a CSP directive name as defined by spec. The value must be an array of
   // strings that form a CSP directive value, most likely a source list, e.g.
@@ -77,6 +81,7 @@ export default function(environment) {
   return {
     delivery: ['header'],
     enabled: true,
+    failTests: true,
     policy: {
       'default-src':  ["'none'"],
       'script-src':   ["'self'"],

index.js

@@ -1,6 +1,7 @@
 'use strict';
-let chalk = require('chalk');
 
+const chalk = require('chalk');
+const VersionChecker = require('ember-cli-version-checker');
 const {
   buildPolicyString,
   calculateConfig,
@@ -197,6 +198,31 @@ module.exports = {
       }
     }
 
+    if (type === 'test-body' && this._config.failTests) {
+      let qunitDependency = (new VersionChecker(this)).for('qunit');
+      if (qunitDependency.exists() && qunitDependency.lt('2.9.2')) {
+        this.ui.writeWarnLine(
+          'QUnit < 2.9.2 violates a strict Content Security Policy (CSP) by itself. ' +
+          `You are using QUnit ${qunitDependency.version}. You should upgrade the ` +
+          'dependency to avoid issues.\n' +
+          'Your project might not depend directly on QUnit but on ember-qunit. ' +
+          'In that case you might want to upgrade ember-qunit to > 4.4.1.'
+        );
+      }
+
+      return `
+        <script nonce="${STATIC_TEST_NONCE}">
+          document.addEventListener('securitypolicyviolation', function(event) {
+            throw new Error(
+              'Content-Security-Policy violation detected: ' +
+              'Violated directive: ' + event.violatedDirective + '. ' +
+              'Blocked URI: ' + event.blockedURI
+            );
+          });
+        </script>
+      `;
+    }
+
     if (type === 'test-body-footer') {
       // Add nonce to <script> tag inserted by ember-cli to assert that test file was loaded.
       existingContent.forEach((entry, index) => {

lib/utils.js

@@ -72,6 +72,7 @@ function calculateConfig(environment, ownConfig, runConfig, ui) {
   let config = {
     delivery: [DELIVERY_HEADER],
     enabled: true,
+    failTests: true,
     policy: {
       'default-src':  [CSP_NONE],
       'script-src':   [CSP_SELF],
@@ -113,6 +114,14 @@ function calculateConfig(environment, ownConfig, runConfig, ui) {
   // apply configuration
   Object.assign(config, ownConfig);
 
+  // test environment may not use express server and therefore requires CSP
+  // delivery through meta element. Otherwise tests may fail if run via
+  // `http://localhost:4200/tests` or `ember test --server` but not for
+  // `ember test`.
+  if (environment === 'test' && config.failTests && !config.delivery.includes('meta')) {
+    config.delivery.push('meta');
+  }
+
   return config;
 }
 

node-tests/e2e/test-support-test.js

@@ -0,0 +1,112 @@
+const expect = require('chai').expect;
+const AddonTestApp = require('ember-cli-addon-tests').AddonTestApp;
+const fs = require('fs-extra');
+const {
+  removeConfig,
+  setConfig
+} = require('../utils');
+
+describe('e2e: provides test support', function() {
+  this.timeout(300000);
+
+  let app;
+
+  // tests not altering package.json
+  describe('', function() {
+    before(async function() {
+      app = new AddonTestApp();
+
+      await app.create('default', { noFixtures: true });
+
+      // create a simple rendering tests that violates default CSP by using
+      // inline JavaScript
+      let testFolder = 'tests/integration/components';
+      let testFile = `${testFolder}/my-component-test.js`;
+      await fs.ensureDir(app.filePath(testFolder));
+      await fs.writeFile(
+        app.filePath(testFile),
+        `
+          import { module, test } from 'qunit';
+          import { setupRenderingTest } from 'ember-qunit';
+          import { render } from '@ember/test-helpers';
+          import hbs from 'htmlbars-inline-precompile';
+
+          module('Integration | Component | my-component', function(hooks) {
+            setupRenderingTest(hooks);
+
+            test('it renders', async function(assert) {
+              await render(hbs\`<div style='display: none;'></div>\`);
+              assert.ok(true);
+            });
+          });
+        `
+      );
+    });
+
+    afterEach(async function() {
+      await removeConfig(app);
+    });
+
+    it('causes tests to fail on CSP violations', async function() {
+      // runEmberCommand throws result object if command exists with non zero
+      // exit code
+      try {
+        await app.runEmberCommand('test');
+
+        // expect runEmberCommand to throw
+        expect(false).to.be.true;
+      } catch({ code }) {
+        expect(code).to.equal(1);
+      }
+    });
+
+    it('does not cause tests failures if addon is disabled', async function() {
+      await setConfig(app, {
+        enabled: false,
+      });
+      let { code } = await app.runEmberCommand('test');
+
+      expect(code).to.equal(0);
+    });
+
+    it('does not cause tests failures if `failTests` config option is `false`', async function() {
+      await setConfig(app, {
+        failTests: false,
+      });
+      let { code } = await app.runEmberCommand('test');
+
+      expect(code).to.equal(0);
+    });
+  });
+
+  // tests altering package.json
+  describe('', function() {
+    // @ToDo: VersionChecker reports qunit@2.9.2 even so app uses 2.7.1
+    it.skip('warns if QUnit version is to old', async function() {
+      let app = new AddonTestApp();
+
+      await app.create('default', {
+        noFixtures: true,
+        skipNpm: true,
+      });
+
+      app.editPackageJSON(pkg => {
+        // ember-qunit@4.0.0 depends on qunit@~2.7.1, which is less than required >= 2.9.2
+        pkg.devDependencies['ember-qunit'] = "4.0.0";
+      });
+
+      await app.run('npm', 'install');
+
+      try {
+        // throws cause QUnit 4.4.0 violates default CSP
+        await app.runEmberCommand('test');
+
+        // expect runEmberCommand to throw
+        expect(false).to.be.true;
+      } catch ({ output }) {
+        let warning = output.find((_) => _.startsWith('WARNING'));
+        expect(warning).to.include('QUnit < 2.9.2');
+      }
+    });
+  });
+});

package.json

@@ -23,7 +23,8 @@
   },
   "dependencies": {
     "body-parser": "^1.17.0",
-    "chalk": "^2.0.0"
+    "chalk": "^2.0.0",
+    "ember-cli-version-checker": "^3.1.3"
   },
   "devDependencies": {
     "@ember/optional-features": "^0.6.3",
@@ -45,7 +46,7 @@
     "ember-export-application-global": "^2.0.0",
     "ember-load-initializers": "^1.1.0",
     "ember-maybe-import-regenerator": "^0.1.6",
-    "ember-qunit": "^3.4.1",
+    "ember-qunit": "^4.4.1",
     "ember-resolver": "^5.0.1",
     "ember-source": "~3.7.0",
     "ember-source-channel-url": "^1.1.0",