Merge pull request #181 from storybookjs/next

Release 0.6.4

.gitignore

@@ -9,4 +9,5 @@ stories/atoms/StressTest.stories.js
 yarn-error.log
 .nyc_output
 coverage
-test-results.json
+test-results.json
+**/junit.xml

README.md

@@ -10,6 +10,7 @@ Storybook test runner turns all of your stories into executable tests.
   - [Jest compatibility](#jest-compatibility)
 - [CLI Options](#cli-options)
 - [Configuration](#configuration)
+- [Test reporters](#test-reporters)
 - [Running against a deployed Storybook](#running-against-a-deployed-storybook)
   - [Index.json mode](#indexjson-mode)
 - [Running in CI](#running-in-ci)
@@ -31,6 +32,7 @@ Storybook test runner turns all of your stories into executable tests.
   - [The test runner seems flaky and keeps timing out](#the-test-runner-seems-flaky-and-keeps-timing-out)
   - [The test runner reports "No tests found" running on a Windows CI](#the-test-runner-reports-no-tests-found-running-on-a-windows-ci)
   - [Adding the test runner to other CI environments](#adding-the-test-runner-to-other-ci-environments)
+  - [Merging test coverage results in wrong coverage](#merging-test-coverage-results-in-wrong-coverage)
 - [Future work](#future-work)
 
 ## Features
@@ -135,6 +137,7 @@ Usage: test-storybook [options]
 | `--eject`                       | Creates a local configuration file to override defaults of the test-runner <br/>`test-storybook --eject`                             |
 | `--json`                        | Prints the test results in JSON. This mode will send all other test output and user messages to stderr. <br/>`test-storybook --json` |
 | `--outputFile`                  | Write test results to a file when the --json option is also specified. <br/>`test-storybook --json --outputFile results.json`        |
+| `--junit`                       | Indicates that test information should be reported in a junit file. <br/>`test-storybook --**junit**`                                |
 
 ## Configuration
 
@@ -146,6 +149,12 @@ The test runner uses [jest-playwright](https://github.com/playwright-community/j
 
 > **NOTE:** The Jest options relate to the version of Jest that come in the test runner. You can check the [Jest compatibility table](#jest-compatibility) for reference.
 
+## Test reporters
+
+The test runner uses default Jest reporters, but you can add additional reporters by ejecting the configuration as explained above and overriding (or merging with) the `reporters` property.
+
+Additionally, if you pass `--junit` to `test-storybook`, the test runner will add `jest-junit` to the reporters list and generate a test report in a JUnit XML format. You can further configure the behavior of `jest-junit` by either setting specific `JEST_JUNIT_*` environment variables or by defining a `jest-junit` field in your package.json with the options you want, which will be respected when generating the report. You can look at all available options here: https://github.com/jest-community/jest-junit#configuration
+
 ## Running against a deployed Storybook
 
 By default, the test runner assumes that you're running it against a locally served Storybook on port 6006.
@@ -330,7 +339,7 @@ The test runner will report the results in the CLI and generate a `coverage/stor
 
 ![](.github/assets/coverage-result.png)
 
-If you want to generate reports with [different reporters](https://istanbul.js.org/docs/advanced/alternative-reporters/), you can use `nyc` and point it to the folder which contains the Storybook coverage file. `nyc` is a dependency of the test runner so you will already have it in your project.
+If you want to generate coverage reports with [different reporters](https://istanbul.js.org/docs/advanced/alternative-reporters/), you can use `nyc` and point it to the folder which contains the Storybook coverage file. `nyc` is a dependency of the test runner so you will already have it in your project.
 
 Here's an example generating an `lcov` report:
 
@@ -369,6 +378,8 @@ Here's an example on how to achieve that:
 }
 ```
 
+> NOTE: If your other tests (e.g. Jest) are using a different coverageProvider than `babel`, you will have issue when merging the coverage files. [More info here](#merging-test-coverage-results-in-wrong-coverage).
+
 ## Experimental test hook API
 
 The test runner renders a story and executes its [play function](https://storybook.js.org/docs/react/writing-stories/play-function) if one exists. However, there are certain behaviors that are not possible to achieve via the play function, which executes in the browser. For example, if you want the test runner to take visual snapshots for you, this is something that is possible via Playwright/Jest, but must be executed in Node.
@@ -535,6 +546,18 @@ env:
 
 As the test runner is based on playwright, depending on your CI setup you might need to use specific docker images or other configuration. In that case, you can refer to the [Playwright CI docs](https://playwright.dev/docs/ci) for more information.
 
+#### Merging test coverage results in wrong coverage
+
+After merging test coverage reports coming from the test runner with reports from other tools (e.g. Jest), if the end result is **not** what you expected. Here's why:
+
+The test runner uses `babel` as coverage provider, which behaves in a certain way when evaluating code coverage. If your other reports happen to use a different coverage provider than `babel`, such as `v8`, they will evaluate the coverage differently. Once merged, the results will likely be wrong.
+
+Example: in `v8`, import and export lines are counted as coverable pieces of code, however in `babel`, they are not. This impacts the percentage of coverage calculation.
+
+While the test runner does not provide `v8` as an option for coverage provider, it is recommended that you set your application's Jest config to use `coverageProvider: 'babel'` if you can, so that the reports line up as expected and get merged correctly.
+
+For more context, [here's some explanation](https://github.com/facebook/jest/issues/11188#issue-830796941) why `v8` is not a 1:1 replacement for Babel/Istanbul coverage.
+
 ## Future work
 
 Future plans involve adding support for the following features:

bin/test-storybook.js

@@ -245,6 +245,10 @@ const main = async () => {
     process.env.STORYBOOK_COLLECT_COVERAGE = 'true';
   }
 
+  if (runnerOptions.junit) {
+    process.env.STORYBOOK_JUNIT = 'true';
+  }
+
   if (process.env.REFERENCE_URL) {
     process.env.REFERENCE_URL = sanitizeURL(process.env.REFERENCE_URL);
   }

package.json

@@ -113,6 +113,7 @@
     "commander": "^9.0.0",
     "global": "^4.4.0",
     "jest": "^26.6.3 || ^27.0.0",
+    "jest-junit": "^14.0.0",
     "jest-playwright-preset": "^1.7.2",
     "jest-serializer-html": "^7.1.0",
     "jest-watch-typeahead": "^1.0.0",

src/config/jest-playwright.ts

@@ -5,11 +5,15 @@ export const getJestConfig = () => {
     STORYBOOK_STORIES_PATTERN,
     TEST_BROWSERS,
     STORYBOOK_COLLECT_COVERAGE,
+    STORYBOOK_JUNIT,
   } = process.env;
 
+  const reporters = STORYBOOK_JUNIT ? ['default', 'jest-junit'] : ['default'];
+
   let config = {
     rootDir: process.cwd(),
     roots: TEST_ROOT ? [TEST_ROOT] : undefined,
+    reporters,
     testMatch: STORYBOOK_STORIES_PATTERN && STORYBOOK_STORIES_PATTERN.split(';'),
     transform: {
       '^.+\\.stories\\.[jt]sx?$': '@storybook/test-runner/playwright/transform',

src/util/getCliOptions.ts

@@ -8,6 +8,7 @@ type CliOptions = {
     configDir?: string;
     eject?: boolean;
     coverage?: boolean;
+    junit?: boolean;
     browsers?: BrowserType | BrowserType[];
   };
   jestOptions: string[];
@@ -22,6 +23,7 @@ const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [
   'eject',
   'url',
   'coverage',
+  'junit',
 ];
 
 export const getCliOptions = () => {

src/util/getParsedCliOptions.ts

@@ -52,6 +52,7 @@ export const getParsedCliOptions = () => {
       '--coverage',
       'Indicates that test coverage information should be collected and reported in the output'
     )
+    .option('--junit', 'Indicates that test information should be reported in a junit file')
     .option(
       '--eject',
       'Creates a local configuration file to override defaults of the test-runner. Use it only if you want to have better control over the runner configurations'

yarn.lock

@@ -8622,6 +8622,16 @@ jest-jasmine2@^27.5.1:
     pretty-format "^27.5.1"
     throat "^6.0.1"
 
+jest-junit@^14.0.0:
+  version "14.0.0"
+  resolved "https://registry.yarnpkg.com/jest-junit/-/jest-junit-14.0.0.tgz#f69fc31bab32224848f443480c2c808fccb2a802"
+  integrity sha512-kALvBDegstTROfDGXH71UGD7k5g7593Y1wuX1wpWT+QTYcBbmtuGOA8UlAt56zo/B2eMIOcaOVEON3j0VXVa4g==
+  dependencies:
+    mkdirp "^1.0.4"
+    strip-ansi "^6.0.1"
+    uuid "^8.3.2"
+    xml "^1.0.1"
+
 jest-leak-detector@^27.5.1:
   version "27.5.1"
   resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-27.5.1.tgz#6ec9d54c3579dd6e3e66d70e3498adf80fde3fb8"
@@ -13540,6 +13550,11 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
+xml@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
+  integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"