From 8ce19054270c2145a1d1bf66530ab75ae0200d6a Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 18 Aug 2022 11:05:54 +0200 Subject: [PATCH 1/4] add intended failure stories --- .storybook/main.js | 6 ++++- package.json | 1 + stories/expected-failures/Page.stories.jsx | 27 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 stories/expected-failures/Page.stories.jsx diff --git a/.storybook/main.js b/.storybook/main.js index e472b171..c8c2874d 100644 --- a/.storybook/main.js +++ b/.storybook/main.js @@ -1,4 +1,4 @@ -const stories = [ +let stories = [ '../stories/docs/**/*.stories.mdx', // default title prefix { @@ -18,6 +18,10 @@ if (process.env.STRESS_TEST) { stories.push('../stories/stress-test/*.stories.@(js|jsx|ts|tsx)'); } +if(process.env.TEST_FAILURES) { + stories = ['../stories/expected-failures/*.stories.@(js|jsx|ts|tsx)']; +} + const addons = [ process.env.WITHOUT_DOCS ? { diff --git a/package.json b/package.json index afa4a121..9449ddb7 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "build-storybook": "build-storybook", "release": "yarn build && auto shipit", "test-storybook": "node bin/test-storybook.js", + "test-storybook:failures": "TEST_FAILURES=1 yarn test-storybook", "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"", diff --git a/stories/expected-failures/Page.stories.jsx b/stories/expected-failures/Page.stories.jsx new file mode 100644 index 00000000..82b9a308 --- /dev/null +++ b/stories/expected-failures/Page.stories.jsx @@ -0,0 +1,27 @@ +import React from 'react'; +import { within, userEvent } from '@storybook/testing-library'; + +import { Page } from '../pages/Page'; + +export default { + title: 'Stories with failures', + component: Page, +}; + +const Template = (args) => ; + +export const ComponentThrowsErrors = () => { + throw new Error('Component has a failure'); +} + +export const PlayFnThrowsErrors = Template.bind({}); +PlayFnThrowsErrors.play = () => { + throw new Error('Play function has a failure'); +}; + +export const PlayFnAssertionFails = Template.bind({}); +PlayFnAssertionFails.play = async ({ canvasElement }) => { + const canvas = within(canvasElement); + const unexistentButton = await canvas.getByRole('button', { name: /I do not exist/i }); + await userEvent.click(unexistentButton); +}; \ No newline at end of file From 686132c03ed79d1c4cb3c771869d9c944a59d139 Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 18 Aug 2022 14:57:32 +0200 Subject: [PATCH 2/4] rename story --- .../{Page.stories.jsx => Failure.stories.jsx} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename stories/expected-failures/{Page.stories.jsx => Failure.stories.jsx} (91%) diff --git a/stories/expected-failures/Page.stories.jsx b/stories/expected-failures/Failure.stories.jsx similarity index 91% rename from stories/expected-failures/Page.stories.jsx rename to stories/expected-failures/Failure.stories.jsx index 82b9a308..47532960 100644 --- a/stories/expected-failures/Page.stories.jsx +++ b/stories/expected-failures/Failure.stories.jsx @@ -11,7 +11,8 @@ export default { const Template = (args) => ; export const ComponentThrowsErrors = () => { - throw new Error('Component has a failure'); + // throw new Error('Component has a failure'); + return
Oi
} export const PlayFnThrowsErrors = Template.bind({}); From 1f732664bd360cb68dcd37a82ee8bc7a264adfbe Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 18 Aug 2022 14:58:12 +0200 Subject: [PATCH 3/4] add skip snapshots for internal example --- .storybook/test-runner.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts index 76d89258..c36b5b0d 100644 --- a/.storybook/test-runner.ts +++ b/.storybook/test-runner.ts @@ -4,6 +4,7 @@ import type { TestRunnerConfig } from '../dist/ts'; const snapshotsDir = process.env.SNAPSHOTS_DIR || '__snapshots__'; const customSnapshotsDir = `${process.cwd()}/${snapshotsDir}`; +const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true'; const config: TestRunnerConfig = { setup() { @@ -17,6 +18,10 @@ const config: TestRunnerConfig = { return; } + if (skipSnapshots) { + return; + } + // Visual snapshot tests const image = await page.screenshot({ fullPage: true }); expect(image).toMatchImageSnapshot({ From 2e94457c89bf918be83831107c2c8234d9e8615c Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Mon, 22 Aug 2022 13:50:23 +0200 Subject: [PATCH 4/4] add failure test to nightly check [skip-release] --- .github/workflows/nightly.yml | 105 ++++++++++++++++++++++++++++++-- .gitignore | 3 +- package.json | 3 +- src/util/getParsedCliOptions.ts | 8 +++ 4 files changed, 113 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d347e0cb..fa06bb32 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,7 @@ name: Nightly checks # runs every day at midnight on: schedule: - - cron: "0 0 * * *" + - cron: '0 0 * * *' workflow_dispatch: # To test fixes on push rather than wait for the scheduling push: @@ -11,7 +11,7 @@ on: - fix/nightly jobs: - test_storybook_prerelease: + assert_test_runner: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -37,7 +37,6 @@ jobs: - name: Fix local @storybook/csf version run: | yarn add @storybook/csf@0.0.2--canary.4566f4d.1 - - name: Run test runner uses: mathiasvr/command-output@v1 id: tests @@ -57,7 +56,7 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.19.0 with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID }}" + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' payload: | { "blocks": [ @@ -97,3 +96,101 @@ jobs: } env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + assert_test_runner_failures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Install dependencies + uses: bahmutov/npm-install@v1 + + - name: Get current version of Storybook + run: | + echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Upgrade to storybook@future + run: | + npx storybook@future upgrade --prerelease --yes + + # TODO: This should not be necessary once @storybook/csf is properly updated + - name: Fix local @storybook/csf version + run: | + yarn add @storybook/csf@0.0.2--canary.4566f4d.1 + + - name: Run test runner and expect failure + uses: mathiasvr/command-output@v1 + with: + run: | + yarn build + yarn test-storybook:ci-failures + + - name: Process test results + if: ${{ always() }} + id: tests + uses: sergeysova/jq-action@v2 + with: + cmd: 'jq .numPassedTests test-results.json -r' + + - name: Set failure check to env + if: ${{ always() }} + run: | + echo "FAILED=${{ steps.tests.outputs.value > 0 }}" >> $GITHUB_ENV + + - name: Get prerelease version of Storybook + if: ${{ always() && env.FAILED == 'true' }} + run: | + echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Report if any test passes + if: ${{ always() && env.FAILED == 'true' }} + id: slack + uses: slackapi/slack-github-action@v1.19.0 + with: + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:", + "emoji": true + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}" + }, + { + "type": "mrkdwn", + "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}" + } + ], + "accessory": { + "type": "button", + "text": { + "type": "plain_text", + "text": "View failure", + "emoji": true + }, + "value": "view_failure", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "action_id": "button-action" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore index 46710295..86ff7083 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ build-storybook.log stories/atoms/StressTest.stories.js yarn-error.log .nyc_output -coverage \ No newline at end of file +coverage +test-results.json \ No newline at end of file diff --git a/package.json b/package.json index 9449ddb7..2a9c5279 100644 --- a/package.json +++ b/package.json @@ -42,10 +42,11 @@ "build-storybook": "build-storybook", "release": "yarn build && auto shipit", "test-storybook": "node bin/test-storybook.js", - "test-storybook:failures": "TEST_FAILURES=1 yarn test-storybook", + "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json", "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"", + "test-storybook:ci-failures": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"TEST_FAILURES=1 yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:failures\"", "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"", "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"", "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js" diff --git a/src/util/getParsedCliOptions.ts b/src/util/getParsedCliOptions.ts index 04013486..d0862bf5 100644 --- a/src/util/getParsedCliOptions.ts +++ b/src/util/getParsedCliOptions.ts @@ -44,6 +44,14 @@ export const getParsedCliOptions = () => { '-u, --updateSnapshot', 'Use this flag to re-record every snapshot that fails during this test run' ) + .option( + '--json', + 'Prints the test results in JSON. This mode will send all other test output and user messages to stderr.' + ) + .option( + '--outputFile', + 'Write test results to a file when the --json option is also specified.' + ) .option( '--coverage', 'Indicates that test coverage information should be collected and reported in the output'