Benchmark #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| old: | |
| description: "Old version to compare (e.g. '6.5', 'latest', 'beta', 'nightly', or ZIP URL)" | |
| type: string | |
| default: "latest" | |
| required: true | |
| new: | |
| description: "New version to compare (e.g. '6.5', 'latest', 'beta', 'nightly', or ZIP URL)" | |
| type: string | |
| default: "nightly" | |
| required: true | |
| theme: | |
| description: "Theme to test" | |
| type: choice | |
| options: | |
| - twentytwentyone | |
| - twentytwentythree | |
| - twentytwentyfour | |
| default: "twentytwentyone" | |
| required: true | |
| locale: | |
| description: 'Locale to use' | |
| type: 'string' | |
| default: 'en_US' | |
| required: false | |
| jobs: | |
| benchmarks: | |
| name: "Benchmarks" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: GoogleChromeLabs/wpp-research | |
| path: wpp-research | |
| - name: Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version-file: ".nvmrc" | |
| cache: npm | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install wpp-research dependencies | |
| working-directory: wpp-research | |
| run: npm ci | |
| - name: Run tests | |
| run: | | |
| OLD_ARGS=() | |
| OLD_ARGS+=("--mount=./wp-content/mu-plugins:/wordpress/wp-content/mu-plugins") | |
| OLD_ARGS+=(--blueprint=./blueprint-old.json) | |
| cp blueprint.json blueprint-shared.json | |
| jq --argjson step "{ \"step\": \"installTheme\", \"themeZipFile\": { \"resource\": \"wordpress.org/themes\", \"slug\": \"$THEME\" }, \"options\": { \"activate\": true, \"importStarterContent\": false } }" '.steps += [$step]' blueprint-shared.json > blueprint-tmp.json | |
| mv blueprint-tmp.json blueprint-shared.json | |
| if [[ $WP_LOCALE != 'en_US' ]]; then | |
| jq --argjson step "{ \"step\": \"setSiteLanguage\", \"language\": \"$WP_LOCALE\" }" '.steps += [$step]' blueprint-shared.json > blueprint-tmp.json | |
| mv blueprint-tmp.json blueprint-shared.json | |
| fi; | |
| echo "::group::Prepare blueprint for old version" | |
| echo "Old version: $OLD_VERSION" | |
| cp blueprint-shared.json blueprint-old.json | |
| if [[ $OLD_VERSION != 'latest' ]]; then | |
| if [[ "$OLD_VERSION" == *".zip"* ]]; then | |
| jq --argjson step "{ \"step\": \"importWordPressFiles\", \"wordPressFilesZip\": { \"resource\": \"url\", \"url\": \"$OLD_VERSION\" } }" '.steps += [$step]' blueprint-old.json > blueprint-tmp.json | |
| mv blueprint-tmp.json blueprint-old.json | |
| else | |
| OLD_ARGS+=(--wp=$OLD_VERSION) | |
| fi | |
| fi | |
| echo "Final blueprint:" | |
| cat ./blueprint-old.json | jq '.' | |
| echo "::endgroup::" | |
| echo "::group::Start Playground server" | |
| ./node_modules/@wp-playground/cli/wp-playground.js server "${OLD_ARGS[@]}" & | |
| bash -c 'echo -n "Waiting port 9400 .."; for _ in `seq 1 400`; do echo -n .; sleep 0.25; curl --fail -LI http://localhost:9400 -o /dev/null -w '%{http_code}\n' -s > /dev/null && echo " Open." && exit ; done; echo " Timeout!" >&2; exit 1' | |
| # The port being open does not mean the blueprint has already finished. Wait a little bit longer. | |
| sleep 15 | |
| echo "::endgroup::" | |
| echo "::group::Run tests for old version" | |
| npm run --prefix wpp-research research --silent -- benchmark-web-vitals -u http://localhost:9400/ -n 100 -p -v -o csv > before_cwv.csv | |
| npm run --prefix wpp-research research --silent -- benchmark-server-timing -u http://localhost:9400/ -n 100 -p -v -o csv > before_st.csv | |
| # Stop Playground server again | |
| npx --yes kill-port 9400 | |
| echo "::endgroup::" | |
| NEW_ARGS=() | |
| NEW_ARGS+=("--mount=./wp-content/mu-plugins:/wordpress/wp-content/mu-plugins") | |
| NEW_ARGS+=(--blueprint=./blueprint-new.json) | |
| echo "::group::Prepare blueprint for new version" | |
| echo "New version: $NEW_VERSION" | |
| cp blueprint-shared.json blueprint-new.json | |
| if [[ "$NEW_VERSION" == *".zip"* ]]; then | |
| jq --argjson step "{ \"step\": \"importWordPressFiles\", \"wordPressFilesZip\": { \"resource\": \"url\", \"url\": \"$NEW_VERSION\" } }" '.steps += [$step]' blueprint-new.json > tee blueprint-tmp.json | |
| mv blueprint-tmp.json blueprint-new.json | |
| else | |
| NEW_ARGS+=(--wp=$NEW_VERSION) | |
| fi | |
| echo "Final blueprint:" | |
| cat ./blueprint-new.json | jq '.' | |
| echo "::endgroup::" | |
| echo "::group::Start Playground server" | |
| ./node_modules/@wp-playground/cli/wp-playground.js server "${NEW_ARGS[@]}" & | |
| bash -c 'echo -n "Waiting port 9400 .."; for _ in `seq 1 400`; do echo -n .; sleep 0.25; curl --fail -LI http://localhost:9400 -o /dev/null -w '%{http_code}\n' -s > /dev/null && echo " Open." && exit ; done; echo " Timeout!" >&2; exit 1' | |
| # The port being open does not mean the blueprint has already finished. Wait a little bit longer. | |
| sleep 15 | |
| echo "::endgroup::" | |
| echo "::group::Run tests for new version" | |
| npm run --prefix wpp-research research --silent -- benchmark-web-vitals -u http://localhost:9400/ -n 100 -p -v -o csv > after_cwv.csv | |
| npm run --prefix wpp-research research --silent -- benchmark-server-timing -u http://localhost:9400/ -n 100 -p -v -o csv > after_st.csv | |
| echo "::endgroup::" | |
| # Stop Playground server again | |
| npx --yes kill-port 9400 | |
| echo "::group::Summarizing results" | |
| node ./scripts/results.js "Server-Timing ($THEME)" before_cwv.csv after_cwv.csv > summary.md | |
| cat summary.md >> $GITHUB_STEP_SUMMARY | |
| node ./scripts/results.js "Web Vitals ($THEME)" before_st.csv after_st.csv > summary.md | |
| cat summary.md >> $GITHUB_STEP_SUMMARY | |
| echo "::endgroup::" | |
| env: | |
| OLD_VERSION: ${{ inputs.old == 'trunk' && 'master' || inputs.old }} | |
| NEW_VERSION: ${{ inputs.new == 'trunk' && 'master' || inputs.new }} | |
| THEME: ${{ inputs.theme }} | |
| WP_LOCALE: ${{ inputs.locale }} |