diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8c00924c1a41df..5d70305ad73e5f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -28,19 +28,9 @@ concurrency: cancel-in-progress: true jobs: - benchmark: - name: Benchmark + build: + name: Build Benchmark runs-on: ubuntu-latest - strategy: - matrix: - # Run each benchmark in own job. - # Linter benchmark is by far the slowest, so split each fixture into own job. - component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap] - include: - - component: linter - fixture: 0 - - component: linter - fixture: 1 steps: - name: Checkout Branch uses: taiki-e/checkout-action@v1 @@ -51,69 +41,49 @@ jobs: shared-key: 'benchmark' save-cache: ${{ github.ref_name == 'main' }} - - name: Install codspeed - uses: taiki-e/install-action@v2 - with: - tool: cargo-codspeed - - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - registry-url: 'https://registry.npmjs.org' - - - name: Start bench results interceptor server - working-directory: ./tasks/benchmark/codspeed - env: - COMPONENT: ${{ matrix.component }} - FIXTURE: ${{ matrix.fixture }} - run: | - corepack enable - pnpm install - node capture.mjs & - - name: Build benchmark env: - RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed" + RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed" shell: bash run: | - cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} --features codspeed + cargo build --release -p oxc_benchmark --benches --features codspeed + rm target/release/deps/*.d mkdir -p target/codspeed/oxc_benchmark/ - mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark - rm -rf target/codspeed/oxc_benchmark/*.d - - - name: Run benchmark - uses: CodSpeedHQ/action@v2 - timeout-minutes: 30 - env: - FIXTURE: ${{ matrix.fixture }} - with: - run: cargo codspeed run - # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`) - token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }} - upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload - - - name: Upload bench data artefact + mv target/release/deps/lexer-* target/codspeed/oxc_benchmark/lexer + mv target/release/deps/parser-* target/codspeed/oxc_benchmark/parser + mv target/release/deps/transformer-* target/codspeed/oxc_benchmark/transformer + mv target/release/deps/semantic-* target/codspeed/oxc_benchmark/semantic + mv target/release/deps/minifier-* target/codspeed/oxc_benchmark/minifier + mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark/codegen_sourcemap + mv target/release/deps/sourcemap-* target/codspeed/oxc_benchmark/sourcemap + mv target/release/deps/linter-* target/codspeed/oxc_benchmark/linter + + - name: Upload Binary uses: actions/upload-artifact@v4 with: - name: ${{ matrix.component }}${{ matrix.fixture }} - path: ${{ env.DATA_DIR }} if-no-files-found: error - retention-days: 1 + name: benchmark + path: ./target/codspeed/oxc_benchmark - benchmark-napi: - name: Benchmark NAPI parser + benchmark: + needs: build + name: Benchmark runs-on: ubuntu-latest - if: false + strategy: + fail-fast: true + matrix: + # component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap, linter] + component: [lexer] + steps: - name: Checkout Branch uses: taiki-e/checkout-action@v1 - - name: Install Rust Toolchain - uses: ./.github/actions/rustup + - name: Download Artifacts + uses: actions/download-artifact@v4 with: - shared-key: 'benchmark_napi' - save-cache: ${{ github.ref_name == 'main' }} + name: benchmark + path: ./target/codspeed/oxc_benchmark - name: Install codspeed uses: taiki-e/install-action@v2 @@ -129,55 +99,107 @@ jobs: - name: Start bench results interceptor server working-directory: ./tasks/benchmark/codspeed env: - COMPONENT: 'parser_napi' + COMPONENT: ${{ matrix.component }} run: | + ls ./target/codspeed/oxc_benchmark corepack enable pnpm install node capture.mjs & - # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`, - # so instead run real benchmark without CodSpeed's instrumentation and save the results. - # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number - # of times required to take same amount of time as the real benchmark took. - # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96 - - name: Build NAPI Benchmark - working-directory: ./napi/parser - run: | - corepack enable - pnpm install - pnpm build - - - name: Run NAPI Benchmark - working-directory: ./napi/parser - run: node parse.bench.mjs - - - name: Build Rust benchmark - env: - RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed" - shell: bash - run: | - cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi - mkdir -p target/codspeed/oxc_benchmark/ - mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark - rm -rf target/codspeed/oxc_benchmark/*.d - - - name: Run Rust benchmark + - name: Run benchmark uses: CodSpeedHQ/action@v2 timeout-minutes: 30 with: - run: cargo codspeed run # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`) token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }} upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload + run: cargo codspeed run ${{ matrix.component }} - name: Upload bench data artefact uses: actions/upload-artifact@v4 with: - name: 'parser_napi' + name: ${{ matrix.component }} path: ${{ env.DATA_DIR }} if-no-files-found: error retention-days: 1 + # benchmark-napi: + # name: Benchmark NAPI parser + # runs-on: ubuntu-latest + # if: false + # steps: + # - name: Checkout Branch + # uses: taiki-e/checkout-action@v1 + + # - name: Install Rust Toolchain + # uses: ./.github/actions/rustup + # with: + # shared-key: 'benchmark_napi' + # save-cache: ${{ github.ref_name == 'main' }} + + # - name: Install codspeed + # uses: taiki-e/install-action@v2 + # with: + # tool: cargo-codspeed + + # - name: Install Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 20 + # registry-url: 'https://registry.npmjs.org' + + # - name: Start bench results interceptor server + # working-directory: ./tasks/benchmark/codspeed + # env: + # COMPONENT: 'parser_napi' + # run: | + # corepack enable + # pnpm install + # node capture.mjs & + + # # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`, + # # so instead run real benchmark without CodSpeed's instrumentation and save the results. + # # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number + # # of times required to take same amount of time as the real benchmark took. + # # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96 + # - name: Build NAPI Benchmark + # working-directory: ./napi/parser + # run: | + # corepack enable + # pnpm install + # pnpm build + + # - name: Run NAPI Benchmark + # working-directory: ./napi/parser + # run: node parse.bench.mjs + + # - name: Build Rust benchmark + # env: + # RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed" + # shell: bash + # run: | + # cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi + # mkdir -p target/codspeed/oxc_benchmark/ + # mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark + # rm -rf target/codspeed/oxc_benchmark/*.d + + # - name: Run Rust benchmark + # uses: CodSpeedHQ/action@v2 + # timeout-minutes: 30 + # with: + # run: cargo codspeed run + # # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`) + # token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }} + # upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload + + # - name: Upload bench data artefact + # uses: actions/upload-artifact@v4 + # with: + # name: 'parser_napi' + # path: ${{ env.DATA_DIR }} + # if-no-files-found: error + # retention-days: 1 + upload: name: Upload benchmarks # needs: [benchmark, benchmark-napi]