ci: improve benchmark build time by only building once (#3201)

relates #3200
oxc-project · May 8, 2024 · 5683ace · 5683ace
1 parent cba1e7f
commit 5683ace
Show file tree

Hide file tree

Showing 3 changed files with 134 additions and 103 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -14,7 +14,6 @@ on:
   push:
     branches:
       - main
-      - bench-*
     paths:
       - '**/*.rs'
       - 'napi/parser/**/*.js'
@@ -28,19 +27,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  benchmark:
-    name: Benchmark
+  build:
+    name: Build Benchmark
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # Run each benchmark in own job.
-        # Linter benchmark is by far the slowest, so split each fixture into own job.
-        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap]
-        include:
-          - component: linter
-            fixture: 0
-          - component: linter
-            fixture: 1
     steps:
       - name: Checkout Branch
         uses: taiki-e/checkout-action@v1
@@ -51,69 +40,62 @@ jobs:
           shared-key: 'benchmark'
           save-cache: ${{ github.ref_name == 'main' }}
 
-      - name: Install codspeed
-        uses: taiki-e/install-action@v2
-        with:
-          tool: cargo-codspeed
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          registry-url: 'https://registry.npmjs.org'
-
-      - name: Start bench results interceptor server
-        working-directory: ./tasks/benchmark/codspeed
-        env:
-          COMPONENT: ${{ matrix.component }}
-          FIXTURE: ${{ matrix.fixture }}
-        run: |
-          corepack enable
-          pnpm install
-          node capture.mjs &
-
       - name: Build benchmark
         env:
-          RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
+          RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed"
         shell: bash
         run: |
-          cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} --features codspeed
+          cargo build --release -p oxc_benchmark --benches --features codspeed
+          rm target/release/deps/*.d
           mkdir -p target/codspeed/oxc_benchmark/
-          mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
-          rm -rf target/codspeed/oxc_benchmark/*.d
-
-      - name: Run benchmark
-        uses: CodSpeedHQ/action@v2
-        timeout-minutes: 30
-        env:
-          FIXTURE: ${{ matrix.fixture }}
-        with:
-          run: cargo codspeed run
-          # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
-          token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
-          upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
-
-      - name: Upload bench data artefact
+          mv target/release/deps/lexer-* target/codspeed/oxc_benchmark/lexer
+          mv target/release/deps/parser-* target/codspeed/oxc_benchmark/parser
+          mv target/release/deps/transformer-* target/codspeed/oxc_benchmark/transformer
+          mv target/release/deps/semantic-* target/codspeed/oxc_benchmark/semantic
+          mv target/release/deps/minifier-* target/codspeed/oxc_benchmark/minifier
+          mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark/codegen_sourcemap
+          mv target/release/deps/sourcemap-* target/codspeed/oxc_benchmark/sourcemap
+          mv target/release/deps/linter-* target/codspeed/oxc_benchmark/linter
+
+      - name: Upload Binary
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.component }}${{ matrix.fixture }}
-          path: ${{ env.DATA_DIR }}
           if-no-files-found: error
-          retention-days: 1
+          name: benchmark
+          path: ./target/codspeed/oxc_benchmark
 
-  benchmark-napi:
-    name: Benchmark NAPI parser
+  benchmark:
+    needs: build
+    name: Benchmark
     runs-on: ubuntu-latest
-    if: false
+    strategy:
+      fail-fast: true
+      matrix:
+        component:
+          - lexer
+          - parser
+          - transformer
+          - semantic
+          - minifier
+          - codegen_sourcemap
+          - sourcemap
+          - linter
+
     steps:
       - name: Checkout Branch
         uses: taiki-e/checkout-action@v1
 
-      - name: Install Rust Toolchain
-        uses: ./.github/actions/rustup
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
         with:
-          shared-key: 'benchmark_napi'
-          save-cache: ${{ github.ref_name == 'main' }}
+          name: benchmark
+          path: ./target/codspeed/oxc_benchmark
+
+      - name: Fix permission loss
+        shell: bash
+        run: |
+          ls ./target/codspeed/oxc_benchmark
+          chmod +x ./target/codspeed/oxc_benchmark/${{ matrix.component }}
 
       - name: Install codspeed
         uses: taiki-e/install-action@v2
@@ -129,55 +111,106 @@ jobs:
       - name: Start bench results interceptor server
         working-directory: ./tasks/benchmark/codspeed
         env:
-          COMPONENT: 'parser_napi'
+          COMPONENT: ${{ matrix.component }}
         run: |
           corepack enable
           pnpm install
           node capture.mjs &
 
-      # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
-      # so instead run real benchmark without CodSpeed's instrumentation and save the results.
-      # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
-      # of times required to take same amount of time as the real benchmark took.
-      # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
-      - name: Build NAPI Benchmark
-        working-directory: ./napi/parser
-        run: |
-          corepack enable
-          pnpm install
-          pnpm build
-
-      - name: Run NAPI Benchmark
-        working-directory: ./napi/parser
-        run: node parse.bench.mjs
-
-      - name: Build Rust benchmark
-        env:
-          RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
-        shell: bash
-        run: |
-          cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
-          mkdir -p target/codspeed/oxc_benchmark/
-          mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
-          rm -rf target/codspeed/oxc_benchmark/*.d
-
-      - name: Run Rust benchmark
+      - name: Run benchmark
         uses: CodSpeedHQ/action@v2
         timeout-minutes: 30
         with:
-          run: cargo codspeed run
           # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
           token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
           upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
+          run: cargo codspeed run ${{ matrix.component }}
 
       - name: Upload bench data artefact
         uses: actions/upload-artifact@v4
         with:
-          name: 'parser_napi'
-          path: ${{ env.DATA_DIR }}
+          name: result-${{ matrix.component }}
+          path: ${{ env.DATA_DIR }} # # env.DEFAULT_PORT from capture.mjs
           if-no-files-found: error
           retention-days: 1
 
+  # benchmark-napi:
+    # name: Benchmark NAPI parser
+    # runs-on: ubuntu-latest
+    # if: false
+    # steps:
+      # - name: Checkout Branch
+        # uses: taiki-e/checkout-action@v1
+
+      # - name: Install Rust Toolchain
+        # uses: ./.github/actions/rustup
+        # with:
+          # shared-key: 'benchmark_napi'
+          # save-cache: ${{ github.ref_name == 'main' }}
+
+      # - name: Install codspeed
+        # uses: taiki-e/install-action@v2
+        # with:
+          # tool: cargo-codspeed
+
+      # - name: Install Node.js
+        # uses: actions/setup-node@v4
+        # with:
+          # node-version: 20
+          # registry-url: 'https://registry.npmjs.org'
+
+      # - name: Start bench results interceptor server
+        # working-directory: ./tasks/benchmark/codspeed
+        # env:
+          # COMPONENT: 'parser_napi'
+        # run: |
+          # corepack enable
+          # pnpm install
+          # node capture.mjs &
+
+      # # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
+      # # so instead run real benchmark without CodSpeed's instrumentation and save the results.
+      # # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
+      # # of times required to take same amount of time as the real benchmark took.
+      # # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
+      # - name: Build NAPI Benchmark
+        # working-directory: ./napi/parser
+        # run: |
+          # corepack enable
+          # pnpm install
+          # pnpm build
+
+      # - name: Run NAPI Benchmark
+        # working-directory: ./napi/parser
+        # run: node parse.bench.mjs
+
+      # - name: Build Rust benchmark
+        # env:
+          # RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
+        # shell: bash
+        # run: |
+          # cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
+          # mkdir -p target/codspeed/oxc_benchmark/
+          # mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
+          # rm -rf target/codspeed/oxc_benchmark/*.d
+
+      # - name: Run Rust benchmark
+        # uses: CodSpeedHQ/action@v2
+        # timeout-minutes: 30
+        # with:
+          # run: cargo codspeed run
+          # # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
+          # token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
+          # upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
+
+      # - name: Upload bench data artefact
+        # uses: actions/upload-artifact@v4
+        # with:
+          # name: 'parser_napi'
+          # path: ${{ env.DATA_DIR }}
+          # if-no-files-found: error
+          # retention-days: 1
+
   upload:
     name: Upload benchmarks
     # needs: [benchmark, benchmark-napi]
@@ -203,8 +236,9 @@ jobs:
       - name: Download artefacts
         uses: actions/download-artifact@v4
         with:
-          path: ${{ env.DATA_DIR }}
           merge-multiple: true
+          pattern: result-*
+          path: ${{ env.DATA_DIR }} # env.DEFAULT_PORT from create_temp_dir.mjs
 
       - name: Upload to Codspeed
         working-directory: ./tasks/benchmark/codspeed

diff --git a/tasks/benchmark/benches/linter.rs b/tasks/benchmark/benches/linter.rs
@@ -1,4 +1,4 @@
-use std::{env, path::PathBuf, rc::Rc};
+use std::{path::PathBuf, rc::Rc};
 
 use oxc_allocator::Allocator;
 use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion};
@@ -11,15 +11,7 @@ use oxc_tasks_common::TestFiles;
 fn bench_linter(criterion: &mut Criterion) {
     let mut group = criterion.benchmark_group("linter");
 
-    // If `FIXTURE` env is set, only run the specified benchmark. This is used for sharding in CI.
-    let test_files = if let Ok(fixture_index) = env::var("FIXTURE") {
-        let fixture_index = fixture_index.parse::<usize>().unwrap();
-        TestFiles::complicated_one(fixture_index)
-    } else {
-        TestFiles::complicated()
-    };
-
-    for file in test_files.files() {
+    for file in TestFiles::linter().files() {
         let source_type = SourceType::from_path(&file.file_name).unwrap();
         group.bench_with_input(
             BenchmarkId::from_parameter(&file.file_name),

diff --git a/tasks/common/src/test_file.rs b/tasks/common/src/test_file.rs
@@ -52,6 +52,11 @@ impl TestFiles {
         Self { files: vec![file] }
     }
 
+    pub fn linter() -> Self {
+        let files = Self::complicated_urls().into_iter().take(2).map(TestFile::new).collect();
+        Self { files }
+    }
+
     fn complicated_urls() -> [&'static str; 5] {
         [
             // TypeScript syntax (2.81MB)