TEMP: Only run NAPI benchmarks on CI

.github/workflows/benchmark.yml

@@ -28,93 +28,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  benchmark:
-    name: Benchmark
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # Run each benchmark in own job.
-        # Linter benchmark is by far the slowest, so split each fixture into own job.
-        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap]
-        include:
-          - component: linter
-            fixture: 0
-          - component: linter
-            fixture: 1
-          - component: linter
-            fixture: 2
-          - component: linter
-            fixture: 3
-          - component: linter
-            fixture: 4
-    steps:
-      - name: Checkout Branch
-        uses: actions/checkout@v4
-        with:
-          show-progress: false
-          persist-credentials: false
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Install Rust Toolchain
-        uses: ./.github/actions/rustup
-        with:
-          shared-key: 'benchmark'
-          save-cache: ${{ github.ref_name == 'main' }}
-
-      - name: Install codspeed
-        uses: taiki-e/install-action@v2
-        with:
-          tool: cargo-codspeed
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          registry-url: 'https://registry.npmjs.org'
-
-      - name: Start bench results interceptor server
-        working-directory: ./tasks/benchmark/codspeed
-        env:
-          COMPONENT: ${{ matrix.component }}
-          FIXTURE: ${{ matrix.fixture }}
-        run: |
-          corepack enable
-          pnpm install
-          node capture.mjs &
-
-      - name: Build benchmark
-        env:
-          RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
-        shell: bash
-        run: |
-          cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} --features codspeed
-          mkdir -p target/codspeed/oxc_benchmark/
-          mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
-          rm -rf target/codspeed/oxc_benchmark/*.d
-
-      - name: Run benchmark
-        uses: CodSpeedHQ/action@v2
-        timeout-minutes: 30
-        env:
-          FIXTURE: ${{ matrix.fixture }}
-        with:
-          run: cargo codspeed run
-          # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
-          token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
-          upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
-
-      - name: Upload bench data artefact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.component }}${{ matrix.fixture }}
-          path: ${{ env.DATA_DIR }}
-          if-no-files-found: error
-          retention-days: 1
-
   benchmark-napi:
     name: Benchmark NAPI parser
     runs-on: ubuntu-latest
-    if: false
     steps:
       - name: Checkout Branch
         uses: actions/checkout@v4
@@ -192,8 +108,7 @@ jobs:
 
   upload:
     name: Upload benchmarks
-    # needs: [benchmark, benchmark-napi]
-    needs: [benchmark]
+    needs: benchmark-napi
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Branch

tasks/benchmark/codspeed/cacheBenches.mjs

@@ -0,0 +1,13 @@
+/*
+ * Create cache of benchmark profile files.
+ */
+
+import {join as pathJoin, dirname} from 'path';
+import {fileURLToPath} from 'url';
+import {create as createTar} from 'tar';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const filesDir = process.env.DATA_DIR;
+const archivePath = pathJoin(__dirname, 'cachedBenches.tar.gz');
+await createTar({file: archivePath, gzip: true, cwd: filesDir}, ['./']);

tasks/benchmark/codspeed/upload.mjs

@@ -4,25 +4,30 @@
 
 import {createReadStream} from 'fs';
 import fs from 'fs/promises';
-import {join as pathJoin} from 'path';
+import {join as pathJoin, dirname} from 'path';
+import {fileURLToPath} from 'url';
 import {createHash} from 'crypto';
 import assert from 'assert';
-import {create as createTar} from 'tar';
+import {create as createTar, extract as extractTar} from 'tar';
 import axios from 'axios';
 
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
 const METADATA_SUFFIX = '_metadata.json',
     CODSPEED_UPLOAD_URL = 'https://api.codspeed.io/upload';
 
 const dataDir = process.env.DATA_DIR,
     token = process.env.CODSPEED_TOKEN;
 
 // Find profile files and first metadata file
-const profileFiles = [];
+const profileFiles = [],
+    components = new Set();
 let metadataPath;
 for (const filename of await fs.readdir(dataDir)) {
     const path = pathJoin(dataDir, filename);
     if (filename.endsWith(METADATA_SUFFIX)) {
         if (!metadataPath) metadataPath = path;
+        components.add(metadataPath.slice(0, -METADATA_SUFFIX.length));
     } else {
         const match = filename.match(/_(\d+)\.out$/);
         assert(match, `Unexpected file: ${filename}`);
@@ -32,6 +37,23 @@ for (const filename of await fs.readdir(dataDir)) {
     }
 }
 
+// Add cached results for benchmarks which weren't run
+const cacheZipPath = pathJoin(__dirname, 'cachedBenches.tar.gz'),
+    cacheDir = pathJoin(dataDir, 'cache');
+await fs.mkdir(cacheDir);
+await extractTar({file: cacheZipPath, cwd: cacheDir});
+
+for (const filename of await fs.readdir(cacheDir)) {
+    const match = filename.match(/^(.+)_(\d+)\.out$/);
+    assert(match, `Unexpected file in cache: ${filename}`);
+    const [, component, pid] = match;
+    if (components.has(component)) continue;
+
+    const outPath = pathJoin(dataDir, filename);
+    await fs.rename(pathJoin(cacheDir, filename), outPath);
+    profileFiles.push({pid: +pid, path: outPath});
+}
+
 // Move all `.out` files to one directory
 console.log('Combining profiles');