feat(tasks): shard benchmarks in CI

.github/workflows/benchmark.yml

@@ -8,6 +8,7 @@ on:
       - '**/*.rs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
+      - 'tasks/benchmark/codspeed/*.mjs'
   push:
     branches:
       - main
@@ -16,6 +17,7 @@ on:
       - '**/*.rs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
+      - 'tasks/benchmark/codspeed/*.mjs'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -25,11 +27,14 @@ jobs:
   benchmark:
     name: Benchmark
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        component: [lexer, parser, transformer, semantic, linter, minifier, codegen_sourcemap]
     steps:
       - name: Checkout Branch
         uses: actions/checkout@v4
         with:
-          progress: false
+          show-progress: false
           persist-credentials: false
 
       - name: Install Rust Toolchain
@@ -43,20 +48,29 @@ jobs:
         with:
           tool: cargo-codspeed
 
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Start bench results interceptor server
+        working-directory: ./tasks/benchmark/codspeed
+        env:
+          COMPONENT: ${{ matrix.component }}
+        run: |
+          corepack enable
+          pnpm install
+          node capture.mjs &
+
       - name: Build Benchmark
         env:
           RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
         shell: bash
         run: |
-          cargo build --release -p oxc_benchmark --features codspeed --bench lexer --bench parser --bench transformer --bench semantic --bench linter --bench minifier --bench codegen_sourcemap
+          cargo build --release -p oxc_benchmark --features codspeed --bench ${{ matrix.component }}
           mkdir -p target/codspeed/oxc_benchmark/
-          mv target/release/deps/lexer-* target/codspeed/oxc_benchmark
-          mv target/release/deps/parser-* target/codspeed/oxc_benchmark
-          mv target/release/deps/transformer-* target/codspeed/oxc_benchmark
-          mv target/release/deps/semantic-* target/codspeed/oxc_benchmark
-          mv target/release/deps/linter-* target/codspeed/oxc_benchmark
-          mv target/release/deps/minifier-* target/codspeed/oxc_benchmark
-          mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark
+          mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
           rm -rf target/codspeed/oxc_benchmark/*.d
 
       - name: Run benchmark
@@ -65,3 +79,48 @@ jobs:
         with:
           run: cargo codspeed run
           token: ${{ secrets.CODSPEED_TOKEN }}
+          upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
+
+      - name: Upload bench data artefact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.component }}
+          path: ${{ env.DATA_DIR }}
+          if-no-files-found: error
+          retention-days: 1
+
+  upload:
+    name: Upload benchmarks
+    needs: benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v4
+        with:
+          show-progress: false
+          persist-credentials: false
+
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Create temp dir
+        working-directory: ./tasks/benchmark/codspeed
+        run: |
+          corepack enable
+          pnpm install
+          node create_temp_dir.mjs
+
+      - name: Download artefacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.DATA_DIR }}
+          merge-multiple: true
+
+      - name: Upload to Codspeed
+        working-directory: ./tasks/benchmark/codspeed
+        env:
+          CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }}
+        run: node upload.mjs

.gitignore

@@ -4,6 +4,7 @@ target/
 /node_modules/
 /website/node_modules/
 /benchmark/node_modules/
+/tasks/benchmark/codspeed/node_modules/
 /editors/vscode/node_modules/
 /editors/vscode/icon.png
 

tasks/benchmark/codspeed/capture.mjs

@@ -0,0 +1,73 @@
+/*
+ * HTTP server to intercept benchmark data from Codspeed runner.
+ * 
+ * Codspeed runner makes 2 API calls:
+ * 1. Uploading metadata
+ * 2. Uploading archive of CPU profile files
+ * 
+ * Server starts on an available port, saves the files sent by Codspeed runner to a directory,
+ * then shuts itself down.
+ */
+
+import fs from 'fs';
+import {pipeline} from 'stream/promises';
+import express from 'express';
+
+const DEFAULT_PORT = 3000,
+    LISTEN_ATTEMPTS = 10;
+
+// Create directory for saving assets
+const rand = Math.round(Math.random() * 1000000000000000000).toString(16),
+    dataDir = `/tmp/oxc_bench_data_${rand}`;
+fs.mkdirSync(dataDir);
+
+const component = process.env.COMPONENT;
+
+const app = express();
+
+app.post('/upload', (req, res, next) => {
+    saveBody(req, 'metadata.json', next, () => {
+        res.json({
+            status: 'success',
+            uploadUrl: `http://localhost:${port}/upload_archive`,
+            runId: 'dummy_value',
+        });
+    });
+});
+
+app.put('/upload_archive', (req, res, next) => {
+    saveBody(req, 'archive.tar.gz', next, () => {
+        res.send('OK');
+        server.close(() => {});
+    });
+});
+
+function saveBody(req, filename, onError, done) {
+    (async () => {
+        const stream = fs.createWriteStream(`${dataDir}/${component}_${filename}`);
+        await pipeline(req, stream);
+        done();
+    })().catch(onError);
+}
+
+// Open server on a port which is not already in use
+let server,
+    port = DEFAULT_PORT;
+for (let i = 0; i < LISTEN_ATTEMPTS; i++) {
+    console.log(`Starting server on port ${port}`);
+    try {
+        await new Promise((resolve, reject) => {
+            server = app.listen(port, resolve);
+            server.on('error', reject);
+        });
+        break;
+    } catch (err) {
+        if (err?.code !== 'EADDRINUSE') throw err;
+        console.log(`Port ${port} in use. Trying again.`);
+        port = DEFAULT_PORT + Math.round(Math.random() * 5000);
+    }
+}
+console.log(`Server listening on port ${port}`);
+
+// Output data dir path + port to env vars
+fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\nINTERCEPT_PORT=${port}\n`);

tasks/benchmark/codspeed/create_temp_dir.mjs

@@ -0,0 +1,13 @@
+/*
+ * Create temp dir for downloading artefacts to.
+ */
+
+import fs from 'fs';
+
+// Create directory for saving assets
+const rand = Math.round(Math.random() * 1000000000000000000).toString(16),
+    dataDir = `/tmp/oxc_bench_data_${rand}`;
+fs.mkdirSync(dataDir);
+
+// Output dir path to env var
+fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\n`);

tasks/benchmark/codspeed/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "oxc_benchmark",
+  "version": "0.0.0",
+  "devDependencies": {
+    "axios": "^1.6.8",
+    "express": "^4.18.3",
+    "tar": "^6.2.0"
+  }
+}