feat: support bench rspack pr

.github/workflows/bench_rspack_pr.yml

@@ -0,0 +1,89 @@
+name: bench-rspack-pr
+
+on:
+  workflow_dispatch:
+    inputs:
+      prNumber:
+        description: "PR number (e.g. 9887)"
+        required: true
+        type: string
+      reportComment:
+        description: "Create a comment to display benchmark results"
+        required: false
+        type: string
+        default: "false"
+
+jobs:
+  create-comment:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.reportComment == "true" }}
+    outputs:
+      comment-id: ${{ steps.create-comment.outputs.result }}
+    steps:
+      - id: create-comment
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.BENCHMARK_ACCESS_TOKEN }}
+          result-encoding: string
+          script: |
+            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            const urlLink = `[Open](${url})`
+
+            const { data: comment } = await github.rest.issues.createComment({
+              issue_number: context.payload.inputs.prNumber,
+              owner: context.repo.owner,
+              repo: 'rspack',
+              body: `⏳ Triggered benchmark: ${urlLink}`
+            })
+            return comment.id
+
+  run-bench:
+    runs-on: [self-hosted, rspack-bench]
+    needs: create-comment
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Init env
+        uses: ./.github/actions/env
+      - name: Build rspack
+        run: node bin/build-rspack.js origin pull/${{ inputs.prNumber }}/head
+      - name: Run benchmark
+        run: node bin/bench.js
+      - name: Print results
+        run: node bin/compare-bench.js latest current
+
+  update-comment:
+    runs-on: ubuntu-latest
+    needs: [create-comment, run-bench]
+    if: ${{ inputs.reportComment == "true" }}
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.ECOSYSTEM_CI_ACCESS_TOKEN }}
+          script: |
+            const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+              per_page: 100
+            });
+
+            const result = jobs.find(job => job.name === "run-bench")
+
+            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            const urlLink = `[Open](${url})`
+            console.log(result)
+            const body = `
+            📝 Benchmark detail: ${urlLink}
+
+            | suite | result |
+            |-------|--------|
+            ${result.map(r => `| [${r.suite}](${r.html_url}) | ${r.conclusion} ${r.conclusion} |`).join("\n")}
+            `
+
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: 'rspack',
+              comment_id: ${{ needs.init.outputs.comment-id }},
+              body
+            })

bin/bench.js

@@ -22,7 +22,7 @@ const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
 	for (const item of benchmarks) {
 		const result = await run(item);
 		console.log(`${item} result is:`);
-		console.log(formatResultTable(result, { colors: true, verbose: true }));
+		console.log(formatResultTable(result, { verbose: true }));
 		await writeFile(
 			resolve(rootDir, `output/${item}.json`),
 			JSON.stringify(result, null, 2)

bin/compare-bench.js

@@ -0,0 +1,94 @@
+import { resolve } from "path";
+import { fileURLToPath } from "url";
+import { readFile, readdir } from "fs/promises";
+import { compare, formatDiffTable } from "../lib/index.js";
+
+const [, , baseDate, currentDate] = process.argv;
+const compareMetric = ["stats"];
+const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
+const outputDir = resolve(rootDir, "output");
+const fetchPrefix =
+	"https://raw.githubusercontent.com/web-infra-dev/rspack-ecosystem-benchmark/data";
+
+function getOverThresholdTags(diff) {
+	return Object.entries(diff)
+		.map(([tag, data]) => {
+			if (!tag.endsWith(" memory") && !tag.endsWith(" size")) {
+				// time type
+				if (data.currentMean < 300) {
+					return null;
+				}
+			}
+			if (data.currentMean / data.baseMean < 1.05) {
+				return null;
+			}
+			return tag;
+		})
+		.filter(item => !!item);
+}
+
+// get the result by date
+// `current` will get ../output data
+// `latest` will get the latest data on the data branch
+// `2023-08-08` will get the data from `2023-08-08` on the data branch
+async function getResults(date) {
+	if (date === "current") {
+		const outputFiles = await readdir(outputDir);
+		return await Promise.all(
+			outputFiles.map(async item => {
+				return {
+					// remove .json
+					name: item.slice(0, -5),
+					result: JSON.parse(await readFile(resolve(outputDir, item)))
+				};
+			})
+		);
+	}
+
+	const indexFile = await (await fetch(`${fetchPrefix}/index.txt`)).text();
+	const dataPaths = indexFile.split("\n").filter(item => !!item);
+	if (date === "latest") {
+		date = dataPaths[dataPaths.length - 1].split("/")[0];
+	}
+	return await Promise.all(
+		dataPaths
+			.filter(item => item.startsWith(date))
+			.map(async item => {
+				return {
+					name: item.split("/")[1].slice(0, -5),
+					result: await (await fetch(`${fetchPrefix}/${item}`)).json()
+				};
+			})
+	);
+}
+
+(async () => {
+	const baseResults = await getResults(baseDate);
+	const currentResults = await getResults(currentDate);
+	const baseData = {};
+	const currentData = {};
+	for (const metric of compareMetric) {
+		for (const { name, result } of baseResults) {
+			const tag = `${name} + ${metric}`;
+			baseData[tag] = result[metric];
+		}
+
+		for (const { name, result } of currentResults) {
+			const tag = `${name} + ${metric}`;
+			currentData[tag] = result[metric];
+		}
+	}
+
+	const diff = compare(baseData, currentData);
+	const formatedTable = formatDiffTable(diff, { verbose: 1 });
+	const overThresholdTags = getOverThresholdTags(diff);
+	console.log(formatedTable);
+	if (overThresholdTags.length > 0) {
+		const message = "Threshold exceeded: " + JSON.stringify(overThresholdTags);
+		console.log(message);
+		throw new Error(message);
+	}
+})().catch(err => {
+	process.exitCode = 1;
+	console.error(err.stack);
+});

lib/compare.js

@@ -1,59 +1,18 @@
-import { getType } from "./utils.js";
-
-function compareStatistics(
-	base,
-	current,
-	totals = {
-		time: current.stats.mean,
-		size: Object.keys(current)
-			.filter(n => getType(n) === "size")
-			.reduce((sum, key) => sum + current[key].mean, 0),
-		"gzip size": Object.keys(current)
-			.filter(n => getType(n) === "gzip size")
-			.reduce((sum, key) => sum + current[key].mean, 0),
-		memory: Object.keys(current)
-			.filter(n => getType(n) === "memory")
-			.reduce((sum, key) => sum + current[key].mean, 0)
-	}
-) {
+export function compare(base, current) {
 	const diff = {};
 	for (const key of new Set([...Object.keys(base), ...Object.keys(current)])) {
 		const baseValue = base[key];
 		const currentValue = current[key];
-		if (baseValue === undefined || currentValue === undefined) continue;
-		if ("mean" in baseValue) {
-			const type = getType(key);
-			diff[key] = {
-				type,
-				relevance: currentValue.mean / totals[type]
-			};
-			for (const k of Object.keys(baseValue)) {
-				diff[key][k] =
-					baseValue[k] === 0 && currentValue[k] === 0
-						? 1
-						: currentValue[k] / baseValue[k];
-			}
-			diff[key].lowHigh = currentValue.high / baseValue.low;
-			diff[key].highLow = currentValue.low / baseValue.high;
-			diff[key].baseStdDev = baseValue.stdDev / baseValue.mean;
-			diff[key].currentStdDev = currentValue.stdDev / currentValue.mean;
-			diff[key].baseConfidence = baseValue.confidence / baseValue.mean;
-			diff[key].currentConfidence = currentValue.confidence / currentValue.mean;
-		} else {
-			diff[key] = compareStatistics(baseValue, currentValue, totals);
+		if (baseValue === undefined || currentValue === undefined) {
+			continue;
 		}
+
+		diff[key] = {
+			baseMean: baseValue.mean,
+			baseConfidence: baseValue.confidence,
+			currentMean: currentValue.mean,
+			currentConfidence: currentValue.confidence
+		};
 	}
 	return diff;
 }
-
-export function getResult(benchmarkName, date) {
-	// TODO use fetch to get data
-}
-
-export function compare(baselineResult, currentResult) {
-	return {
-		diff: compareStatistics(baselineResult, currentResult),
-		result: currentResult,
-		baseResult: baselineResult
-	};
-}