Skip to content

Commit

Permalink
perf: Benchmark expression evaluation
Browse files Browse the repository at this point in the history
  • Loading branch information
ivov committed May 13, 2024
1 parent 2445205 commit a9ea2fe
Show file tree
Hide file tree
Showing 7 changed files with 173 additions and 42 deletions.
44 changes: 44 additions & 0 deletions .github/workflows/benchmark-workflow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: Benchmark workflow package

on:
pull_request:
paths:
- 'packages/workflow/**'
workflow_dispatch:

jobs:
benchmark:
name: Benchmark
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1

- run: corepack enable

- uses: actions/setup-node@v4.0.1
with:
node-version: 18.x
cache: pnpm

- run: pnpm install --frozen-lockfile

- name: Build
if: ${{ inputs.cacheKey == '' }}
run: pnpm build:backend

- name: Restore cached build artifacts
if: ${{ inputs.cacheKey != '' }}
uses: actions/cache/restore@v4.0.0
with:
path: ./packages/**/dist
key: ${{ inputs.cacheKey }}

- name: Build benchmarks
working-directory: packages/workflow
run: pnpm benchmark:build

- name: Benchmark workflow package
uses: CodSpeedHQ/action@v2
with:
working-directory: packages/workflow
run: pnpm benchmark:run
10 changes: 7 additions & 3 deletions packages/workflow/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
"./*": "./*"
},
"scripts": {
"benchmark:build": "tsc -p tsconfig.benchmark.json > /dev/null; tsc-alias -p tsconfig.benchmark.json",
"benchmark:run": "node dist/test/benchmark.js",
"clean": "rimraf dist .turbo",
"dev": "pnpm watch",
"typecheck": "tsc",
Expand All @@ -39,15 +41,18 @@
"dist/**/*"
],
"devDependencies": {
"@codspeed/tinybench-plugin": "^3.1.0",
"@types/deep-equal": "^1.0.1",
"@types/express": "^4.17.21",
"@types/jmespath": "^0.15.0",
"@types/lodash": "^4.14.195",
"@types/luxon": "^3.2.0",
"@types/md5": "^2.3.5",
"@types/xml2js": "^0.4.14"
"@types/xml2js": "^0.4.14",
"tinybench": "^2.8.0"
},
"dependencies": {
"@langchain/core": "0.1.61",
"@n8n/tournament": "1.0.2",
"@n8n_io/riot-tmpl": "4.0.0",
"ast-types": "0.15.2",
Expand All @@ -65,7 +70,6 @@
"recast": "0.21.5",
"title-case": "3.0.3",
"transliteration": "2.3.5",
"xml2js": "0.6.2",
"@langchain/core": "0.1.61"
"xml2js": "0.6.2"
}
}
34 changes: 1 addition & 33 deletions packages/workflow/test/Expression.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,44 +3,19 @@
*/

import { DateTime, Duration, Interval } from 'luxon';
import { Workflow } from '@/Workflow';
import * as Helpers from './Helpers';
import type { ExpressionTestEvaluation, ExpressionTestTransform } from './ExpressionFixtures/base';
import { baseFixtures } from './ExpressionFixtures/base';
import type { INodeExecutionData } from '@/Interfaces';
import { extendSyntax } from '@/Extensions/ExpressionExtension';
import { ExpressionError } from '@/errors/expression.error';
import { setDifferEnabled, setEvaluator } from '@/ExpressionEvaluatorProxy';
import { workflow } from './ExpressionExtensions/Helpers';
import { evaluate } from './evaluate';

setDifferEnabled(true);

for (const evaluator of ['tmpl', 'tournament'] as const) {
setEvaluator(evaluator);
describe(`Expression (with ${evaluator})`, () => {
describe('getParameterValue()', () => {
const nodeTypes = Helpers.NodeTypes();
const workflow = new Workflow({
id: '1',
nodes: [
{
name: 'node',
typeVersion: 1,
type: 'test.set',
id: 'uuid-1234',
position: [0, 0],
parameters: {},
},
],
connections: {},
active: false,
nodeTypes,
});
const expression = workflow.expression;

const evaluate = (value: string) =>
expression.getParameterValue(value, null, 0, 0, 'node', [], 'manual', {});

it('should not be able to use global built-ins from denylist', () => {
expect(evaluate('={{document}}')).toEqual({});
expect(evaluate('={{window}}')).toEqual({});
Expand Down Expand Up @@ -173,13 +148,6 @@ for (const evaluator of ['tmpl', 'tournament'] as const) {
});

describe('Test all expression value fixtures', () => {
const expression = workflow.expression;

const evaluate = (value: string, data: INodeExecutionData[]) => {
const itemIndex = data.length === 0 ? -1 : 0;
return expression.getParameterValue(value, null, 0, itemIndex, 'node', data, 'manual', {});
};

for (const t of baseFixtures) {
if (!t.tests.some((test) => test.type === 'evaluation')) {
continue;
Expand Down
31 changes: 31 additions & 0 deletions packages/workflow/test/benchmark.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { Bench } from 'tinybench';
import { withCodSpeed } from '@codspeed/tinybench-plugin';
import { baseFixtures } from './ExpressionFixtures/base';
import { evaluate } from './evaluate';
import type { INodeExecutionData } from '@/index';

function addExpressionEvaluationTasks(bench: Bench) {
for (const fixture of baseFixtures) {
for (const test of fixture.tests) {
if ('error' in test) continue;

if (test.type === 'evaluation') {
const input = test.input.map((d) => ({ json: d })) as INodeExecutionData[];
bench.add(fixture.expression, () => evaluate(fixture.expression, input));
}
}
}
}

async function main() {
const bench = withCodSpeed(new Bench());

addExpressionEvaluationTasks(bench);

await bench.warmup();
await bench.run();

console.table(bench.table());
}

void main();
6 changes: 6 additions & 0 deletions packages/workflow/test/evaluate.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { workflow } from './ExpressionExtensions/Helpers';
import type { INodeExecutionData } from '@/Interfaces';

export const evaluate = (expression: string, data: INodeExecutionData[] = []) => {
return workflow.expression.getParameterValue(expression, null, 0, 0, 'node', data, 'manual', {});
};
13 changes: 13 additions & 0 deletions packages/workflow/tsconfig.benchmark.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"extends": ["./tsconfig.json", "../../tsconfig.build.json"],
"compilerOptions": {
"outDir": "dist",
"tsBuildInfoFile": "dist/benchmark.tsbuildinfo",
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["test/**/*.ts"],
"exclude": ["test/**/*.test.ts"]
}
77 changes: 71 additions & 6 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit a9ea2fe

Please sign in to comment.