diff --git a/.github/workflows/benchmark-workflow.yml b/.github/workflows/benchmark-workflow.yml new file mode 100644 index 0000000000000..0aae1263783bb --- /dev/null +++ b/.github/workflows/benchmark-workflow.yml @@ -0,0 +1,44 @@ +name: Benchmark workflow package + +on: + pull_request: + paths: + - 'packages/workflow/**' + workflow_dispatch: + +jobs: + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + + - run: corepack enable + + - uses: actions/setup-node@v4.0.1 + with: + node-version: 18.x + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Build + if: ${{ inputs.cacheKey == '' }} + run: pnpm build:backend + + - name: Restore cached build artifacts + if: ${{ inputs.cacheKey != '' }} + uses: actions/cache/restore@v4.0.0 + with: + path: ./packages/**/dist + key: ${{ inputs.cacheKey }} + + - name: Build benchmarks + working-directory: packages/workflow + run: pnpm benchmark:build + + - name: Benchmark workflow package + uses: CodSpeedHQ/action@v2 + with: + working-directory: packages/workflow + run: pnpm benchmark:run diff --git a/packages/workflow/package.json b/packages/workflow/package.json index d3839c66b71b9..0cd0d39311d34 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -24,6 +24,8 @@ "./*": "./*" }, "scripts": { + "benchmark:build": "tsc -p tsconfig.benchmark.json > /dev/null; tsc-alias -p tsconfig.benchmark.json", + "benchmark:run": "node dist/test/benchmark.js", "clean": "rimraf dist .turbo", "dev": "pnpm watch", "typecheck": "tsc", @@ -39,15 +41,18 @@ "dist/**/*" ], "devDependencies": { + "@codspeed/tinybench-plugin": "^3.1.0", "@types/deep-equal": "^1.0.1", "@types/express": "^4.17.21", "@types/jmespath": "^0.15.0", "@types/lodash": "^4.14.195", "@types/luxon": "^3.2.0", "@types/md5": "^2.3.5", - "@types/xml2js": "^0.4.14" + "@types/xml2js": "^0.4.14", + "tinybench": "^2.8.0" }, "dependencies": { + "@langchain/core": "0.1.61", "@n8n/tournament": "1.0.2", "@n8n_io/riot-tmpl": "4.0.0", "ast-types": "0.15.2", @@ -65,7 +70,6 @@ "recast": "0.21.5", "title-case": "3.0.3", "transliteration": "2.3.5", - "xml2js": "0.6.2", - "@langchain/core": "0.1.61" + "xml2js": "0.6.2" } } diff --git a/packages/workflow/test/Expression.test.ts b/packages/workflow/test/Expression.test.ts index f9d15fb0af5f3..2326b96593894 100644 --- a/packages/workflow/test/Expression.test.ts +++ b/packages/workflow/test/Expression.test.ts @@ -3,15 +3,12 @@ */ import { DateTime, Duration, Interval } from 'luxon'; -import { Workflow } from '@/Workflow'; -import * as Helpers from './Helpers'; import type { ExpressionTestEvaluation, ExpressionTestTransform } from './ExpressionFixtures/base'; import { baseFixtures } from './ExpressionFixtures/base'; -import type { INodeExecutionData } from '@/Interfaces'; import { extendSyntax } from '@/Extensions/ExpressionExtension'; import { ExpressionError } from '@/errors/expression.error'; import { setDifferEnabled, setEvaluator } from '@/ExpressionEvaluatorProxy'; -import { workflow } from './ExpressionExtensions/Helpers'; +import { evaluate } from './evaluate'; setDifferEnabled(true); @@ -19,28 +16,6 @@ for (const evaluator of ['tmpl', 'tournament'] as const) { setEvaluator(evaluator); describe(`Expression (with ${evaluator})`, () => { describe('getParameterValue()', () => { - const nodeTypes = Helpers.NodeTypes(); - const workflow = new Workflow({ - id: '1', - nodes: [ - { - name: 'node', - typeVersion: 1, - type: 'test.set', - id: 'uuid-1234', - position: [0, 0], - parameters: {}, - }, - ], - connections: {}, - active: false, - nodeTypes, - }); - const expression = workflow.expression; - - const evaluate = (value: string) => - expression.getParameterValue(value, null, 0, 0, 'node', [], 'manual', {}); - it('should not be able to use global built-ins from denylist', () => { expect(evaluate('={{document}}')).toEqual({}); expect(evaluate('={{window}}')).toEqual({}); @@ -173,13 +148,6 @@ for (const evaluator of ['tmpl', 'tournament'] as const) { }); describe('Test all expression value fixtures', () => { - const expression = workflow.expression; - - const evaluate = (value: string, data: INodeExecutionData[]) => { - const itemIndex = data.length === 0 ? -1 : 0; - return expression.getParameterValue(value, null, 0, itemIndex, 'node', data, 'manual', {}); - }; - for (const t of baseFixtures) { if (!t.tests.some((test) => test.type === 'evaluation')) { continue; diff --git a/packages/workflow/test/benchmark.ts b/packages/workflow/test/benchmark.ts new file mode 100644 index 0000000000000..c9739a6f884f5 --- /dev/null +++ b/packages/workflow/test/benchmark.ts @@ -0,0 +1,31 @@ +import { Bench } from 'tinybench'; +import { withCodSpeed } from '@codspeed/tinybench-plugin'; +import { baseFixtures } from './ExpressionFixtures/base'; +import { evaluate } from './evaluate'; +import type { INodeExecutionData } from '@/index'; + +function addExpressionEvaluationTasks(bench: Bench) { + for (const fixture of baseFixtures) { + for (const test of fixture.tests) { + if ('error' in test) continue; + + if (test.type === 'evaluation') { + const input = test.input.map((d) => ({ json: d })) as INodeExecutionData[]; + bench.add(fixture.expression, () => evaluate(fixture.expression, input)); + } + } + } +} + +async function main() { + const bench = withCodSpeed(new Bench()); + + addExpressionEvaluationTasks(bench); + + await bench.warmup(); + await bench.run(); + + console.table(bench.table()); +} + +void main(); diff --git a/packages/workflow/test/evaluate.ts b/packages/workflow/test/evaluate.ts new file mode 100644 index 0000000000000..7d573632771bf --- /dev/null +++ b/packages/workflow/test/evaluate.ts @@ -0,0 +1,6 @@ +import { workflow } from './ExpressionExtensions/Helpers'; +import type { INodeExecutionData } from '@/Interfaces'; + +export const evaluate = (expression: string, data: INodeExecutionData[] = []) => { + return workflow.expression.getParameterValue(expression, null, 0, 0, 'node', data, 'manual', {}); +}; diff --git a/packages/workflow/tsconfig.benchmark.json b/packages/workflow/tsconfig.benchmark.json new file mode 100644 index 0000000000000..a7dcfb1d90298 --- /dev/null +++ b/packages/workflow/tsconfig.benchmark.json @@ -0,0 +1,13 @@ +{ + "extends": ["./tsconfig.json", "../../tsconfig.build.json"], + "compilerOptions": { + "outDir": "dist", + "tsBuildInfoFile": "dist/benchmark.tsbuildinfo", + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["test/**/*.ts"], + "exclude": ["test/**/*.test.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2888cfe8070c..56418d62bb9a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1652,6 +1652,9 @@ importers: specifier: 0.6.2 version: 0.6.2 devDependencies: + '@codspeed/tinybench-plugin': + specifier: ^3.1.0 + version: 3.1.0(tinybench@2.8.0) '@types/deep-equal': specifier: ^1.0.1 version: 1.0.1 @@ -1673,6 +1676,9 @@ importers: '@types/xml2js': specifier: ^0.4.14 version: 0.4.14 + tinybench: + specifier: ^2.8.0 + version: 2.8.0 packages: @@ -4740,6 +4746,29 @@ packages: w3c-keyname: 2.2.6 dev: false + /@codspeed/core@3.1.0: + resolution: {integrity: sha512-oYd7X46QhnRkgRbZkqAoX9i3Fwm17FpunK4Ee5RdrvRYR0Xr93ewH8/O5g6uyTPDOOqDEv1v2KRYtWhVgN+2VQ==} + dependencies: + axios: 1.6.7 + find-up: 6.3.0 + form-data: 4.0.0 + node-gyp-build: 4.7.0 + transitivePeerDependencies: + - debug + dev: true + + /@codspeed/tinybench-plugin@3.1.0(tinybench@2.8.0): + resolution: {integrity: sha512-yl0WzzUGIXkZzWaw7+2U+xGkuIal1Rs9hS09DtlDZGGAcGRoMMU5d2vyCS8nBrna4hrPQZ5Sx/hIKerO+lqWaw==} + peerDependencies: + tinybench: ^2.3.0 + dependencies: + '@codspeed/core': 3.1.0 + stack-trace: 1.0.0-pre2 + tinybench: 2.8.0 + transitivePeerDependencies: + - debug + dev: true + /@colors/colors@1.5.0: resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} engines: {node: '>=0.1.90'} @@ -11518,7 +11547,6 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - dev: false /axios@1.6.7(debug@3.2.7): resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} @@ -14921,6 +14949,14 @@ packages: path-exists: 4.0.0 dev: true + /find-up@6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + locate-path: 7.2.0 + path-exists: 5.0.0 + dev: true + /first-match@0.0.1: resolution: {integrity: sha512-VvKbnaxrC0polTFDC+teKPTdl2mn6B/KUW+WB3C9RzKDeNwbzfLdnUz3FxC+tnjvus6bI0jWrWicQyVIPdS37A==} dev: false @@ -14960,7 +14996,6 @@ packages: optional: true dependencies: debug: 3.2.7(supports-color@5.5.0) - dev: false /follow-redirects@1.15.6(debug@4.3.4): resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} @@ -18133,6 +18168,13 @@ packages: p-locate: 5.0.0 dev: true + /locate-path@7.2.0: + resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-locate: 6.0.0 + dev: true + /lodash-es@4.17.21: resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} dev: false @@ -19246,7 +19288,6 @@ packages: /node-gyp-build@4.7.0: resolution: {integrity: sha512-PbZERfeFdrHQOOXiAKOY0VPbykZy90ndPKk0d+CFDegTKmWp1VgOTz2xACVbr1BjCWxrQp68CXtvNsveFhqDJg==} hasBin: true - dev: false /node-gyp@8.4.1: resolution: {integrity: sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==} @@ -19769,6 +19810,13 @@ packages: dependencies: yocto-queue: 0.1.0 + /p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + yocto-queue: 1.0.0 + dev: true + /p-limit@5.0.0: resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==} engines: {node: '>=18'} @@ -19796,6 +19844,13 @@ packages: p-limit: 3.1.0 dev: true + /p-locate@6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-limit: 4.0.0 + dev: true + /p-map@4.0.0: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} @@ -19933,6 +19988,11 @@ packages: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} + /path-exists@5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: true + /path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} @@ -22466,6 +22526,11 @@ packages: resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} dev: false + /stack-trace@1.0.0-pre2: + resolution: {integrity: sha512-2ztBJRek8IVofG9DBJqdy2N5kulaacX30Nz7xmkYF6ale9WBVmIy6mFBchvGX7Vx/MyjBhx+Rcxqrj+dbOnQ6A==} + engines: {node: '>=16'} + dev: true + /stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -23158,8 +23223,8 @@ packages: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} dev: true - /tinybench@2.5.1: - resolution: {integrity: sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==} + /tinybench@2.8.0: + resolution: {integrity: sha512-1/eK7zUnIklz4JUUlL+658n58XO2hHLQfSk1Zf2LKieUjxidN16eKFEoDEfjHc3ohofSSqK3X5yO6VGb6iW8Lw==} dev: true /tinypool@0.8.2: @@ -24236,7 +24301,7 @@ packages: picocolors: 1.0.0 std-env: 3.6.0 strip-literal: 2.0.0 - tinybench: 2.5.1 + tinybench: 2.8.0 tinypool: 0.8.2 vite: 5.1.6(sass@1.64.1) vite-node: 1.3.1