Merge master, fix conflicts

.github/pull_request_title_conventions.md

@@ -11,19 +11,19 @@ A PR title consists of these elements:
   |       |                        Capitalized
   |       |                        No period at the end.
   │       │
-  │       └─⫸ Scope: API|core|editor|* Node
+  │       └─⫸ Scope: API|core|editor|* Node|benchmark
   │
   └─⫸ Type: build|ci|docs|feat|fix|perf|refactor|test
 ```
 
 - PR title
-    - type
-    - scope (*optional*)
-    - summary
+  - type
+  - scope (_optional_)
+  - summary
 - PR description
-    - body (optional)
-    - blank line
-    - footer (optional)
+  - body (optional)
+  - blank line
+  - footer (optional)
 
 The structure looks like this:
 
@@ -46,22 +46,23 @@ If the prefix is `feat`, `fix` or `perf`, it will appear in the changelog. H
 
 The scope should specify the place of the commit change as long as the commit clearly addresses one of the following supported scopes. (Otherwise, omit the scope!)
 
-- `API` - changes to the *public* API
+- `API` - changes to the _public_ API
 - `core` - changes to the core / private API / backend of n8n
 - `editor` - changes to the Editor UI
 - `* Node` - changes to a specific node or trigger node (”`*`” to be replaced with the node name, not its display name), e.g.
-    - mattermost → Mattermost Node
-    - microsoftToDo  → Microsoft To Do Node
-    - n8n → n8n Node
+  - mattermost → Mattermost Node
+  - microsoftToDo → Microsoft To Do Node
+  - n8n → n8n Node
+- `benchmark` - changes to the Benchmark cli
 
 ### **Summary**
 
 The summary contains succinct description of the change:
 
 - use the imperative, present tense: "change" not "changed" nor "changes"
 - capitalize the first letter
-- *no* dot (.) at the end
-- do *not* include Linear ticket IDs etc. (e.g. N8N-1234)
+- _no_ dot (.) at the end
+- do _not_ include Linear ticket IDs etc. (e.g. N8N-1234)
 - suffix with “(no-changelog)” for commits / PRs that should not get mentioned in the changelog.
 
 ### **Body (optional)**
@@ -95,7 +96,7 @@ Closes #<pr number>
 A Breaking Change section should start with the phrase "`BREAKING CHANGE:` " followed by a summary of the breaking change, a blank line, and a detailed description of the breaking change that also includes migration instructions.
 
 > 💡 A breaking change can additionally also be marked by adding a “`!`” to the header, right before the “`:`”, e.g. `feat(editor)!: Remove support for dark mode`
-> 
+>
 > This makes locating breaking changes easier when just skimming through commit messages.
 
 > 💡 The breaking changes must also be added to the [packages/cli/BREAKING-CHANGES.md](https://github.com/n8n-io/n8n/blob/master/packages/cli/BREAKING-CHANGES.md) file located in the n8n repository.
@@ -109,4 +110,4 @@ If the commit reverts a previous commit, it should begin with `revert:` , foll
 The content of the commit message body should contain:
 
 - information about the SHA of the commit being reverted in the following format: `This reverts commit <SHA>`,
-- a clear description of the reason for reverting the commit message.
+- a clear description of the reason for reverting the commit message.

.github/workflows/benchmark-destroy-nightly.yml

@@ -0,0 +1,40 @@
+name: Destroy Benchmark Env
+
+on:
+  schedule:
+    - cron: '30 4 * * *'
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: benchmarking
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - name: Azure login
+        uses: azure/login@v2.1.1
+        with:
+          client-id: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
+          tenant-id: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
+          subscription-id: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
+
+      - run: Setup node
+      - uses: actions/setup-node@v4.0.2
+        with:
+          node-version: 20.x
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Destroy cloud env
+        if: github.event.inputs.debug == 'true'
+        run: pnpm destroy-cloud-env
+        working-directory: packages/@n8n/benchmark

.github/workflows/benchmark-nightly.yml

@@ -0,0 +1,69 @@
+name: Run Nightly Benchmark
+run-name: Benchmark ${{ inputs.n8n_tag }}
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    inputs:
+      debug:
+        description: 'Use debug logging'
+        required: true
+        default: 'false'
+      n8n_tag:
+        description: 'Name of the n8n docker tag to run the benchmark against.'
+        required: true
+        default: 'nightly'
+      benchmark_tag:
+        description: 'Name of the benchmark cli docker tag to run the benchmark with.'
+        required: true
+        default: 'latest'
+
+env:
+  ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
+  ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
+  ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
+  K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: benchmarking
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: '1.8.5'
+
+      - run: corepack enable
+      - uses: actions/setup-node@v4.0.2
+        with:
+          node-version: 20.x
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Azure login
+        uses: azure/login@v2.1.1
+        with:
+          client-id: ${{ env.ARM_CLIENT_ID }}
+          tenant-id: ${{ env.ARM_TENANT_ID }}
+          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+
+      - name: Run the benchmark with debug logging
+        if: github.event.inputs.debug == 'true'
+        run: pnpm run-in-cloud sqlite --debug
+        working-directory: packages/@n8n/benchmark
+
+      - name: Run the benchmark
+        if: github.event.inputs.debug != 'true'
+        run: pnpm run-in-cloud sqlite
+        working-directory: packages/@n8n/benchmark

.github/workflows/docker-images-benchmark.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - main
+      - master
     paths:
       - 'packages/@n8n/benchmark/**'
       - 'pnpm-lock.yaml'

packages/@n8n/benchmark/.eslintrc.js

@@ -12,7 +12,7 @@ module.exports = {
 		project: './tsconfig.json',
 	},
 
-	ignorePatterns: ['scenarios/**'],
+	ignorePatterns: ['scenarios/**', 'scripts/**'],
 
 	rules: {
 		'n8n-local-rules/no-plain-errors': 'off',

packages/@n8n/benchmark/.gitignore

@@ -0,0 +1,4 @@
+**/.terraform/*
+**/*.tfstate*
+**/*.tfvars
+privatekey.pem

packages/@n8n/benchmark/README.md

@@ -40,6 +40,14 @@ N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
 K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
 ```
 
+## Running in the cloud
+
+There's a script to run the performance tests in a cloud environment. The script provisions a cloud environment, sets up n8n in the environment, runs the tests and destroys the environment.
+
+```sh
+pnpm run-in-cloud
+```
+
 ## Configuration
 
 The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)

packages/@n8n/benchmark/infra/benchmark-env.tf

@@ -0,0 +1,54 @@
+
+data "azurerm_resource_group" "main" {
+  name = var.resource_group_name
+}
+
+# Random prefix for the resources
+resource "random_string" "prefix" {
+  length  = 8
+  special = false
+}
+
+# SSH key pair
+resource "tls_private_key" "ssh_key" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+# Dedicated Host Group & Hosts
+
+resource "azurerm_dedicated_host_group" "main" {
+  name                        = "${random_string.prefix.result}-hostgroup"
+  location                    = var.location
+  resource_group_name         = data.azurerm_resource_group.main.name
+  platform_fault_domain_count = 1
+  automatic_placement_enabled = false
+  zone                        = 1
+
+	tags = local.common_tags
+}
+
+resource "azurerm_dedicated_host" "hosts" {
+  name                    = "${random_string.prefix.result}-host"
+  location                = var.location
+  dedicated_host_group_id = azurerm_dedicated_host_group.main.id
+  sku_name                = var.host_size_family
+  platform_fault_domain   = 0
+
+	tags = local.common_tags
+}
+
+# VM
+
+module "test_vm" {
+  source = "./modules/benchmark-vm"
+
+  location            = var.location
+  resource_group_name = data.azurerm_resource_group.main.name
+  prefix              = random_string.prefix.result
+  dedicated_host_id   = azurerm_dedicated_host.hosts.id
+  ssh_public_key      = tls_private_key.ssh_key.public_key_openssh
+  vm_size             = var.vm_size
+
+  tags = local.common_tags
+}

packages/@n8n/benchmark/infra/modules/benchmark-vm/output.tf

@@ -0,0 +1,7 @@
+output "vm_name" {
+  value = azurerm_linux_virtual_machine.main.name
+}
+
+output "ip" {
+  value = azurerm_public_ip.main.ip_address
+}

packages/@n8n/benchmark/infra/modules/benchmark-vm/vars.tf

@@ -0,0 +1,31 @@
+variable "location" {
+  description = "Region to deploy resources"
+  default     = "East US"
+}
+
+variable "resource_group_name" {
+  description = "Name of the resource group"
+}
+
+variable "prefix" {
+  description = "Prefix to append to resources"
+}
+
+variable "dedicated_host_id" {
+  description = "Dedicated Host ID"
+}
+
+variable "ssh_public_key" {
+  description = "SSH Public Key"
+}
+
+variable "vm_size" {
+  description = "VM Size"
+  # 4 vCPUs, 16 GiB memory
+  default = "Standard_DC4s_v2"
+}
+
+variable "tags" {
+  description = "Tags to apply to all resources created by this module"
+  type        = map(string)
+}