From c1dda76328719781e8a092f7529d4dea3a0c4416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 20 Jun 2024 12:08:28 +0200 Subject: [PATCH 01/15] perf(core): Speed up common execution queries --- .../1717498465932-RefactorExecutionIndices.ts | 44 +++++++++++++++++++ .../src/databases/migrations/mysqldb/index.ts | 2 + .../databases/migrations/postgresdb/index.ts | 2 + .../src/databases/migrations/sqlite/index.ts | 2 + .../repositories/execution.repository.ts | 4 +- packages/cli/src/databases/types.ts | 2 + .../src/databases/utils/migrationHelpers.ts | 4 ++ .../cli/src/executions/execution.service.ts | 4 +- .../cli/src/executions/execution.types.ts | 2 +- 9 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts new file mode 100644 index 0000000000000..43ed641e4ea33 --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -0,0 +1,44 @@ +import type { MigrationContext, ReversibleMigration } from '@/databases/types'; + +export class RefactorExecutionIndices1717498465932 implements ReversibleMigration { + transaction = false as const; + + async up({ schemaBuilder, isPostgres, isSqlite }: MigrationContext) { + await schemaBuilder.dropIndex( + 'execution_entity', + ['waitTill', 'id'], // covered by `waitTill, status` - `id` is unused + isPostgres ? 'IDX_85b981df7b444f905f8bf50747' : 'IDX_b94b45ce2c73ce46c54f20b5f9', + ); + + if (isSqlite) { + await schemaBuilder.dropIndex( + 'execution_entity', + ['stoppedAt'], + 'idx_execution_entity_stopped_at', // duplicate of `IDX_execution_entity_stoppedAt` + ); + } + + await schemaBuilder.createIndex( + 'execution_entity', + ['status', 'startedAt'], // for default query at `GET /executions` + ); + + await schemaBuilder.createIndex( + 'execution_entity', + ['workflowId', 'status', 'startedAt'], // for query with filters at `GET /executions` + ); + + await schemaBuilder.createIndex( + 'execution_entity', + ['waitTill', 'status'], // for waiting executions queries on interval + ); + } + + async down({ schemaBuilder }: MigrationContext) { + await schemaBuilder.dropIndex('execution_entity', ['status', 'startedAt']); + await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'status', 'startedAt']); + await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); + + await schemaBuilder.createIndex('execution_entity', ['waitTill', 'id']); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 51c514dca5240..745b8fb75e73d 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -58,6 +58,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -119,4 +120,5 @@ export const mysqlMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, + RefactorExecutionIndices1717498465932, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index dc2b14edff355..09ae891014594 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -57,6 +57,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -117,4 +118,5 @@ export const postgresMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, + RefactorExecutionIndices1717498465932, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 6bda48f6f4d37..735b78b61dc93 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -55,6 +55,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; +import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -113,6 +114,7 @@ const sqliteMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, + RefactorExecutionIndices1717498465932, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 03a2a8ae82e73..36cb61eb5994e 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -735,8 +735,8 @@ export class ExecutionRepository extends Repository { if (firstId) qb.andWhere('execution.id > :firstId', { firstId }); if (lastId) qb.andWhere('execution.id < :lastId', { lastId }); - if (query.order?.stoppedAt === 'DESC') { - qb.orderBy({ 'execution.stoppedAt': 'DESC' }); + if (query.order?.startedAt === 'DESC') { + qb.orderBy({ 'execution.startedAt': 'DESC' }); } else if (query.order?.top) { qb.orderBy(`(CASE WHEN execution.status = '${query.order.top}' THEN 0 ELSE 1 END)`); } else { diff --git a/packages/cli/src/databases/types.ts b/packages/cli/src/databases/types.ts index e1dc8f20e9b2d..05d95d61877da 100644 --- a/packages/cli/src/databases/types.ts +++ b/packages/cli/src/databases/types.ts @@ -11,6 +11,8 @@ export interface MigrationContext { tablePrefix: string; dbType: DatabaseType; isMysql: boolean; + isSqlite: boolean; + isPostgres: boolean; dbName: string; migrationName: string; nodeTypes: INodeTypes; diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts index 1dc4b45ec32ef..3181e46fd2201 100644 --- a/packages/cli/src/databases/utils/migrationHelpers.ts +++ b/packages/cli/src/databases/utils/migrationHelpers.ts @@ -91,6 +91,8 @@ function parseJson(data: string | T): T { const dbType = config.getEnv('database.type'); const isMysql = ['mariadb', 'mysqldb'].includes(dbType); +const isSqlite = dbType === 'sqlite'; +const isPostgres = dbType === 'postgresdb'; const dbName = config.getEnv(`database.${dbType === 'mariadb' ? 'mysqldb' : dbType}.database`); const tablePrefix = config.getEnv('database.tablePrefix'); @@ -99,6 +101,8 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio tablePrefix, dbType, isMysql, + isSqlite, + isPostgres, dbName, migrationName: migration.name, queryRunner, diff --git a/packages/cli/src/executions/execution.service.ts b/packages/cli/src/executions/execution.service.ts index b0c4f11e164b0..9e34c12d6bb36 100644 --- a/packages/cli/src/executions/execution.service.ts +++ b/packages/cli/src/executions/execution.service.ts @@ -368,7 +368,7 @@ export class ExecutionService { /** * Return: * - * - the latest summaries of current and completed executions that satisfy a query, + * - the summaries of latest current and completed executions that satisfy a query, * - the total count of all completed executions that satisfy the query, and * - whether the total of completed executions is an estimate. * @@ -389,7 +389,7 @@ export class ExecutionService { this.findRangeWithCount({ ...query, status: completedStatuses, - order: { stoppedAt: 'DESC' }, + order: { startedAt: 'DESC' }, }), ]); diff --git a/packages/cli/src/executions/execution.types.ts b/packages/cli/src/executions/execution.types.ts index 95b9a3cdec4f1..36d9ce2bf65a8 100644 --- a/packages/cli/src/executions/execution.types.ts +++ b/packages/cli/src/executions/execution.types.ts @@ -80,7 +80,7 @@ export namespace ExecutionSummaries { type OrderFields = { order?: { top?: ExecutionStatus; - stoppedAt?: 'DESC'; + startedAt?: 'DESC'; }; }; } From c2d0761ea86e17777302b4dc3bf5ba0804ec39fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 20 Jun 2024 12:18:53 +0200 Subject: [PATCH 02/15] Remove disabling transactions --- .../migrations/common/1717498465932-RefactorExecutionIndices.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts index 43ed641e4ea33..396b0aff249f0 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -1,8 +1,6 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; export class RefactorExecutionIndices1717498465932 implements ReversibleMigration { - transaction = false as const; - async up({ schemaBuilder, isPostgres, isSqlite }: MigrationContext) { await schemaBuilder.dropIndex( 'execution_entity', From dbb1c2e1117fc46333b06ef1c31e6caa08315751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 20 Jun 2024 14:35:59 +0200 Subject: [PATCH 03/15] Remove unneeded sorting on FE --- packages/editor-ui/src/stores/executions.store.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/editor-ui/src/stores/executions.store.ts b/packages/editor-ui/src/stores/executions.store.ts index 756fdff6c4d97..bb0e9420c6f20 100644 --- a/packages/editor-ui/src/stores/executions.store.ts +++ b/packages/editor-ui/src/stores/executions.store.ts @@ -58,15 +58,7 @@ export const useExecutionsStore = defineStore('executions', () => { ); const currentExecutionsById = ref>({}); - const currentExecutions = computed(() => { - const data = Object.values(currentExecutionsById.value); - - data.sort((a, b) => { - return new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime(); - }); - - return data; - }); + const currentExecutions = computed(() => Object.values(currentExecutionsById.value)); const currentExecutionsByWorkflowId = computed(() => currentExecutions.value.reduce>((acc, execution) => { From 956ceead53dcb9372fa3586b7e0c1a26b3b00cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 20 Jun 2024 14:49:43 +0200 Subject: [PATCH 04/15] Skip if missing on drop index --- packages/cli/src/databases/dsl/Indices.ts | 13 +++++++++++-- packages/cli/src/databases/dsl/index.ts | 10 ++++++++-- .../1717498465932-RefactorExecutionIndices.ts | 16 ++++++++++------ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/databases/dsl/Indices.ts b/packages/cli/src/databases/dsl/Indices.ts index a5ec95e1012ed..2fa2ccaa7590a 100644 --- a/packages/cli/src/databases/dsl/Indices.ts +++ b/packages/cli/src/databases/dsl/Indices.ts @@ -19,9 +19,18 @@ abstract class IndexOperation extends LazyPromise { protected tablePrefix: string, queryRunner: QueryRunner, protected customIndexName?: string, + protected skipIfMissing = false, ) { - super((resolve) => { - void this.execute(queryRunner).then(resolve); + super((resolve, reject) => { + void this.execute(queryRunner) + .then(resolve) + .catch((error) => { + if (this.skipIfMissing) { + resolve(); + } else { + reject(error); + } + }); }); } } diff --git a/packages/cli/src/databases/dsl/index.ts b/packages/cli/src/databases/dsl/index.ts index bb5ef859ecf8f..a51e61bd174ba 100644 --- a/packages/cli/src/databases/dsl/index.ts +++ b/packages/cli/src/databases/dsl/index.ts @@ -32,8 +32,14 @@ export const createSchemaBuilder = (tablePrefix: string, queryRunner: QueryRunne customIndexName?: string, ) => new CreateIndex(tableName, columnNames, isUnique, tablePrefix, queryRunner, customIndexName), - dropIndex: (tableName: string, columnNames: string[], customIndexName?: string) => - new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName), + dropIndex: ( + tableName: string, + columnNames: string[], + { customIndexName, skipIfMissing }: { customIndexName?: string; skipIfMissing?: boolean } = { + skipIfMissing: false, + }, + ) => + new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName, skipIfMissing), addForeignKey: ( tableName: string, diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts index 396b0aff249f0..e3506bc9ab426 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -5,15 +5,19 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio await schemaBuilder.dropIndex( 'execution_entity', ['waitTill', 'id'], // covered by `waitTill, status` - `id` is unused - isPostgres ? 'IDX_85b981df7b444f905f8bf50747' : 'IDX_b94b45ce2c73ce46c54f20b5f9', + { + customIndexName: isPostgres + ? 'IDX_85b981df7b444f905f8bf50747' + : 'IDX_b94b45ce2c73ce46c54f20b5f9', + skipIfMissing: true, + }, ); if (isSqlite) { - await schemaBuilder.dropIndex( - 'execution_entity', - ['stoppedAt'], - 'idx_execution_entity_stopped_at', // duplicate of `IDX_execution_entity_stoppedAt` - ); + await schemaBuilder.dropIndex('execution_entity', ['stoppedAt'], { + customIndexName: 'idx_execution_entity_stopped_at', // duplicate of `IDX_execution_entity_stoppedAt` + skipIfMissing: true, + }); } await schemaBuilder.createIndex( From cd2ab81f460fd17df27b96922cbc1623455fab8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 20 Jun 2024 15:21:53 +0200 Subject: [PATCH 05/15] Revert "Remove unneeded sorting on FE" This reverts commit dbb1c2e1117fc46333b06ef1c31e6caa08315751. --- packages/editor-ui/src/stores/executions.store.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/editor-ui/src/stores/executions.store.ts b/packages/editor-ui/src/stores/executions.store.ts index bb0e9420c6f20..756fdff6c4d97 100644 --- a/packages/editor-ui/src/stores/executions.store.ts +++ b/packages/editor-ui/src/stores/executions.store.ts @@ -58,7 +58,15 @@ export const useExecutionsStore = defineStore('executions', () => { ); const currentExecutionsById = ref>({}); - const currentExecutions = computed(() => Object.values(currentExecutionsById.value)); + const currentExecutions = computed(() => { + const data = Object.values(currentExecutionsById.value); + + data.sort((a, b) => { + return new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime(); + }); + + return data; + }); const currentExecutionsByWorkflowId = computed(() => currentExecutions.value.reduce>((acc, execution) => { From 714546295d0d6fae5a937024e7c82c587821a960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 28 Jun 2024 10:15:32 +0200 Subject: [PATCH 06/15] Apply feedback --- packages/cli/src/databases/dsl/Indices.ts | 2 +- .../1717498465932-RefactorExecutionIndices.ts | 89 ++++++++++++++----- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/databases/dsl/Indices.ts b/packages/cli/src/databases/dsl/Indices.ts index 2fa2ccaa7590a..5791f13f55b3f 100644 --- a/packages/cli/src/databases/dsl/Indices.ts +++ b/packages/cli/src/databases/dsl/Indices.ts @@ -25,7 +25,7 @@ abstract class IndexOperation extends LazyPromise { void this.execute(queryRunner) .then(resolve) .catch((error) => { - if (this.skipIfMissing) { + if (error instanceof Error && error.message.includes('not found') && this.skipIfMissing) { resolve(); } else { reject(error); diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts index e3506bc9ab426..42afe5b82d167 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -1,39 +1,78 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; +/** + * Add new indices: + * + * - `status, startedAt` for default query at `ExecutionRepository.findManyByRangeQuery` + * - `workflowId, status, startedAt` for filtered query at `ExecutionRepository.findManyByRangeQuery` + * - `waitTill, status` for regular queries at `ExecutionRepository.getWaitingExecutions` + * + * Remove unused indices in sqlite: + * + * - `stoppedAt` (duplicate with different casing) + * - `waitTill` + * - `status, workflowId` + * + * Remove unused indices in MySql: + * + * - `status` + * + * Remove unused indices in all DBs: + * + * - `stopped_at` + * - `waitTill, id` + * - `workflowId, id` + * + * Keep index as is: + * + * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions` + */ export class RefactorExecutionIndices1717498465932 implements ReversibleMigration { - async up({ schemaBuilder, isPostgres, isSqlite }: MigrationContext) { - await schemaBuilder.dropIndex( - 'execution_entity', - ['waitTill', 'id'], // covered by `waitTill, status` - `id` is unused - { - customIndexName: isPostgres - ? 'IDX_85b981df7b444f905f8bf50747' - : 'IDX_b94b45ce2c73ce46c54f20b5f9', - skipIfMissing: true, - }, - ); + async up({ schemaBuilder, isPostgres, isSqlite, isMysql }: MigrationContext) { + await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); + await schemaBuilder.createIndex('execution_entity', ['workflowId', 'status', 'startedAt']); + await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status']); if (isSqlite) { await schemaBuilder.dropIndex('execution_entity', ['stoppedAt'], { - customIndexName: 'idx_execution_entity_stopped_at', // duplicate of `IDX_execution_entity_stoppedAt` + customIndexName: 'idx_execution_entity_stopped_at', + skipIfMissing: true, + }); + + await schemaBuilder.dropIndex('execution_entity', ['waitTill'], { + customIndexName: 'idx_execution_entity_wait_till', + skipIfMissing: true, + }); + + await schemaBuilder.dropIndex('execution_entity', ['status', 'workflowId'], { + customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584', skipIfMissing: true, }); } - await schemaBuilder.createIndex( - 'execution_entity', - ['status', 'startedAt'], // for default query at `GET /executions` - ); + if (isMysql) { + await schemaBuilder.dropIndex('execution_entity', ['status'], { + customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584', + skipIfMissing: true, + }); + } - await schemaBuilder.createIndex( - 'execution_entity', - ['workflowId', 'status', 'startedAt'], // for query with filters at `GET /executions` - ); + // all DBs - await schemaBuilder.createIndex( - 'execution_entity', - ['waitTill', 'status'], // for waiting executions queries on interval - ); + await schemaBuilder.dropIndex('execution_entity', ['stoppedAt']); + await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'id'], { + customIndexName: isPostgres + ? 'IDX_85b981df7b444f905f8bf50747' + : 'IDX_b94b45ce2c73ce46c54f20b5f9', + skipIfMissing: true, + }); + await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'id'], { + customIndexName: + isPostgres || isMysql + ? 'idx_execution_entity_workflow_id_id' + : 'IDX_81fc04c8a17de15835713505e4', + skipIfMissing: true, + }); } async down({ schemaBuilder }: MigrationContext) { @@ -42,5 +81,7 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); await schemaBuilder.createIndex('execution_entity', ['waitTill', 'id']); + await schemaBuilder.createIndex('execution_entity', ['stoppedAt']); + await schemaBuilder.createIndex('execution_entity', ['workflowId', 'id']); } } From dedb12ab46f410acfc1b29df6faf8306d31dee6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 11 Jul 2024 12:16:18 +0200 Subject: [PATCH 07/15] Convert `stoppedAt` into composite index for soft deletions --- .../1717498465932-RefactorExecutionIndices.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts index 42afe5b82d167..4c63f982e5076 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -3,9 +3,10 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; /** * Add new indices: * - * - `status, startedAt` for default query at `ExecutionRepository.findManyByRangeQuery` - * - `workflowId, status, startedAt` for filtered query at `ExecutionRepository.findManyByRangeQuery` - * - `waitTill, status` for regular queries at `ExecutionRepository.getWaitingExecutions` + * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) + * - `workflowId, status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (filter query) + * - `waitTill, status` for `ExecutionRepository.getWaitingExecutions` + * - `stoppedAt, deletedAt, status` for `ExecutionRepository.softDeletePrunableExecutions` * * Remove unused indices in sqlite: * @@ -13,16 +14,19 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; * - `waitTill` * - `status, workflowId` * - * Remove unused indices in MySql: + * Remove unused indices in MySQL: * * - `status` * * Remove unused indices in all DBs: * - * - `stopped_at` * - `waitTill, id` * - `workflowId, id` * + * Remove incomplete index in all DBs: + * + * - `stopped_at` (replaced with composite index) + * * Keep index as is: * * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions` @@ -32,6 +36,7 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); await schemaBuilder.createIndex('execution_entity', ['workflowId', 'status', 'startedAt']); await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status']); + await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); if (isSqlite) { await schemaBuilder.dropIndex('execution_entity', ['stoppedAt'], { @@ -79,6 +84,7 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio await schemaBuilder.dropIndex('execution_entity', ['status', 'startedAt']); await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'status', 'startedAt']); await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); + await schemaBuilder.dropIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); await schemaBuilder.createIndex('execution_entity', ['waitTill', 'id']); await schemaBuilder.createIndex('execution_entity', ['stoppedAt']); From 059ddf984c067a050e03f6bd2ab008319208df5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 11 Jul 2024 12:21:43 +0200 Subject: [PATCH 08/15] Combine when dropping `stoppedAt` --- .../common/1717498465932-RefactorExecutionIndices.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts index 4c63f982e5076..2bb3ea2647ef2 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts @@ -39,11 +39,6 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); if (isSqlite) { - await schemaBuilder.dropIndex('execution_entity', ['stoppedAt'], { - customIndexName: 'idx_execution_entity_stopped_at', - skipIfMissing: true, - }); - await schemaBuilder.dropIndex('execution_entity', ['waitTill'], { customIndexName: 'idx_execution_entity_wait_till', skipIfMissing: true, @@ -64,7 +59,11 @@ export class RefactorExecutionIndices1717498465932 implements ReversibleMigratio // all DBs - await schemaBuilder.dropIndex('execution_entity', ['stoppedAt']); + await schemaBuilder.dropIndex( + 'execution_entity', + ['stoppedAt'], + isSqlite ? { customIndexName: 'idx_execution_entity_stopped_at', skipIfMissing: true } : {}, + ); await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'id'], { customIndexName: isPostgres ? 'IDX_85b981df7b444f905f8bf50747' From dd177968c47dba5a0952de5ff09e95e395a7c6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 11 Jul 2024 12:22:00 +0200 Subject: [PATCH 09/15] Move `skipIfMissing` to `DropIndex` --- packages/cli/src/databases/dsl/Indices.ts | 36 +++++++++++++---------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/databases/dsl/Indices.ts b/packages/cli/src/databases/dsl/Indices.ts index 5791f13f55b3f..9f91cd0214ab0 100644 --- a/packages/cli/src/databases/dsl/Indices.ts +++ b/packages/cli/src/databases/dsl/Indices.ts @@ -19,18 +19,9 @@ abstract class IndexOperation extends LazyPromise { protected tablePrefix: string, queryRunner: QueryRunner, protected customIndexName?: string, - protected skipIfMissing = false, ) { - super((resolve, reject) => { - void this.execute(queryRunner) - .then(resolve) - .catch((error) => { - if (error instanceof Error && error.message.includes('not found') && this.skipIfMissing) { - resolve(); - } else { - reject(error); - } - }); + super((resolve) => { + void this.execute(queryRunner).then(resolve); }); } } @@ -57,10 +48,25 @@ export class CreateIndex extends IndexOperation { } export class DropIndex extends IndexOperation { + constructor( + tableName: string, + columnNames: string[], + tablePrefix: string, + queryRunner: QueryRunner, + customIndexName?: string, + protected skipIfMissing = false, + ) { + super(tableName, columnNames, tablePrefix, queryRunner, customIndexName); + } + async execute(queryRunner: QueryRunner) { - return await queryRunner.dropIndex( - this.fullTableName, - this.customIndexName ?? this.fullIndexName, - ); + return await queryRunner + .dropIndex(this.fullTableName, this.customIndexName ?? this.fullIndexName) + .catch((error) => { + if (error instanceof Error && error.message.includes('not found') && this.skipIfMissing) { + return; + } + throw error; + }); } } From fdc5909b9b7104d8d48f5a0fa9afc8a8d8dbd6c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 16 Aug 2024 10:15:41 +0200 Subject: [PATCH 10/15] Use `TypeORMError` --- packages/cli/src/databases/dsl/Indices.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/databases/dsl/Indices.ts b/packages/cli/src/databases/dsl/Indices.ts index 9f91cd0214ab0..040ab041057df 100644 --- a/packages/cli/src/databases/dsl/Indices.ts +++ b/packages/cli/src/databases/dsl/Indices.ts @@ -1,5 +1,5 @@ import type { QueryRunner } from '@n8n/typeorm'; -import { TableIndex } from '@n8n/typeorm'; +import { TableIndex, TypeORMError } from '@n8n/typeorm'; import LazyPromise from 'p-lazy'; abstract class IndexOperation extends LazyPromise { @@ -63,7 +63,11 @@ export class DropIndex extends IndexOperation { return await queryRunner .dropIndex(this.fullTableName, this.customIndexName ?? this.fullIndexName) .catch((error) => { - if (error instanceof Error && error.message.includes('not found') && this.skipIfMissing) { + if ( + error instanceof TypeORMError && + error.message.includes('not found') && + this.skipIfMissing + ) { return; } throw error; From 43e18a38f696b11373a12a5aceeaafa343b2f58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 16 Aug 2024 10:18:40 +0200 Subject: [PATCH 11/15] Update migration timestamp --- ...onIndices.ts => 1723796243146-RefactorExecutionIndices.ts} | 2 +- packages/cli/src/databases/migrations/mysqldb/index.ts | 4 ++-- packages/cli/src/databases/migrations/postgresdb/index.ts | 4 ++-- packages/cli/src/databases/migrations/sqlite/index.ts | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) rename packages/cli/src/databases/migrations/common/{1717498465932-RefactorExecutionIndices.ts => 1723796243146-RefactorExecutionIndices.ts} (98%) diff --git a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts similarity index 98% rename from packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts rename to packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts index 2bb3ea2647ef2..fe214f3ac2066 100644 --- a/packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts @@ -31,7 +31,7 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; * * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions` */ -export class RefactorExecutionIndices1717498465932 implements ReversibleMigration { +export class RefactorExecutionIndices1723796243146 implements ReversibleMigration { async up({ schemaBuilder, isPostgres, isSqlite, isMysql }: MigrationContext) { await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); await schemaBuilder.createIndex('execution_entity', ['workflowId', 'status', 'startedAt']); diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 2d886d3b69cc5..2431c390013ff 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -58,7 +58,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; -import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; export const mysqlMigrations: Migration[] = [ @@ -121,6 +121,6 @@ export const mysqlMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, - RefactorExecutionIndices1717498465932, AddConstraintToExecutionMetadata1720101653148, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 33823c5e28036..c902496237551 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -57,7 +57,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; -import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; import { FixExecutionMetadataSequence1721377157740 } from './1721377157740-FixExecutionMetadataSequence'; @@ -120,7 +120,7 @@ export const postgresMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, - RefactorExecutionIndices1717498465932, AddConstraintToExecutionMetadata1720101653148, FixExecutionMetadataSequence1721377157740, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index d2bcda161b6c7..8440824937dc4 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -55,7 +55,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess'; import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable'; import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting'; -import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices'; +import { RefactorExecutionIndices1723796243146 } from '../common/1723796243146-RefactorExecutionIndices'; import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata'; const sqliteMigrations: Migration[] = [ @@ -115,8 +115,8 @@ const sqliteMigrations: Migration[] = [ CreateProject1714133768519, MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, - RefactorExecutionIndices1717498465932, AddConstraintToExecutionMetadata1720101653148, + RefactorExecutionIndices1723796243146, ]; export { sqliteMigrations }; From 21212e28d7afe9e3aa1f138f1a46cdbc42c8316c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 16 Aug 2024 10:22:08 +0200 Subject: [PATCH 12/15] Use `status, startedAt` for default and filter query --- .../common/1723796243146-RefactorExecutionIndices.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts index fe214f3ac2066..ecb6055c95df4 100644 --- a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts @@ -3,8 +3,7 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; /** * Add new indices: * - * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) - * - `workflowId, status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (filter query) + * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) and for `ExecutionRepository.findManyByRangeQuery` (filter query) * - `waitTill, status` for `ExecutionRepository.getWaitingExecutions` * - `stoppedAt, deletedAt, status` for `ExecutionRepository.softDeletePrunableExecutions` * @@ -34,7 +33,6 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; export class RefactorExecutionIndices1723796243146 implements ReversibleMigration { async up({ schemaBuilder, isPostgres, isSqlite, isMysql }: MigrationContext) { await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); - await schemaBuilder.createIndex('execution_entity', ['workflowId', 'status', 'startedAt']); await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status']); await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); @@ -81,7 +79,6 @@ export class RefactorExecutionIndices1723796243146 implements ReversibleMigratio async down({ schemaBuilder }: MigrationContext) { await schemaBuilder.dropIndex('execution_entity', ['status', 'startedAt']); - await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'status', 'startedAt']); await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); await schemaBuilder.dropIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); From 90af0bab325314b21ce0950199d70f5574c5c1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Fri, 16 Aug 2024 11:07:49 +0200 Subject: [PATCH 13/15] Use partial indexes for PG and MySQL --- .../1723796243146-RefactorExecutionIndices.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts index ecb6055c95df4..68b6db179f1b7 100644 --- a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts @@ -4,8 +4,8 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; * Add new indices: * * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) and for `ExecutionRepository.findManyByRangeQuery` (filter query) - * - `waitTill, status` for `ExecutionRepository.getWaitingExecutions` - * - `stoppedAt, deletedAt, status` for `ExecutionRepository.softDeletePrunableExecutions` + * - `waitTill, status, deletedAt` for `ExecutionRepository.getWaitingExecutions` + * - `stoppedAt, status, deletedAt` for `ExecutionRepository.softDeletePrunableExecutions` * * Remove unused indices in sqlite: * @@ -31,10 +31,32 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions` */ export class RefactorExecutionIndices1723796243146 implements ReversibleMigration { - async up({ schemaBuilder, isPostgres, isSqlite, isMysql }: MigrationContext) { + async up({ schemaBuilder, isPostgres, isSqlite, isMysql, runQuery, escape }: MigrationContext) { await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); - await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status']); - await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']); + + if (isSqlite || isPostgres) { + const executionEntity = escape.tableName('execution_entity'); + + const waitTill = escape.columnName('waitTill'); + const status = escape.columnName('status'); + const deletedAt = escape.columnName('deletedAt'); + const stoppedAt = escape.columnName('stoppedAt'); + + await runQuery(` + CREATE INDEX idx_execution_entity_wait_till_status_deleted_at + ON ${executionEntity} (${waitTill}, ${status}, ${deletedAt}) + WHERE ${waitTill} IS NOT NULL AND ${deletedAt} IS NULL; + `); + + await runQuery(` + CREATE INDEX idx_execution_entity_stopped_at_status_deleted_at + ON ${executionEntity} (${stoppedAt}, ${status}, ${deletedAt}) + WHERE ${stoppedAt} IS NOT NULL AND ${deletedAt} IS NULL; + `); + } else if (isMysql) { + await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status', 'deletedAt']); + await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'status', 'deletedAt']); + } if (isSqlite) { await schemaBuilder.dropIndex('execution_entity', ['waitTill'], { From 9af725d02fd23336c290eea4f1b4f6af7e547761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 22 Aug 2024 11:25:03 +0200 Subject: [PATCH 14/15] Sort migrations by date --- packages/cli/src/databases/migrations/mysqldb/index.ts | 2 +- packages/cli/src/databases/migrations/postgresdb/index.ts | 2 +- packages/cli/src/databases/migrations/sqlite/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 42d1755a5a38a..ed08c6b2f2916 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -123,6 +123,6 @@ export const mysqlMigrations: Migration[] = [ MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, - RefactorExecutionIndices1723796243146, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index dd156dde4d80c..3a37f369360c3 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -123,6 +123,6 @@ export const postgresMigrations: Migration[] = [ AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, FixExecutionMetadataSequence1721377157740, - RefactorExecutionIndices1723796243146, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 20c525be776d2..e55a36c1c1c34 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -117,8 +117,8 @@ const sqliteMigrations: Migration[] = [ MakeExecutionStatusNonNullable1714133768521, AddActivatedAtUserSetting1717498465931, AddConstraintToExecutionMetadata1720101653148, - RefactorExecutionIndices1723796243146, CreateInvalidAuthTokenTable1723627610222, + RefactorExecutionIndices1723796243146, ]; export { sqliteMigrations }; From 40681750bc9174e3e8456a770ecd0d01bed0ab85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 22 Aug 2024 11:38:17 +0200 Subject: [PATCH 15/15] Apply feedback re: `workflowId, startedAt` index --- .../1723796243146-RefactorExecutionIndices.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts index 68b6db179f1b7..dcf558f2022bd 100644 --- a/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts +++ b/packages/cli/src/databases/migrations/common/1723796243146-RefactorExecutionIndices.ts @@ -3,7 +3,7 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; /** * Add new indices: * - * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) and for `ExecutionRepository.findManyByRangeQuery` (filter query) + * - `workflowId, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query) and for `ExecutionRepository.findManyByRangeQuery` (filter query) * - `waitTill, status, deletedAt` for `ExecutionRepository.getWaitingExecutions` * - `stoppedAt, status, deletedAt` for `ExecutionRepository.softDeletePrunableExecutions` * @@ -32,16 +32,22 @@ import type { MigrationContext, ReversibleMigration } from '@/databases/types'; */ export class RefactorExecutionIndices1723796243146 implements ReversibleMigration { async up({ schemaBuilder, isPostgres, isSqlite, isMysql, runQuery, escape }: MigrationContext) { - await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']); - if (isSqlite || isPostgres) { const executionEntity = escape.tableName('execution_entity'); + const workflowId = escape.columnName('workflowId'); + const startedAt = escape.columnName('startedAt'); const waitTill = escape.columnName('waitTill'); const status = escape.columnName('status'); const deletedAt = escape.columnName('deletedAt'); const stoppedAt = escape.columnName('stoppedAt'); + await runQuery(` + CREATE INDEX idx_execution_entity_workflow_id_started_at + ON ${executionEntity} (${workflowId}, ${startedAt}) + WHERE ${startedAt} IS NOT NULL AND ${deletedAt} IS NULL; + `); + await runQuery(` CREATE INDEX idx_execution_entity_wait_till_status_deleted_at ON ${executionEntity} (${waitTill}, ${status}, ${deletedAt}) @@ -54,6 +60,7 @@ export class RefactorExecutionIndices1723796243146 implements ReversibleMigratio WHERE ${stoppedAt} IS NOT NULL AND ${deletedAt} IS NULL; `); } else if (isMysql) { + await schemaBuilder.createIndex('execution_entity', ['workflowId', 'startedAt']); await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status', 'deletedAt']); await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'status', 'deletedAt']); } @@ -100,7 +107,7 @@ export class RefactorExecutionIndices1723796243146 implements ReversibleMigratio } async down({ schemaBuilder }: MigrationContext) { - await schemaBuilder.dropIndex('execution_entity', ['status', 'startedAt']); + await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'startedAt']); await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']); await schemaBuilder.dropIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']);