perf(core): Make execution queries faster

packages/cli/src/databases/dsl/Indices.ts

@@ -48,10 +48,25 @@ export class CreateIndex extends IndexOperation {
 }
 
 export class DropIndex extends IndexOperation {
+	constructor(
+		tableName: string,
+		columnNames: string[],
+		tablePrefix: string,
+		queryRunner: QueryRunner,
+		customIndexName?: string,
+		protected skipIfMissing = false,
+	) {
+		super(tableName, columnNames, tablePrefix, queryRunner, customIndexName);
+	}
+
 	async execute(queryRunner: QueryRunner) {
-		return await queryRunner.dropIndex(
-			this.fullTableName,
-			this.customIndexName ?? this.fullIndexName,
-		);
+		return await queryRunner
+			.dropIndex(this.fullTableName, this.customIndexName ?? this.fullIndexName)
+			.catch((error) => {
+				if (error instanceof Error && error.message.includes('not found') && this.skipIfMissing) {
+					return;
+				}
+				throw error;
+			});
 	}
 }

packages/cli/src/databases/dsl/index.ts

@@ -32,8 +32,14 @@ export const createSchemaBuilder = (tablePrefix: string, queryRunner: QueryRunne
 		customIndexName?: string,
 	) => new CreateIndex(tableName, columnNames, isUnique, tablePrefix, queryRunner, customIndexName),
 
-	dropIndex: (tableName: string, columnNames: string[], customIndexName?: string) =>
-		new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName),
+	dropIndex: (
+		tableName: string,
+		columnNames: string[],
+		{ customIndexName, skipIfMissing }: { customIndexName?: string; skipIfMissing?: boolean } = {
+			skipIfMissing: false,
+		},
+	) =>
+		new DropIndex(tableName, columnNames, tablePrefix, queryRunner, customIndexName, skipIfMissing),
 
 	addForeignKey: (
 		tableName: string,

packages/cli/src/databases/migrations/common/1717498465932-RefactorExecutionIndices.ts

@@ -0,0 +1,92 @@
+import type { MigrationContext, ReversibleMigration } from '@/databases/types';
+
+/**
+ * Add new indices:
+ *
+ * - `status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (default query)
+ * - `workflowId, status, startedAt` for `ExecutionRepository.findManyByRangeQuery` (filter query)
+ * - `waitTill, status` for `ExecutionRepository.getWaitingExecutions`
+ * - `stoppedAt, deletedAt, status` for `ExecutionRepository.softDeletePrunableExecutions`
+ *
+ * Remove unused indices in sqlite:
+ *
+ * - `stoppedAt` (duplicate with different casing)
+ * - `waitTill`
+ * - `status, workflowId`
+ *
+ * Remove unused indices in MySQL:
+ *
+ * - `status`
+ *
+ * Remove unused indices in all DBs:
+ *
+ * - `waitTill, id`
+ * - `workflowId, id`
+ *
+ * Remove incomplete index in all DBs:
+ *
+ * - `stopped_at` (replaced with composite index)
+ *
+ * Keep index as is:
+ *
+ * - `deletedAt` for query at `ExecutionRepository.hardDeleteSoftDeletedExecutions`
+ */
+export class RefactorExecutionIndices1717498465932 implements ReversibleMigration {
+	async up({ schemaBuilder, isPostgres, isSqlite, isMysql }: MigrationContext) {
+		await schemaBuilder.createIndex('execution_entity', ['status', 'startedAt']);
+		await schemaBuilder.createIndex('execution_entity', ['workflowId', 'status', 'startedAt']);
+		await schemaBuilder.createIndex('execution_entity', ['waitTill', 'status']);
+		await schemaBuilder.createIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']);
+
+		if (isSqlite) {
+			await schemaBuilder.dropIndex('execution_entity', ['waitTill'], {
+				customIndexName: 'idx_execution_entity_wait_till',
+				skipIfMissing: true,
+			});
+
+			await schemaBuilder.dropIndex('execution_entity', ['status', 'workflowId'], {
+				customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584',
+				skipIfMissing: true,
+			});
+		}
+
+		if (isMysql) {
+			await schemaBuilder.dropIndex('execution_entity', ['status'], {
+				customIndexName: 'IDX_8b6f3f9ae234f137d707b98f3bf43584',
+				skipIfMissing: true,
+			});
+		}
+
+		// all DBs
+
+		await schemaBuilder.dropIndex(
+			'execution_entity',
+			['stoppedAt'],
+			isSqlite ? { customIndexName: 'idx_execution_entity_stopped_at', skipIfMissing: true } : {},
+		);
+		await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'id'], {
+			customIndexName: isPostgres
+				? 'IDX_85b981df7b444f905f8bf50747'
+				: 'IDX_b94b45ce2c73ce46c54f20b5f9',
+			skipIfMissing: true,
+		});
+		await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'id'], {
+			customIndexName:
+				isPostgres || isMysql
+					? 'idx_execution_entity_workflow_id_id'
+					: 'IDX_81fc04c8a17de15835713505e4',
+			skipIfMissing: true,
+		});
+	}
+
+	async down({ schemaBuilder }: MigrationContext) {
+		await schemaBuilder.dropIndex('execution_entity', ['status', 'startedAt']);
+		await schemaBuilder.dropIndex('execution_entity', ['workflowId', 'status', 'startedAt']);
+		await schemaBuilder.dropIndex('execution_entity', ['waitTill', 'status']);
+		await schemaBuilder.dropIndex('execution_entity', ['stoppedAt', 'deletedAt', 'status']);
+
+		await schemaBuilder.createIndex('execution_entity', ['waitTill', 'id']);
+		await schemaBuilder.createIndex('execution_entity', ['stoppedAt']);
+		await schemaBuilder.createIndex('execution_entity', ['workflowId', 'id']);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -58,6 +58,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move
 import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess';
 import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable';
 import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting';
+import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices';
 import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata';
 
 export const mysqlMigrations: Migration[] = [
@@ -120,5 +121,6 @@ export const mysqlMigrations: Migration[] = [
 	CreateProject1714133768519,
 	MakeExecutionStatusNonNullable1714133768521,
 	AddActivatedAtUserSetting1717498465931,
+	RefactorExecutionIndices1717498465932,
 	AddConstraintToExecutionMetadata1720101653148,
 ];

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -57,6 +57,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move
 import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess';
 import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable';
 import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting';
+import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices';
 import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata';
 
 export const postgresMigrations: Migration[] = [
@@ -118,5 +119,6 @@ export const postgresMigrations: Migration[] = [
 	CreateProject1714133768519,
 	MakeExecutionStatusNonNullable1714133768521,
 	AddActivatedAtUserSetting1717498465931,
+	RefactorExecutionIndices1717498465932,
 	AddConstraintToExecutionMetadata1720101653148,
 ];

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -55,6 +55,7 @@ import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-Move
 import { RemoveNodesAccess1712044305787 } from '../common/1712044305787-RemoveNodesAccess';
 import { MakeExecutionStatusNonNullable1714133768521 } from '../common/1714133768521-MakeExecutionStatusNonNullable';
 import { AddActivatedAtUserSetting1717498465931 } from './1717498465931-AddActivatedAtUserSetting';
+import { RefactorExecutionIndices1717498465932 } from '../common/1717498465932-RefactorExecutionIndices';
 import { AddConstraintToExecutionMetadata1720101653148 } from '../common/1720101653148-AddConstraintToExecutionMetadata';
 
 const sqliteMigrations: Migration[] = [
@@ -114,6 +115,7 @@ const sqliteMigrations: Migration[] = [
 	CreateProject1714133768519,
 	MakeExecutionStatusNonNullable1714133768521,
 	AddActivatedAtUserSetting1717498465931,
+	RefactorExecutionIndices1717498465932,
 	AddConstraintToExecutionMetadata1720101653148,
 ];
 

packages/cli/src/databases/repositories/execution.repository.ts

@@ -780,8 +780,8 @@ export class ExecutionRepository extends Repository<ExecutionEntity> {
 			if (firstId) qb.andWhere('execution.id > :firstId', { firstId });
 			if (lastId) qb.andWhere('execution.id < :lastId', { lastId });
 
-			if (query.order?.stoppedAt === 'DESC') {
-				qb.orderBy({ 'execution.stoppedAt': 'DESC' });
+			if (query.order?.startedAt === 'DESC') {
+				qb.orderBy({ 'execution.startedAt': 'DESC' });
 			} else if (query.order?.top) {
 				qb.orderBy(`(CASE WHEN execution.status = '${query.order.top}' THEN 0 ELSE 1 END)`);
 			} else {

packages/cli/src/databases/types.ts

@@ -11,6 +11,8 @@ export interface MigrationContext {
 	tablePrefix: string;
 	dbType: DatabaseType;
 	isMysql: boolean;
+	isSqlite: boolean;
+	isPostgres: boolean;
 	dbName: string;
 	migrationName: string;
 	nodeTypes: INodeTypes;

packages/cli/src/databases/utils/migrationHelpers.ts

@@ -93,6 +93,8 @@ function parseJson<T>(data: string | T): T {
 const globalConfig = Container.get(GlobalConfig);
 const dbType = globalConfig.database.type;
 const isMysql = ['mariadb', 'mysqldb'].includes(dbType);
+const isSqlite = dbType === 'sqlite';
+const isPostgres = dbType === 'postgresdb';
 const dbName = globalConfig.database[dbType === 'mariadb' ? 'mysqldb' : dbType].database;
 const tablePrefix = globalConfig.database.tablePrefix;
 
@@ -101,6 +103,8 @@ const createContext = (queryRunner: QueryRunner, migration: Migration): Migratio
 	tablePrefix,
 	dbType,
 	isMysql,
+	isSqlite,
+	isPostgres,
 	dbName,
 	migrationName: migration.name,
 	queryRunner,

packages/cli/src/executions/execution.service.ts

@@ -360,7 +360,7 @@ export class ExecutionService {
 	/**
 	 * Return:
 	 *
-	 * - the latest summaries of current and completed executions that satisfy a query,
+	 * - the summaries of latest current and completed executions that satisfy a query,
 	 * - the total count of all completed executions that satisfy the query, and
 	 * - whether the total of completed executions is an estimate.
 	 *
@@ -381,7 +381,7 @@ export class ExecutionService {
 			this.findRangeWithCount({
 				...query,
 				status: completedStatuses,
-				order: { stoppedAt: 'DESC' },
+				order: { startedAt: 'DESC' },
 			}),
 		]);
 

packages/cli/src/executions/execution.types.ts

@@ -80,7 +80,7 @@ export namespace ExecutionSummaries {
 	type OrderFields = {
 		order?: {
 			top?: ExecutionStatus;
-			stoppedAt?: 'DESC';
+			startedAt?: 'DESC';
 		};
 	};
 }