Historical by timestamp

subquery · Nov 7, 2024 · e22fda5 · e22fda5
1 parent cdfa98e
commit e22fda5
Show file tree

Hide file tree

Showing 43 changed files with 455 additions and 454 deletions.
diff --git a/packages/common/src/constants.ts b/packages/common/src/constants.ts
@@ -39,12 +39,6 @@ export const runnerMapping = {
   '@subql/node-concordium': NETWORK_FAMILY.concordium,
 };
 
-// DATABASE TYPE
-export enum SUPPORT_DB {
-  cockRoach = 'CockroachDB',
-  postgres = 'PostgreSQL',
-}
-
 // DATABASE ERROR REGEX
 export const CONNECTION_SSL_ERROR_REGEX = 'not support SSL';
 

diff --git a/packages/common/src/project/database/databaseUtil.ts b/packages/common/src/project/database/databaseUtil.ts
diff --git a/packages/common/src/project/database/index.ts b/packages/common/src/project/database/index.ts
diff --git a/packages/common/src/project/index.ts b/packages/common/src/project/index.ts
@@ -4,6 +4,5 @@
 export * from './load';
 export * from './versioned';
 export * from './readers';
-export * from './database';
 export * from './utils';
 export * from './IpfsHttpClientLite';
diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts
@@ -8,6 +8,7 @@ import {getFileContent, loadFromJsonOrYaml, normalizeNetworkEndpoints} from '@su
 import {IEndpointConfig} from '@subql/types-core';
 import {last} from 'lodash';
 import {LevelWithSilent} from 'pino';
+import {HistoricalMode} from '../indexer';
 import {getLogger} from '../logger';
 import {assign} from '../utils/object';
 
@@ -38,7 +39,7 @@ export interface IConfig {
   readonly profiler?: boolean;
   readonly unsafe?: boolean;
   readonly subscription: boolean;
-  readonly disableHistorical: boolean;
+  readonly historical: HistoricalMode;
   readonly multiChain: boolean;
   readonly reindex?: number;
   readonly unfinalizedBlocks?: boolean;
@@ -74,7 +75,7 @@ const DEFAULT_CONFIG = {
   dictionaryQuerySize: 10000,
   profiler: false,
   subscription: false,
-  disableHistorical: false,
+  historical: 'height',
   multiChain: false,
   unfinalizedBlocks: false,
   storeCacheThreshold: 1000,
@@ -256,8 +257,8 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     return this._config.subscription;
   }
 
-  get disableHistorical(): boolean {
-    return this._isTest ? true : this._config.disableHistorical;
+  get historical(): HistoricalMode {
+    return this._isTest ? false : this._config.historical;
   }
 
   get multiChain(): boolean {
@@ -325,7 +326,7 @@ export class NodeConfig<C extends IConfig = IConfig> implements IConfig {
     const defaultMonitorFileSize = 200;
     // If user passed though yarg, we will record monitor file by this size, no matter poi or not
     // if user didn't pass through yarg, we will record monitor file by this default size only when poi is enabled
-    return this._config.monitorFileSize ?? this._config.proofOfIndex ? defaultMonitorFileSize : 0;
+    return (this._config.monitorFileSize ?? this._config.proofOfIndex) ? defaultMonitorFileSize : 0;
   }
 
   merge(config: Partial<IConfig>): this {

diff --git a/packages/node-core/src/configure/configure.module.ts b/packages/node-core/src/configure/configure.module.ts
@@ -67,10 +67,13 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record<string, string>
         value = [value];
       }
       if (Array.isArray(value)) {
-        value = value.reduce((acc, endpoint, index) => {
-          acc[endpoint] = endpointConfig[index] ?? {};
-          return acc;
-        }, {} as Record<string, IEndpointConfig>);
+        value = value.reduce(
+          (acc, endpoint, index) => {
+            acc[endpoint] = endpointConfig[index] ?? {};
+            return acc;
+          },
+          {} as Record<string, IEndpointConfig>
+        );
       }
     }
     if (key === 'primary-network-endpoint') {
@@ -79,6 +82,21 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record<string, string>
     }
     if (['network-endpoint-config', 'primary-network-endpoint-config'].includes(key)) return acc;
 
+    if (key === 'disable-historical' && value) {
+      acc.historical = false;
+    }
+    if (key === 'historical' && value === 'false') {
+      value = false;
+      // acc.historical = false;
+    }
+
+    if (key === 'historical') {
+      console.log('HISTORICAL KEY', typeof value, value);
+    }
+    if (key === 'disable-historical') {
+      console.log('HISTORICAL KEY DIS', typeof value, value);
+    }
+
     acc[nameMapping[key] ?? camelCase(key)] = value;
     return acc;
   }, {} as any);

diff --git a/packages/node-core/src/db/migration-service/SchemaMigration.service.ts b/packages/node-core/src/db/migration-service/SchemaMigration.service.ts
@@ -1,9 +1,8 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {SUPPORT_DB} from '@subql/common';
 import {getAllEntitiesRelations, GraphQLModelsType, GraphQLRelationsType} from '@subql/utils';
-import {ModelStatic, Sequelize, Transaction} from '@subql/x-sequelize';
+import {Sequelize, Transaction} from '@subql/x-sequelize';
 import {GraphQLSchema} from 'graphql';
 import {NodeConfig} from '../../configure';
 import {StoreService} from '../../indexer';
@@ -31,8 +30,7 @@ export class SchemaMigrationService {
     private storeService: StoreService,
     private flushCache: (flushAll?: boolean) => Promise<void>,
     private dbSchema: string,
-    private config: NodeConfig,
-    private dbType: SUPPORT_DB = SUPPORT_DB.postgres
+    private config: NodeConfig
   ) {}
 
   static validateSchemaChanges(currentSchema: GraphQLSchema, nextSchema: GraphQLSchema): boolean {
@@ -118,13 +116,7 @@ export class SchemaMigrationService {
     // Flush any pending data before running the migration
     await this.flushCache(true);
 
-    const migrationAction = await Migration.create(
-      this.sequelize,
-      this.storeService,
-      this.dbSchema,
-      this.config,
-      this.dbType
-    );
+    const migrationAction = await Migration.create(this.sequelize, this.storeService, this.dbSchema, this.config);
 
     if (this.config.debug) {
       logger.debug(`${schemaChangesLoggerMessage(schemaDifference)}`);

diff --git a/packages/node-core/src/db/migration-service/migration.ts b/packages/node-core/src/db/migration-service/migration.ts
@@ -2,15 +2,13 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import assert from 'node:assert';
-import {SUPPORT_DB} from '@subql/common';
 import {
   GraphQLEntityField,
   GraphQLEntityIndex,
   GraphQLEnumsType,
   GraphQLModelsType,
   GraphQLRelationsType,
   hashName,
-  IndexType,
 } from '@subql/utils';
 import {
   IndexesOptions,
@@ -23,7 +21,7 @@ import {
 } from '@subql/x-sequelize';
 import {isEqual, uniq} from 'lodash';
 import {NodeConfig} from '../../configure/NodeConfig';
-import {StoreService} from '../../indexer';
+import {HistoricalMode, StoreService} from '../../indexer';
 import {getLogger} from '../../logger';
 import {EnumType, getColumnOption, modelsTypeToModelAttributes} from '../../utils';
 import {formatAttributes, formatColumnName, modelToTableName} from '../sequelizeUtil';
@@ -43,21 +41,19 @@ export class Migration {
    */
   private mainQueries: syncHelper.QueryString[] = [];
   private extraQueries: syncHelper.QueryString[] = [];
-  private readonly historical: boolean;
+  private readonly historical: HistoricalMode;
   private readonly useSubscription: boolean;
   private foreignKeyMap: Map<string, Map<string, syncHelper.SmartTags>> = new Map<
     string,
     Map<string, syncHelper.SmartTags>
   >();
   private enumTypeMap: Map<string, EnumType>;
-  private removedIndexes: RemovedIndexes = {};
 
   private constructor(
     private sequelize: Sequelize,
     private storeService: StoreService,
     private readonly schemaName: string,
     private readonly config: NodeConfig,
-    private readonly dbType: SUPPORT_DB,
     private readonly existingForeignKeys: string[], // this the source of truth from the db
     private initEnumTypeMap: Map<string, EnumType>,
     private existingIndexes: {indexname: string}[]
@@ -67,11 +63,6 @@ export class Migration {
     this.historical = storeService.historical;
     this.useSubscription = config.subscription;
 
-    if (this.useSubscription && dbType === SUPPORT_DB.cockRoach) {
-      this.useSubscription = false;
-      logger.warn(`Subscription is not support with ${this.dbType}`);
-    }
-
     this.enumTypeMap = this.initEnumTypeMap;
 
     if (this.useSubscription) {
@@ -83,8 +74,7 @@ export class Migration {
     sequelize: Sequelize,
     storeService: StoreService,
     schemaName: string,
-    config: NodeConfig,
-    dbType: SUPPORT_DB
+    config: NodeConfig
   ): Promise<Migration> {
     const existingForeignKeys = await syncHelper.getExistingForeignKeys(schemaName, sequelize);
     const enumTypeMap = await syncHelper.getExistingEnums(schemaName, sequelize);
@@ -93,16 +83,7 @@ export class Migration {
     })) as {
       indexname: string;
     }[];
-    return new Migration(
-      sequelize,
-      storeService,
-      schemaName,
-      config,
-      dbType,
-      existingForeignKeys,
-      enumTypeMap,
-      indexesResult
-    );
+    return new Migration(sequelize, storeService, schemaName, config, existingForeignKeys, enumTypeMap, indexesResult);
   }
 
   async run(transaction?: Transaction): Promise<{modifiedModels: ModelStatic<any>[]; removedModels: string[]}> {
@@ -137,8 +118,6 @@ export class Migration {
       throw e;
     }
 
-    this.afterHandleCockroachIndex();
-
     return {
       modifiedModels: this.modifiedModels,
       removedModels: this.removedModels,
@@ -189,8 +168,6 @@ export class Migration {
     }
 
     syncHelper.updateIndexesName(model.name, indexes, existedIndexes as string[]);
-    // Update index query for cockroach db
-    this.beforeHandleCockroachIndex(model.name, indexes, existedIndexes as string[]);
 
     const sequelizeModel = this.storeService.defineModel(model, attributes, indexes, this.schemaName);
 
@@ -220,14 +197,6 @@ export class Migration {
         syncHelper.validateNotifyTriggers(triggerName, notifyTriggers as syncHelper.NotifyTriggerPayload[]);
       }
     } else {
-      //TODO: DROP TRIGGER IF EXIST is not valid syntax for cockroach, better check trigger exist at first.
-      if (this.dbType !== SUPPORT_DB.cockRoach) {
-        // trigger drop should be prioritized
-        this.extraQueries.unshift(syncHelper.dropNotifyTrigger(this.schemaName, sequelizeModel.tableName));
-      }
-    }
-
-    if (!this.useSubscription && this.dbType !== SUPPORT_DB.cockRoach) {
       this.extraQueries.push(syncHelper.dropNotifyFunction(this.schemaName));
     }
 
@@ -311,9 +280,7 @@ export class Migration {
           const rel = model.belongsTo(relatedModel, {foreignKey: relation.foreignKey});
           const fkConstraint = syncHelper.getFkConstraint(rel.source.tableName, rel.foreignKey);
           if (this.existingForeignKeys.includes(fkConstraint)) break;
-          if (this.dbType !== SUPPORT_DB.cockRoach) {
-            this.extraQueries.push(syncHelper.constraintDeferrableQuery(model.getTableName().toString(), fkConstraint));
-          }
+          this.extraQueries.push(syncHelper.constraintDeferrableQuery(model.getTableName().toString(), fkConstraint));
           break;
         }
         case 'hasOne': {
@@ -415,17 +382,9 @@ export class Migration {
       queries.unshift(syncHelper.createEnumQuery(type, escapedEnumValues));
     }
 
-    if (this.dbType === SUPPORT_DB.cockRoach) {
-      logger.warn(
-        `Comment on enum ${e.description} is not supported with ${this.dbType}, enum name may display incorrectly in query service`
-      );
-    } else {
-      const comment = this.sequelize.escape(
-        `@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}`
-      );
+    const comment = this.sequelize.escape(`@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}`);
 
-      queries.push(syncHelper.commentOnEnumQuery(type, comment));
-    }
+    queries.push(syncHelper.commentOnEnumQuery(type, comment));
     this.mainQueries.unshift(...queries);
     this.enumTypeMap.set(enumTypeName, {
       enumValues: e.values,
@@ -473,44 +432,4 @@ export class Migration {
 
     this.mainQueries.push(...queries);
   }
-
-  // Sequelize model will generate follow query to create hash indexes
-  // Example SQL:  CREATE INDEX "accounts_person_id" ON "polkadot-starter"."accounts" USING hash ("person_id")
-  // This will be rejected from cockroach db due to syntax error
-  // To avoid this we need to create index manually and add to extraQueries in order to create index in db
-  private beforeHandleCockroachIndex(modelName: string, indexes: IndexesOptions[], existedIndexes: string[]): void {
-    if (this.dbType !== SUPPORT_DB.cockRoach) {
-      return;
-    }
-    indexes.forEach((index, i) => {
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      if (index.using === IndexType.HASH && !existedIndexes.includes(index.name!)) {
-        // TODO double check with idempotent on cockroach
-        const cockroachDbIndexQuery = `CREATE INDEX "${index.name}" ON "${this.schemaName}"."${modelToTableName(
-          modelName
-        )}"(${index.fields}) USING HASH;`;
-        this.extraQueries.push(cockroachDbIndexQuery);
-        if (this.removedIndexes[modelName] === undefined) {
-          this.removedIndexes[modelName] = [];
-        }
-        this.removedIndexes[modelName].push(indexes[i]);
-        delete indexes[i];
-      }
-    });
-  }
-
-  // Due to we have removed hash index, it will be missing from the model, we need temp store it under `this.removedIndexes`
-  // And force add back to the model use `afterHandleCockroachIndex()` after db is synced
-  private afterHandleCockroachIndex(): void {
-    if (this.dbType !== SUPPORT_DB.cockRoach) {
-      return;
-    }
-    const removedIndexes = Object.entries(this.removedIndexes);
-    if (removedIndexes.length > 0) {
-      for (const [model, indexes] of removedIndexes) {
-        const sqModel = this.sequelize.model(model);
-        (sqModel as any)._indexes = (sqModel as any)._indexes.concat(indexes);
-      }
-    }
-  }
 }