From cbfc6b04e305f1ecb13a65b3f4552cabe43b2dfa Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 20 Jul 2022 19:52:20 +1200 Subject: [PATCH 1/2] Insert a config bundle records in dynamodb to refference the config file in s3. --- packages/cli/src/cli/config/action.import.ts | 17 +++++++++++++++-- packages/config/src/base.config.ts | 6 ++++++ packages/config/src/config/config.bundle.ts | 9 +++++++++ packages/config/src/dynamo/dynamo.config.ts | 2 ++ packages/config/src/index.ts | 1 + 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 packages/config/src/config/config.bundle.ts diff --git a/packages/cli/src/cli/config/action.import.ts b/packages/cli/src/cli/config/action.import.ts index a83c5f585..e753f4ba5 100644 --- a/packages/cli/src/cli/config/action.import.ts +++ b/packages/cli/src/cli/config/action.import.ts @@ -1,5 +1,5 @@ import { Env, fsa, LogConfig } from '@basemaps/shared'; -import { BaseConfig, ConfigBundled, ConfigProviderMemory } from '@basemaps/config'; +import { BaseConfig, Config, ConfigBundle, ConfigBundled, ConfigProviderMemory } from '@basemaps/config'; import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; import { Q, Updater } from './config.update.js'; import { invalidateCache } from '../util.js'; @@ -27,7 +27,7 @@ export class CommandImport extends CommandLineAction { this.config = this.defineStringParameter({ argumentName: 'CONFIG', parameterLongName: '--config', - description: 'Path of config json', + description: 'Path of config json, this can be both a local path or s3 location', required: true, }); this.backup = this.defineStringParameter({ @@ -49,6 +49,8 @@ export class CommandImport extends CommandLineAction { const config = this.config.value; const backup = this.backup.value; if (config == null) throw new Error('Please provide a config json'); + if (commit && !config.startsWith('s3://')) + throw new Error('To acturally import into dynamo has to use the config file from s3.'); const HostPrefix = Env.isProduction() ? '' : 'dev.'; const healthEndpoint = `https://${HostPrefix}basemaps.linz.govt.nz/v1/health`; @@ -68,6 +70,17 @@ export class CommandImport extends CommandLineAction { for (const config of mem.objects.values()) this.update(config, commit); await Promise.all(this.promises); + if (commit) { + const configBundle: ConfigBundle = { + id: Config.ConfigBundle.id(configJson.id), + name: Config.ConfigBundle.id(`config-${configJson.hash}.json`), + path: config, + hash: configJson.hash, + }; + logger.info({ config }, 'Import:ConfigBundle'); + if (Config.ConfigBundle.isWriteable()) await Config.ConfigBundle.put(configBundle); + } + if (commit && this.invalidations.length > 0) { // Lots of invalidations just invalidate everything if (this.invalidations.length > 10) { diff --git a/packages/config/src/base.config.ts b/packages/config/src/base.config.ts index dc4507db9..a48b807e9 100644 --- a/packages/config/src/base.config.ts +++ b/packages/config/src/base.config.ts @@ -1,5 +1,6 @@ import { Epsg } from '@basemaps/geo'; import { BaseConfig } from './config/base.js'; +import { ConfigBundle } from './config/config.bundle.js'; import { ConfigPrefix, ConfigPrefixes } from './config/prefix.js'; import { ConfigLayer, ConfigTileSet, TileSetType } from './config/tile.set.js'; import { @@ -34,6 +35,10 @@ export class ConfigInstance { return this.cfg.ProcessingJob; } + get ConfigBundle(): BasemapsConfigObject { + return this.cfg.ConfigBundle; + } + setConfigProvider(cfg: BasemapsConfigProvider): void { this.cfg = cfg; } @@ -93,6 +98,7 @@ export abstract class BasemapsConfigProvider { abstract Style: BasemapsConfigObject; abstract Provider: BasemapsConfigObject; abstract ProcessingJob: BasemapsConfigObject; + abstract ConfigBundle: BasemapsConfigObject; } export abstract class BasemapsConfigObject { diff --git a/packages/config/src/config/config.bundle.ts b/packages/config/src/config/config.bundle.ts new file mode 100644 index 000000000..53434f059 --- /dev/null +++ b/packages/config/src/config/config.bundle.ts @@ -0,0 +1,9 @@ +import { BaseConfig } from './base.js'; + +export interface ConfigBundle extends BaseConfig { + /** Path for the config bundle file */ + path: string; + + /** Hash of the config bundle file */ + hash: string; +} diff --git a/packages/config/src/dynamo/dynamo.config.ts b/packages/config/src/dynamo/dynamo.config.ts index f81205425..fbeba9ee4 100644 --- a/packages/config/src/dynamo/dynamo.config.ts +++ b/packages/config/src/dynamo/dynamo.config.ts @@ -9,6 +9,7 @@ import { ConfigTileSet } from '../config/tile.set.js'; import { ConfigVectorStyle } from '../config/vector.style.js'; import { ConfigDynamoBase } from './dynamo.config.base.js'; import { ConfigDynamoCached } from './dynamo.config.cached.js'; +import { ConfigBundle } from '../config/config.bundle.js'; export class ConfigProviderDynamo extends BasemapsConfigProvider { Prefix = ConfigPrefix; @@ -22,6 +23,7 @@ export class ConfigProviderDynamo extends BasemapsConfigProvider { TileSet = new ConfigDynamoBase(this, ConfigPrefix.TileSet); Provider = new ConfigDynamoCached(this, ConfigPrefix.Provider); ProcessingJob = new ConfigDynamoBase(this, ConfigPrefix.ProcessingJob); + ConfigBundle = new ConfigDynamoBase(this, ConfigPrefix.ConfigBundle); constructor(tableName: string) { super(); diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index e16a630aa..95920bd05 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -3,6 +3,7 @@ export { BaseConfig } from './config/base.js'; export { ConfigImagery } from './config/imagery.js'; export { ConfigPrefix } from './config/prefix.js'; export { ConfigProvider } from './config/provider.js'; +export { ConfigBundle } from './config/config.bundle.js'; export { JobStatus, ProcessingJob, From 59c521391b1f21b0962176c7bef54192c02f3efe Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 21 Jul 2022 09:07:56 +1200 Subject: [PATCH 2/2] Add configbunle to config provider memory --- packages/config/src/memory/memory.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index eb4718740..ce86e0f82 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -9,6 +9,7 @@ import { ConfigProcessingJob } from '../config/processing.job.js'; import { ConfigProvider } from '../config/provider.js'; import { ConfigTileSet, TileSetType } from '../config/tile.set.js'; import { ConfigVectorStyle } from '../config/vector.style.js'; +import { ConfigBundle } from '../config/config.bundle.js'; import { standardizeLayerName } from '../json/name.convertor.js'; /** bundle the configuration as a single JSON object */ @@ -51,6 +52,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { TileSet = new MemoryConfigObject(this, ConfigPrefix.TileSet); Provider = new MemoryConfigObject(this, ConfigPrefix.Provider); ProcessingJob = new MemoryConfigObject(this, ConfigPrefix.ProcessingJob); + ConfigBundle = new MemoryConfigObject(this, ConfigPrefix.ConfigBundle); /** Memory cache of all objects */ objects = new Map();