From 8ab3f5ebb262c69101b2e61d84ecbfca317af14f Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 9 Jun 2023 13:22:42 +0800 Subject: [PATCH 1/2] fix(extension-store-canner): support adding workspace sql name prefix to summary and operationId in spec of artifact. - display indicator files of each workspace when log debug info at "CannerPersistenceStore" and "CannerProfileReader" - support adding workspace sql name prefix to summary and operationId in spec of artifact in the "CannerPersistenceStore" --- .../src/lib/canner/persistenceStore.ts | 21 ++++++++++-- .../src/lib/canner/profileReader.ts | 10 ++++-- .../src/test/cannerPersistenceStore.spec.ts | 32 +++++++++++++++---- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/packages/extension-store-canner/src/lib/canner/persistenceStore.ts b/packages/extension-store-canner/src/lib/canner/persistenceStore.ts index 2ed3d1a1..ca2e81fd 100644 --- a/packages/extension-store-canner/src/lib/canner/persistenceStore.ts +++ b/packages/extension-store-canner/src/lib/canner/persistenceStore.ts @@ -70,7 +70,11 @@ export class CannerPersistenceStore extends PersistentStore { }); // get the indicator files path of each workspaces const files = await getIndicatorFilesOfWorkspaces(filesInfo); - this.logger.debug('Succeed to get the indicator files of each workspaces'); + this.logger.debug( + `Succeed to get the indicator files of each workspaces: ${JSON.stringify( + files + )}` + ); // get the latest artifacts of each workspaces const artifacts = await this.getLatestArtifactsOfWorkspaces( storageService, @@ -78,6 +82,7 @@ export class CannerPersistenceStore extends PersistentStore { ); // merge the artifacts of each workspaces to one artifact const artifact = await this.mergeArtifactsOfWorkspaces(artifacts); + this.logger.debug(`Succeed to merge the artifacts: ${artifact}`); return Buffer.from(JSON.stringify(artifact), 'utf-8'); } @@ -150,11 +155,21 @@ export class CannerPersistenceStore extends PersistentStore { if (artifact.specs['oas3']['paths']) Object.entries(artifact.specs['oas3']['paths']).forEach( ([apiEndpoint, endpointInfo]) => { - // concat the workspace sql name prefix to original api endpoint - // ths api endpoint has the "/" prefix, so concat directly + // concat the workspace sql name prefix to original api endpoint, the "apiEndpoint" has the "/" prefix, so concat directly const endpoint = `${workspaceSqlName}${apiEndpoint}`; merged.specs['oas3']['paths'][endpoint] = endpointInfo as oas3.PathItemObject; + // Add workspace sql name prefix to original operationId & summary + const { summary, operationId } = merged.specs['oas3']['paths'][ + endpoint + ]['get'] as oas3.OperationObject; + merged.specs['oas3']['paths'][endpoint]['get'] = { + ...merged.specs['oas3']['paths'][endpoint]['get'], + // e.g: get/xxx => get/{workspaceSqlName}/xxx + operationId: `get/${workspaceSqlName}/${operationId?.slice(4)}`, + // e.g: /xxx => /{workspaceSqlName}/xxx + summary: `/${workspaceSqlName}${summary}`, + }; } ); return merged; diff --git a/packages/extension-store-canner/src/lib/canner/profileReader.ts b/packages/extension-store-canner/src/lib/canner/profileReader.ts index dbd06c97..3c1a29cd 100644 --- a/packages/extension-store-canner/src/lib/canner/profileReader.ts +++ b/packages/extension-store-canner/src/lib/canner/profileReader.ts @@ -3,7 +3,7 @@ import { ProfileReader, VulcanExtensionId, VulcanInternalExtension, - ConfigurationError + ConfigurationError, } from '@vulcan-sql/core'; import { CannerStoreConfig, getEnvConfig } from '../config'; import { createStorageService } from '../storageService'; @@ -38,7 +38,11 @@ export class CannerProfileReader extends ProfileReader { }); // get the indicator files path of each workspaces const files = await getIndicatorFilesOfWorkspaces(filesInfo); - this.logger.debug('Succeed to get the indicator files of each workspaces'); + this.logger.debug( + `Succeed to get the indicator files of each workspaces: ${JSON.stringify( + files + )}` + ); // generate profiles from the indicator files of each workspaces const { user, password, host, port } = this.envConfig.profile; @@ -65,7 +69,7 @@ export class CannerProfileReader extends ProfileReader { }, allow: '*', } as Profile>; - this.logger.debug(`created ${profile.name}.`); + this.logger.debug(`created "${profile.name}".`); return profile; }) ); diff --git a/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts b/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts index 323d77b5..8325faef 100644 --- a/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts +++ b/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts @@ -139,7 +139,6 @@ describe('Test CannerPersistenceStore', () => { it('Should load successfully when "specs" field is "oas3" format ', async () => { // Arrange - // test artifacts from each workspaces downloaded from storage const artifacts = { ws1: { @@ -156,7 +155,12 @@ describe('Test CannerPersistenceStore', () => { oas3: { ...sinon.stubInterface(), paths: { - '/orders': {}, + '/orders': { + get: { + operationId: 'get/orders', + summary: '/orders', + }, + }, }, }, }, @@ -175,7 +179,12 @@ describe('Test CannerPersistenceStore', () => { oas3: { ...sinon.stubInterface(), paths: { - '/products/:id': {}, + '/products/:id': { + get: { + operationId: 'get/products/:id', + summary: '/products/:id', + }, + }, }, }, }, @@ -209,10 +218,19 @@ describe('Test CannerPersistenceStore', () => { description: 'Data API for Canner Enterprise', }, paths: { - [`${fakeWorkspaces.ws1.sqlName}/orders`]: - artifacts['ws1'].specs['oas3'].paths['/orders'], - [`${fakeWorkspaces.ws2.sqlName}/products/:id`]: - artifacts['ws2'].specs['oas3'].paths['/products/:id'], + [`${fakeWorkspaces.ws1.sqlName}/orders`]: { + get: { + operationId: `get/${fakeWorkspaces.ws1.sqlName}/orders`, + summary: `/${fakeWorkspaces.ws1.sqlName}/orders`, + }, + }, + + [`${fakeWorkspaces.ws2.sqlName}/products/:id`]: { + get: { + operationId: `get/${fakeWorkspaces.ws2.sqlName}/products/:id`, + summary: `/${fakeWorkspaces.ws2.sqlName}/products/:id`, + }, + }, }, }, }, From 2d530cca2284752935d334ca33da94e7880ccfcf Mon Sep 17 00:00:00 2001 From: kokokuo Date: Mon, 12 Jun 2023 15:29:15 +0800 Subject: [PATCH 2/2] fix(extension-store-canner): fix non-consistency canner profile name - add some debug logger for providing information about "Evaluator" and "CannerPersistenceStore" - fix the non-consistency canner profile name in "CannerProfileReader". - fix missing the root path when download buffer stream of the artifact - update th test cases --- .../src/lib/canner/persistenceStore.ts | 17 ++++---- .../src/lib/canner/profileReader.ts | 4 +- .../src/test/cannerPersistenceStore.spec.ts | 39 +++++++++++++------ .../src/test/cannerProfileReader.spec.ts | 4 +- packages/serve/src/lib/evaluator/evaluator.ts | 1 + 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/packages/extension-store-canner/src/lib/canner/persistenceStore.ts b/packages/extension-store-canner/src/lib/canner/persistenceStore.ts index ca2e81fd..29298efe 100644 --- a/packages/extension-store-canner/src/lib/canner/persistenceStore.ts +++ b/packages/extension-store-canner/src/lib/canner/persistenceStore.ts @@ -4,7 +4,6 @@ import { PersistentStore, TYPES, VulcanExtensionId, - VulcanInternalExtension, } from '@vulcan-sql/core'; import { inject } from 'inversify'; import * as oas3 from 'openapi3-ts'; @@ -37,7 +36,7 @@ interface WorkspaceArtifact { * Used the string to identify the extension Id not by the enum "ArtifactBuilderProviderType". * Because if we create another enum to extend the 'ArtifactBuilderProviderType', it seems unnecessary to give the new enum only has 'Canner' as its type." * */ -@VulcanInternalExtension() + @VulcanExtensionId('Canner') export class CannerPersistenceStore extends PersistentStore { private filePath: string; @@ -92,14 +91,16 @@ export class CannerPersistenceStore extends PersistentStore { ): Promise { return await Promise.all( // download latest artifact buffer content of each workspace by viewing the indicator.json of the each workspace - indicators.map(async ({ workspaceId, name }) => { - const buffer = await storageService.downObjectAsBuffer({ name }); + indicators.map(async ({ workspaceId, name: indicatorPath }) => { + const buffer = await storageService.downObjectAsBuffer({ + name: indicatorPath, + }); const indicator = JSON.parse( buffer.toString('utf-8') ) as ArtifactIndicator; const artifact = await this.getWorkspaceArtifact( storageService, - workspaceId, + indicatorPath, indicator ); this.logger.debug('Succeed to download latest artifacts of workspaces'); @@ -113,11 +114,13 @@ export class CannerPersistenceStore extends PersistentStore { private async getWorkspaceArtifact( storageService: BaseStorageService, - workspaceId: string, + indicatorPath: string, indicator: ArtifactIndicator ): Promise { const latestArtifactFolder = indicator[indicator.master]; - const path = `${workspaceId}/vulcansql/${latestArtifactFolder}/result.json`; + const vulcanFolderPath = indicatorPath.replace('/indicator.json', ''); + const path = `${vulcanFolderPath}/${latestArtifactFolder}/result.json`; + this.logger.debug(`Download the artifact from path: ${path}`); // download from artifact path name const buffer = await storageService.downObjectAsBuffer({ name: path, diff --git a/packages/extension-store-canner/src/lib/canner/profileReader.ts b/packages/extension-store-canner/src/lib/canner/profileReader.ts index 3c1a29cd..970d0b05 100644 --- a/packages/extension-store-canner/src/lib/canner/profileReader.ts +++ b/packages/extension-store-canner/src/lib/canner/profileReader.ts @@ -2,7 +2,6 @@ import { Profile, ProfileReader, VulcanExtensionId, - VulcanInternalExtension, ConfigurationError, } from '@vulcan-sql/core'; import { CannerStoreConfig, getEnvConfig } from '../config'; @@ -19,7 +18,6 @@ export interface CannerProfileReaderOptions { * Used the string to identify the extension Id not by the enum "LocalFileProfileReader". * Because if we create another enum to extend the 'LocalFileProfileReader', it seems unnecessary to give the new enum only has 'Canner' as its type." * */ -@VulcanInternalExtension() @VulcanExtensionId('Canner') export class CannerProfileReader extends ProfileReader { private envConfig: CannerStoreConfig = getEnvConfig(); @@ -58,7 +56,7 @@ export class CannerProfileReader extends ProfileReader { ) as ArtifactIndicator; const workspaceSqlName = indicator[workspaceId]; const profile = { - name: `profile-${workspaceSqlName}`, + name: `canner-${workspaceSqlName}`, type: 'canner', connection: { user, diff --git a/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts b/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts index 8325faef..7fa0e2af 100644 --- a/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts +++ b/packages/extension-store-canner/src/test/cannerPersistenceStore.spec.ts @@ -11,6 +11,7 @@ import { } from '../lib/canner/persistenceStore'; describe('Test CannerPersistenceStore', () => { + const fakePath = 'fake-path'; // fake workspaceId const fakeWorkspaces = { // fake workspace id and sql name @@ -41,21 +42,25 @@ describe('Test CannerPersistenceStore', () => { }; const fakeStorageListedObjectsInfo = { ws1: { - indicator: { name: `${fakeWorkspaces.ws1.id}/vulcansql/indicator.json` }, + indicator: { + name: `${fakePath}/${fakeWorkspaces.ws1.id}/vulcansql/indicator.json`, + }, artifact: { - name: `${fakeWorkspaces.ws1.id}/vulcansql/${fakeWorkspaces.ws1.folders[1]}/result.json`, + name: `${fakePath}/${fakeWorkspaces.ws1.id}/vulcansql/${fakeWorkspaces.ws1.folders[1]}/result.json`, }, others: [ - { name: `${fakeWorkspaces.ws1.id}/notebook` }, - { name: `${fakeWorkspaces.ws1.id}/notebook_output` }, + { name: `${fakePath}/${fakeWorkspaces.ws1.id}/notebook` }, + { name: `${fakePath}/${fakeWorkspaces.ws1.id}/notebook_output` }, ], }, ws2: { - indicator: { name: `${fakeWorkspaces.ws2.id}/vulcansql/indicator.json` }, + indicator: { + name: `${fakePath}/${fakeWorkspaces.ws2.id}/vulcansql/indicator.json`, + }, artifact: { - name: `${fakeWorkspaces.ws2.id}/vulcansql/${fakeWorkspaces.ws2.folders[1]}/result.json`, + name: `${fakePath}/${fakeWorkspaces.ws2.id}/vulcansql/${fakeWorkspaces.ws2.folders[1]}/result.json`, }, - others: [{ name: `${fakeWorkspaces.ws2.id}/data` }], + others: [{ name: `${fakePath}/${fakeWorkspaces.ws2.id}/data` }], }, }; @@ -128,8 +133,14 @@ describe('Test CannerPersistenceStore', () => { .stub(storageServiceModule, 'createStorageService') .resolves(stubStorageService); - const stubOption = sinon.stubInterface(); - const store = new CannerPersistenceStore(stubOption, {}, 'Canner'); + const store = new CannerPersistenceStore( + { + ...sinon.stubInterface(), + filePath: fakePath, + }, + {}, + 'Canner' + ); // Act, Assert await expect(store.load()).rejects.toThrowError( @@ -269,8 +280,14 @@ describe('Test CannerPersistenceStore', () => { .stub(storageServiceModule, 'createStorageService') .resolves(stubStorageService); - const stubOption = sinon.stubInterface(); - const store = new CannerPersistenceStore(stubOption, {}, 'Canner'); + const store = new CannerPersistenceStore( + { + ...sinon.stubInterface(), + filePath: fakePath, + }, + {}, + 'Canner' + ); const mergedArtifact = JSON.parse((await store.load()).toString('utf-8')); // Act, Assert diff --git a/packages/extension-store-canner/src/test/cannerProfileReader.spec.ts b/packages/extension-store-canner/src/test/cannerProfileReader.spec.ts index 8046f7ce..68b7c1f3 100644 --- a/packages/extension-store-canner/src/test/cannerProfileReader.spec.ts +++ b/packages/extension-store-canner/src/test/cannerProfileReader.spec.ts @@ -122,7 +122,7 @@ describe('Test CannerProfileReader', () => { }; const expected = [ { - name: `profile-${fakeWorkspaces.ws1.sqlName}`, + name: `canner-${fakeWorkspaces.ws1.sqlName}`, type: 'canner', connection: { ...connectionInfo, @@ -131,7 +131,7 @@ describe('Test CannerProfileReader', () => { allow: '*', }, { - name: `profile-${fakeWorkspaces.ws2.sqlName}`, + name: `canner-${fakeWorkspaces.ws2.sqlName}`, type: 'canner', connection: { ...connectionInfo, diff --git a/packages/serve/src/lib/evaluator/evaluator.ts b/packages/serve/src/lib/evaluator/evaluator.ts index 468e06ab..4c74e1d1 100644 --- a/packages/serve/src/lib/evaluator/evaluator.ts +++ b/packages/serve/src/lib/evaluator/evaluator.ts @@ -53,6 +53,7 @@ export class Evaluator { ); continue; } + logger.debug(`profile: ${profile.name}, allow: ${profile.allow}`); this.profiles.set(profile.name, this.getConstraints(profile.allow)); } }