From f6a6539462b9546ae9e1b087071fd6c511924b19 Mon Sep 17 00:00:00 2001 From: "danilo.spinelli" Date: Sat, 7 Mar 2020 01:23:59 +0100 Subject: [PATCH 1/3] renamed logos to assets --- README.md | 2 +- UploadServiceLogo/function.json | 2 +- env.example | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4c12f883..20efaf45 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ they may be customized as needed. | Variable name | Description | type | | -------------------------------------- | -------------------------------------------------------------------------------------------------- | ------- | | LOGOS_URL | The url of the service logos storage | string | -| LogosStorageConnection | The connection string used to connect to Azure Blob Storage containing the service logos | string | +| AssetsStorageConnection | The connection string used to connect to Azure Blob Storage containing the service cache | string | | SERVICE_PRINCIPAL_CLIENT_ID | The service principal name used to get the token credentials to connect to the APIM | string | | SERVICE_PRINCIPAL_SECRET | The service principal secret used to get the token credentials to connect to the APIM | string | | SERVICE_PRINCIPAL_TENANT_ID | The service principal tenant id used to get the token credentials to connect to the APIM | string | diff --git a/UploadServiceLogo/function.json b/UploadServiceLogo/function.json index ba19560f..be6822fd 100644 --- a/UploadServiceLogo/function.json +++ b/UploadServiceLogo/function.json @@ -19,7 +19,7 @@ "type": "blob", "name": "logo", "path": "services/{serviceId}.png", - "connection": "LogosStorageConnection", + "connection": "AssetsStorageConnection", "direction": "out" } ], diff --git a/env.example b/env.example index af414903..a2fb666b 100644 --- a/env.example +++ b/env.example @@ -1,6 +1,6 @@ AZURE_APIM_HOST=localhost LOGOS_URL=localhost -LogosStorageConnection=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1; +AssetsStorageConnection=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1; SERVICE_PRINCIPAL_CLIENT_ID=service-principal-client-id SERVICE_PRINCIPAL_SECRET=service-principal-secret From 5b1903999c7c73523069ea6fa502c6ddf506f80e Mon Sep 17 00:00:00 2001 From: "danilo.spinelli" Date: Sat, 7 Mar 2020 16:20:48 +0100 Subject: [PATCH 2/3] add scheduled function to update service cache --- README.md | 4 ++ UpdateVisibleServicesCache/function.json | 38 ++++++++++ UpdateVisibleServicesCache/index.ts | 71 +++++++++++++++++++ .../function.json | 18 +++++ UpdateVisibleServicesCacheActivity/index.ts | 38 ++++++++++ .../function.json | 12 ++++ .../index.ts | 35 +++++++++ package.json | 5 +- utils/conversions.ts | 30 ++++---- yarn.lock | 8 +-- 10 files changed, 238 insertions(+), 21 deletions(-) create mode 100644 UpdateVisibleServicesCache/function.json create mode 100644 UpdateVisibleServicesCache/index.ts create mode 100644 UpdateVisibleServicesCacheActivity/function.json create mode 100644 UpdateVisibleServicesCacheActivity/index.ts create mode 100644 UpdateVisibleServicesCacheOrchestrator/function.json create mode 100644 UpdateVisibleServicesCacheOrchestrator/index.ts diff --git a/README.md b/README.md index 20efaf45..3facdc09 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ they may be customized as needed. | Variable name | Description | type | | -------------------------------------- | -------------------------------------------------------------------------------------------------- | ------- | +| StorageConnection | Storage connection string to store computed visible-service.json (retrieved by io-functions-app) | string | +| CUSTOMCONNSTR_COSMOSDB_URI | CosmosDB connection URI | string | +| CUSTOMCONNSTR_COSMOSDB_KEY | CosmoDB connection key | string | +| COSMOSDB_NAME | CosmosDB database name | string | | LOGOS_URL | The url of the service logos storage | string | | AssetsStorageConnection | The connection string used to connect to Azure Blob Storage containing the service cache | string | | SERVICE_PRINCIPAL_CLIENT_ID | The service principal name used to get the token credentials to connect to the APIM | string | diff --git a/UpdateVisibleServicesCache/function.json b/UpdateVisibleServicesCache/function.json new file mode 100644 index 00000000..90581ccc --- /dev/null +++ b/UpdateVisibleServicesCache/function.json @@ -0,0 +1,38 @@ +{ + "disabled": false, + "bindings": [ + { + "schedule": "0 0 * * * *", + "name": "updateVisibleServiceCacheTimer", + "type": "timerTrigger", + "direction": "in" + }, + { + "name": "starter", + "type": "orchestrationClient", + "direction": "in" + }, + { + "name": "visibleServicesBlob", + "type": "blob", + "path": "cached/visible-services.json", + "connection": "StorageConnection", + "direction": "in" + }, + { + "name": "visibleServicesCacheBlob", + "type": "blob", + "path": "services/visible-services.json", + "connection": "AssetsStorageConnection", + "direction": "out" + }, + { + "name": "visibleServicesByScopeCacheBlob", + "type": "blob", + "path": "services/visible-services-by-scope.json", + "connection": "AssetsStorageConnection", + "direction": "out" + } + ], + "scriptFile": "../dist/UpdateVisibleServicesCache/index.js" +} diff --git a/UpdateVisibleServicesCache/index.ts b/UpdateVisibleServicesCache/index.ts new file mode 100644 index 00000000..873f85f4 --- /dev/null +++ b/UpdateVisibleServicesCache/index.ts @@ -0,0 +1,71 @@ +/** + * This time triggered function creates a cache for visible services: + * + * - read the cached visible-service.json (input binding) + * - create a version of services/visible-services.json suitable to be consumed by the mobile APP + * - put the generated JSON into the assets storage (which is reachable behind the CDN) + * - loop on visible services and store services/.json (output binding) + * + * The tuple stored is (serviceId, version, scope). + * + * TODO: delete blobs for services that aren't visible anymore. + */ +import { Context } from "@azure/functions"; + +import { isLeft } from "fp-ts/lib/Either"; +import { StrMap } from "fp-ts/lib/StrMap"; +import { VisibleService } from "io-functions-commons/dist/src/models/visible_service"; + +import * as df from "durable-functions"; +import * as t from "io-ts"; + +export const VisibleServices = t.record(t.string, VisibleService); + +async function UpdateVisibleServiceCache(context: Context): Promise { + const errorOrVisibleServices = VisibleServices.decode( + context.bindings.visibleServicesBlob + ); + + if (isLeft(errorOrVisibleServices)) { + context.log.info( + "UpdateVisibleServiceCache|Cannot decode visible services" + ); + return; + } + + const visibleServiceJson = errorOrVisibleServices.value; + const visibleServices = new StrMap(visibleServiceJson); + + const visibleServicesTuples = visibleServices.mapWithKey((k, v) => ({ + id: v.id, + scope: v.serviceMetadata ? v.serviceMetadata.scope : undefined, + version: v.version + })); + + // store visible services in the blob + // tslint:disable-next-line: no-object-mutation + context.bindings.visibleServicesCacheBlob = visibleServicesTuples; + + const { left: NATIONAL, right: LOCAL } = visibleServices.partition( + s => s.serviceMetadata && s.serviceMetadata.scope === "LOCAL" + ); + + // store visible services partitioned by scope + // tslint:disable-next-line: no-object-mutation + context.bindings.visibleServicesByScopeCacheBlob = { + LOCAL: LOCAL.map(_ => _.serviceId).reduce([], (p, c) => [...p, c]), + NATIONAL: NATIONAL.map(_ => _.serviceId).reduce([], (p, c) => [...p, c]) + }; + + // start orchestrator to loop on every visible service + // and to store it in a blob + await df + .getClient(context) + .startNew( + "UpdateVisibleServicesCacheOrchestrator", + undefined, + visibleServiceJson + ); +} + +export { UpdateVisibleServiceCache as index }; diff --git a/UpdateVisibleServicesCacheActivity/function.json b/UpdateVisibleServicesCacheActivity/function.json new file mode 100644 index 00000000..927200b5 --- /dev/null +++ b/UpdateVisibleServicesCacheActivity/function.json @@ -0,0 +1,18 @@ +{ + "disabled": false, + "bindings": [ + { + "name": "visibleServiceJson", + "type": "activityTrigger", + "direction": "in" + }, + { + "name": "visibleServiceCacheBlob", + "type": "blob", + "path": "services/{serviceId}.json", + "connection": "AssetsStorageConnection", + "direction": "out" + } + ], + "scriptFile": "../dist/UpdateVisibleServicesCacheActivity/index.js" +} diff --git a/UpdateVisibleServicesCacheActivity/index.ts b/UpdateVisibleServicesCacheActivity/index.ts new file mode 100644 index 00000000..c8b74862 --- /dev/null +++ b/UpdateVisibleServicesCacheActivity/index.ts @@ -0,0 +1,38 @@ +/** + * Take service json as input and store the JSON into + * services/.json through output binding. + */ +import { Context } from "@azure/functions"; +import { isLeft } from "fp-ts/lib/Either"; +import { VisibleService } from "io-functions-commons/dist/src/models/visible_service"; +import { toApiServiceMetadata } from "../utils/conversions"; + +async function UpdateVisibleServiceCacheActivity( + context: Context +): Promise { + const visibleServiceJson = context.bindings.visibleServiceJson; + const errorOrVisibleService = VisibleService.decode(visibleServiceJson); + + if (isLeft(errorOrVisibleService)) { + context.log.error( + "UpdateVisibleServiceCacheActivity|Cannot decode visible service JSON" + ); + return; + } + const visibleService = errorOrVisibleService.value; + + context.log.info( + "UpdateVisibleServiceCacheActivity|SERVICE_ID=", + visibleService.serviceId + ); + // we don't want to pollute the table storage + // (where the activity result is saved), + // so we return void from this method and + // use context bindings + // tslint:disable-next-line: no-object-mutation + context.bindings.visibleServiceCacheBlob = toApiServiceMetadata( + visibleService + ); +} + +export default UpdateVisibleServiceCacheActivity; diff --git a/UpdateVisibleServicesCacheOrchestrator/function.json b/UpdateVisibleServicesCacheOrchestrator/function.json new file mode 100644 index 00000000..d4af1dd3 --- /dev/null +++ b/UpdateVisibleServicesCacheOrchestrator/function.json @@ -0,0 +1,12 @@ +{ + "disabled": false, + "bindings": [ + { + "name": "context", + "type": "orchestrationTrigger", + "direction": "in" + } + ], + "scriptFile": "../dist/UpdateVisibleServicesCacheOrchestrator/index.js" +} + diff --git a/UpdateVisibleServicesCacheOrchestrator/index.ts b/UpdateVisibleServicesCacheOrchestrator/index.ts new file mode 100644 index 00000000..f4dc732b --- /dev/null +++ b/UpdateVisibleServicesCacheOrchestrator/index.ts @@ -0,0 +1,35 @@ +/* + * Calls an activity function for each visible service + * found into the stored JSON. The activity function will store + * one JSON into the blob storage for each visible service found. + */ + +import { IFunctionContext, Task } from "durable-functions/lib/src/classes"; + +import * as df from "durable-functions"; +import { isLeft } from "fp-ts/lib/Either"; +import { VisibleServices } from "../UpdateVisibleServicesCache"; + +const UpdateVisibleServicesCacheOrchestrator = df.orchestrator(function*( + context: IFunctionContext +): IterableIterator { + const visibleServicesJson = context.df.getInput(); + const errorOrVisibleServices = VisibleServices.decode(visibleServicesJson); + + if (isLeft(errorOrVisibleServices)) { + context.log.error( + "UpdateVisibleServicesCacheOrchestrator|Error decoding visible services JSON." + ); + return; + } + const visibleServices = errorOrVisibleServices.value; + + for (const visibleServiceId of Object.keys(visibleServices)) { + yield context.df.callActivity( + "UpdateVisibleServicesCacheActivity", + visibleServices[visibleServiceId] + ); + } +}); + +export default UpdateVisibleServicesCacheOrchestrator; diff --git a/package.json b/package.json index d13eca84..91851fe2 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "documentdb": "^1.12.2", "durable-functions": "^1.2.4", "express": "^4.15.3", - "fp-ts": "1.12.0", + "fp-ts": "1.17.0", "io-functions-commons": "^1.12.1", "io-functions-express": "^0.1.0", "io-ts": "1.8.5", @@ -59,6 +59,7 @@ "winston": "^3.2.1" }, "resolutions": { - "handlebars": "~4.5.3" + "handlebars": "~4.5.3", + "fp-ts": "1.17.0" } } diff --git a/utils/conversions.ts b/utils/conversions.ts index 8c034fff..cdb7035a 100644 --- a/utils/conversions.ts +++ b/utils/conversions.ts @@ -55,22 +55,22 @@ export function apiServiceToService(service: ApiService): Service { } // Returns an API Service Metadata from an internal Service model -function toApiServiceMetadata( - retrievedService: RetrievedService +export function toApiServiceMetadata( + service: RetrievedService | VisibleService ): ApiServiceMetadata { - return retrievedService.serviceMetadata + return service.serviceMetadata ? { - address: retrievedService.serviceMetadata.address, - app_android: retrievedService.serviceMetadata.appAndroid, - app_ios: retrievedService.serviceMetadata.appIos, - description: retrievedService.serviceMetadata.description, - email: retrievedService.serviceMetadata.email, - pec: retrievedService.serviceMetadata.pec, - phone: retrievedService.serviceMetadata.phone, - privacy_url: retrievedService.serviceMetadata.privacyUrl, - scope: retrievedService.serviceMetadata.scope, - tos_url: retrievedService.serviceMetadata.tosUrl, - web_url: retrievedService.serviceMetadata.webUrl + address: service.serviceMetadata.address, + app_android: service.serviceMetadata.appAndroid, + app_ios: service.serviceMetadata.appIos, + description: service.serviceMetadata.description, + email: service.serviceMetadata.email, + pec: service.serviceMetadata.pec, + phone: service.serviceMetadata.phone, + privacy_url: service.serviceMetadata.privacyUrl, + scope: service.serviceMetadata.scope, + tos_url: service.serviceMetadata.tosUrl, + web_url: service.serviceMetadata.webUrl } : undefined; } @@ -111,7 +111,7 @@ export function retrievedServiceToVisibleService( organizationFiscalCode: retrievedService.organizationFiscalCode, organizationName: retrievedService.organizationName, serviceId: retrievedService.serviceId, - serviceMetadata: toApiServiceMetadata(retrievedService), + serviceMetadata: retrievedService.serviceMetadata, serviceName: retrievedService.serviceName, version: retrievedService.version }; diff --git a/yarn.lock b/yarn.lock index 0b5a2478..dd22197c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2707,10 +2707,10 @@ forwarded@~0.1.2: resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ= -fp-ts@1.12.0, fp-ts@^1.0.0: - version "1.12.0" - resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-1.12.0.tgz#d333310e4ac104cdcb6bea47908e381bb09978e7" - integrity sha512-fWwnAgVlTsV26Ruo9nx+fxNHIm6l1puE1VJ/C0XJ3nRQJJJIgRHYw6sigB3MuNFZL1o4fpGlhwFhcbxHK0RsOA== +fp-ts@1.12.0, fp-ts@1.17.0, fp-ts@^1.0.0: + version "1.17.0" + resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-1.17.0.tgz#289127353ddbb4622ada1920d4ad6643182c1f1f" + integrity sha512-nBq25aCAMbCwVLobUUuM/MZihPKyjn0bCVBf6xMAGriHlf8W8Ze9UhyfLnbmfp0ekFTxMuTfLXrCzpJ34px7PQ== fragment-cache@^0.2.1: version "0.2.1" From 0b3e0e37eeb581aa9ce794726f9f6bbbc3add0cd Mon Sep 17 00:00:00 2001 From: "danilo.spinelli" Date: Sat, 7 Mar 2020 16:27:15 +0100 Subject: [PATCH 3/3] fix linting --- UpdateVisibleServicesCache/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UpdateVisibleServicesCache/index.ts b/UpdateVisibleServicesCache/index.ts index 873f85f4..0307725e 100644 --- a/UpdateVisibleServicesCache/index.ts +++ b/UpdateVisibleServicesCache/index.ts @@ -36,7 +36,7 @@ async function UpdateVisibleServiceCache(context: Context): Promise { const visibleServiceJson = errorOrVisibleServices.value; const visibleServices = new StrMap(visibleServiceJson); - const visibleServicesTuples = visibleServices.mapWithKey((k, v) => ({ + const visibleServicesTuples = visibleServices.mapWithKey((_, v) => ({ id: v.id, scope: v.serviceMetadata ? v.serviceMetadata.scope : undefined, version: v.version