From cec9bc94ed0848c415bd3d720b3fb71beaf350a8 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 6 Jul 2020 13:51:49 +0200 Subject: [PATCH 01/34] user data download orchestrator --- .../handler.ts | 4 +- .../handler.ts | 6 +- .../__tests__/handler.test.ts | 303 ++++++++++++++++++ UserDataDownloadOrchestrator/function.json | 10 + UserDataDownloadOrchestrator/handler.ts | 173 ++++++++++ UserDataDownloadOrchestrator/index.ts | 26 +- __mocks__/durable-functions.ts | 76 ++++- __mocks__/mocks.ts | 9 + 8 files changed, 572 insertions(+), 35 deletions(-) create mode 100644 UserDataDownloadOrchestrator/__tests__/handler.test.ts create mode 100644 UserDataDownloadOrchestrator/handler.ts diff --git a/SendUserDataDownloadMessageActivity/handler.ts b/SendUserDataDownloadMessageActivity/handler.ts index 39e5b003..b0011c99 100644 --- a/SendUserDataDownloadMessageActivity/handler.ts +++ b/SendUserDataDownloadMessageActivity/handler.ts @@ -114,11 +114,11 @@ async function sendMessage( } // Activity result -const ActivityResultSuccess = t.interface({ +export const ActivityResultSuccess = t.interface({ kind: t.literal("SUCCESS") }); -type ActivityResultSuccess = t.TypeOf; +export type ActivityResultSuccess = t.TypeOf; const ActivityResultFailure = t.interface({ kind: t.literal("FAILURE"), diff --git a/SetUserDataProcessingStatusActivity/handler.ts b/SetUserDataProcessingStatusActivity/handler.ts index 992e26aa..e0f6451b 100644 --- a/SetUserDataProcessingStatusActivity/handler.ts +++ b/SetUserDataProcessingStatusActivity/handler.ts @@ -25,14 +25,14 @@ export const ActivityInput = t.interface({ export type ActivityInput = t.TypeOf; // Activity result -const ActivityResultSuccess = t.interface({ +export const ActivityResultSuccess = t.interface({ kind: t.literal("SUCCESS"), value: UserDataProcessing }); export type ActivityResultSuccess = t.TypeOf; // Activity failed because of invalid input -const ActivityResultInvalidInputFailure = t.interface({ +export const ActivityResultInvalidInputFailure = t.interface({ kind: t.literal("INVALID_INPUT_FAILURE"), reason: t.string }); @@ -41,7 +41,7 @@ export type ActivityResultInvalidInputFailure = t.TypeOf< >; // Activity failed because of an error on a query -const ActivityResultQueryFailure = t.intersection([ +export const ActivityResultQueryFailure = t.intersection([ t.interface({ kind: t.literal("QUERY_FAILURE"), reason: t.string diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts new file mode 100644 index 00000000..8de3cb9c --- /dev/null +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -0,0 +1,303 @@ +import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { + mockOrchestratorCallActivity, + mockOrchestratorCallActivityWithRetry, + mockOrchestratorContext +} from "../../__mocks__/durable-functions"; +import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; +import { + handler, + InvalidInputFailure, + OrchestratorSuccess, + SkippedDocument, + ActivityFailure +} from "../handler"; + +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; + +const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) +); +const extractUserDataActivity = jest.fn().mockImplementation(() => + ExtractUserDataActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aArchiveInfo + }) +); +const sendUserDataDownloadMessageActivity = jest + .fn() + .mockImplementation(() => + SendUserDataDownloadMessageActivityResultSuccess.encode({ kind: "SUCCESS" }) + ); + +// A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation +const switchMockImplementation = (name: string, ...args: readonly unknown[]) => + (name === "setUserDataProcessingStatusActivity" + ? setUserDataProcessingStatusActivity + : name === "extractUserDataActivity" + ? extractUserDataActivity + : name === "sendUserDataDownloadMessageActivity" + ? sendUserDataDownloadMessageActivity + : jest.fn())(name, ...args); + +// I assign switchMockImplementation to both because +// I don't want tests to depend on implementation details +// such as which activity is called with retry and which is not +mockOrchestratorCallActivity.mockImplementation(switchMockImplementation); +mockOrchestratorCallActivityWithRetry.mockImplementation( + switchMockImplementation +); + +/** + * Util function that takes an orchestrator and executes each step until is done + * @param orch an orchestrator + * + * @returns the last value yielded by the orchestrator + */ +const consumeOrchestrator = (orch: any) => { + // tslint:disable-next-line: no-let + let prevValue: unknown; + while (true) { + const { done, value } = orch.next(prevValue); + if (done) { + return value; + } + prevValue = value; + } +}; + +// just a convenient cast, good for every test case +const context = (mockOrchestratorContext as unknown) as IFunctionContext; + +describe("handler", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should fail on invalid input", () => { + const documents: ReadonlyArray = ["invalid"]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it.each` + name | status + ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} + ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} + `("should skip if the status is $name", ({ status }) => { + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(SkippedDocument.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should success if everything goes well", () => { + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); + // first, set as WIP + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.WIP + } + ); + // then, set as CLOSED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.CLOSED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when data extraction fails", () => { + extractUserDataActivity.mockImplementationOnce( + () => "any non-success value" + ); + + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("extractUserDataActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when send message fails", () => { + sendUserDataDownloadMessageActivity.mockImplementationOnce( + () => "any non-success value" + ); + + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("sendUserDataDownloadMessageActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => "any non-success value" + ); + + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => "any non-success value" + ); + + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.WIP + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to CLOSED fails", () => { + // the first time is called is for WIP + setUserDataProcessingStatusActivity.mockImplementationOnce(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) + ); + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => "any non-success value" + ); + + const documents: ReadonlyArray = [ + { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + } + ]; + + const result = consumeOrchestrator(handler(context, documents)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.CLOSED + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); +}); diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index dc59add8..74fcb54c 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -4,6 +4,16 @@ "name": "context", "type": "orchestrationTrigger", "direction": "in" + }, + { + "type": "cosmosDBTrigger", + "name": "documents", + "direction": "in", + "leaseCollectionName": "leases", + "connectionStringSetting": "COSMOSDB_CONNECTION_STRING", + "databaseName": "%COSMOSDB_BONUS_DATABASE_NAME%", + "collectionName": "user-data-processing", + "createLeaseCollectionIfNotExists": true } ], "scriptFile": "../dist/UserDataDownloadOrchestrator/index.js" diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts new file mode 100644 index 00000000..99bcf982 --- /dev/null +++ b/UserDataDownloadOrchestrator/handler.ts @@ -0,0 +1,173 @@ +import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { toString, identity } from "fp-ts/lib/function"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; +import { readableReport } from "italia-ts-commons/lib/reporters"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; + +import * as t from "io-ts"; +import { left, right, isLeft, Either } from "fp-ts/lib/Either"; + +const logPrefix = ""; + +export type InvalidInputFailure = t.TypeOf; +export const InvalidInputFailure = t.interface({ + kind: t.literal("INVALID_INPUT"), + reason: t.string +}); + +export type UnhanldedFailure = t.TypeOf; +export const UnhanldedFailure = t.interface({ + kind: t.literal("UNHANDLED"), + reason: t.string +}); + +export type ActivityFailure = t.TypeOf; +export const ActivityFailure = t.intersection([ + t.interface({ + activityName: t.string, + kind: t.literal("ACTIVITY"), + reason: t.string + }), + t.partial({ extra: t.object }) +]); + +type OrchestratorFailure = t.TypeOf; +const OrchestratorFailure = t.taggedUnion("kind", [ + InvalidInputFailure, + UnhanldedFailure, + ActivityFailure +]); + +export type OrchestratorSuccess = t.TypeOf; +export const OrchestratorSuccess = t.interface({ + kind: t.literal("SUCCESS") +}); + +export type SkippedDocument = t.TypeOf; +export const SkippedDocument = t.interface({ + kind: t.literal("SKIPPED") +}); + +type OrchestratorResult = t.TypeOf; +const OrchestratorResult = t.union([ + OrchestratorFailure, + SkippedDocument, + OrchestratorSuccess +]); + +export const handler = function*( + context: IFunctionContext, + documents: readonly unknown[] +): IterableIterator { + const document = documents[0]; + + const earlyReturnOrCurrentUserDataProcessing = UserDataProcessing.decode( + document + ) + .mapLeft(err => { + context.log.warn( + `${logPrefix}|WARN|Cannot decode UserDataProcessing document: ${readableReport( + err + )}` + ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) + }); + }) + .fold>(left, decoded => + [ + // we are already working on it + UserDataProcessingStatusEnum.WIP, + // it's done already + UserDataProcessingStatusEnum.CLOSED + ].includes(decoded.status) + ? left(SkippedDocument.encode({ kind: "SKIPPED" })) + : right(decoded) + ); + + if (isLeft(earlyReturnOrCurrentUserDataProcessing)) { + return earlyReturnOrCurrentUserDataProcessing.value; + } + + const currentUserDataProcessing = + earlyReturnOrCurrentUserDataProcessing.value; + + try { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.WIP + }) + ).getOrElseL(err => { + throw ActivityFailure.encode({ + activityName: "setUserDataProcessingStatusActivity", + extra: { status: UserDataProcessingStatusEnum.WIP }, + kind: "ACTIVITY", + reason: readableReport(err) + }); + }); + + const bundle = ExtractUserDataActivityResultSuccess.decode( + yield context.df.callActivity("extractUserDataActivity", { + fiscalCode: currentUserDataProcessing.fiscalCode + }) + ).getOrElseL(err => { + throw ActivityFailure.encode({ + activityName: "extractUserDataActivity", + kind: "ACTIVITY", + reason: readableReport(err) + }); + }); + + SendUserDataDownloadMessageActivityResultSuccess.decode( + yield context.df.callActivity("sendUserDataDownloadMessageActivity", { + blobName: bundle.value.blobName, + fiscalCode: currentUserDataProcessing.fiscalCode, + password: bundle.value.password + }) + ).getOrElseL(err => { + throw ActivityFailure.encode({ + activityName: "sendUserDataDownloadMessageActivity", + kind: "ACTIVITY", + reason: readableReport(err) + }); + }); + + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.CLOSED + }) + ).getOrElseL(err => { + throw ActivityFailure.encode({ + activityName: "setUserDataProcessingStatusActivity", + extra: { status: UserDataProcessingStatusEnum.CLOSED }, + kind: "ACTIVITY", + reason: readableReport(err) + }); + }); + + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); + } catch (error) { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.FAILED + }) + ).getOrElseL(err => { + throw new Error( + `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + err + )}` + ); + }); + + return OrchestratorFailure.is(error) + ? error + : UnhanldedFailure.encode({ kind: "UNHANDLED", reason: error.message }); + } +}; diff --git a/UserDataDownloadOrchestrator/index.ts b/UserDataDownloadOrchestrator/index.ts index 8a04ea28..be7758ad 100644 --- a/UserDataDownloadOrchestrator/index.ts +++ b/UserDataDownloadOrchestrator/index.ts @@ -1,27 +1,7 @@ -/* - * This function is not intended to be invoked directly. Instead it will be - * triggered by an HTTP starter function. - * - * Before running this sample, please: - * - create a Durable activity function (default name is "Hello") - * - create a Durable HTTP starter function - * - run 'npm install durable-functions' from the wwwroot folder of your - * function app in Kudu - */ - -import { IFunctionContext, Task } from "durable-functions/lib/src/classes"; - import * as df from "durable-functions"; -const orchestrator = df.orchestrator(function*( - context: IFunctionContext -): IterableIterator { - const input = context.df.getInput(); - const xx = yield context.df.callActivity("ExtractUserDataActivity", { - // tslint:disable-next-line: no-any - fiscalCode: (input as any).fiscalCode - }); - context.log.info(JSON.stringify(xx)); -}); +import { handler } from "./handler"; + +const orchestrator = df.orchestrator(handler); export default orchestrator; diff --git a/__mocks__/durable-functions.ts b/__mocks__/durable-functions.ts index ec2196e9..c6744125 100644 --- a/__mocks__/durable-functions.ts +++ b/__mocks__/durable-functions.ts @@ -1,21 +1,83 @@ // tslint:disable: no-any import { Context } from "@azure/functions"; +import * as df from "durable-functions"; -export const mockStartNew = jest.fn(); +export const mockStatusRunning = { + runtimeStatus: df.OrchestrationRuntimeStatus.Running +}; +export const mockStatusCompleted = { + runtimeStatus: df.OrchestrationRuntimeStatus.Completed +}; + +export const OrchestrationRuntimeStatus = df.OrchestrationRuntimeStatus; + +export const mockStartNew = jest.fn((_, __, ___) => + Promise.resolve("instanceId") +); +export const mockGetStatus = jest + .fn() + .mockImplementation(async () => mockStatusCompleted); +export const mockTerminate = jest.fn(async (_, __) => { + return; +}); export const getClient = jest.fn(() => ({ - startNew: mockStartNew + getStatus: mockGetStatus, + startNew: mockStartNew, + terminate: mockTerminate })); -export const orchestrator = jest.fn(); - export const RetryOptions = jest.fn(() => ({})); export const context = ({ + bindings: {}, log: { - error: jest.fn(), - verbose: jest.fn(), - warn: jest.fn() + // tslint:disable-next-line: no-console + error: jest.fn().mockImplementation(console.log), + // tslint:disable-next-line: no-console + info: jest.fn().mockImplementation(console.log), + // tslint:disable-next-line: no-console + verbose: jest.fn().mockImplementation(console.log), + // tslint:disable-next-line: no-console + warn: jest.fn().mockImplementation(console.log) } } as any) as Context; + +// +// Orchestrator context +// + +export const mockOrchestratorGetInput = jest.fn(); +export const mockOrchestratorCallActivity = jest + .fn() + .mockImplementation((name: string, input?: unknown) => ({ + input, + name + })); +export const mockOrchestratorCallActivityWithRetry = jest + .fn() + .mockImplementation( + (name: string, retryOptions: df.RetryOptions, input?: unknown) => ({ + input, + name, + retryOptions + }) + ); +export const mockOrchestratorSetCustomStatus = jest.fn(); +export const mockOrchestratorCreateTimer = jest.fn(); + +export const mockOrchestratorContext = { + ...context, + df: { + callActivity: mockOrchestratorCallActivity, + callActivityWithRetry: mockOrchestratorCallActivityWithRetry, + createTimer: mockOrchestratorCreateTimer, + getInput: mockOrchestratorGetInput, + setCustomStatus: mockOrchestratorSetCustomStatus + } +}; + +export const orchestrator = jest + .fn() + .mockImplementation(fn => () => fn(mockOrchestratorContext)); diff --git a/__mocks__/mocks.ts b/__mocks__/mocks.ts index 7efc31d6..5502b698 100644 --- a/__mocks__/mocks.ts +++ b/__mocks__/mocks.ts @@ -60,6 +60,7 @@ import { } from "io-functions-commons/dist/src/models/notification_status"; import { readableReport } from "italia-ts-commons/lib/reporters"; import { EmailAddress } from "../generated/definitions/EmailAddress"; +import { ArchiveInfo } from "../ExtractUserDataActivity/handler"; export const aFiscalCode = "SPNDNL80A13Y555X" as FiscalCode; @@ -280,3 +281,11 @@ export const aRetrievedMessageStatus = RetrievedMessageStatus.decode( const error = readableReport(errs); throw new Error("Fix MessageStatus mock: " + error); }); + +export const aArchiveInfo = ArchiveInfo.decode({ + blobName: "blobname", + password: "A".repeat(18) +}).getOrElseL(errs => { + const error = readableReport(errs); + throw new Error("Fix ArchiveInfo mock: " + error); +}); From e358919a99f9d5462bacea0d9e6cf6bfd4c8edd0 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 6 Jul 2020 15:08:57 +0200 Subject: [PATCH 02/34] better modeling --- .../__tests__/handler.test.ts | 90 ++++++++----------- UserDataDownloadOrchestrator/handler.ts | 71 ++++++++------- 2 files changed, 73 insertions(+), 88 deletions(-) diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index 8de3cb9c..a43bbbe5 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -81,9 +81,9 @@ describe("handler", () => { }); it("should fail on invalid input", () => { - const documents: ReadonlyArray = ["invalid"]; + const document = "invalid"; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(InvalidInputFailure.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); @@ -96,30 +96,26 @@ describe("handler", () => { ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} `("should skip if the status is $name", ({ status }) => { - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status - } - ]; + const document = { + ...aUserDataProcessing, + status + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); - expect(SkippedDocument.decode(result).isRight()).toBe(true); + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); expect(extractUserDataActivity).not.toHaveBeenCalled(); expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); }); it("should success if everything goes well", () => { - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); @@ -148,14 +144,12 @@ describe("handler", () => { () => "any non-success value" ); - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(ActivityFailure.decode(result).isRight()).toBe(true); expect(result.activityName).toBe("extractUserDataActivity"); @@ -177,14 +171,12 @@ describe("handler", () => { () => "any non-success value" ); - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(ActivityFailure.decode(result).isRight()).toBe(true); expect(result.activityName).toBe("sendUserDataDownloadMessageActivity"); @@ -206,14 +198,12 @@ describe("handler", () => { () => "any non-success value" ); - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(ActivityFailure.decode(result).isRight()).toBe(true); expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); @@ -235,14 +225,12 @@ describe("handler", () => { () => "any non-success value" ); - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(ActivityFailure.decode(result).isRight()).toBe(true); expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); @@ -274,14 +262,12 @@ describe("handler", () => { () => "any non-success value" ); - const documents: ReadonlyArray = [ - { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - } - ]; + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; - const result = consumeOrchestrator(handler(context, documents)); + const result = consumeOrchestrator(handler(context, document)); expect(ActivityFailure.decode(result).isRight()).toBe(true); expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 99bcf982..7a049cf5 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -1,17 +1,29 @@ import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { toString, identity } from "fp-ts/lib/function"; +import { Either, isLeft, left, right } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; +import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; -import * as t from "io-ts"; -import { left, right, isLeft, Either } from "fp-ts/lib/Either"; - const logPrefix = ""; +// models the subset of UserDataProcessing documents that this orchestrator accepts +export type ProcessableUserDataProcessing = t.TypeOf< + typeof ProcessableUserDataProcessing +>; +export const ProcessableUserDataProcessing = t.intersection([ + UserDataProcessing, + t.interface({ + status: t.union([ + t.literal(UserDataProcessingStatusEnum.PENDING), + t.literal(UserDataProcessingStatusEnum.FAILED) + ]) + }) +]); + export type InvalidInputFailure = t.TypeOf; export const InvalidInputFailure = t.interface({ kind: t.literal("INVALID_INPUT"), @@ -34,8 +46,8 @@ export const ActivityFailure = t.intersection([ t.partial({ extra: t.object }) ]); -type OrchestratorFailure = t.TypeOf; -const OrchestratorFailure = t.taggedUnion("kind", [ +export type OrchestratorFailure = t.TypeOf; +export const OrchestratorFailure = t.taggedUnion("kind", [ InvalidInputFailure, UnhanldedFailure, ActivityFailure @@ -51,8 +63,8 @@ export const SkippedDocument = t.interface({ kind: t.literal("SKIPPED") }); -type OrchestratorResult = t.TypeOf; -const OrchestratorResult = t.union([ +export type OrchestratorResult = t.TypeOf; +export const OrchestratorResult = t.union([ OrchestratorFailure, SkippedDocument, OrchestratorSuccess @@ -60,41 +72,28 @@ const OrchestratorResult = t.union([ export const handler = function*( context: IFunctionContext, - documents: readonly unknown[] + document: unknown ): IterableIterator { - const document = documents[0]; - - const earlyReturnOrCurrentUserDataProcessing = UserDataProcessing.decode( + const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( document - ) - .mapLeft(err => { - context.log.warn( - `${logPrefix}|WARN|Cannot decode UserDataProcessing document: ${readableReport( - err - )}` - ); - return InvalidInputFailure.encode({ - kind: "INVALID_INPUT", - reason: readableReport(err) - }); - }) - .fold>(left, decoded => - [ - // we are already working on it - UserDataProcessingStatusEnum.WIP, - // it's done already - UserDataProcessingStatusEnum.CLOSED - ].includes(decoded.status) - ? left(SkippedDocument.encode({ kind: "SKIPPED" })) - : right(decoded) + ).mapLeft(err => { + context.log.error( + `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( + err + )}` ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) + }); + }); - if (isLeft(earlyReturnOrCurrentUserDataProcessing)) { - return earlyReturnOrCurrentUserDataProcessing.value; + if (isLeft(invalidInputOrCurrentUserDataProcessing)) { + return invalidInputOrCurrentUserDataProcessing.value; } const currentUserDataProcessing = - earlyReturnOrCurrentUserDataProcessing.value; + invalidInputOrCurrentUserDataProcessing.value; try { SetUserDataProcessingStatusActivityResultSuccess.decode( From 81f2f56ce724d2817d1d3888a0f613a0e4a6706d Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 7 Jul 2020 10:01:40 +0200 Subject: [PATCH 03/34] sub orchestrator --- .../__tests__/handler.test.ts | 280 ++--------------- UserDataDownloadOrchestrator/function.json | 2 +- UserDataDownloadOrchestrator/handler.ts | 184 +++-------- UserDataDownloadOrchestrator/index.ts | 8 +- .../__tests__/handler.test.ts | 291 ++++++++++++++++++ UserDataDownloadSubOrchestrator/function.json | 10 + UserDataDownloadSubOrchestrator/handler.ts | 158 ++++++++++ UserDataDownloadSubOrchestrator/index.ts | 11 + __mocks__/durable-functions.ts | 11 + __mocks__/mocks.ts | 2 +- 10 files changed, 560 insertions(+), 397 deletions(-) create mode 100644 UserDataDownloadSubOrchestrator/__tests__/handler.test.ts create mode 100644 UserDataDownloadSubOrchestrator/function.json create mode 100644 UserDataDownloadSubOrchestrator/handler.ts create mode 100644 UserDataDownloadSubOrchestrator/index.ts diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index a43bbbe5..616bbe4e 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -1,58 +1,23 @@ +// tslint:disable: no-any + import { IFunctionContext } from "durable-functions/lib/src/classes"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { - mockOrchestratorCallActivity, - mockOrchestratorCallActivityWithRetry, + mockCallSubOrchestrator, mockOrchestratorContext } from "../../__mocks__/durable-functions"; -import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; -import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; -import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; -import { - handler, - InvalidInputFailure, - OrchestratorSuccess, - SkippedDocument, - ActivityFailure -} from "../handler"; - -import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; +import { aUserDataProcessing } from "../../__mocks__/mocks"; +import { handler } from "../handler"; -const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => - SetUserDataProcessingStatusActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aUserDataProcessing - }) -); -const extractUserDataActivity = jest.fn().mockImplementation(() => - ExtractUserDataActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aArchiveInfo - }) -); -const sendUserDataDownloadMessageActivity = jest - .fn() - .mockImplementation(() => - SendUserDataDownloadMessageActivityResultSuccess.encode({ kind: "SUCCESS" }) - ); - -// A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation -const switchMockImplementation = (name: string, ...args: readonly unknown[]) => - (name === "setUserDataProcessingStatusActivity" - ? setUserDataProcessingStatusActivity - : name === "extractUserDataActivity" - ? extractUserDataActivity - : name === "sendUserDataDownloadMessageActivity" - ? sendUserDataDownloadMessageActivity - : jest.fn())(name, ...args); +const aProcessableDocument = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING +}; -// I assign switchMockImplementation to both because -// I don't want tests to depend on implementation details -// such as which activity is called with retry and which is not -mockOrchestratorCallActivity.mockImplementation(switchMockImplementation); -mockOrchestratorCallActivityWithRetry.mockImplementation( - switchMockImplementation -); +const aNonProcessableDocument = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.WIP +}; /** * Util function that takes an orchestrator and executes each step until is done @@ -81,209 +46,26 @@ describe("handler", () => { }); it("should fail on invalid input", () => { - const document = "invalid"; - - const result = consumeOrchestrator(handler(context, document)); - - expect(InvalidInputFailure.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it.each` - name | status - ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} - ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} - `("should skip if the status is $name", ({ status }) => { - const document = { - ...aUserDataProcessing, - status - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(InvalidInputFailure.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should success if everything goes well", () => { - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); - // first, set as WIP - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.WIP - } - ); - // then, set as CLOSED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.CLOSED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); - }); - - it("should set as FAILED when data extraction fails", () => { - extractUserDataActivity.mockImplementationOnce( - () => "any non-success value" - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); + const input = "invalid"; - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("extractUserDataActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should set as FAILED when send message fails", () => { - sendUserDataDownloadMessageActivity.mockImplementationOnce( - () => "any non-success value" - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("sendUserDataDownloadMessageActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); - }); - - it("should set as FAILED when status update to WIP fails", () => { - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => "any non-success value" - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should set as FAILED when status update to WIP fails", () => { - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => "any non-success value" - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); - expect(result.extra).toEqual({ - status: UserDataProcessingStatusEnum.WIP - }); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + try { + consumeOrchestrator(handler(context, input)); + fail("it should throw"); + } catch (error) { + expect(mockCallSubOrchestrator).not.toHaveBeenCalled(); + } }); - it("should set as FAILED when status update to CLOSED fails", () => { - // the first time is called is for WIP - setUserDataProcessingStatusActivity.mockImplementationOnce(() => - SetUserDataProcessingStatusActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aUserDataProcessing - }) - ); - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => "any non-success value" - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - - const result = consumeOrchestrator(handler(context, document)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); - expect(result.extra).toEqual({ - status: UserDataProcessingStatusEnum.CLOSED - }); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + it("should process every processable document", () => { + const input: ReadonlyArray = [ + aProcessableDocument, + aProcessableDocument, + aProcessableDocument, + aNonProcessableDocument, + aNonProcessableDocument + ]; + + consumeOrchestrator(handler(context, input)); + expect(mockCallSubOrchestrator).toHaveBeenCalledTimes(3); }); }); diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index 74fcb54c..d5961aab 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -11,7 +11,7 @@ "direction": "in", "leaseCollectionName": "leases", "connectionStringSetting": "COSMOSDB_CONNECTION_STRING", - "databaseName": "%COSMOSDB_BONUS_DATABASE_NAME%", + "databaseName": "%COSMOSDB_NAME%", "collectionName": "user-data-processing", "createLeaseCollectionIfNotExists": true } diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 7a049cf5..6322fd15 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -1,14 +1,11 @@ import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { Either, isLeft, left, right } from "fp-ts/lib/Either"; +import { isLeft } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; -import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; -import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; -const logPrefix = ""; +const logPrefix = "UserDataDownloadOrchestrator"; // models the subset of UserDataProcessing documents that this orchestrator accepts export type ProcessableUserDataProcessing = t.TypeOf< @@ -24,149 +21,48 @@ export const ProcessableUserDataProcessing = t.intersection([ }) ]); -export type InvalidInputFailure = t.TypeOf; -export const InvalidInputFailure = t.interface({ - kind: t.literal("INVALID_INPUT"), - reason: t.string -}); - -export type UnhanldedFailure = t.TypeOf; -export const UnhanldedFailure = t.interface({ - kind: t.literal("UNHANDLED"), - reason: t.string -}); - -export type ActivityFailure = t.TypeOf; -export const ActivityFailure = t.intersection([ - t.interface({ - activityName: t.string, - kind: t.literal("ACTIVITY"), - reason: t.string - }), - t.partial({ extra: t.object }) -]); - -export type OrchestratorFailure = t.TypeOf; -export const OrchestratorFailure = t.taggedUnion("kind", [ - InvalidInputFailure, - UnhanldedFailure, - ActivityFailure -]); - -export type OrchestratorSuccess = t.TypeOf; -export const OrchestratorSuccess = t.interface({ - kind: t.literal("SUCCESS") -}); - -export type SkippedDocument = t.TypeOf; -export const SkippedDocument = t.interface({ - kind: t.literal("SKIPPED") -}); - -export type OrchestratorResult = t.TypeOf; -export const OrchestratorResult = t.union([ - OrchestratorFailure, - SkippedDocument, - OrchestratorSuccess -]); +const CosmosDbDocumentCollection = t.readonlyArray(t.readonly(t.UnknownRecord)); +type CosmosDbDocumentCollection = t.TypeOf; export const handler = function*( context: IFunctionContext, - document: unknown + input: unknown ): IterableIterator { - const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( - document - ).mapLeft(err => { - context.log.error( - `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( - err - )}` + const subTasks = CosmosDbDocumentCollection.decode(input) + .fold( + err => { + throw Error( + `${logPrefix}: cannot decode input [${readableReport(err)}]` + ); + }, + documents => + documents.map(doc => ProcessableUserDataProcessing.decode(doc)) + ) + .reduce( + (documents, maybeProcessable) => { + if (isLeft(maybeProcessable)) { + context.log.warn( + `${logPrefix}: skipping document [${readableReport( + maybeProcessable.value + )}]` + ); + return documents; + } + return [...documents, maybeProcessable.value]; + }, + [] as readonly ProcessableUserDataProcessing[] + ) + .map(processableDoc => + context.df.callSubOrchestrator( + "UserDataDownloadSubOrchestrator", + processableDoc + ) ); - return InvalidInputFailure.encode({ - kind: "INVALID_INPUT", - reason: readableReport(err) - }); - }); - - if (isLeft(invalidInputOrCurrentUserDataProcessing)) { - return invalidInputOrCurrentUserDataProcessing.value; - } - - const currentUserDataProcessing = - invalidInputOrCurrentUserDataProcessing.value; - - try { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.WIP - }) - ).getOrElseL(err => { - throw ActivityFailure.encode({ - activityName: "setUserDataProcessingStatusActivity", - extra: { status: UserDataProcessingStatusEnum.WIP }, - kind: "ACTIVITY", - reason: readableReport(err) - }); - }); - - const bundle = ExtractUserDataActivityResultSuccess.decode( - yield context.df.callActivity("extractUserDataActivity", { - fiscalCode: currentUserDataProcessing.fiscalCode - }) - ).getOrElseL(err => { - throw ActivityFailure.encode({ - activityName: "extractUserDataActivity", - kind: "ACTIVITY", - reason: readableReport(err) - }); - }); - - SendUserDataDownloadMessageActivityResultSuccess.decode( - yield context.df.callActivity("sendUserDataDownloadMessageActivity", { - blobName: bundle.value.blobName, - fiscalCode: currentUserDataProcessing.fiscalCode, - password: bundle.value.password - }) - ).getOrElseL(err => { - throw ActivityFailure.encode({ - activityName: "sendUserDataDownloadMessageActivity", - kind: "ACTIVITY", - reason: readableReport(err) - }); - }); - - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.CLOSED - }) - ).getOrElseL(err => { - throw ActivityFailure.encode({ - activityName: "setUserDataProcessingStatusActivity", - extra: { status: UserDataProcessingStatusEnum.CLOSED }, - kind: "ACTIVITY", - reason: readableReport(err) - }); - }); - - return OrchestratorSuccess.encode({ kind: "SUCCESS" }); - } catch (error) { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.FAILED - }) - ).getOrElseL(err => { - throw new Error( - `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( - err - )}` - ); - }); - return OrchestratorFailure.is(error) - ? error - : UnhanldedFailure.encode({ kind: "UNHANDLED", reason: error.message }); - } + context.log.info( + `${logPrefix}: processing ${subTasks.length} document${ + subTasks.length === 1 ? "" : "s" + }` + ); + yield context.df.Task.all(subTasks); }; diff --git a/UserDataDownloadOrchestrator/index.ts b/UserDataDownloadOrchestrator/index.ts index be7758ad..f4df85b6 100644 --- a/UserDataDownloadOrchestrator/index.ts +++ b/UserDataDownloadOrchestrator/index.ts @@ -1,7 +1,11 @@ import * as df from "durable-functions"; - +import { IFunctionContext } from "durable-functions/lib/src/classes"; import { handler } from "./handler"; -const orchestrator = df.orchestrator(handler); +const orchestrator = df.orchestrator(function*( + context: IFunctionContext +): IterableIterator { + yield handler(context, context.df.getInput()); +}); export default orchestrator; diff --git a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts new file mode 100644 index 00000000..7ca8fbee --- /dev/null +++ b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts @@ -0,0 +1,291 @@ +// tslint:disable: no-any + +import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { + mockOrchestratorCallActivity, + mockOrchestratorCallActivityWithRetry, + mockOrchestratorContext +} from "../../__mocks__/durable-functions"; +import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; +import { + ActivityFailure, + handler, + InvalidInputFailure, + OrchestratorSuccess +} from "../handler"; + +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; + +const aNonSuccess = "any non-success value"; + +const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) +); +const extractUserDataActivity = jest.fn().mockImplementation(() => + ExtractUserDataActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aArchiveInfo + }) +); +const sendUserDataDownloadMessageActivity = jest + .fn() + .mockImplementation(() => + SendUserDataDownloadMessageActivityResultSuccess.encode({ kind: "SUCCESS" }) + ); + +// A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation +const switchMockImplementation = (name: string, ...args: readonly unknown[]) => + (name === "setUserDataProcessingStatusActivity" + ? setUserDataProcessingStatusActivity + : name === "extractUserDataActivity" + ? extractUserDataActivity + : name === "sendUserDataDownloadMessageActivity" + ? sendUserDataDownloadMessageActivity + : jest.fn())(name, ...args); + +// I assign switchMockImplementation to both because +// I don't want tests to depend on implementation details +// such as which activity is called with retry and which is not +mockOrchestratorCallActivity.mockImplementation(switchMockImplementation); +mockOrchestratorCallActivityWithRetry.mockImplementation( + switchMockImplementation +); + +/** + * Util function that takes an orchestrator and executes each step until is done + * @param orch an orchestrator + * + * @returns the last value yielded by the orchestrator + */ +const consumeOrchestrator = (orch: any) => { + // tslint:disable-next-line: no-let + let prevValue: unknown; + while (true) { + const { done, value } = orch.next(prevValue); + if (done) { + return value; + } + prevValue = value; + } +}; + +// just a convenient cast, good for every test case +const context = (mockOrchestratorContext as unknown) as IFunctionContext; + +// tslint:disable-next-line: no-big-function +describe("handler", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should fail on invalid input", () => { + const document = "invalid"; + + const result = consumeOrchestrator(handler(context, document)); + + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it.each` + name | status + ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} + ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} + `("should skip if the status is $name", ({ status }) => { + const document = { + ...aUserDataProcessing, + status + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should success if everything goes well", () => { + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); + // first, set as WIP + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.WIP + } + ); + // then, set as CLOSED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.CLOSED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when data extraction fails", () => { + extractUserDataActivity.mockImplementationOnce(() => aNonSuccess); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("extractUserDataActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when send message fails", () => { + sendUserDataDownloadMessageActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("sendUserDataDownloadMessageActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.WIP + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to CLOSED fails", () => { + // the first time is called is for WIP + setUserDataProcessingStatusActivity.mockImplementationOnce(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) + ); + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + + const result = consumeOrchestrator(handler(context, document)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.CLOSED + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); +}); diff --git a/UserDataDownloadSubOrchestrator/function.json b/UserDataDownloadSubOrchestrator/function.json new file mode 100644 index 00000000..ea76d6d4 --- /dev/null +++ b/UserDataDownloadSubOrchestrator/function.json @@ -0,0 +1,10 @@ +{ + "bindings": [ + { + "name": "context", + "type": "orchestrationTrigger", + "direction": "in" + } + ], + "scriptFile": "../dist/UserDataDownloadSubOrchestrator/index.js" +} \ No newline at end of file diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts new file mode 100644 index 00000000..69ee6e7e --- /dev/null +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -0,0 +1,158 @@ +import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { isLeft } from "fp-ts/lib/Either"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import * as t from "io-ts"; +import { readableReport } from "italia-ts-commons/lib/reporters"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; +import { ProcessableUserDataProcessing } from "../UserDataDownloadOrchestrator/handler"; + +const logPrefix = "UserDataDownloadSubOrchestrator"; + +export type InvalidInputFailure = t.TypeOf; +export const InvalidInputFailure = t.interface({ + kind: t.literal("INVALID_INPUT"), + reason: t.string +}); + +export type UnhanldedFailure = t.TypeOf; +export const UnhanldedFailure = t.interface({ + kind: t.literal("UNHANDLED"), + reason: t.string +}); + +export type ActivityFailure = t.TypeOf; +export const ActivityFailure = t.intersection([ + t.interface({ + activityName: t.string, + kind: t.literal("ACTIVITY"), + reason: t.string + }), + t.partial({ extra: t.object }) +]); + +export type OrchestratorFailure = t.TypeOf; +export const OrchestratorFailure = t.taggedUnion("kind", [ + InvalidInputFailure, + UnhanldedFailure, + ActivityFailure +]); + +export type OrchestratorSuccess = t.TypeOf; +export const OrchestratorSuccess = t.interface({ + kind: t.literal("SUCCESS") +}); + +export type SkippedDocument = t.TypeOf; +export const SkippedDocument = t.interface({ + kind: t.literal("SKIPPED") +}); + +export type OrchestratorResult = t.TypeOf; +export const OrchestratorResult = t.union([ + OrchestratorFailure, + SkippedDocument, + OrchestratorSuccess +]); + +const toActivityFailure = ( + err: t.Errors, + activityName: string, + extra?: object +) => + ActivityFailure.encode({ + activityName, + extra, + kind: "ACTIVITY", + reason: readableReport(err) + }); + +export const handler = function*( + context: IFunctionContext, + document: unknown +): IterableIterator { + // This check has been done on the parent orchestrator, so it should never fail. + // However, it's maybe woth the effort of check it twice + const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( + document + ).mapLeft(err => { + context.log.error( + `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( + err + )}` + ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) + }); + }); + + if (isLeft(invalidInputOrCurrentUserDataProcessing)) { + return invalidInputOrCurrentUserDataProcessing.value; + } + + const currentUserDataProcessing = + invalidInputOrCurrentUserDataProcessing.value; + + try { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.WIP + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.WIP + }); + }); + + const bundle = ExtractUserDataActivityResultSuccess.decode( + yield context.df.callActivity("extractUserDataActivity", { + fiscalCode: currentUserDataProcessing.fiscalCode + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "extractUserDataActivity"); + }); + + SendUserDataDownloadMessageActivityResultSuccess.decode( + yield context.df.callActivity("sendUserDataDownloadMessageActivity", { + blobName: bundle.value.blobName, + fiscalCode: currentUserDataProcessing.fiscalCode, + password: bundle.value.password + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "sendUserDataDownloadMessageActivity"); + }); + + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.CLOSED + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.CLOSED + }); + }); + + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); + } catch (error) { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.FAILED + }) + ).getOrElseL(err => { + throw new Error( + `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + err + )}` + ); + }); + + return OrchestratorFailure.is(error) + ? error + : UnhanldedFailure.encode({ kind: "UNHANDLED", reason: error.message }); + } +}; diff --git a/UserDataDownloadSubOrchestrator/index.ts b/UserDataDownloadSubOrchestrator/index.ts new file mode 100644 index 00000000..f4df85b6 --- /dev/null +++ b/UserDataDownloadSubOrchestrator/index.ts @@ -0,0 +1,11 @@ +import * as df from "durable-functions"; +import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { handler } from "./handler"; + +const orchestrator = df.orchestrator(function*( + context: IFunctionContext +): IterableIterator { + yield handler(context, context.df.getInput()); +}); + +export default orchestrator; diff --git a/__mocks__/durable-functions.ts b/__mocks__/durable-functions.ts index c6744125..2e85d7b8 100644 --- a/__mocks__/durable-functions.ts +++ b/__mocks__/durable-functions.ts @@ -64,14 +64,25 @@ export const mockOrchestratorCallActivityWithRetry = jest retryOptions }) ); +export const mockCallSubOrchestrator = jest + .fn() + .mockImplementation((name: string, input?: unknown) => ({ + input, + name + })); export const mockOrchestratorSetCustomStatus = jest.fn(); export const mockOrchestratorCreateTimer = jest.fn(); export const mockOrchestratorContext = { ...context, df: { + Task: { + all: jest.fn(), + any: jest.fn() + }, callActivity: mockOrchestratorCallActivity, callActivityWithRetry: mockOrchestratorCallActivityWithRetry, + callSubOrchestrator: mockCallSubOrchestrator, createTimer: mockOrchestratorCreateTimer, getInput: mockOrchestratorGetInput, setCustomStatus: mockOrchestratorSetCustomStatus diff --git a/__mocks__/mocks.ts b/__mocks__/mocks.ts index 5502b698..debb5675 100644 --- a/__mocks__/mocks.ts +++ b/__mocks__/mocks.ts @@ -59,8 +59,8 @@ import { NotificationStatusId } from "io-functions-commons/dist/src/models/notification_status"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { EmailAddress } from "../generated/definitions/EmailAddress"; import { ArchiveInfo } from "../ExtractUserDataActivity/handler"; +import { EmailAddress } from "../generated/definitions/EmailAddress"; export const aFiscalCode = "SPNDNL80A13Y555X" as FiscalCode; From 98210963de6107d29d5473c88a2ba527c66cee59 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 7 Jul 2020 10:11:17 +0200 Subject: [PATCH 04/34] timer --- UserDataDownloadSubOrchestrator/handler.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index 69ee6e7e..315ceab6 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -1,4 +1,4 @@ -import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { IFunctionContext, Task } from "durable-functions/lib/src/classes"; import { isLeft } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import * as t from "io-ts"; @@ -95,6 +95,9 @@ export const handler = function*( const currentUserDataProcessing = invalidInputOrCurrentUserDataProcessing.value; + // start this operation tomorrow + yield context.df.createTimer(new Date(Date.now() + 24 * 60 * 60 * 100)); + try { SetUserDataProcessingStatusActivityResultSuccess.decode( yield context.df.callActivity("setUserDataProcessingStatusActivity", { From 78a20636dc1cec6601428fca7e214872f25e46c0 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 7 Jul 2020 10:20:05 +0200 Subject: [PATCH 05/34] lint fix --- UserDataDownloadSubOrchestrator/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index 315ceab6..eeff7508 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -1,4 +1,4 @@ -import { IFunctionContext, Task } from "durable-functions/lib/src/classes"; +import { IFunctionContext } from "durable-functions/lib/src/classes"; import { isLeft } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import * as t from "io-ts"; From c0b2a50dc33914347b20e275c3afb8f8384968d4 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 7 Jul 2020 10:47:24 +0200 Subject: [PATCH 06/34] exlude DELETE choices --- .../__tests__/handler.test.ts | 26 ++++++++++++++----- UserDataDownloadOrchestrator/handler.ts | 17 ++++++------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index 616bbe4e..2441e356 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -1,6 +1,7 @@ // tslint:disable: no-any import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { mockCallSubOrchestrator, @@ -8,17 +9,24 @@ import { } from "../../__mocks__/durable-functions"; import { aUserDataProcessing } from "../../__mocks__/mocks"; import { handler } from "../handler"; +import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; const aProcessableDocument = { ...aUserDataProcessing, + choice: UserDataProcessingChoiceEnum.DOWNLOAD, status: UserDataProcessingStatusEnum.PENDING }; -const aNonProcessableDocument = { +const aNonProcessableDocumentWrongStatus = { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.WIP }; +const aNonProcessableDocumentWrongChoice = { + ...aUserDataProcessing, + choice: UserDataProcessingChoiceEnum.DELETE +}; + /** * Util function that takes an orchestrator and executes each step until is done * @param orch an orchestrator @@ -57,15 +65,21 @@ describe("handler", () => { }); it("should process every processable document", () => { - const input: ReadonlyArray = [ + const processableDocs: ReadonlyArray = [ aProcessableDocument, aProcessableDocument, - aProcessableDocument, - aNonProcessableDocument, - aNonProcessableDocument + aProcessableDocument + ]; + + const input: ReadonlyArray = [ + ...processableDocs, + aNonProcessableDocumentWrongStatus, + aNonProcessableDocumentWrongChoice ]; consumeOrchestrator(handler(context, input)); - expect(mockCallSubOrchestrator).toHaveBeenCalledTimes(3); + expect(mockCallSubOrchestrator).toHaveBeenCalledTimes( + processableDocs.length + ); }); }); diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 6322fd15..e47b1f89 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -1,5 +1,6 @@ import { IFunctionContext } from "durable-functions/lib/src/classes"; import { isLeft } from "fp-ts/lib/Either"; +import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import * as t from "io-ts"; @@ -13,7 +14,10 @@ export type ProcessableUserDataProcessing = t.TypeOf< >; export const ProcessableUserDataProcessing = t.intersection([ UserDataProcessing, + // ony the subset of UserDataProcessing documents + // with the following characteristics must be processed t.interface({ + choice: t.literal(UserDataProcessingChoiceEnum.DOWNLOAD), status: t.union([ t.literal(UserDataProcessingStatusEnum.PENDING), t.literal(UserDataProcessingStatusEnum.FAILED) @@ -29,15 +33,10 @@ export const handler = function*( input: unknown ): IterableIterator { const subTasks = CosmosDbDocumentCollection.decode(input) - .fold( - err => { - throw Error( - `${logPrefix}: cannot decode input [${readableReport(err)}]` - ); - }, - documents => - documents.map(doc => ProcessableUserDataProcessing.decode(doc)) - ) + .getOrElseL(err => { + throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); + }) + .map(doc => ProcessableUserDataProcessing.decode(doc)) .reduce( (documents, maybeProcessable) => { if (isLeft(maybeProcessable)) { From 52d17ed7f6dbf8f4fc7f1733bb1efb1ac5941512 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 7 Jul 2020 11:30:50 +0200 Subject: [PATCH 07/34] lint fix --- UserDataDownloadOrchestrator/__tests__/handler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index 2441e356..b5f4b586 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -3,13 +3,13 @@ import { IFunctionContext } from "durable-functions/lib/src/classes"; import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import { mockCallSubOrchestrator, mockOrchestratorContext } from "../../__mocks__/durable-functions"; import { aUserDataProcessing } from "../../__mocks__/mocks"; import { handler } from "../handler"; -import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; const aProcessableDocument = { ...aUserDataProcessing, From bb6c54872c94331c7f77a2327ec9baf1771df2f8 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Wed, 8 Jul 2020 10:10:08 +0200 Subject: [PATCH 08/34] Update UserDataDownloadSubOrchestrator/handler.ts Co-authored-by: Danilo Spinelli --- UserDataDownloadSubOrchestrator/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index eeff7508..5e4053be 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -73,7 +73,7 @@ export const handler = function*( document: unknown ): IterableIterator { // This check has been done on the parent orchestrator, so it should never fail. - // However, it's maybe woth the effort of check it twice + // However, it's worth the effort to check it twice const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( document ).mapLeft(err => { From 2e5ce860330f271120935d88973946bfc411ca84 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Wed, 8 Jul 2020 12:11:56 +0200 Subject: [PATCH 09/34] fix delay --- UserDataDownloadOrchestrator/function.json | 3 +- .../__tests__/handler.test.ts | 9 +- UserDataDownloadSubOrchestrator/handler.ts | 172 +++++++++--------- UserDataDownloadSubOrchestrator/index.ts | 11 +- __mocks__/durable-functions.ts | 1 + package.json | 2 +- yarn.lock | 8 +- 7 files changed, 115 insertions(+), 91 deletions(-) diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index d5961aab..2315a3fe 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -9,7 +9,8 @@ "type": "cosmosDBTrigger", "name": "documents", "direction": "in", - "leaseCollectionName": "leases", + "leaseCollectionName": "change-feed-leases", + "leaseCollectionPrefix": "userDataDownload", "connectionStringSetting": "COSMOSDB_CONNECTION_STRING", "databaseName": "%COSMOSDB_NAME%", "collectionName": "user-data-processing", diff --git a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts index 7ca8fbee..a844de26 100644 --- a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts @@ -12,13 +12,16 @@ import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from ". import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; import { ActivityFailure, - handler, + getHandler, InvalidInputFailure, OrchestratorSuccess } from "../handler"; +import { Millisecond } from "italia-ts-commons/lib/units"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; +const DELAY = 1 as Millisecond; + const aNonSuccess = "any non-success value"; const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => @@ -78,8 +81,10 @@ const consumeOrchestrator = (orch: any) => { // just a convenient cast, good for every test case const context = (mockOrchestratorContext as unknown) as IFunctionContext; +const handler = getHandler(DELAY); + // tslint:disable-next-line: no-big-function -describe("handler", () => { +describe("handler(DELAY)", () => { beforeEach(() => { jest.clearAllMocks(); }); diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index 5e4053be..c06f5bff 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -3,6 +3,7 @@ import { isLeft } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; +import { Millisecond } from "italia-ts-commons/lib/units"; import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; @@ -68,94 +69,101 @@ const toActivityFailure = ( reason: readableReport(err) }); -export const handler = function*( - context: IFunctionContext, - document: unknown -): IterableIterator { - // This check has been done on the parent orchestrator, so it should never fail. - // However, it's worth the effort to check it twice - const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( - document - ).mapLeft(err => { - context.log.error( - `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( - err - )}` - ); - return InvalidInputFailure.encode({ - kind: "INVALID_INPUT", - reason: readableReport(err) +export const getHandler = (delay: Millisecond = 0 as Millisecond) => + function*( + context: IFunctionContext, + document: unknown + ): IterableIterator { + // This check has been done on the parent orchestrator, so it should never fail. + // However, it's worth the effort to check it twice + const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( + document + ).mapLeft(err => { + context.log.error( + `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( + err + )}` + ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) + }); }); - }); - if (isLeft(invalidInputOrCurrentUserDataProcessing)) { - return invalidInputOrCurrentUserDataProcessing.value; - } - - const currentUserDataProcessing = - invalidInputOrCurrentUserDataProcessing.value; - - // start this operation tomorrow - yield context.df.createTimer(new Date(Date.now() + 24 * 60 * 60 * 100)); - - try { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.WIP - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.WIP + if (isLeft(invalidInputOrCurrentUserDataProcessing)) { + return invalidInputOrCurrentUserDataProcessing.value; + } + + const currentUserDataProcessing = + invalidInputOrCurrentUserDataProcessing.value; + + // start this operation tomorrow + yield context.df.createTimer( + // tslint:disable-next-line: restrict-plus-operands + new Date(context.df.currentUtcDateTime.getTime() + delay) + ); + + try { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.WIP + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.WIP + }); }); - }); - const bundle = ExtractUserDataActivityResultSuccess.decode( - yield context.df.callActivity("extractUserDataActivity", { - fiscalCode: currentUserDataProcessing.fiscalCode - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "extractUserDataActivity"); - }); + const bundle = ExtractUserDataActivityResultSuccess.decode( + yield context.df.callActivity("extractUserDataActivity", { + fiscalCode: currentUserDataProcessing.fiscalCode + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "extractUserDataActivity"); + }); - SendUserDataDownloadMessageActivityResultSuccess.decode( - yield context.df.callActivity("sendUserDataDownloadMessageActivity", { - blobName: bundle.value.blobName, - fiscalCode: currentUserDataProcessing.fiscalCode, - password: bundle.value.password - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "sendUserDataDownloadMessageActivity"); - }); + SendUserDataDownloadMessageActivityResultSuccess.decode( + yield context.df.callActivity("sendUserDataDownloadMessageActivity", { + blobName: bundle.value.blobName, + fiscalCode: currentUserDataProcessing.fiscalCode, + password: bundle.value.password + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "sendUserDataDownloadMessageActivity"); + }); - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.CLOSED - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.CLOSED + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.CLOSED + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.CLOSED + }); }); - }); - return OrchestratorSuccess.encode({ kind: "SUCCESS" }); - } catch (error) { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.FAILED - }) - ).getOrElseL(err => { - throw new Error( - `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( - err - )}` - ); - }); + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); + } catch (error) { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("setUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.FAILED + }) + ).getOrElseL(err => { + throw new Error( + `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + err + )}` + ); + }); - return OrchestratorFailure.is(error) - ? error - : UnhanldedFailure.encode({ kind: "UNHANDLED", reason: error.message }); - } -}; + return OrchestratorFailure.is(error) + ? error + : UnhanldedFailure.encode({ + kind: "UNHANDLED", + reason: error.message + }); + } + }; diff --git a/UserDataDownloadSubOrchestrator/index.ts b/UserDataDownloadSubOrchestrator/index.ts index f4df85b6..3502addc 100644 --- a/UserDataDownloadSubOrchestrator/index.ts +++ b/UserDataDownloadSubOrchestrator/index.ts @@ -1,6 +1,15 @@ import * as df from "durable-functions"; import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { handler } from "./handler"; +import { Hour, Millisecond } from "italia-ts-commons/lib/units"; +import { getHandler } from "./handler"; + +const delayInHours = (typeof process.env.USER_DATA_DOWNLOAD_DELAY_HOURS === +"undefined" + ? 24 + : process.env.USER_DATA_DOWNLOAD_DELAY_HOURS) as Hour; +const delay = (delayInHours * 60 * 60 * 1000) as Millisecond; + +const handler = getHandler(delay); const orchestrator = df.orchestrator(function*( context: IFunctionContext diff --git a/__mocks__/durable-functions.ts b/__mocks__/durable-functions.ts index 2e85d7b8..2c8c10ed 100644 --- a/__mocks__/durable-functions.ts +++ b/__mocks__/durable-functions.ts @@ -84,6 +84,7 @@ export const mockOrchestratorContext = { callActivityWithRetry: mockOrchestratorCallActivityWithRetry, callSubOrchestrator: mockCallSubOrchestrator, createTimer: mockOrchestratorCreateTimer, + currentUtcDateTime: new Date(), getInput: mockOrchestratorGetInput, setCustomStatus: mockOrchestratorSetCustomStatus } diff --git a/package.json b/package.json index e50b2738..bbc2297f 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "io-functions-commons": "^10.0.0", "io-functions-express": "^0.1.0", "io-ts": "1.8.5", - "italia-ts-commons": "^8.1.0", + "italia-ts-commons": "^8.5.0", "randomstring": "^1.1.5", "redis": "^3.0.2", "redis-clustr": "^1.7.0", diff --git a/yarn.lock b/yarn.lock index 52a537da..165c43a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4804,10 +4804,10 @@ italia-ts-commons@^7.0.1: node-fetch "^2.6.0" validator "^10.1.0" -italia-ts-commons@^8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/italia-ts-commons/-/italia-ts-commons-8.1.0.tgz#929c805a7d9e8877fef26436946bf3ee76a76c82" - integrity sha512-d03UKxjeIX+4otNgZaDoS8t1l+gVEHD/q/JsQox1CC3nAiKHQ9TFNv4BdHIiUIswgpx1szjT6lzeJQUCUL++YA== +italia-ts-commons@^8.5.0: + version "8.5.0" + resolved "https://registry.yarnpkg.com/italia-ts-commons/-/italia-ts-commons-8.5.0.tgz#9967ec1b94745f482d5cc942a1bacf7de6f6461b" + integrity sha512-pTX90/WW987t7pdcbQngCFm35c3gQlDFIUCnDZ6VCo5vNMl8VwdZztGIcYyEaJw6xMjb1WjSRlltnDdSe0B9Eg== dependencies: abort-controller "^3.0.0" agentkeepalive "^4.1.2" From 3db3333b6ccaba7fa75852e54bed63764ed2e603 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Wed, 8 Jul 2020 15:10:17 +0200 Subject: [PATCH 10/34] pause after wip --- UserDataDownloadSubOrchestrator/handler.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index c06f5bff..8c7d61e9 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -97,12 +97,6 @@ export const getHandler = (delay: Millisecond = 0 as Millisecond) => const currentUserDataProcessing = invalidInputOrCurrentUserDataProcessing.value; - // start this operation tomorrow - yield context.df.createTimer( - // tslint:disable-next-line: restrict-plus-operands - new Date(context.df.currentUtcDateTime.getTime() + delay) - ); - try { SetUserDataProcessingStatusActivityResultSuccess.decode( yield context.df.callActivity("setUserDataProcessingStatusActivity", { @@ -115,6 +109,12 @@ export const getHandler = (delay: Millisecond = 0 as Millisecond) => }); }); + // pause this operation for a while + yield context.df.createTimer( + // tslint:disable-next-line: restrict-plus-operands + new Date(context.df.currentUtcDateTime.getTime() + delay) + ); + const bundle = ExtractUserDataActivityResultSuccess.decode( yield context.df.callActivity("extractUserDataActivity", { fiscalCode: currentUserDataProcessing.fiscalCode From a1458114480ed859e81098ab3e0472fc37992b71 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 15:35:36 +0200 Subject: [PATCH 11/34] reviewed orchestrator workflow --- .../__tests__/handler.test.ts | 9 ++-- UserDataDownloadOrchestrator/function.json | 11 ----- UserDataDownloadOrchestrator/handler.ts | 12 ++--- UserDataDownloadOrchestrator/index.ts | 7 +-- .../function.json | 21 +++++++++ UserDataDownloadOrchestratorTrigger/index.ts | 7 +++ .../__tests__/handler.test.ts | 46 +++++++++++-------- UserDataDownloadSubOrchestrator/handler.ts | 30 ++++++------ UserDataDownloadSubOrchestrator/index.ts | 6 +-- 9 files changed, 84 insertions(+), 65 deletions(-) create mode 100644 UserDataDownloadOrchestratorTrigger/function.json create mode 100644 UserDataDownloadOrchestratorTrigger/index.ts diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index b5f4b586..b98eaf08 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -6,7 +6,8 @@ import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generate import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import { mockCallSubOrchestrator, - mockOrchestratorContext + mockOrchestratorContext, + mockOrchestratorGetInput } from "../../__mocks__/durable-functions"; import { aUserDataProcessing } from "../../__mocks__/mocks"; import { handler } from "../handler"; @@ -55,9 +56,10 @@ describe("handler", () => { it("should fail on invalid input", () => { const input = "invalid"; + mockOrchestratorGetInput.mockReturnValueOnce(input); try { - consumeOrchestrator(handler(context, input)); + consumeOrchestrator(handler(context)); fail("it should throw"); } catch (error) { expect(mockCallSubOrchestrator).not.toHaveBeenCalled(); @@ -76,8 +78,9 @@ describe("handler", () => { aNonProcessableDocumentWrongStatus, aNonProcessableDocumentWrongChoice ]; + mockOrchestratorGetInput.mockReturnValueOnce(input); - consumeOrchestrator(handler(context, input)); + consumeOrchestrator(handler(context)); expect(mockCallSubOrchestrator).toHaveBeenCalledTimes( processableDocs.length ); diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index 2315a3fe..dc59add8 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -4,17 +4,6 @@ "name": "context", "type": "orchestrationTrigger", "direction": "in" - }, - { - "type": "cosmosDBTrigger", - "name": "documents", - "direction": "in", - "leaseCollectionName": "change-feed-leases", - "leaseCollectionPrefix": "userDataDownload", - "connectionStringSetting": "COSMOSDB_CONNECTION_STRING", - "databaseName": "%COSMOSDB_NAME%", - "collectionName": "user-data-processing", - "createLeaseCollectionIfNotExists": true } ], "scriptFile": "../dist/UserDataDownloadOrchestrator/index.js" diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index e47b1f89..3fd2735e 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -18,10 +18,7 @@ export const ProcessableUserDataProcessing = t.intersection([ // with the following characteristics must be processed t.interface({ choice: t.literal(UserDataProcessingChoiceEnum.DOWNLOAD), - status: t.union([ - t.literal(UserDataProcessingStatusEnum.PENDING), - t.literal(UserDataProcessingStatusEnum.FAILED) - ]) + status: t.literal(UserDataProcessingStatusEnum.PENDING) }) ]); @@ -29,9 +26,9 @@ const CosmosDbDocumentCollection = t.readonlyArray(t.readonly(t.UnknownRecord)); type CosmosDbDocumentCollection = t.TypeOf; export const handler = function*( - context: IFunctionContext, - input: unknown + context: IFunctionContext ): IterableIterator { + const input = context.df.getInput(); const subTasks = CosmosDbDocumentCollection.decode(input) .getOrElseL(err => { throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); @@ -63,5 +60,6 @@ export const handler = function*( subTasks.length === 1 ? "" : "s" }` ); - yield context.df.Task.all(subTasks); + const result = yield context.df.Task.all(subTasks); + context.log.info(`${logPrefix}: processed ${JSON.stringify(result)}`); }; diff --git a/UserDataDownloadOrchestrator/index.ts b/UserDataDownloadOrchestrator/index.ts index f4df85b6..8a754b91 100644 --- a/UserDataDownloadOrchestrator/index.ts +++ b/UserDataDownloadOrchestrator/index.ts @@ -1,11 +1,6 @@ import * as df from "durable-functions"; -import { IFunctionContext } from "durable-functions/lib/src/classes"; import { handler } from "./handler"; -const orchestrator = df.orchestrator(function*( - context: IFunctionContext -): IterableIterator { - yield handler(context, context.df.getInput()); -}); +const orchestrator = df.orchestrator(handler); export default orchestrator; diff --git a/UserDataDownloadOrchestratorTrigger/function.json b/UserDataDownloadOrchestratorTrigger/function.json new file mode 100644 index 00000000..7b52ec60 --- /dev/null +++ b/UserDataDownloadOrchestratorTrigger/function.json @@ -0,0 +1,21 @@ +{ + "bindings": [ + { + "type": "cosmosDBTrigger", + "name": "documents", + "direction": "in", + "leaseCollectionName": "change-feed-leases", + "leaseCollectionPrefix": "userDataDownload", + "connectionStringSetting": "COSMOSDB_CONNECTION_STRING", + "databaseName": "%COSMOSDB_NAME%", + "collectionName": "user-data-processing", + "createLeaseCollectionIfNotExists": true + }, + { + "name": "starter", + "type": "orchestrationClient", + "direction": "in" + } + ], + "scriptFile": "../dist/UserDataDownloadOrchestratorTrigger/index.js" +} \ No newline at end of file diff --git a/UserDataDownloadOrchestratorTrigger/index.ts b/UserDataDownloadOrchestratorTrigger/index.ts new file mode 100644 index 00000000..0a07d1d1 --- /dev/null +++ b/UserDataDownloadOrchestratorTrigger/index.ts @@ -0,0 +1,7 @@ +import { Context } from "@azure/functions"; +import * as df from "durable-functions"; + +export default (context: Context, input: unknown) => + df + .getClient(context) + .startNew("UserDataDownloadOrchestrator", undefined, input); diff --git a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts index a844de26..90262582 100644 --- a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts @@ -5,7 +5,8 @@ import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generate import { mockOrchestratorCallActivity, mockOrchestratorCallActivityWithRetry, - mockOrchestratorContext + mockOrchestratorContext, + mockOrchestratorGetInput } from "../../__mocks__/durable-functions"; import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; @@ -44,11 +45,11 @@ const sendUserDataDownloadMessageActivity = jest // A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation const switchMockImplementation = (name: string, ...args: readonly unknown[]) => - (name === "setUserDataProcessingStatusActivity" + (name === "SetUserDataProcessingStatusActivity" ? setUserDataProcessingStatusActivity - : name === "extractUserDataActivity" + : name === "ExtractUserDataActivity" ? extractUserDataActivity - : name === "sendUserDataDownloadMessageActivity" + : name === "SendUserDataDownloadMessageActivity" ? sendUserDataDownloadMessageActivity : jest.fn())(name, ...args); @@ -84,15 +85,16 @@ const context = (mockOrchestratorContext as unknown) as IFunctionContext; const handler = getHandler(DELAY); // tslint:disable-next-line: no-big-function -describe("handler(DELAY)", () => { +describe("handler", () => { beforeEach(() => { jest.clearAllMocks(); }); it("should fail on invalid input", () => { const document = "invalid"; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(InvalidInputFailure.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); @@ -109,8 +111,9 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(InvalidInputFailure.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); @@ -123,8 +126,9 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); @@ -155,11 +159,12 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("extractUserDataActivity"); + expect(result.activityName).toBe("ExtractUserDataActivity"); expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one // then, set as FAILED expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( @@ -182,11 +187,12 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("sendUserDataDownloadMessageActivity"); + expect(result.activityName).toBe("SendUserDataDownloadMessageActivity"); expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one // then, set as FAILED expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( @@ -209,11 +215,12 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one // then, set as FAILED expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( @@ -236,11 +243,12 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); expect(result.extra).toEqual({ status: UserDataProcessingStatusEnum.WIP }); @@ -265,6 +273,7 @@ describe("handler(DELAY)", () => { value: aUserDataProcessing }) ); + setUserDataProcessingStatusActivity.mockImplementationOnce( () => aNonSuccess ); @@ -273,11 +282,12 @@ describe("handler(DELAY)", () => { ...aUserDataProcessing, status: UserDataProcessingStatusEnum.PENDING }; + mockOrchestratorGetInput.mockReturnValueOnce(document); - const result = consumeOrchestrator(handler(context, document)); + const result = consumeOrchestrator(handler(context)); expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("setUserDataProcessingStatusActivity"); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); expect(result.extra).toEqual({ status: UserDataProcessingStatusEnum.CLOSED }); diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index 8c7d61e9..de1cbcb1 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -70,10 +70,8 @@ const toActivityFailure = ( }); export const getHandler = (delay: Millisecond = 0 as Millisecond) => - function*( - context: IFunctionContext, - document: unknown - ): IterableIterator { + function*(context: IFunctionContext): IterableIterator { + const document = context.df.getInput(); // This check has been done on the parent orchestrator, so it should never fail. // However, it's worth the effort to check it twice const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( @@ -99,12 +97,12 @@ export const getHandler = (delay: Millisecond = 0 as Millisecond) => try { SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { currentRecord: currentUserDataProcessing, nextStatus: UserDataProcessingStatusEnum.WIP }) ).getOrElseL(err => { - throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { status: UserDataProcessingStatusEnum.WIP }); }); @@ -116,44 +114,46 @@ export const getHandler = (delay: Millisecond = 0 as Millisecond) => ); const bundle = ExtractUserDataActivityResultSuccess.decode( - yield context.df.callActivity("extractUserDataActivity", { + yield context.df.callActivity("ExtractUserDataActivity", { fiscalCode: currentUserDataProcessing.fiscalCode }) ).getOrElseL(err => { - throw toActivityFailure(err, "extractUserDataActivity"); + throw toActivityFailure(err, "ExtractUserDataActivity"); }); SendUserDataDownloadMessageActivityResultSuccess.decode( - yield context.df.callActivity("sendUserDataDownloadMessageActivity", { + yield context.df.callActivity("SendUserDataDownloadMessageActivity", { blobName: bundle.value.blobName, fiscalCode: currentUserDataProcessing.fiscalCode, password: bundle.value.password }) ).getOrElseL(err => { - throw toActivityFailure(err, "sendUserDataDownloadMessageActivity"); + throw toActivityFailure(err, "SendUserDataDownloadMessageActivity"); }); SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { currentRecord: currentUserDataProcessing, nextStatus: UserDataProcessingStatusEnum.CLOSED }) ).getOrElseL(err => { - throw toActivityFailure(err, "setUserDataProcessingStatusActivity", { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { status: UserDataProcessingStatusEnum.CLOSED }); }); - return OrchestratorSuccess.encode({ kind: "SUCCESS" }); } catch (error) { + context.log.error( + `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` + ); SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("setUserDataProcessingStatusActivity", { + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { currentRecord: currentUserDataProcessing, nextStatus: UserDataProcessingStatusEnum.FAILED }) ).getOrElseL(err => { throw new Error( - `Activity setUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( err )}` ); diff --git a/UserDataDownloadSubOrchestrator/index.ts b/UserDataDownloadSubOrchestrator/index.ts index 3502addc..1ae07151 100644 --- a/UserDataDownloadSubOrchestrator/index.ts +++ b/UserDataDownloadSubOrchestrator/index.ts @@ -11,10 +11,6 @@ const delay = (delayInHours * 60 * 60 * 1000) as Millisecond; const handler = getHandler(delay); -const orchestrator = df.orchestrator(function*( - context: IFunctionContext -): IterableIterator { - yield handler(context, context.df.getInput()); -}); +const orchestrator = df.orchestrator(handler); export default orchestrator; From 9cccdf1a5e9e1c87916c0b144935c1ae39bbb620 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 15:39:49 +0200 Subject: [PATCH 12/34] fix lint --- UserDataDownloadSubOrchestrator/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/index.ts b/UserDataDownloadSubOrchestrator/index.ts index 1ae07151..cd47e75e 100644 --- a/UserDataDownloadSubOrchestrator/index.ts +++ b/UserDataDownloadSubOrchestrator/index.ts @@ -1,5 +1,4 @@ import * as df from "durable-functions"; -import { IFunctionContext } from "durable-functions/lib/src/classes"; import { Hour, Millisecond } from "italia-ts-commons/lib/units"; import { getHandler } from "./handler"; From f30eb1eb1ff625b1e485f61e5a3a9410158f7ead Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 16:48:39 +0200 Subject: [PATCH 13/34] Update UserDataDownloadSubOrchestrator/handler.ts Co-authored-by: Danilo Spinelli --- UserDataDownloadSubOrchestrator/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts index de1cbcb1..7c8cc43b 100644 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ b/UserDataDownloadSubOrchestrator/handler.ts @@ -69,7 +69,7 @@ const toActivityFailure = ( reason: readableReport(err) }); -export const getHandler = (delay: Millisecond = 0 as Millisecond) => +export const getHandler = (delayMs: Millisecond = 0 as Millisecond) => function*(context: IFunctionContext): IterableIterator { const document = context.df.getInput(); // This check has been done on the parent orchestrator, so it should never fail. From 79087fa7e54bd494cf4841e67ac878e2ecee2f27 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 16:51:07 +0200 Subject: [PATCH 14/34] Update UserDataDownloadSubOrchestrator/__tests__/handler.test.ts Co-authored-by: Danilo Spinelli --- UserDataDownloadSubOrchestrator/__tests__/handler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts index 90262582..e38d19ef 100644 --- a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts @@ -85,7 +85,7 @@ const context = (mockOrchestratorContext as unknown) as IFunctionContext; const handler = getHandler(DELAY); // tslint:disable-next-line: no-big-function -describe("handler", () => { +describe(" UserDataDownloadSubOrchestrator", () => { beforeEach(() => { jest.clearAllMocks(); }); From 63b8af61907617bc02a743e1d94fd0d95a1ada07 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 16:51:19 +0200 Subject: [PATCH 15/34] Update UserDataDownloadOrchestrator/__tests__/handler.test.ts Co-authored-by: Danilo Spinelli --- UserDataDownloadOrchestrator/__tests__/handler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index b98eaf08..2e4e73e7 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -49,7 +49,7 @@ const consumeOrchestrator = (orch: any) => { // just a convenient cast, good for every test case const context = (mockOrchestratorContext as unknown) as IFunctionContext; -describe("handler", () => { +describe(" UserDataDownloadOrchestrator", () => { beforeEach(() => { jest.clearAllMocks(); }); From 56f26a6f247b81ebbf911623f77d0e6d20aec95a Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 9 Jul 2020 17:41:57 +0200 Subject: [PATCH 16/34] refactor --- .../__tests__/handler.test.ts | 306 +++++++++++++++--- UserDataDownloadOrchestrator/function.json | 2 +- UserDataDownloadOrchestrator/handler.ts | 223 +++++++++---- UserDataDownloadOrchestrator/index.ts | 11 +- UserDataDownloadOrchestratorTrigger/index.ts | 7 - .../__tests__/handler.test.ts | 306 ------------------ UserDataDownloadSubOrchestrator/function.json | 10 - UserDataDownloadSubOrchestrator/handler.ts | 169 ---------- UserDataDownloadSubOrchestrator/index.ts | 15 - .../__tests__/index.test.ts | 59 ++++ .../function.json | 2 +- UserDataProcessingTrigger/index.ts | 69 ++++ __mocks__/durable-functions.ts | 3 +- 13 files changed, 571 insertions(+), 611 deletions(-) delete mode 100644 UserDataDownloadOrchestratorTrigger/index.ts delete mode 100644 UserDataDownloadSubOrchestrator/__tests__/handler.test.ts delete mode 100644 UserDataDownloadSubOrchestrator/function.json delete mode 100644 UserDataDownloadSubOrchestrator/handler.ts delete mode 100644 UserDataDownloadSubOrchestrator/index.ts create mode 100644 UserDataProcessingTrigger/__tests__/index.test.ts rename {UserDataDownloadOrchestratorTrigger => UserDataProcessingTrigger}/function.json (88%) create mode 100644 UserDataProcessingTrigger/index.ts diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index b98eaf08..90262582 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -1,32 +1,65 @@ // tslint:disable: no-any import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; -import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import { - mockCallSubOrchestrator, + mockOrchestratorCallActivity, + mockOrchestratorCallActivityWithRetry, mockOrchestratorContext, mockOrchestratorGetInput } from "../../__mocks__/durable-functions"; -import { aUserDataProcessing } from "../../__mocks__/mocks"; -import { handler } from "../handler"; +import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; +import { + ActivityFailure, + getHandler, + InvalidInputFailure, + OrchestratorSuccess +} from "../handler"; -const aProcessableDocument = { - ...aUserDataProcessing, - choice: UserDataProcessingChoiceEnum.DOWNLOAD, - status: UserDataProcessingStatusEnum.PENDING -}; +import { Millisecond } from "italia-ts-commons/lib/units"; +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; -const aNonProcessableDocumentWrongStatus = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.WIP -}; +const DELAY = 1 as Millisecond; -const aNonProcessableDocumentWrongChoice = { - ...aUserDataProcessing, - choice: UserDataProcessingChoiceEnum.DELETE -}; +const aNonSuccess = "any non-success value"; + +const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) +); +const extractUserDataActivity = jest.fn().mockImplementation(() => + ExtractUserDataActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aArchiveInfo + }) +); +const sendUserDataDownloadMessageActivity = jest + .fn() + .mockImplementation(() => + SendUserDataDownloadMessageActivityResultSuccess.encode({ kind: "SUCCESS" }) + ); + +// A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation +const switchMockImplementation = (name: string, ...args: readonly unknown[]) => + (name === "SetUserDataProcessingStatusActivity" + ? setUserDataProcessingStatusActivity + : name === "ExtractUserDataActivity" + ? extractUserDataActivity + : name === "SendUserDataDownloadMessageActivity" + ? sendUserDataDownloadMessageActivity + : jest.fn())(name, ...args); + +// I assign switchMockImplementation to both because +// I don't want tests to depend on implementation details +// such as which activity is called with retry and which is not +mockOrchestratorCallActivity.mockImplementation(switchMockImplementation); +mockOrchestratorCallActivityWithRetry.mockImplementation( + switchMockImplementation +); /** * Util function that takes an orchestrator and executes each step until is done @@ -49,40 +82,225 @@ const consumeOrchestrator = (orch: any) => { // just a convenient cast, good for every test case const context = (mockOrchestratorContext as unknown) as IFunctionContext; +const handler = getHandler(DELAY); + +// tslint:disable-next-line: no-big-function describe("handler", () => { beforeEach(() => { jest.clearAllMocks(); }); it("should fail on invalid input", () => { - const input = "invalid"; - mockOrchestratorGetInput.mockReturnValueOnce(input); - - try { - consumeOrchestrator(handler(context)); - fail("it should throw"); - } catch (error) { - expect(mockCallSubOrchestrator).not.toHaveBeenCalled(); - } + const document = "invalid"; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it.each` + name | status + ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} + ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} + `("should skip if the status is $name", ({ status }) => { + const document = { + ...aUserDataProcessing, + status + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(InvalidInputFailure.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should success if everything goes well", () => { + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); + // first, set as WIP + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.WIP + } + ); + // then, set as CLOSED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.CLOSED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when data extraction fails", () => { + extractUserDataActivity.mockImplementationOnce(() => aNonSuccess); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("ExtractUserDataActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); }); - it("should process every processable document", () => { - const processableDocs: ReadonlyArray = [ - aProcessableDocument, - aProcessableDocument, - aProcessableDocument - ]; - - const input: ReadonlyArray = [ - ...processableDocs, - aNonProcessableDocumentWrongStatus, - aNonProcessableDocumentWrongChoice - ]; - mockOrchestratorGetInput.mockReturnValueOnce(input); - - consumeOrchestrator(handler(context)); - expect(mockCallSubOrchestrator).toHaveBeenCalledTimes( - processableDocs.length + it("should set as FAILED when send message fails", () => { + sendUserDataDownloadMessageActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("SendUserDataDownloadMessageActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to WIP fails", () => { + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.WIP + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } + ); + expect(extractUserDataActivity).not.toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); + }); + + it("should set as FAILED when status update to CLOSED fails", () => { + // the first time is called is for WIP + setUserDataProcessingStatusActivity.mockImplementationOnce(() => + SetUserDataProcessingStatusActivityResultSuccess.encode({ + kind: "SUCCESS", + value: aUserDataProcessing + }) + ); + + setUserDataProcessingStatusActivity.mockImplementationOnce( + () => aNonSuccess + ); + + const document = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.PENDING + }; + mockOrchestratorGetInput.mockReturnValueOnce(document); + + const result = consumeOrchestrator(handler(context)); + + expect(ActivityFailure.decode(result).isRight()).toBe(true); + expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); + expect(result.extra).toEqual({ + status: UserDataProcessingStatusEnum.CLOSED + }); + expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one + // then, set as FAILED + expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( + expect.any(String), + { + currentRecord: expect.any(Object), + nextStatus: UserDataProcessingStatusEnum.FAILED + } ); + expect(extractUserDataActivity).toHaveBeenCalled(); + expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); }); }); diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index dc59add8..ea76d6d4 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -6,5 +6,5 @@ "direction": "in" } ], - "scriptFile": "../dist/UserDataDownloadOrchestrator/index.js" + "scriptFile": "../dist/UserDataDownloadSubOrchestrator/index.js" } \ No newline at end of file diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 3fd2735e..ee20848b 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -1,65 +1,176 @@ -import { IFunctionContext } from "durable-functions/lib/src/classes"; +import { + IFunctionContext, + RetryOptions +} from "durable-functions/lib/src/classes"; import { isLeft } from "fp-ts/lib/Either"; -import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; -import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; +import { Millisecond } from "italia-ts-commons/lib/units"; +import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; +import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; +import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; +import { ProcessableUserDataDownload } from "../UserDataProcessingTrigger"; -const logPrefix = "UserDataDownloadOrchestrator"; +const logPrefix = "UserDataDownloadSubOrchestrator"; -// models the subset of UserDataProcessing documents that this orchestrator accepts -export type ProcessableUserDataProcessing = t.TypeOf< - typeof ProcessableUserDataProcessing ->; -export const ProcessableUserDataProcessing = t.intersection([ - UserDataProcessing, - // ony the subset of UserDataProcessing documents - // with the following characteristics must be processed +export type InvalidInputFailure = t.TypeOf; +export const InvalidInputFailure = t.interface({ + kind: t.literal("INVALID_INPUT"), + reason: t.string +}); + +export type UnhanldedFailure = t.TypeOf; +export const UnhanldedFailure = t.interface({ + kind: t.literal("UNHANDLED"), + reason: t.string +}); + +export type ActivityFailure = t.TypeOf; +export const ActivityFailure = t.intersection([ t.interface({ - choice: t.literal(UserDataProcessingChoiceEnum.DOWNLOAD), - status: t.literal(UserDataProcessingStatusEnum.PENDING) - }) + activityName: t.string, + kind: t.literal("ACTIVITY"), + reason: t.string + }), + t.partial({ extra: t.object }) ]); -const CosmosDbDocumentCollection = t.readonlyArray(t.readonly(t.UnknownRecord)); -type CosmosDbDocumentCollection = t.TypeOf; - -export const handler = function*( - context: IFunctionContext -): IterableIterator { - const input = context.df.getInput(); - const subTasks = CosmosDbDocumentCollection.decode(input) - .getOrElseL(err => { - throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); - }) - .map(doc => ProcessableUserDataProcessing.decode(doc)) - .reduce( - (documents, maybeProcessable) => { - if (isLeft(maybeProcessable)) { - context.log.warn( - `${logPrefix}: skipping document [${readableReport( - maybeProcessable.value - )}]` - ); - return documents; - } - return [...documents, maybeProcessable.value]; - }, - [] as readonly ProcessableUserDataProcessing[] - ) - .map(processableDoc => - context.df.callSubOrchestrator( - "UserDataDownloadSubOrchestrator", - processableDoc - ) - ); - - context.log.info( - `${logPrefix}: processing ${subTasks.length} document${ - subTasks.length === 1 ? "" : "s" - }` - ); - const result = yield context.df.Task.all(subTasks); - context.log.info(`${logPrefix}: processed ${JSON.stringify(result)}`); -}; +export type OrchestratorFailure = t.TypeOf; +export const OrchestratorFailure = t.taggedUnion("kind", [ + InvalidInputFailure, + UnhanldedFailure, + ActivityFailure +]); + +export type OrchestratorSuccess = t.TypeOf; +export const OrchestratorSuccess = t.interface({ + kind: t.literal("SUCCESS") +}); + +export type SkippedDocument = t.TypeOf; +export const SkippedDocument = t.interface({ + kind: t.literal("SKIPPED") +}); + +export type OrchestratorResult = t.TypeOf; +export const OrchestratorResult = t.union([ + OrchestratorFailure, + SkippedDocument, + OrchestratorSuccess +]); + +const toActivityFailure = ( + err: t.Errors, + activityName: string, + extra?: object +) => + ActivityFailure.encode({ + activityName, + extra, + kind: "ACTIVITY", + reason: readableReport(err) + }); + +export const getHandler = (delay: Millisecond = 0 as Millisecond) => + function*(context: IFunctionContext): IterableIterator { + const document = context.df.getInput(); + // This check has been done on the trigger, so it should never fail. + // However, it's worth the effort to check it twice + const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataDownload.decode( + document + ).mapLeft(err => { + context.log.error( + `${logPrefix}|WARN|Cannot decode ProcessableUserDataDownload document: ${readableReport( + err + )}` + ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) + }); + }); + + if (isLeft(invalidInputOrCurrentUserDataProcessing)) { + return invalidInputOrCurrentUserDataProcessing.value; + } + + const currentUserDataProcessing = + invalidInputOrCurrentUserDataProcessing.value; + + try { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.WIP + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.WIP + }); + }); + + // pause this operation for a while + yield context.df.createTimer( + // tslint:disable-next-line: restrict-plus-operands + new Date(context.df.currentUtcDateTime.getTime() + delay) + ); + + const bundle = ExtractUserDataActivityResultSuccess.decode( + yield context.df.callActivity("ExtractUserDataActivity", { + fiscalCode: currentUserDataProcessing.fiscalCode + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "ExtractUserDataActivity"); + }); + + SendUserDataDownloadMessageActivityResultSuccess.decode( + yield context.df.callActivityWithRetry( + "SendUserDataDownloadMessageActivity", + new RetryOptions(5000, 10), + { + blobName: bundle.value.blobName, + fiscalCode: currentUserDataProcessing.fiscalCode, + password: bundle.value.password + } + ) + ).getOrElseL(err => { + throw toActivityFailure(err, "SendUserDataDownloadMessageActivity"); + }); + + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.CLOSED + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.CLOSED + }); + }); + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); + } catch (error) { + context.log.error( + `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` + ); + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.FAILED + }) + ).getOrElseL(err => { + throw new Error( + `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + err + )}` + ); + }); + + return OrchestratorFailure.is(error) + ? error + : UnhanldedFailure.encode({ + kind: "UNHANDLED", + reason: error.message + }); + } + }; diff --git a/UserDataDownloadOrchestrator/index.ts b/UserDataDownloadOrchestrator/index.ts index 8a754b91..b70667e3 100644 --- a/UserDataDownloadOrchestrator/index.ts +++ b/UserDataDownloadOrchestrator/index.ts @@ -1,5 +1,14 @@ import * as df from "durable-functions"; -import { handler } from "./handler"; +import { NonNegativeInteger } from "italia-ts-commons/lib/numbers"; +import { Millisecond } from "italia-ts-commons/lib/units"; +import { getHandler } from "./handler"; + +const delayInHours = NonNegativeInteger.decode( + process.env.USER_DATA_DOWNLOAD_DELAY_HOURS +).getOrElse(24 as NonNegativeInteger); +const delay = (delayInHours * 60 * 60 * 1000) as Millisecond; + +const handler = getHandler(delay); const orchestrator = df.orchestrator(handler); diff --git a/UserDataDownloadOrchestratorTrigger/index.ts b/UserDataDownloadOrchestratorTrigger/index.ts deleted file mode 100644 index 0a07d1d1..00000000 --- a/UserDataDownloadOrchestratorTrigger/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Context } from "@azure/functions"; -import * as df from "durable-functions"; - -export default (context: Context, input: unknown) => - df - .getClient(context) - .startNew("UserDataDownloadOrchestrator", undefined, input); diff --git a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts b/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts deleted file mode 100644 index 90262582..00000000 --- a/UserDataDownloadSubOrchestrator/__tests__/handler.test.ts +++ /dev/null @@ -1,306 +0,0 @@ -// tslint:disable: no-any - -import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; -import { - mockOrchestratorCallActivity, - mockOrchestratorCallActivityWithRetry, - mockOrchestratorContext, - mockOrchestratorGetInput -} from "../../__mocks__/durable-functions"; -import { aArchiveInfo, aUserDataProcessing } from "../../__mocks__/mocks"; -import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../../ExtractUserDataActivity/handler"; -import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; -import { - ActivityFailure, - getHandler, - InvalidInputFailure, - OrchestratorSuccess -} from "../handler"; - -import { Millisecond } from "italia-ts-commons/lib/units"; -import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; - -const DELAY = 1 as Millisecond; - -const aNonSuccess = "any non-success value"; - -const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => - SetUserDataProcessingStatusActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aUserDataProcessing - }) -); -const extractUserDataActivity = jest.fn().mockImplementation(() => - ExtractUserDataActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aArchiveInfo - }) -); -const sendUserDataDownloadMessageActivity = jest - .fn() - .mockImplementation(() => - SendUserDataDownloadMessageActivityResultSuccess.encode({ kind: "SUCCESS" }) - ); - -// A mock implementation proxy for df.callActivity/df.df.callActivityWithRetry that routes each call to the correct mock implentation -const switchMockImplementation = (name: string, ...args: readonly unknown[]) => - (name === "SetUserDataProcessingStatusActivity" - ? setUserDataProcessingStatusActivity - : name === "ExtractUserDataActivity" - ? extractUserDataActivity - : name === "SendUserDataDownloadMessageActivity" - ? sendUserDataDownloadMessageActivity - : jest.fn())(name, ...args); - -// I assign switchMockImplementation to both because -// I don't want tests to depend on implementation details -// such as which activity is called with retry and which is not -mockOrchestratorCallActivity.mockImplementation(switchMockImplementation); -mockOrchestratorCallActivityWithRetry.mockImplementation( - switchMockImplementation -); - -/** - * Util function that takes an orchestrator and executes each step until is done - * @param orch an orchestrator - * - * @returns the last value yielded by the orchestrator - */ -const consumeOrchestrator = (orch: any) => { - // tslint:disable-next-line: no-let - let prevValue: unknown; - while (true) { - const { done, value } = orch.next(prevValue); - if (done) { - return value; - } - prevValue = value; - } -}; - -// just a convenient cast, good for every test case -const context = (mockOrchestratorContext as unknown) as IFunctionContext; - -const handler = getHandler(DELAY); - -// tslint:disable-next-line: no-big-function -describe("handler", () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it("should fail on invalid input", () => { - const document = "invalid"; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(InvalidInputFailure.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it.each` - name | status - ${"WIP"} | ${UserDataProcessingStatusEnum.WIP} - ${"CLOSED"} | ${UserDataProcessingStatusEnum.CLOSED} - `("should skip if the status is $name", ({ status }) => { - const document = { - ...aUserDataProcessing, - status - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(InvalidInputFailure.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).not.toHaveBeenCalled(); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should success if everything goes well", () => { - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(OrchestratorSuccess.decode(result).isRight()).toBe(true); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledTimes(2); - // first, set as WIP - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.WIP - } - ); - // then, set as CLOSED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.CLOSED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); - }); - - it("should set as FAILED when data extraction fails", () => { - extractUserDataActivity.mockImplementationOnce(() => aNonSuccess); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("ExtractUserDataActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should set as FAILED when send message fails", () => { - sendUserDataDownloadMessageActivity.mockImplementationOnce( - () => aNonSuccess - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("SendUserDataDownloadMessageActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); - }); - - it("should set as FAILED when status update to WIP fails", () => { - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => aNonSuccess - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should set as FAILED when status update to WIP fails", () => { - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => aNonSuccess - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); - expect(result.extra).toEqual({ - status: UserDataProcessingStatusEnum.WIP - }); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).not.toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).not.toHaveBeenCalled(); - }); - - it("should set as FAILED when status update to CLOSED fails", () => { - // the first time is called is for WIP - setUserDataProcessingStatusActivity.mockImplementationOnce(() => - SetUserDataProcessingStatusActivityResultSuccess.encode({ - kind: "SUCCESS", - value: aUserDataProcessing - }) - ); - - setUserDataProcessingStatusActivity.mockImplementationOnce( - () => aNonSuccess - ); - - const document = { - ...aUserDataProcessing, - status: UserDataProcessingStatusEnum.PENDING - }; - mockOrchestratorGetInput.mockReturnValueOnce(document); - - const result = consumeOrchestrator(handler(context)); - - expect(ActivityFailure.decode(result).isRight()).toBe(true); - expect(result.activityName).toBe("SetUserDataProcessingStatusActivity"); - expect(result.extra).toEqual({ - status: UserDataProcessingStatusEnum.CLOSED - }); - expect(setUserDataProcessingStatusActivity).toHaveBeenCalled(); // any times, at least one - // then, set as FAILED - expect(setUserDataProcessingStatusActivity).toHaveBeenCalledWith( - expect.any(String), - { - currentRecord: expect.any(Object), - nextStatus: UserDataProcessingStatusEnum.FAILED - } - ); - expect(extractUserDataActivity).toHaveBeenCalled(); - expect(sendUserDataDownloadMessageActivity).toHaveBeenCalled(); - }); -}); diff --git a/UserDataDownloadSubOrchestrator/function.json b/UserDataDownloadSubOrchestrator/function.json deleted file mode 100644 index ea76d6d4..00000000 --- a/UserDataDownloadSubOrchestrator/function.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "bindings": [ - { - "name": "context", - "type": "orchestrationTrigger", - "direction": "in" - } - ], - "scriptFile": "../dist/UserDataDownloadSubOrchestrator/index.js" -} \ No newline at end of file diff --git a/UserDataDownloadSubOrchestrator/handler.ts b/UserDataDownloadSubOrchestrator/handler.ts deleted file mode 100644 index de1cbcb1..00000000 --- a/UserDataDownloadSubOrchestrator/handler.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { IFunctionContext } from "durable-functions/lib/src/classes"; -import { isLeft } from "fp-ts/lib/Either"; -import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; -import * as t from "io-ts"; -import { readableReport } from "italia-ts-commons/lib/reporters"; -import { Millisecond } from "italia-ts-commons/lib/units"; -import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; -import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; -import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; -import { ProcessableUserDataProcessing } from "../UserDataDownloadOrchestrator/handler"; - -const logPrefix = "UserDataDownloadSubOrchestrator"; - -export type InvalidInputFailure = t.TypeOf; -export const InvalidInputFailure = t.interface({ - kind: t.literal("INVALID_INPUT"), - reason: t.string -}); - -export type UnhanldedFailure = t.TypeOf; -export const UnhanldedFailure = t.interface({ - kind: t.literal("UNHANDLED"), - reason: t.string -}); - -export type ActivityFailure = t.TypeOf; -export const ActivityFailure = t.intersection([ - t.interface({ - activityName: t.string, - kind: t.literal("ACTIVITY"), - reason: t.string - }), - t.partial({ extra: t.object }) -]); - -export type OrchestratorFailure = t.TypeOf; -export const OrchestratorFailure = t.taggedUnion("kind", [ - InvalidInputFailure, - UnhanldedFailure, - ActivityFailure -]); - -export type OrchestratorSuccess = t.TypeOf; -export const OrchestratorSuccess = t.interface({ - kind: t.literal("SUCCESS") -}); - -export type SkippedDocument = t.TypeOf; -export const SkippedDocument = t.interface({ - kind: t.literal("SKIPPED") -}); - -export type OrchestratorResult = t.TypeOf; -export const OrchestratorResult = t.union([ - OrchestratorFailure, - SkippedDocument, - OrchestratorSuccess -]); - -const toActivityFailure = ( - err: t.Errors, - activityName: string, - extra?: object -) => - ActivityFailure.encode({ - activityName, - extra, - kind: "ACTIVITY", - reason: readableReport(err) - }); - -export const getHandler = (delay: Millisecond = 0 as Millisecond) => - function*(context: IFunctionContext): IterableIterator { - const document = context.df.getInput(); - // This check has been done on the parent orchestrator, so it should never fail. - // However, it's worth the effort to check it twice - const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataProcessing.decode( - document - ).mapLeft(err => { - context.log.error( - `${logPrefix}|WARN|Cannot decode ProcessableUserDataProcessing document: ${readableReport( - err - )}` - ); - return InvalidInputFailure.encode({ - kind: "INVALID_INPUT", - reason: readableReport(err) - }); - }); - - if (isLeft(invalidInputOrCurrentUserDataProcessing)) { - return invalidInputOrCurrentUserDataProcessing.value; - } - - const currentUserDataProcessing = - invalidInputOrCurrentUserDataProcessing.value; - - try { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.WIP - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.WIP - }); - }); - - // pause this operation for a while - yield context.df.createTimer( - // tslint:disable-next-line: restrict-plus-operands - new Date(context.df.currentUtcDateTime.getTime() + delay) - ); - - const bundle = ExtractUserDataActivityResultSuccess.decode( - yield context.df.callActivity("ExtractUserDataActivity", { - fiscalCode: currentUserDataProcessing.fiscalCode - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "ExtractUserDataActivity"); - }); - - SendUserDataDownloadMessageActivityResultSuccess.decode( - yield context.df.callActivity("SendUserDataDownloadMessageActivity", { - blobName: bundle.value.blobName, - fiscalCode: currentUserDataProcessing.fiscalCode, - password: bundle.value.password - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "SendUserDataDownloadMessageActivity"); - }); - - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.CLOSED - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.CLOSED - }); - }); - return OrchestratorSuccess.encode({ kind: "SUCCESS" }); - } catch (error) { - context.log.error( - `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` - ); - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.FAILED - }) - ).getOrElseL(err => { - throw new Error( - `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( - err - )}` - ); - }); - - return OrchestratorFailure.is(error) - ? error - : UnhanldedFailure.encode({ - kind: "UNHANDLED", - reason: error.message - }); - } - }; diff --git a/UserDataDownloadSubOrchestrator/index.ts b/UserDataDownloadSubOrchestrator/index.ts deleted file mode 100644 index cd47e75e..00000000 --- a/UserDataDownloadSubOrchestrator/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import * as df from "durable-functions"; -import { Hour, Millisecond } from "italia-ts-commons/lib/units"; -import { getHandler } from "./handler"; - -const delayInHours = (typeof process.env.USER_DATA_DOWNLOAD_DELAY_HOURS === -"undefined" - ? 24 - : process.env.USER_DATA_DOWNLOAD_DELAY_HOURS) as Hour; -const delay = (delayInHours * 60 * 60 * 1000) as Millisecond; - -const handler = getHandler(delay); - -const orchestrator = df.orchestrator(handler); - -export default orchestrator; diff --git a/UserDataProcessingTrigger/__tests__/index.test.ts b/UserDataProcessingTrigger/__tests__/index.test.ts new file mode 100644 index 00000000..c3e735ca --- /dev/null +++ b/UserDataProcessingTrigger/__tests__/index.test.ts @@ -0,0 +1,59 @@ +// tslint:disable: no-any + +import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; +import { context, mockStartNew } from "../../__mocks__/durable-functions"; +import { aUserDataProcessing } from "../../__mocks__/mocks"; +import handler from "../index"; + +const aProcessableDownload = { + ...aUserDataProcessing, + choice: UserDataProcessingChoiceEnum.DOWNLOAD, + status: UserDataProcessingStatusEnum.PENDING +}; + +const aNonProcessableDownloadWrongStatus = { + ...aUserDataProcessing, + status: UserDataProcessingStatusEnum.WIP +}; + +const aNonProcessableDownloadWrongChoice = { + ...aUserDataProcessing, + choice: UserDataProcessingChoiceEnum.DELETE +}; + +describe("UserDataProcessingTrigger", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("should fail on invalid input", () => { + const input = "invalid"; + + try { + handler(context, input); + fail("it should throw"); + } catch (error) { + expect(mockStartNew).not.toHaveBeenCalled(); + } + }); + + it("should process every processable document", () => { + const processableDocs: ReadonlyArray = [ + aProcessableDownload, + aProcessableDownload, + aProcessableDownload + ]; + + const input: ReadonlyArray = [ + ...processableDocs, + aNonProcessableDownloadWrongStatus, + aNonProcessableDownloadWrongChoice + ]; + + handler(context, input); + + expect(mockStartNew).toHaveBeenCalledTimes(processableDocs.length); + }); +}); diff --git a/UserDataDownloadOrchestratorTrigger/function.json b/UserDataProcessingTrigger/function.json similarity index 88% rename from UserDataDownloadOrchestratorTrigger/function.json rename to UserDataProcessingTrigger/function.json index 7b52ec60..c53c927a 100644 --- a/UserDataDownloadOrchestratorTrigger/function.json +++ b/UserDataProcessingTrigger/function.json @@ -17,5 +17,5 @@ "direction": "in" } ], - "scriptFile": "../dist/UserDataDownloadOrchestratorTrigger/index.js" + "scriptFile": "../dist/UserDataProcessingTrigger/index.js" } \ No newline at end of file diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts new file mode 100644 index 00000000..702eeb24 --- /dev/null +++ b/UserDataProcessingTrigger/index.ts @@ -0,0 +1,69 @@ +import { Context } from "@azure/functions"; +import * as df from "durable-functions"; +import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; +import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; +import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; +import * as t from "io-ts"; +import { readableReport } from "italia-ts-commons/lib/reporters"; + +const logPrefix = "UserDataProcessingTrigger"; + +// models the subset of UserDataProcessing documents that this orchestrator accepts +export type ProcessableUserDataDownload = t.TypeOf< + typeof ProcessableUserDataDownload +>; +export const ProcessableUserDataDownload = t.intersection([ + UserDataProcessing, + // ony the subset of UserDataProcessing documents + // with the following characteristics must be processed + t.interface({ + choice: t.literal(UserDataProcessingChoiceEnum.DOWNLOAD), + status: t.literal(UserDataProcessingStatusEnum.PENDING) + }) +]); + +const CosmosDbDocumentCollection = t.readonlyArray(t.readonly(t.UnknownRecord)); +type CosmosDbDocumentCollection = t.TypeOf; + +interface ITaskDescriptor { + orchestrator: string; + input: ProcessableUserDataDownload; +} + +export default (context: Context, input: unknown) => { + const tasksDescriptors = CosmosDbDocumentCollection.decode(input) + .getOrElseL(err => { + throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); + }) + .reduce( + (tasks, maybeProcessable) => { + if (ProcessableUserDataDownload.is(maybeProcessable)) { + return [ + ...tasks, + { + input: maybeProcessable, + orchestrator: "UserDataDownloadOrchestrator" + } + ]; + } else { + context.log.warn( + `${logPrefix}: skipping document [${JSON.stringify( + maybeProcessable + )}]` + ); + return tasks; + } + }, + [] as readonly ITaskDescriptor[] + ); + + context.log.info( + `${logPrefix}: processing ${tasksDescriptors.length} document${ + tasksDescriptors.length === 1 ? "" : "s" + }` + ); + + tasksDescriptors.forEach(task => + df.getClient(context).startNew(task.orchestrator, undefined, task.input) + ); +}; diff --git a/__mocks__/durable-functions.ts b/__mocks__/durable-functions.ts index 2c8c10ed..d34331bf 100644 --- a/__mocks__/durable-functions.ts +++ b/__mocks__/durable-functions.ts @@ -22,7 +22,7 @@ export const mockTerminate = jest.fn(async (_, __) => { return; }); -export const getClient = jest.fn(() => ({ +export const getClient = jest.fn().mockImplementation(() => ({ getStatus: mockGetStatus, startNew: mockStartNew, terminate: mockTerminate @@ -85,6 +85,7 @@ export const mockOrchestratorContext = { callSubOrchestrator: mockCallSubOrchestrator, createTimer: mockOrchestratorCreateTimer, currentUtcDateTime: new Date(), + getClient, getInput: mockOrchestratorGetInput, setCustomStatus: mockOrchestratorSetCustomStatus } From 8e28f3f4128e34737d7c87be08787f6312fcc3d4 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Fri, 10 Jul 2020 12:53:57 +0200 Subject: [PATCH 17/34] remove delay --- .../__tests__/handler.test.ts | 7 +- UserDataDownloadOrchestrator/handler.ts | 181 +++++++++--------- UserDataDownloadOrchestrator/index.ts | 11 +- 3 files changed, 90 insertions(+), 109 deletions(-) diff --git a/UserDataDownloadOrchestrator/__tests__/handler.test.ts b/UserDataDownloadOrchestrator/__tests__/handler.test.ts index 794e29dd..92be435f 100644 --- a/UserDataDownloadOrchestrator/__tests__/handler.test.ts +++ b/UserDataDownloadOrchestrator/__tests__/handler.test.ts @@ -13,16 +13,13 @@ import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from ". import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../../SendUserDataDownloadMessageActivity/handler"; import { ActivityFailure, - getHandler, + handler, InvalidInputFailure, OrchestratorSuccess } from "../handler"; -import { Millisecond } from "italia-ts-commons/lib/units"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../../SetUserDataProcessingStatusActivity/handler"; -const DELAY = 1 as Millisecond; - const aNonSuccess = "any non-success value"; const setUserDataProcessingStatusActivity = jest.fn().mockImplementation(() => @@ -82,8 +79,6 @@ const consumeOrchestrator = (orch: any) => { // just a convenient cast, good for every test case const context = (mockOrchestratorContext as unknown) as IFunctionContext; -const handler = getHandler(DELAY); - // tslint:disable-next-line: no-big-function describe("UserDataDownloadOrchestrator", () => { beforeEach(() => { diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index ee20848b..ae84b587 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -72,105 +72,100 @@ const toActivityFailure = ( reason: readableReport(err) }); -export const getHandler = (delay: Millisecond = 0 as Millisecond) => - function*(context: IFunctionContext): IterableIterator { - const document = context.df.getInput(); - // This check has been done on the trigger, so it should never fail. - // However, it's worth the effort to check it twice - const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataDownload.decode( - document - ).mapLeft(err => { - context.log.error( - `${logPrefix}|WARN|Cannot decode ProcessableUserDataDownload document: ${readableReport( - err - )}` - ); - return InvalidInputFailure.encode({ - kind: "INVALID_INPUT", - reason: readableReport(err) - }); +export const handler = function*( + context: IFunctionContext +): IterableIterator { + const document = context.df.getInput(); + // This check has been done on the trigger, so it should never fail. + // However, it's worth the effort to check it twice + const invalidInputOrCurrentUserDataProcessing = ProcessableUserDataDownload.decode( + document + ).mapLeft(err => { + context.log.error( + `${logPrefix}|WARN|Cannot decode ProcessableUserDataDownload document: ${readableReport( + err + )}` + ); + return InvalidInputFailure.encode({ + kind: "INVALID_INPUT", + reason: readableReport(err) }); + }); - if (isLeft(invalidInputOrCurrentUserDataProcessing)) { - return invalidInputOrCurrentUserDataProcessing.value; - } - - const currentUserDataProcessing = - invalidInputOrCurrentUserDataProcessing.value; - - try { - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.WIP - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.WIP - }); + if (isLeft(invalidInputOrCurrentUserDataProcessing)) { + return invalidInputOrCurrentUserDataProcessing.value; + } + + const currentUserDataProcessing = + invalidInputOrCurrentUserDataProcessing.value; + + try { + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.WIP + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.WIP }); + }); - // pause this operation for a while - yield context.df.createTimer( - // tslint:disable-next-line: restrict-plus-operands - new Date(context.df.currentUtcDateTime.getTime() + delay) - ); - - const bundle = ExtractUserDataActivityResultSuccess.decode( - yield context.df.callActivity("ExtractUserDataActivity", { - fiscalCode: currentUserDataProcessing.fiscalCode - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "ExtractUserDataActivity"); - }); + const bundle = ExtractUserDataActivityResultSuccess.decode( + yield context.df.callActivity("ExtractUserDataActivity", { + fiscalCode: currentUserDataProcessing.fiscalCode + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "ExtractUserDataActivity"); + }); - SendUserDataDownloadMessageActivityResultSuccess.decode( - yield context.df.callActivityWithRetry( - "SendUserDataDownloadMessageActivity", - new RetryOptions(5000, 10), - { - blobName: bundle.value.blobName, - fiscalCode: currentUserDataProcessing.fiscalCode, - password: bundle.value.password - } - ) - ).getOrElseL(err => { - throw toActivityFailure(err, "SendUserDataDownloadMessageActivity"); - }); + SendUserDataDownloadMessageActivityResultSuccess.decode( + yield context.df.callActivityWithRetry( + "SendUserDataDownloadMessageActivity", + new RetryOptions(5000, 10), + { + blobName: bundle.value.blobName, + fiscalCode: currentUserDataProcessing.fiscalCode, + password: bundle.value.password + } + ) + ).getOrElseL(err => { + throw toActivityFailure(err, "SendUserDataDownloadMessageActivity"); + }); - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.CLOSED - }) - ).getOrElseL(err => { - throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { - status: UserDataProcessingStatusEnum.CLOSED - }); + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.CLOSED + }) + ).getOrElseL(err => { + throw toActivityFailure(err, "SetUserDataProcessingStatusActivity", { + status: UserDataProcessingStatusEnum.CLOSED }); - return OrchestratorSuccess.encode({ kind: "SUCCESS" }); - } catch (error) { - context.log.error( - `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` + }); + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); + } catch (error) { + context.log.error( + `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` + ); + SetUserDataProcessingStatusActivityResultSuccess.decode( + yield context.df.callActivity("SetUserDataProcessingStatusActivity", { + currentRecord: currentUserDataProcessing, + nextStatus: UserDataProcessingStatusEnum.FAILED + }) + ).getOrElseL(err => { + throw new Error( + `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( + err + )}` ); - SetUserDataProcessingStatusActivityResultSuccess.decode( - yield context.df.callActivity("SetUserDataProcessingStatusActivity", { - currentRecord: currentUserDataProcessing, - nextStatus: UserDataProcessingStatusEnum.FAILED - }) - ).getOrElseL(err => { - throw new Error( - `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( - err - )}` - ); - }); + }); - return OrchestratorFailure.is(error) - ? error - : UnhanldedFailure.encode({ - kind: "UNHANDLED", - reason: error.message - }); - } - }; + return OrchestratorFailure.is(error) + ? error + : UnhanldedFailure.encode({ + kind: "UNHANDLED", + reason: error.message + }); + } +}; diff --git a/UserDataDownloadOrchestrator/index.ts b/UserDataDownloadOrchestrator/index.ts index b70667e3..8a754b91 100644 --- a/UserDataDownloadOrchestrator/index.ts +++ b/UserDataDownloadOrchestrator/index.ts @@ -1,14 +1,5 @@ import * as df from "durable-functions"; -import { NonNegativeInteger } from "italia-ts-commons/lib/numbers"; -import { Millisecond } from "italia-ts-commons/lib/units"; -import { getHandler } from "./handler"; - -const delayInHours = NonNegativeInteger.decode( - process.env.USER_DATA_DOWNLOAD_DELAY_HOURS -).getOrElse(24 as NonNegativeInteger); -const delay = (delayInHours * 60 * 60 * 1000) as Millisecond; - -const handler = getHandler(delay); +import { handler } from "./handler"; const orchestrator = df.orchestrator(handler); From 7977438dbf67ec89f9ac38d6b481d4911192ae50 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Fri, 10 Jul 2020 13:04:38 +0200 Subject: [PATCH 18/34] fix lint --- UserDataDownloadOrchestrator/handler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index ae84b587..2aab91db 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -6,7 +6,6 @@ import { isLeft } from "fp-ts/lib/Either"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { Millisecond } from "italia-ts-commons/lib/units"; import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from "../ExtractUserDataActivity/handler"; import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; From d1e0a4a97a4500c89d904458988107e438bd636a Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Fri, 10 Jul 2020 16:41:29 +0200 Subject: [PATCH 19/34] env vars --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index cd41a219..9709d61f 100644 --- a/README.md +++ b/README.md @@ -26,3 +26,11 @@ they may be customized as needed. | AZURE_APIM_HOST | The host name of the API Management service | string | | AZURE_APIM_RESOURCE_GROUP | The name of the resource group used to get the subscriptions | string | | AZURE_SUBSCRIPTION_ID | Credentials which identify the Azure subscription, used to init the APIM client | string | +| UserDataArchiveStorageConnection | Storage connection string to store zip file for user to download their data | string | +| USER_DATA_CONTAINER_NAME | Name of the container on which zip files with usr data are stored | string | +| MessageContentStorageConnection | Storage connection string o where message content is stored | string | +| MESSAGE_CONTAINER_NAME | name of the container which stores message content | string | +| PUBLIC_API_URL | url of the public api | string | +| PUBLIC_API_KEY | API Managment access key for public api | string | +| PUBLIC_DOWNLOAD_BASE_URL | Url of user data zip bundle storage | string | + From 4226fb9d9ed406d56bdfba0c8d5f29ac0a34a5af Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 13 Jul 2020 12:42:33 +0200 Subject: [PATCH 20/34] await all --- UserDataProcessingTrigger/index.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index 702eeb24..c86c6f3e 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -1,5 +1,6 @@ import { Context } from "@azure/functions"; import * as df from "durable-functions"; +import { task } from "fp-ts/lib/Task"; import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; @@ -31,6 +32,7 @@ interface ITaskDescriptor { } export default (context: Context, input: unknown) => { + const dfClient = df.getClient(context); const tasksDescriptors = CosmosDbDocumentCollection.decode(input) .getOrElseL(err => { throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); @@ -63,7 +65,10 @@ export default (context: Context, input: unknown) => { }` ); - tasksDescriptors.forEach(task => - df.getClient(context).startNew(task.orchestrator, undefined, task.input) - ); + const startAllNew = () => + tasksDescriptors.map(({ orchestrator, input: orchestratorInput }) => + dfClient.startNew(orchestrator, undefined, orchestratorInput) + ); + + return Promise.all(startAllNew()); }; From 479a359573f9a17edcfeff14afbadce76cdc3ea6 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 13 Jul 2020 12:57:38 +0200 Subject: [PATCH 21/34] fix lint --- UserDataProcessingTrigger/__tests__/index.test.ts | 8 ++++---- UserDataProcessingTrigger/index.ts | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/UserDataProcessingTrigger/__tests__/index.test.ts b/UserDataProcessingTrigger/__tests__/index.test.ts index c3e735ca..5383de4f 100644 --- a/UserDataProcessingTrigger/__tests__/index.test.ts +++ b/UserDataProcessingTrigger/__tests__/index.test.ts @@ -28,18 +28,18 @@ describe("UserDataProcessingTrigger", () => { jest.clearAllMocks(); }); - it("should fail on invalid input", () => { + it("should fail on invalid input", async () => { const input = "invalid"; try { - handler(context, input); + await handler(context, input); fail("it should throw"); } catch (error) { expect(mockStartNew).not.toHaveBeenCalled(); } }); - it("should process every processable document", () => { + it("should process every processable document", async () => { const processableDocs: ReadonlyArray = [ aProcessableDownload, aProcessableDownload, @@ -52,7 +52,7 @@ describe("UserDataProcessingTrigger", () => { aNonProcessableDownloadWrongChoice ]; - handler(context, input); + await handler(context, input); expect(mockStartNew).toHaveBeenCalledTimes(processableDocs.length); }); diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index c86c6f3e..88bfdc6f 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -1,6 +1,5 @@ import { Context } from "@azure/functions"; import * as df from "durable-functions"; -import { task } from "fp-ts/lib/Task"; import { UserDataProcessingChoiceEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingChoice"; import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generated/definitions/UserDataProcessingStatus"; import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; From a3ea39566efa78d049e4b61040aa47de3530c6a5 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 13 Jul 2020 14:12:56 +0200 Subject: [PATCH 22/34] use id --- UserDataProcessingTrigger/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index 88bfdc6f..bebc9f64 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -5,6 +5,7 @@ import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generate import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; +import { types } from "italia-ts-commons"; const logPrefix = "UserDataProcessingTrigger"; @@ -27,6 +28,7 @@ type CosmosDbDocumentCollection = t.TypeOf; interface ITaskDescriptor { orchestrator: string; + id: ProcessableUserDataDownload["userDataProcessingId"]; input: ProcessableUserDataDownload; } @@ -42,6 +44,7 @@ export default (context: Context, input: unknown) => { return [ ...tasks, { + id: maybeProcessable.userDataProcessingId, input: maybeProcessable, orchestrator: "UserDataDownloadOrchestrator" } @@ -65,8 +68,8 @@ export default (context: Context, input: unknown) => { ); const startAllNew = () => - tasksDescriptors.map(({ orchestrator, input: orchestratorInput }) => - dfClient.startNew(orchestrator, undefined, orchestratorInput) + tasksDescriptors.map(({ orchestrator, id, input: orchestratorInput }) => + dfClient.startNew(orchestrator, id, orchestratorInput) ); return Promise.all(startAllNew()); From 849b7c93d078fc592b3d9a6103aee809a966e1ee Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 13 Jul 2020 14:32:13 +0200 Subject: [PATCH 23/34] fix lint --- UserDataProcessingTrigger/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index bebc9f64..fbc3a247 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -5,7 +5,6 @@ import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generate import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { types } from "italia-ts-commons"; const logPrefix = "UserDataProcessingTrigger"; From e50ba7fe3ef4c7c47dc39471afafb14eb3c35ad2 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Mon, 13 Jul 2020 17:04:20 +0200 Subject: [PATCH 24/34] use global storage for messages --- ExtractUserDataActivity/index.ts | 2 +- README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ExtractUserDataActivity/index.ts b/ExtractUserDataActivity/index.ts index ffcff2ed..c895f1d4 100644 --- a/ExtractUserDataActivity/index.ts +++ b/ExtractUserDataActivity/index.ts @@ -79,7 +79,7 @@ const userDataBlobService = createBlobService( ); const messageContentBlobService = createBlobService( - getRequiredStringEnv("MessageContentStorageConnection") + getRequiredStringEnv("StorageConnection") ); const userDataContainerName = getRequiredStringEnv("USER_DATA_CONTAINER_NAME"); diff --git a/README.md b/README.md index 9709d61f..3d4d0d4e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@ they may be customized as needed. | AZURE_SUBSCRIPTION_ID | Credentials which identify the Azure subscription, used to init the APIM client | string | | UserDataArchiveStorageConnection | Storage connection string to store zip file for user to download their data | string | | USER_DATA_CONTAINER_NAME | Name of the container on which zip files with usr data are stored | string | -| MessageContentStorageConnection | Storage connection string o where message content is stored | string | | MESSAGE_CONTAINER_NAME | name of the container which stores message content | string | | PUBLIC_API_URL | url of the public api | string | | PUBLIC_API_KEY | API Managment access key for public api | string | From e07b84dd107ce8a8ccfd05ce2cd31cfc5b2db6eb Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 14:39:09 +0200 Subject: [PATCH 25/34] readme fixes and small refactor --- ExtractUserDataActivity/handler.ts | 15 +++++++-------- README.md | 12 +++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index d0ebcd6a..03ac59ae 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -12,6 +12,7 @@ import { array, flatten } from "fp-ts/lib/Array"; import { Either, fromOption, left, right, toError } from "fp-ts/lib/Either"; import { fromEither, + fromLeft, TaskEither, taskEither, taskify, @@ -156,15 +157,13 @@ const fromQueryEither = ( /** * Converts a Promise> that can reject - * into a TaskEither + * into a TaskEither */ -const fromPromiseEither = ( - promise: Promise> -): TaskEither => - tryCatch(() => promise.then(e => e), toError).foldTaskEither( - err => fromEither(left(err)), - _ => fromEither(_.fold(err => left(err), __ => right(__))) - ); +const fromPromiseEither = (promise: Promise>) => + taskEither + .of(void 0) + .chainSecond(tryCatch(() => promise, toError)) + .chain(fromEither); /** * To be used for exhaustive checks diff --git a/README.md b/README.md index 3d4d0d4e..992e9f7e 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,8 @@ they may be customized as needed. | AZURE_APIM_HOST | The host name of the API Management service | string | | AZURE_APIM_RESOURCE_GROUP | The name of the resource group used to get the subscriptions | string | | AZURE_SUBSCRIPTION_ID | Credentials which identify the Azure subscription, used to init the APIM client | string | -| UserDataArchiveStorageConnection | Storage connection string to store zip file for user to download their data | string | -| USER_DATA_CONTAINER_NAME | Name of the container on which zip files with usr data are stored | string | -| MESSAGE_CONTAINER_NAME | name of the container which stores message content | string | -| PUBLIC_API_URL | url of the public api | string | -| PUBLIC_API_KEY | API Managment access key for public api | string | -| PUBLIC_DOWNLOAD_BASE_URL | Url of user data zip bundle storage | string | - +| USER_DATA_CONTAINER_NAME | Name of the container on which zip files with usr data are stored | string | +| MESSAGE_CONTAINER_NAME | Name of the container which stores message content | string | +| PUBLIC_API_URL | Internal URL of the API management used to send messages | string | +| PUBLIC_API_KEY | GDPR service access key for the message API | string | +| PUBLIC_DOWNLOAD_BASE_URL | Public URL of user's data zip bundle storage | string | From 8033bd0a722408a48827f898c649924491c954d9 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 15:27:34 +0200 Subject: [PATCH 26/34] revert readme change --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 992e9f7e..bab99628 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ they may be customized as needed. | AZURE_APIM_HOST | The host name of the API Management service | string | | AZURE_APIM_RESOURCE_GROUP | The name of the resource group used to get the subscriptions | string | | AZURE_SUBSCRIPTION_ID | Credentials which identify the Azure subscription, used to init the APIM client | string | +| UserDataArchiveStorageConnection | Storage connection string to store zip file for user to download their data | string | | USER_DATA_CONTAINER_NAME | Name of the container on which zip files with usr data are stored | string | | MESSAGE_CONTAINER_NAME | Name of the container which stores message content | string | | PUBLIC_API_URL | Internal URL of the API management used to send messages | string | From d6bc106c5e57e4d136afbd3eabc8b212ae6c9f4e Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 16:32:14 +0200 Subject: [PATCH 27/34] add env into readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bab99628..e846e8d4 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ they may be customized as needed. | Variable name | Description | type | | --------------------------- | ------------------------------------------------------------------------------------------------ | ------ | | StorageConnection | Storage connection string to store computed visible-service.json (retrieved by io-functions-app) | string | +| COSMOSDB_CONNECTION_STRING | CosmosDB connection string (needed in triggers) | string | | COSMOSDB_URI | CosmosDB connection URI | string | -| COSMOSDB_KEY | CosmoDB connection key | string | +| COSMOSDB_KEY | CosmosDB connection key | string | | COSMOSDB_NAME | CosmosDB database name | string | | LOGOS_URL | The url of the service logos storage | string | | AssetsStorageConnection | The connection string used to connect to Azure Blob Storage containing the service cache | string | From 97b8e9bbb227ff018ceed6c238fbb6ce0d3411e7 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 14 Jul 2020 16:57:26 +0200 Subject: [PATCH 28/34] fix lint --- ExtractUserDataActivity/handler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index 03ac59ae..4d2e4c73 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -12,7 +12,6 @@ import { array, flatten } from "fp-ts/lib/Array"; import { Either, fromOption, left, right, toError } from "fp-ts/lib/Either"; import { fromEither, - fromLeft, TaskEither, taskEither, taskify, From f6d40a588c3d81b2289e7fb5cccf5212ea11d6a2 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 17:55:53 +0200 Subject: [PATCH 29/34] fix entrypoint --- UserDataProcessingTrigger/__tests__/index.test.ts | 6 +++--- UserDataProcessingTrigger/index.ts | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/UserDataProcessingTrigger/__tests__/index.test.ts b/UserDataProcessingTrigger/__tests__/index.test.ts index 5383de4f..cf931de1 100644 --- a/UserDataProcessingTrigger/__tests__/index.test.ts +++ b/UserDataProcessingTrigger/__tests__/index.test.ts @@ -5,7 +5,7 @@ import { UserDataProcessingStatusEnum } from "io-functions-commons/dist/generate import { UserDataProcessing } from "io-functions-commons/dist/src/models/user_data_processing"; import { context, mockStartNew } from "../../__mocks__/durable-functions"; import { aUserDataProcessing } from "../../__mocks__/mocks"; -import handler from "../index"; +import { index } from "../index"; const aProcessableDownload = { ...aUserDataProcessing, @@ -32,7 +32,7 @@ describe("UserDataProcessingTrigger", () => { const input = "invalid"; try { - await handler(context, input); + await index(context, input); fail("it should throw"); } catch (error) { expect(mockStartNew).not.toHaveBeenCalled(); @@ -52,7 +52,7 @@ describe("UserDataProcessingTrigger", () => { aNonProcessableDownloadWrongChoice ]; - await handler(context, input); + await index(context, input); expect(mockStartNew).toHaveBeenCalledTimes(processableDocs.length); }); diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index fbc3a247..ab11b6c3 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -31,7 +31,10 @@ interface ITaskDescriptor { input: ProcessableUserDataDownload; } -export default (context: Context, input: unknown) => { +export function index( + context: Context, + input: unknown +): Promise { const dfClient = df.getClient(context); const tasksDescriptors = CosmosDbDocumentCollection.decode(input) .getOrElseL(err => { @@ -72,4 +75,4 @@ export default (context: Context, input: unknown) => { ); return Promise.all(startAllNew()); -}; +} From 579bbbd3b5a33f4c52eb850c35372df0c501d653 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 17:56:38 +0200 Subject: [PATCH 30/34] fix path --- UserDataDownloadOrchestrator/function.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/function.json b/UserDataDownloadOrchestrator/function.json index ea76d6d4..dc59add8 100644 --- a/UserDataDownloadOrchestrator/function.json +++ b/UserDataDownloadOrchestrator/function.json @@ -6,5 +6,5 @@ "direction": "in" } ], - "scriptFile": "../dist/UserDataDownloadSubOrchestrator/index.js" + "scriptFile": "../dist/UserDataDownloadOrchestrator/index.js" } \ No newline at end of file From fcd72d017598fc359d90e1214c5f99aaf1ee5a64 Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Tue, 14 Jul 2020 17:58:15 +0200 Subject: [PATCH 31/34] fix typos --- UserDataDownloadOrchestrator/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 2aab91db..aacd74d2 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -11,7 +11,7 @@ import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSucce import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; import { ProcessableUserDataDownload } from "../UserDataProcessingTrigger"; -const logPrefix = "UserDataDownloadSubOrchestrator"; +const logPrefix = "UserDataDownloadOrchestrator"; export type InvalidInputFailure = t.TypeOf; export const InvalidInputFailure = t.interface({ From bb1878b99281aa122b30131effab18109cee3b62 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Tue, 14 Jul 2020 18:20:48 +0200 Subject: [PATCH 32/34] fix decode --- UserDataDownloadOrchestrator/handler.ts | 2 +- UserDataProcessingTrigger/index.ts | 30 ++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 2aab91db..aacd74d2 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -11,7 +11,7 @@ import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSucce import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; import { ProcessableUserDataDownload } from "../UserDataProcessingTrigger"; -const logPrefix = "UserDataDownloadSubOrchestrator"; +const logPrefix = "UserDataDownloadOrchestrator"; export type InvalidInputFailure = t.TypeOf; export const InvalidInputFailure = t.interface({ diff --git a/UserDataProcessingTrigger/index.ts b/UserDataProcessingTrigger/index.ts index ab11b6c3..508a92fe 100644 --- a/UserDataProcessingTrigger/index.ts +++ b/UserDataProcessingTrigger/index.ts @@ -41,25 +41,25 @@ export function index( throw Error(`${logPrefix}: cannot decode input [${readableReport(err)}]`); }) .reduce( - (tasks, maybeProcessable) => { - if (ProcessableUserDataDownload.is(maybeProcessable)) { - return [ + (tasks, maybeProcessable) => + ProcessableUserDataDownload.decode(maybeProcessable).fold( + _ => { + context.log.warn( + `${logPrefix}: skipping document [${JSON.stringify( + maybeProcessable + )}]` + ); + return tasks; + }, + processable => [ ...tasks, { - id: maybeProcessable.userDataProcessingId, - input: maybeProcessable, + id: processable.userDataProcessingId, + input: processable, orchestrator: "UserDataDownloadOrchestrator" } - ]; - } else { - context.log.warn( - `${logPrefix}: skipping document [${JSON.stringify( - maybeProcessable - )}]` - ); - return tasks; - } - }, + ] + ), [] as readonly ITaskDescriptor[] ); From a3dc1188e4101b3ec41f5797e0019f835674be6d Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 16 Jul 2020 12:13:54 +0200 Subject: [PATCH 33/34] adding ai events --- UserDataDownloadOrchestrator/handler.ts | 29 +++++++++++++++++ __mocks__/applicationinsights.ts | 4 +++ utils/appinsights.ts | 41 +++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 __mocks__/applicationinsights.ts create mode 100644 utils/appinsights.ts diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index aacd74d2..e89df03d 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -10,6 +10,7 @@ import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from ". import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; import { ProcessableUserDataDownload } from "../UserDataProcessingTrigger"; +import { trackException, trackEvent } from "../utils/appinsights"; const logPrefix = "UserDataDownloadOrchestrator"; @@ -142,8 +143,28 @@ export const handler = function*( status: UserDataProcessingStatusEnum.CLOSED }); }); + + trackEvent({ + // tslint:disable-next-line: no-duplicate-string + name: "user.data.download", + properties: { + userDataProcessingId: currentUserDataProcessing.userDataProcessingId + }, + tagOverrides: { + "ai.operation.id": currentUserDataProcessing.userDataProcessingId, + "ai.operation.parentId": currentUserDataProcessing.userDataProcessingId + } + }); + return OrchestratorSuccess.encode({ kind: "SUCCESS" }); } catch (error) { + trackException({ + exception: new Error(error), + properties: { + name: "user.data.download", + userDataProcessingId: currentUserDataProcessing.userDataProcessingId + } + }); context.log.error( `${logPrefix}|ERROR|Failed processing user data for download: ${error.message}` ); @@ -153,6 +174,14 @@ export const handler = function*( nextStatus: UserDataProcessingStatusEnum.FAILED }) ).getOrElseL(err => { + trackException({ + exception: new Error(readableReport(err)), + properties: { + name: "user.data.download", + type: "unhandled exception when trying to set document as FAILED", + userDataProcessingId: currentUserDataProcessing.userDataProcessingId + } + }); throw new Error( `Activity SetUserDataProcessingStatusActivity (status=FAILED) failed: ${readableReport( err diff --git a/__mocks__/applicationinsights.ts b/__mocks__/applicationinsights.ts new file mode 100644 index 00000000..301cc526 --- /dev/null +++ b/__mocks__/applicationinsights.ts @@ -0,0 +1,4 @@ +export const defaultClient = { + trackEvent: jest.fn(), + trackException: jest.fn() +}; diff --git a/utils/appinsights.ts b/utils/appinsights.ts new file mode 100644 index 00000000..457161f2 --- /dev/null +++ b/utils/appinsights.ts @@ -0,0 +1,41 @@ +import * as ai from "applicationinsights"; +import { + EventTelemetry, + ExceptionTelemetry +} from "applicationinsights/out/Declarations/Contracts"; +import { fromNullable } from "fp-ts/lib/Option"; +import { tryCatch } from "fp-ts/lib/Option"; +import { initAppInsights } from "italia-ts-commons/lib/appinsights"; +import { IntegerFromString } from "italia-ts-commons/lib/numbers"; +import { NonEmptyString } from "italia-ts-commons/lib/strings"; + +// the internal function runtime has MaxTelemetryItem per second set to 20 by default +// @see https://github.com/Azure/azure-functions-host/blob/master/src/WebJobs.Script/Config/ApplicationInsightsLoggerOptionsSetup.cs#L29 +const DEFAULT_SAMPLING_PERCENTAGE = 20; + +// Avoid to initialize Application Insights more than once +export const initTelemetryClient = (env = process.env) => + ai.defaultClient + ? ai.defaultClient + : NonEmptyString.decode(env.APPINSIGHTS_INSTRUMENTATIONKEY).fold( + _ => undefined, + k => + initAppInsights(k, { + disableAppInsights: env.APPINSIGHTS_DISABLE === "true", + samplingPercentage: IntegerFromString.decode( + env.APPINSIGHTS_SAMPLING_PERCENTAGE + ).getOrElse(DEFAULT_SAMPLING_PERCENTAGE) + }) + ); + +export const trackEvent = (event: EventTelemetry) => { + fromNullable(initTelemetryClient()).map(_ => + tryCatch(() => _.trackEvent(event)) + ); +}; + +export const trackException = (event: ExceptionTelemetry) => { + fromNullable(initTelemetryClient()).map(_ => + tryCatch(() => _.trackException(event)) + ); +}; From 1dbaafb6b1e364cd10efb1aae924a2dad79bd58a Mon Sep 17 00:00:00 2001 From: Emanuele De Cupis Date: Thu, 16 Jul 2020 12:28:55 +0200 Subject: [PATCH 34/34] Merge branch 'master' into 172884826-automatic-download --- UserDataDownloadOrchestrator/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index e89df03d..c997f0e5 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -10,7 +10,7 @@ import { ActivityResultSuccess as ExtractUserDataActivityResultSuccess } from ". import { ActivityResultSuccess as SendUserDataDownloadMessageActivityResultSuccess } from "../SendUserDataDownloadMessageActivity/handler"; import { ActivityResultSuccess as SetUserDataProcessingStatusActivityResultSuccess } from "../SetUserDataProcessingStatusActivity/handler"; import { ProcessableUserDataDownload } from "../UserDataProcessingTrigger"; -import { trackException, trackEvent } from "../utils/appinsights"; +import { trackEvent, trackException } from "../utils/appinsights"; const logPrefix = "UserDataDownloadOrchestrator";