From c71353ebb9dd221888ddd73560b9bb24a3d84ec8 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Fri, 15 May 2020 18:06:35 +0200 Subject: [PATCH 1/4] refactor --- .../__tests__/handler.test.ts | 39 ---------------- ExtractUserDataActivity/handler.ts | 44 ++++++++++-------- utils/__mocks__/zip.ts | 7 --- utils/zip.ts | 45 +++++++------------ 4 files changed, 41 insertions(+), 94 deletions(-) delete mode 100644 utils/__mocks__/zip.ts diff --git a/ExtractUserDataActivity/__tests__/handler.test.ts b/ExtractUserDataActivity/__tests__/handler.test.ts index c298dbd6..44e2cdf6 100644 --- a/ExtractUserDataActivity/__tests__/handler.test.ts +++ b/ExtractUserDataActivity/__tests__/handler.test.ts @@ -1,5 +1,4 @@ /* tslint:disable: no-any */ -jest.mock("../../utils/zip"); import { Either, right } from "fp-ts/lib/Either"; import { fromNullable, Option, some } from "fp-ts/lib/Option"; @@ -34,7 +33,6 @@ import { aRetrievedWebhookNotification } from "../../__mocks__/mocks"; import { AllUserData } from "../../utils/userData"; -import { createCompressedStream } from "../../utils/zip"; import { NotificationModel } from "../notification"; // we use the local-defined model const createMockIterator = (a: ReadonlyArray) => { @@ -120,36 +118,6 @@ describe("createExtractUserDataActivityHandler", () => { ); }); - it("should not export webhook notification data", async () => { - const notificationWebhookModelMock = ({ - findNotificationsForMessage: jest.fn(() => - createMockIterator([aRetrievedWebhookNotification]) - ) - } as any) as NotificationModel; - - const handler = createExtractUserDataActivityHandler( - messageModelMock, - messageStatusModelMock, - notificationWebhookModelMock, - notificationStatusModelMock, - profileModelMock, - senderServiceModelMock, - blobServiceMock, - aUserDataContainerName - ); - const input: ActivityInput = { - fiscalCode: aFiscalCode - }; - - await handler(contextMock, input); - - // @ts-ignore as - const mockCall = createCompressedStream.mock.calls[0]; - const allUserData: AllUserData = mockCall[0][`${aFiscalCode}.json`]; - - expect(allUserData.notifications[0].channels.WEBHOOK).toEqual({}); - }); - it("should query using correct data", async () => { const handler = createExtractUserDataActivityHandler( messageModelMock, @@ -187,12 +155,5 @@ describe("createExtractUserDataActivityHandler", () => { expect( senderServiceModelMock.findSenderServicesForRecipient ).toHaveBeenCalledWith(aFiscalCode); - - expect(createCompressedStream).toHaveBeenCalledWith( - { - [`${aFiscalCode}.json`]: expect.any(Object) - }, - expect.any(String) - ); }); }); diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index 924f898a..c803b307 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -2,6 +2,7 @@ * This activity extracts all the data about a user contained in our db. */ +import * as archiver from "archiver"; import * as t from "io-ts"; import { sequenceS } from "fp-ts/lib/Apply"; @@ -49,7 +50,11 @@ import { readableReport } from "italia-ts-commons/lib/reporters"; import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings"; import { generateStrongPassword, StrongPassword } from "../utils/random"; import { AllUserData, MessageContentWithId } from "../utils/userData"; -import { createCompressedStream } from "../utils/zip"; +import { + DEFAULT_ZIP_ENCRYPTION_METHOD, + DEFAULT_ZLIB_LEVEL, + initArchiverZipEncryptedPlugin +} from "../utils/zip"; import { NotificationModel } from "./notification"; export const ArchiveInfo = t.interface({ @@ -352,15 +357,10 @@ export const createExtractUserDataActivityHandler = ( .reduce( (queries, { id: notificationId }) => [ ...queries, - ...Object.values(NotificationChannelEnum).map(channel => { - switch (channel) { - case NotificationChannelEnum.EMAIL: - case NotificationChannelEnum.WEBHOOK: - return [notificationId, channel]; - default: - assertNever(channel); - } - }) + ...Object.values(NotificationChannelEnum).map(channel => [ + notificationId, + channel + ]) ], [] ) @@ -482,12 +482,15 @@ export const createExtractUserDataActivityHandler = ( }-${Date.now()}.zip` as NonEmptyString; const fileName = `${data.profile.fiscalCode}.json` as NonEmptyString; - const readableZipStream = createCompressedStream( - { - [fileName]: data - }, - password - ); + initArchiverZipEncryptedPlugin.init(); + + const readableZipStream = archiver.create("zip-encrypted", { + encryptionMethod: DEFAULT_ZIP_ENCRYPTION_METHOD, + password, + zlib: { level: DEFAULT_ZLIB_LEVEL } + // following cast due to uncomplete archive typings + // tslint:disable-next-line: no-any + } as any); const writableBlobStream = blobService.createWriteStreamToBlockBlob( userDataContainerName, @@ -512,8 +515,8 @@ export const createExtractUserDataActivityHandler = ( } ); readableZipStream.pipe(writableBlobStream); - - readableZipStream.on("error", err => + // tslint:disable-next-line: no-any + readableZipStream.on("error", (err: any) => cb( ActivityResultArchiveGenerationFailure.encode({ kind: "ARCHIVE_GENERATION_FAILURE", @@ -521,10 +524,13 @@ export const createExtractUserDataActivityHandler = ( }) ) ); + readableZipStream.append(JSON.stringify(data), { name: fileName }); + // TODO: handle this promise correctly + readableZipStream.finalize().catch(); } )(); - // the actual handler© + // the actual handler return (context: Context, input: unknown) => fromEither( ActivityInput.decode(input).mapLeft( diff --git a/utils/__mocks__/zip.ts b/utils/__mocks__/zip.ts deleted file mode 100644 index a1a514b8..00000000 --- a/utils/__mocks__/zip.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Mock implementation of the zip module - */ - -export const createCompressedStream = jest.fn(() => ({ - pipe: jest.fn() -})); diff --git a/utils/zip.ts b/utils/zip.ts index 0d645901..a258a84d 100644 --- a/utils/zip.ts +++ b/utils/zip.ts @@ -1,37 +1,24 @@ import * as archiver from "archiver"; -import * as achiverEncryptedFormat from "archiver-zip-encrypted"; -import { NonEmptyString } from "italia-ts-commons/lib/strings"; -import { Readable } from "stream"; -archiver.registerFormat("zip-encrypted", achiverEncryptedFormat); + +export const initArchiverZipEncryptedPlugin = { + called: false, + init(): void { + if (!initArchiverZipEncryptedPlugin.called) { + // tslint:disable-next-line: no-object-mutation + initArchiverZipEncryptedPlugin.called = true; + // note: only do it once per Node.js process/application, as duplicate registration will throw an error + archiver.registerFormat( + "zip-encrypted", + require("archiver-zip-encrypted") + ); + } + } +}; export enum EncryptionMethodEnum { ZIP20 = "zip20", AES256 = "aes256" } -export const DEFAULT_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20; +export const DEFAULT_ZIP_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20; export const DEFAULT_ZLIB_LEVEL = 8; - -export const createCompressedStream = ( - // tslint:disable-next-line: no-any - data: Record, - password?: NonEmptyString, - encryptionMethod: EncryptionMethodEnum = EncryptionMethodEnum.ZIP20 -): Readable => { - const zipArchive = password - ? archiver("zip-encrypted", { - encryptionMethod, - password, - zlib: { level: DEFAULT_ZLIB_LEVEL } - }) - : archiver("zip", { - zlib: { level: DEFAULT_ZLIB_LEVEL } - }); - - Object.entries(data).forEach(([fileName, content]) => { - zipArchive.append(JSON.stringify(content), { name: fileName }); - }); - zipArchive.finalize(); - - return zipArchive; -}; From 78fbdb3feabf3e21d1c3fceb7837e65435cc7695 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Fri, 15 May 2020 18:18:57 +0200 Subject: [PATCH 2/4] refactors --- .../__tests__/handler.test.ts | 10 +--- ExtractUserDataActivity/handler.ts | 6 ++- package.json | 2 + utils/__tests__/random.test.ts | 27 ---------- utils/random.ts | 50 +++---------------- yarn.lock | 31 ++++++++++++ 6 files changed, 47 insertions(+), 79 deletions(-) delete mode 100644 utils/__tests__/random.test.ts diff --git a/ExtractUserDataActivity/__tests__/handler.test.ts b/ExtractUserDataActivity/__tests__/handler.test.ts index f3f41061..e0f16615 100644 --- a/ExtractUserDataActivity/__tests__/handler.test.ts +++ b/ExtractUserDataActivity/__tests__/handler.test.ts @@ -29,10 +29,8 @@ import { aMessageContent, aRetrievedMessageWithoutContent, aRetrievedNotification, - aRetrievedSenderService, - aRetrievedWebhookNotification + aRetrievedSenderService } from "../../__mocks__/mocks"; -import { AllUserData } from "../../utils/userData"; import { NotificationModel } from "../notification"; // we use the local-defined model const createMockIterator = (a: ReadonlyArray) => { @@ -84,12 +82,6 @@ const blobServiceMock = ({ const aUserDataContainerName = "aUserDataContainerName" as NonEmptyString; -jest.mock("../../utils/zip", () => ({ - createCompressedStream: jest.fn(() => ({ - pipe: jest.fn() - })) -})); - describe("createExtractUserDataActivityHandler", () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index c803b307..315e080a 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -47,7 +47,11 @@ import { } from "io-functions-commons/dist/src/models/sender_service"; import { iteratorToArray } from "io-functions-commons/dist/src/utils/documentdb"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings"; +import { + FiscalCode, + NonEmptyString, + WithinRangeString +} from "italia-ts-commons/lib/strings"; import { generateStrongPassword, StrongPassword } from "../utils/random"; import { AllUserData, MessageContentWithId } from "../utils/userData"; import { diff --git a/package.json b/package.json index c66a9653..5a14617b 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,8 @@ "@azure/arm-apimanagement": "^5.1.1", "@azure/graph": "^4.0.1", "@azure/ms-rest-nodeauth": "^2.0.5", + "@types/archiver": "^3.1.0", + "@types/randomstring": "^1.1.6", "archiver": "^4.0.1", "archiver-zip-encrypted": "^1.0.8", "azure-storage": "^2.10.3", diff --git a/utils/__tests__/random.test.ts b/utils/__tests__/random.test.ts deleted file mode 100644 index 8c6e673c..00000000 --- a/utils/__tests__/random.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { readableReport } from "italia-ts-commons/lib/reporters"; -import { generateStrongPassword, StrongPassword } from "../random"; - -describe("Utils > Random", () => { - describe("generateStrongPassword()", () => { - const passwordBulk = Array.from({ length: 1e5 }).map(_ => - generateStrongPassword() - ); - - it("should not generate the same password twice", () => { - const uniques = new Set(passwordBulk); - expect(passwordBulk.length).toBe(uniques.size); - }); - - it("should generate a password with a good level of entropy", () => { - passwordBulk.forEach(password => { - StrongPassword.decode(password).getOrElseL(err => - fail( - `Provided string did not meet the required strenght. Input: ${password}, Error: ${readableReport( - err - )}` - ) - ); - }); - }); - }); -}); diff --git a/utils/random.ts b/utils/random.ts index 131e1e37..f51b1d4e 100644 --- a/utils/random.ts +++ b/utils/random.ts @@ -4,58 +4,24 @@ import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { NonEmptyString, PatternString } from "italia-ts-commons/lib/strings"; +import { WithinRangeString } from "italia-ts-commons/lib/strings"; import * as randomstring from "randomstring"; -const UPPERCASED_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWYZ"; -const LOWERCASE_LETTERS = "abcdefghijklmnopqrstuvwyz"; -const NUMBERS = "0123456789"; -const SYMBOLS = "!£$%&()?+@=€"; +const RANDOM_CHARSET = + "ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwyz0123456789!£$%&()?+@=€"; -// at least 10 characters, at least one symbol one uppercase, one lowercase, one number -const STRONG_PASSWORD_PATTERN = - "(?=.{10,})(?=.*[!£$%&()?+@=€].*)(?=.*[a-z].*)(?=.*[A-Z].*)(?=.*[0-9].*)"; - -export const StrongPassword = t.intersection([ - PatternString(STRONG_PASSWORD_PATTERN), - NonEmptyString -]); +export const StrongPassword = WithinRangeString(18, 19); export type StrongPassword = t.TypeOf; -const shuffleString = (str: string): string => { - const a = str.split(""); - // tslint:disable-next-line: no-let - for (let i = a.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [a[i], a[j]] = [a[j], a[i]]; - } - return a.join(""); -}; - /** * Generates a randomic passwords with a high variety of characters */ export const generateStrongPassword = (): StrongPassword => StrongPassword.decode( - shuffleString( - // tslint:disable-next-line: restrict-plus-operands - randomstring.generate({ - charset: UPPERCASED_LETTERS, - length: 5 - }) + - randomstring.generate({ - charset: LOWERCASE_LETTERS, - length: 5 - }) + - randomstring.generate({ - charset: NUMBERS, - length: 5 - }) + - randomstring.generate({ - charset: SYMBOLS, - length: 3 - }) - ) + randomstring.generate({ + charset: RANDOM_CHARSET, + length: 18 + }) ).getOrElseL(err => { throw new Error( `Failed generating strong password - ${readableReport(err)}` diff --git a/yarn.lock b/yarn.lock index df42d8f4..eb6ededa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -549,6 +549,13 @@ dependencies: defer-to-connect "^2.0.0" +"@types/archiver@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-3.1.0.tgz#0d5bd922ba5cf06e137cd6793db7942439b1805e" + integrity sha512-nTvHwgWONL+iXG+9CX+gnQ/tTOV+qucAjwpXqeUn4OCRMxP42T29FFP/7XaOo0EqqO3TlENhObeZEe7RUJAriw== + dependencies: + "@types/glob" "*" + "@types/babel__core@^7.1.0": version "7.1.2" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.2.tgz#608c74f55928033fce18b99b213c16be4b3d114f" @@ -619,6 +626,11 @@ dependencies: "@types/node" "*" +"@types/events@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + "@types/express-serve-static-core@*": version "4.16.7" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.16.7.tgz#50ba6f8a691c08a3dd9fa7fba25ef3133d298049" @@ -636,6 +648,15 @@ "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/glob@*": + version "7.1.1" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" + integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w== + dependencies: + "@types/events" "*" + "@types/minimatch" "*" + "@types/node" "*" + "@types/http-cache-semantics@*": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz#9140779736aa2655635ee756e2467d787cfe8a2a" @@ -695,6 +716,11 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d" integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw== +"@types/minimatch@*": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== + "@types/node-fetch@^2.5.6": version "2.5.7" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" @@ -723,6 +749,11 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/randomstring@^1.1.6": + version "1.1.6" + resolved "https://registry.yarnpkg.com/@types/randomstring/-/randomstring-1.1.6.tgz#45cdc060a6f043d610bcd46503a6887db2a209c3" + integrity sha512-XRIZIMTxjcUukqQcYBdpFWGbcRDyNBXrvTEtTYgFMIbBNUVt+9mCKsU+jUUDLeFO/RXopUgR5OLiBqbY18vSHQ== + "@types/range-parser@*": version "1.2.3" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c" From 4e3a3f8effa2e402b2fc98f82796528cf299b5b7 Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Fri, 15 May 2020 18:21:24 +0200 Subject: [PATCH 3/4] fix linter --- ExtractUserDataActivity/handler.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index 315e080a..c803b307 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -47,11 +47,7 @@ import { } from "io-functions-commons/dist/src/models/sender_service"; import { iteratorToArray } from "io-functions-commons/dist/src/utils/documentdb"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { - FiscalCode, - NonEmptyString, - WithinRangeString -} from "italia-ts-commons/lib/strings"; +import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings"; import { generateStrongPassword, StrongPassword } from "../utils/random"; import { AllUserData, MessageContentWithId } from "../utils/userData"; import { From 528d92162f769df2f67056520931b1c300e284fb Mon Sep 17 00:00:00 2001 From: danilo spinelli Date: Fri, 15 May 2020 18:26:16 +0200 Subject: [PATCH 4/4] fix naming --- ExtractUserDataActivity/handler.ts | 4 ++-- utils/zip.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index c803b307..57d808b7 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -482,13 +482,13 @@ export const createExtractUserDataActivityHandler = ( }-${Date.now()}.zip` as NonEmptyString; const fileName = `${data.profile.fiscalCode}.json` as NonEmptyString; - initArchiverZipEncryptedPlugin.init(); + initArchiverZipEncryptedPlugin.run(); const readableZipStream = archiver.create("zip-encrypted", { encryptionMethod: DEFAULT_ZIP_ENCRYPTION_METHOD, password, zlib: { level: DEFAULT_ZLIB_LEVEL } - // following cast due to uncomplete archive typings + // following cast due to incomplete archive typings // tslint:disable-next-line: no-any } as any); diff --git a/utils/zip.ts b/utils/zip.ts index a258a84d..57e33fbe 100644 --- a/utils/zip.ts +++ b/utils/zip.ts @@ -2,7 +2,7 @@ import * as archiver from "archiver"; export const initArchiverZipEncryptedPlugin = { called: false, - init(): void { + run(): void { if (!initArchiverZipEncryptedPlugin.called) { // tslint:disable-next-line: no-object-mutation initArchiverZipEncryptedPlugin.called = true;