diff --git a/ExtractUserDataActivity/__tests__/handler.test.ts b/ExtractUserDataActivity/__tests__/handler.test.ts index 072a4c5b..e0f16615 100644 --- a/ExtractUserDataActivity/__tests__/handler.test.ts +++ b/ExtractUserDataActivity/__tests__/handler.test.ts @@ -29,11 +29,8 @@ import { aMessageContent, aRetrievedMessageWithoutContent, aRetrievedNotification, - aRetrievedSenderService, - aRetrievedWebhookNotification + aRetrievedSenderService } from "../../__mocks__/mocks"; -import { AllUserData } from "../../utils/userData"; -import { createCompressedStream } from "../../utils/zip"; import { NotificationModel } from "../notification"; // we use the local-defined model const createMockIterator = (a: ReadonlyArray) => { @@ -85,12 +82,6 @@ const blobServiceMock = ({ const aUserDataContainerName = "aUserDataContainerName" as NonEmptyString; -jest.mock("../../utils/zip", () => ({ - createCompressedStream: jest.fn(() => ({ - pipe: jest.fn() - })) -})); - describe("createExtractUserDataActivityHandler", () => { beforeEach(() => { jest.clearAllMocks(); @@ -125,36 +116,6 @@ describe("createExtractUserDataActivityHandler", () => { ); }); - it("should not export webhook notification data", async () => { - const notificationWebhookModelMock = ({ - findNotificationsForMessage: jest.fn(() => - createMockIterator([aRetrievedWebhookNotification]) - ) - } as any) as NotificationModel; - - const handler = createExtractUserDataActivityHandler( - messageModelMock, - messageStatusModelMock, - notificationWebhookModelMock, - notificationStatusModelMock, - profileModelMock, - senderServiceModelMock, - blobServiceMock, - aUserDataContainerName - ); - const input: ActivityInput = { - fiscalCode: aFiscalCode - }; - - await handler(contextMock, input); - - // @ts-ignore as - const mockCall = createCompressedStream.mock.calls[0]; - const allUserData: AllUserData = mockCall[0][`${aFiscalCode}.json`]; - - expect(allUserData.notifications[0].channels.WEBHOOK).toEqual({}); - }); - it("should query using correct data", async () => { const handler = createExtractUserDataActivityHandler( messageModelMock, @@ -192,12 +153,5 @@ describe("createExtractUserDataActivityHandler", () => { expect( senderServiceModelMock.findSenderServicesForRecipient ).toHaveBeenCalledWith(aFiscalCode); - - expect(createCompressedStream).toHaveBeenCalledWith( - { - [`${aFiscalCode}.json`]: expect.any(Object) - }, - expect.any(String) - ); }); }); diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts index 924f898a..57d808b7 100644 --- a/ExtractUserDataActivity/handler.ts +++ b/ExtractUserDataActivity/handler.ts @@ -2,6 +2,7 @@ * This activity extracts all the data about a user contained in our db. */ +import * as archiver from "archiver"; import * as t from "io-ts"; import { sequenceS } from "fp-ts/lib/Apply"; @@ -49,7 +50,11 @@ import { readableReport } from "italia-ts-commons/lib/reporters"; import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings"; import { generateStrongPassword, StrongPassword } from "../utils/random"; import { AllUserData, MessageContentWithId } from "../utils/userData"; -import { createCompressedStream } from "../utils/zip"; +import { + DEFAULT_ZIP_ENCRYPTION_METHOD, + DEFAULT_ZLIB_LEVEL, + initArchiverZipEncryptedPlugin +} from "../utils/zip"; import { NotificationModel } from "./notification"; export const ArchiveInfo = t.interface({ @@ -352,15 +357,10 @@ export const createExtractUserDataActivityHandler = ( .reduce( (queries, { id: notificationId }) => [ ...queries, - ...Object.values(NotificationChannelEnum).map(channel => { - switch (channel) { - case NotificationChannelEnum.EMAIL: - case NotificationChannelEnum.WEBHOOK: - return [notificationId, channel]; - default: - assertNever(channel); - } - }) + ...Object.values(NotificationChannelEnum).map(channel => [ + notificationId, + channel + ]) ], [] ) @@ -482,12 +482,15 @@ export const createExtractUserDataActivityHandler = ( }-${Date.now()}.zip` as NonEmptyString; const fileName = `${data.profile.fiscalCode}.json` as NonEmptyString; - const readableZipStream = createCompressedStream( - { - [fileName]: data - }, - password - ); + initArchiverZipEncryptedPlugin.run(); + + const readableZipStream = archiver.create("zip-encrypted", { + encryptionMethod: DEFAULT_ZIP_ENCRYPTION_METHOD, + password, + zlib: { level: DEFAULT_ZLIB_LEVEL } + // following cast due to incomplete archive typings + // tslint:disable-next-line: no-any + } as any); const writableBlobStream = blobService.createWriteStreamToBlockBlob( userDataContainerName, @@ -512,8 +515,8 @@ export const createExtractUserDataActivityHandler = ( } ); readableZipStream.pipe(writableBlobStream); - - readableZipStream.on("error", err => + // tslint:disable-next-line: no-any + readableZipStream.on("error", (err: any) => cb( ActivityResultArchiveGenerationFailure.encode({ kind: "ARCHIVE_GENERATION_FAILURE", @@ -521,10 +524,13 @@ export const createExtractUserDataActivityHandler = ( }) ) ); + readableZipStream.append(JSON.stringify(data), { name: fileName }); + // TODO: handle this promise correctly + readableZipStream.finalize().catch(); } )(); - // the actual handler© + // the actual handler return (context: Context, input: unknown) => fromEither( ActivityInput.decode(input).mapLeft( diff --git a/package.json b/package.json index c66a9653..5a14617b 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,8 @@ "@azure/arm-apimanagement": "^5.1.1", "@azure/graph": "^4.0.1", "@azure/ms-rest-nodeauth": "^2.0.5", + "@types/archiver": "^3.1.0", + "@types/randomstring": "^1.1.6", "archiver": "^4.0.1", "archiver-zip-encrypted": "^1.0.8", "azure-storage": "^2.10.3", diff --git a/utils/__mocks__/zip.ts b/utils/__mocks__/zip.ts deleted file mode 100644 index a1a514b8..00000000 --- a/utils/__mocks__/zip.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Mock implementation of the zip module - */ - -export const createCompressedStream = jest.fn(() => ({ - pipe: jest.fn() -})); diff --git a/utils/__tests__/random.test.ts b/utils/__tests__/random.test.ts deleted file mode 100644 index 8c6e673c..00000000 --- a/utils/__tests__/random.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { readableReport } from "italia-ts-commons/lib/reporters"; -import { generateStrongPassword, StrongPassword } from "../random"; - -describe("Utils > Random", () => { - describe("generateStrongPassword()", () => { - const passwordBulk = Array.from({ length: 1e5 }).map(_ => - generateStrongPassword() - ); - - it("should not generate the same password twice", () => { - const uniques = new Set(passwordBulk); - expect(passwordBulk.length).toBe(uniques.size); - }); - - it("should generate a password with a good level of entropy", () => { - passwordBulk.forEach(password => { - StrongPassword.decode(password).getOrElseL(err => - fail( - `Provided string did not meet the required strenght. Input: ${password}, Error: ${readableReport( - err - )}` - ) - ); - }); - }); - }); -}); diff --git a/utils/random.ts b/utils/random.ts index 131e1e37..f51b1d4e 100644 --- a/utils/random.ts +++ b/utils/random.ts @@ -4,58 +4,24 @@ import * as t from "io-ts"; import { readableReport } from "italia-ts-commons/lib/reporters"; -import { NonEmptyString, PatternString } from "italia-ts-commons/lib/strings"; +import { WithinRangeString } from "italia-ts-commons/lib/strings"; import * as randomstring from "randomstring"; -const UPPERCASED_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWYZ"; -const LOWERCASE_LETTERS = "abcdefghijklmnopqrstuvwyz"; -const NUMBERS = "0123456789"; -const SYMBOLS = "!£$%&()?+@=€"; +const RANDOM_CHARSET = + "ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwyz0123456789!£$%&()?+@=€"; -// at least 10 characters, at least one symbol one uppercase, one lowercase, one number -const STRONG_PASSWORD_PATTERN = - "(?=.{10,})(?=.*[!£$%&()?+@=€].*)(?=.*[a-z].*)(?=.*[A-Z].*)(?=.*[0-9].*)"; - -export const StrongPassword = t.intersection([ - PatternString(STRONG_PASSWORD_PATTERN), - NonEmptyString -]); +export const StrongPassword = WithinRangeString(18, 19); export type StrongPassword = t.TypeOf; -const shuffleString = (str: string): string => { - const a = str.split(""); - // tslint:disable-next-line: no-let - for (let i = a.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [a[i], a[j]] = [a[j], a[i]]; - } - return a.join(""); -}; - /** * Generates a randomic passwords with a high variety of characters */ export const generateStrongPassword = (): StrongPassword => StrongPassword.decode( - shuffleString( - // tslint:disable-next-line: restrict-plus-operands - randomstring.generate({ - charset: UPPERCASED_LETTERS, - length: 5 - }) + - randomstring.generate({ - charset: LOWERCASE_LETTERS, - length: 5 - }) + - randomstring.generate({ - charset: NUMBERS, - length: 5 - }) + - randomstring.generate({ - charset: SYMBOLS, - length: 3 - }) - ) + randomstring.generate({ + charset: RANDOM_CHARSET, + length: 18 + }) ).getOrElseL(err => { throw new Error( `Failed generating strong password - ${readableReport(err)}` diff --git a/utils/zip.ts b/utils/zip.ts index 0d645901..57e33fbe 100644 --- a/utils/zip.ts +++ b/utils/zip.ts @@ -1,37 +1,24 @@ import * as archiver from "archiver"; -import * as achiverEncryptedFormat from "archiver-zip-encrypted"; -import { NonEmptyString } from "italia-ts-commons/lib/strings"; -import { Readable } from "stream"; -archiver.registerFormat("zip-encrypted", achiverEncryptedFormat); + +export const initArchiverZipEncryptedPlugin = { + called: false, + run(): void { + if (!initArchiverZipEncryptedPlugin.called) { + // tslint:disable-next-line: no-object-mutation + initArchiverZipEncryptedPlugin.called = true; + // note: only do it once per Node.js process/application, as duplicate registration will throw an error + archiver.registerFormat( + "zip-encrypted", + require("archiver-zip-encrypted") + ); + } + } +}; export enum EncryptionMethodEnum { ZIP20 = "zip20", AES256 = "aes256" } -export const DEFAULT_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20; +export const DEFAULT_ZIP_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20; export const DEFAULT_ZLIB_LEVEL = 8; - -export const createCompressedStream = ( - // tslint:disable-next-line: no-any - data: Record, - password?: NonEmptyString, - encryptionMethod: EncryptionMethodEnum = EncryptionMethodEnum.ZIP20 -): Readable => { - const zipArchive = password - ? archiver("zip-encrypted", { - encryptionMethod, - password, - zlib: { level: DEFAULT_ZLIB_LEVEL } - }) - : archiver("zip", { - zlib: { level: DEFAULT_ZLIB_LEVEL } - }); - - Object.entries(data).forEach(([fileName, content]) => { - zipArchive.append(JSON.stringify(content), { name: fileName }); - }); - zipArchive.finalize(); - - return zipArchive; -}; diff --git a/yarn.lock b/yarn.lock index df42d8f4..eb6ededa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -549,6 +549,13 @@ dependencies: defer-to-connect "^2.0.0" +"@types/archiver@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-3.1.0.tgz#0d5bd922ba5cf06e137cd6793db7942439b1805e" + integrity sha512-nTvHwgWONL+iXG+9CX+gnQ/tTOV+qucAjwpXqeUn4OCRMxP42T29FFP/7XaOo0EqqO3TlENhObeZEe7RUJAriw== + dependencies: + "@types/glob" "*" + "@types/babel__core@^7.1.0": version "7.1.2" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.2.tgz#608c74f55928033fce18b99b213c16be4b3d114f" @@ -619,6 +626,11 @@ dependencies: "@types/node" "*" +"@types/events@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + "@types/express-serve-static-core@*": version "4.16.7" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.16.7.tgz#50ba6f8a691c08a3dd9fa7fba25ef3133d298049" @@ -636,6 +648,15 @@ "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/glob@*": + version "7.1.1" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" + integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w== + dependencies: + "@types/events" "*" + "@types/minimatch" "*" + "@types/node" "*" + "@types/http-cache-semantics@*": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz#9140779736aa2655635ee756e2467d787cfe8a2a" @@ -695,6 +716,11 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d" integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw== +"@types/minimatch@*": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" + integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== + "@types/node-fetch@^2.5.6": version "2.5.7" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" @@ -723,6 +749,11 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/randomstring@^1.1.6": + version "1.1.6" + resolved "https://registry.yarnpkg.com/@types/randomstring/-/randomstring-1.1.6.tgz#45cdc060a6f043d610bcd46503a6887db2a209c3" + integrity sha512-XRIZIMTxjcUukqQcYBdpFWGbcRDyNBXrvTEtTYgFMIbBNUVt+9mCKsU+jUUDLeFO/RXopUgR5OLiBqbY18vSHQ== + "@types/range-parser@*": version "1.2.3" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c"