[#172621882] small refactor: generate zip with user data (#45)

pagopa · May 15, 2020 · f5388fb · f5388fb
1 parent 47f9123
commit f5388fb
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 171 deletions.
diff --git a/ExtractUserDataActivity/__tests__/handler.test.ts b/ExtractUserDataActivity/__tests__/handler.test.ts
@@ -29,11 +29,8 @@ import {
   aMessageContent,
   aRetrievedMessageWithoutContent,
   aRetrievedNotification,
-  aRetrievedSenderService,
-  aRetrievedWebhookNotification
+  aRetrievedSenderService
 } from "../../__mocks__/mocks";
-import { AllUserData } from "../../utils/userData";
-import { createCompressedStream } from "../../utils/zip";
 import { NotificationModel } from "../notification"; // we use the local-defined model
 
 const createMockIterator = <T>(a: ReadonlyArray<T>) => {
@@ -85,12 +82,6 @@ const blobServiceMock = ({
 
 const aUserDataContainerName = "aUserDataContainerName" as NonEmptyString;
 
-jest.mock("../../utils/zip", () => ({
-  createCompressedStream: jest.fn(() => ({
-    pipe: jest.fn()
-  }))
-}));
-
 describe("createExtractUserDataActivityHandler", () => {
   beforeEach(() => {
     jest.clearAllMocks();
@@ -125,36 +116,6 @@ describe("createExtractUserDataActivityHandler", () => {
     );
   });
 
-  it("should not export webhook notification data", async () => {
-    const notificationWebhookModelMock = ({
-      findNotificationsForMessage: jest.fn(() =>
-        createMockIterator([aRetrievedWebhookNotification])
-      )
-    } as any) as NotificationModel;
-
-    const handler = createExtractUserDataActivityHandler(
-      messageModelMock,
-      messageStatusModelMock,
-      notificationWebhookModelMock,
-      notificationStatusModelMock,
-      profileModelMock,
-      senderServiceModelMock,
-      blobServiceMock,
-      aUserDataContainerName
-    );
-    const input: ActivityInput = {
-      fiscalCode: aFiscalCode
-    };
-
-    await handler(contextMock, input);
-
-    // @ts-ignore as
-    const mockCall = createCompressedStream.mock.calls[0];
-    const allUserData: AllUserData = mockCall[0][`${aFiscalCode}.json`];
-
-    expect(allUserData.notifications[0].channels.WEBHOOK).toEqual({});
-  });
-
   it("should query using correct data", async () => {
     const handler = createExtractUserDataActivityHandler(
       messageModelMock,
@@ -192,12 +153,5 @@ describe("createExtractUserDataActivityHandler", () => {
     expect(
       senderServiceModelMock.findSenderServicesForRecipient
     ).toHaveBeenCalledWith(aFiscalCode);
-
-    expect(createCompressedStream).toHaveBeenCalledWith(
-      {
-        [`${aFiscalCode}.json`]: expect.any(Object)
-      },
-      expect.any(String)
-    );
   });
 });
diff --git a/ExtractUserDataActivity/handler.ts b/ExtractUserDataActivity/handler.ts
@@ -2,6 +2,7 @@
  * This activity extracts all the data about a user contained in our db.
  */
 
+import * as archiver from "archiver";
 import * as t from "io-ts";
 
 import { sequenceS } from "fp-ts/lib/Apply";
@@ -49,7 +50,11 @@ import { readableReport } from "italia-ts-commons/lib/reporters";
 import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings";
 import { generateStrongPassword, StrongPassword } from "../utils/random";
 import { AllUserData, MessageContentWithId } from "../utils/userData";
-import { createCompressedStream } from "../utils/zip";
+import {
+  DEFAULT_ZIP_ENCRYPTION_METHOD,
+  DEFAULT_ZLIB_LEVEL,
+  initArchiverZipEncryptedPlugin
+} from "../utils/zip";
 import { NotificationModel } from "./notification";
 
 export const ArchiveInfo = t.interface({
@@ -352,15 +357,10 @@ export const createExtractUserDataActivityHandler = (
           .reduce(
             (queries, { id: notificationId }) => [
               ...queries,
-              ...Object.values(NotificationChannelEnum).map(channel => {
-                switch (channel) {
-                  case NotificationChannelEnum.EMAIL:
-                  case NotificationChannelEnum.WEBHOOK:
-                    return [notificationId, channel];
-                  default:
-                    assertNever(channel);
-                }
-              })
+              ...Object.values(NotificationChannelEnum).map(channel => [
+                notificationId,
+                channel
+              ])
             ],
             []
           )
@@ -482,12 +482,15 @@ export const createExtractUserDataActivityHandler = (
         }-${Date.now()}.zip` as NonEmptyString;
         const fileName = `${data.profile.fiscalCode}.json` as NonEmptyString;
 
-        const readableZipStream = createCompressedStream(
-          {
-            [fileName]: data
-          },
-          password
-        );
+        initArchiverZipEncryptedPlugin.run();
+
+        const readableZipStream = archiver.create("zip-encrypted", {
+          encryptionMethod: DEFAULT_ZIP_ENCRYPTION_METHOD,
+          password,
+          zlib: { level: DEFAULT_ZLIB_LEVEL }
+          // following cast due to incomplete archive typings
+          // tslint:disable-next-line: no-any
+        } as any);
 
         const writableBlobStream = blobService.createWriteStreamToBlockBlob(
           userDataContainerName,
@@ -512,19 +515,22 @@ export const createExtractUserDataActivityHandler = (
           }
         );
         readableZipStream.pipe(writableBlobStream);
-
-        readableZipStream.on("error", err =>
+        // tslint:disable-next-line: no-any
+        readableZipStream.on("error", (err: any) =>
           cb(
             ActivityResultArchiveGenerationFailure.encode({
               kind: "ARCHIVE_GENERATION_FAILURE",
               reason: err.message
             })
           )
         );
+        readableZipStream.append(JSON.stringify(data), { name: fileName });
+        // TODO: handle this promise correctly
+        readableZipStream.finalize().catch();
       }
     )();
 
-  // the actual handler©
+  // the actual handler
   return (context: Context, input: unknown) =>
     fromEither(
       ActivityInput.decode(input).mapLeft<ActivityResultFailure>(

diff --git a/package.json b/package.json
@@ -49,6 +49,8 @@
     "@azure/arm-apimanagement": "^5.1.1",
     "@azure/graph": "^4.0.1",
     "@azure/ms-rest-nodeauth": "^2.0.5",
+    "@types/archiver": "^3.1.0",
+    "@types/randomstring": "^1.1.6",
     "archiver": "^4.0.1",
     "archiver-zip-encrypted": "^1.0.8",
     "azure-storage": "^2.10.3",

diff --git a/utils/__mocks__/zip.ts b/utils/__mocks__/zip.ts
diff --git a/utils/__tests__/random.test.ts b/utils/__tests__/random.test.ts
diff --git a/utils/random.ts b/utils/random.ts
@@ -4,58 +4,24 @@
 
 import * as t from "io-ts";
 import { readableReport } from "italia-ts-commons/lib/reporters";
-import { NonEmptyString, PatternString } from "italia-ts-commons/lib/strings";
+import { WithinRangeString } from "italia-ts-commons/lib/strings";
 import * as randomstring from "randomstring";
 
-const UPPERCASED_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWYZ";
-const LOWERCASE_LETTERS = "abcdefghijklmnopqrstuvwyz";
-const NUMBERS = "0123456789";
-const SYMBOLS = "!£$%&()?+@=€";
+const RANDOM_CHARSET =
+  "ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwyz0123456789!£$%&()?+@=€";
 
-// at least 10 characters, at least one symbol one uppercase, one lowercase, one number
-const STRONG_PASSWORD_PATTERN =
-  "(?=.{10,})(?=.*[!£$%&()?+@=€].*)(?=.*[a-z].*)(?=.*[A-Z].*)(?=.*[0-9].*)";
-
-export const StrongPassword = t.intersection([
-  PatternString(STRONG_PASSWORD_PATTERN),
-  NonEmptyString
-]);
+export const StrongPassword = WithinRangeString(18, 19);
 export type StrongPassword = t.TypeOf<typeof StrongPassword>;
 
-const shuffleString = (str: string): string => {
-  const a = str.split("");
-  // tslint:disable-next-line: no-let
-  for (let i = a.length - 1; i > 0; i--) {
-    const j = Math.floor(Math.random() * (i + 1));
-    [a[i], a[j]] = [a[j], a[i]];
-  }
-  return a.join("");
-};
-
 /**
  * Generates a randomic passwords with a high variety of characters
  */
 export const generateStrongPassword = (): StrongPassword =>
   StrongPassword.decode(
-    shuffleString(
-      // tslint:disable-next-line: restrict-plus-operands
-      randomstring.generate({
-        charset: UPPERCASED_LETTERS,
-        length: 5
-      }) +
-        randomstring.generate({
-          charset: LOWERCASE_LETTERS,
-          length: 5
-        }) +
-        randomstring.generate({
-          charset: NUMBERS,
-          length: 5
-        }) +
-        randomstring.generate({
-          charset: SYMBOLS,
-          length: 3
-        })
-    )
+    randomstring.generate({
+      charset: RANDOM_CHARSET,
+      length: 18
+    })
   ).getOrElseL(err => {
     throw new Error(
       `Failed generating strong password - ${readableReport(err)}`

diff --git a/utils/zip.ts b/utils/zip.ts
@@ -1,37 +1,24 @@
 import * as archiver from "archiver";
-import * as achiverEncryptedFormat from "archiver-zip-encrypted";
-import { NonEmptyString } from "italia-ts-commons/lib/strings";
-import { Readable } from "stream";
-archiver.registerFormat("zip-encrypted", achiverEncryptedFormat);
+
+export const initArchiverZipEncryptedPlugin = {
+  called: false,
+  run(): void {
+    if (!initArchiverZipEncryptedPlugin.called) {
+      // tslint:disable-next-line: no-object-mutation
+      initArchiverZipEncryptedPlugin.called = true;
+      // note: only do it once per Node.js process/application, as duplicate registration will throw an error
+      archiver.registerFormat(
+        "zip-encrypted",
+        require("archiver-zip-encrypted")
+      );
+    }
+  }
+};
 
 export enum EncryptionMethodEnum {
   ZIP20 = "zip20",
   AES256 = "aes256"
 }
 
-export const DEFAULT_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20;
+export const DEFAULT_ZIP_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20;
 export const DEFAULT_ZLIB_LEVEL = 8;
-
-export const createCompressedStream = (
-  // tslint:disable-next-line: no-any
-  data: Record<string, any>,
-  password?: NonEmptyString,
-  encryptionMethod: EncryptionMethodEnum = EncryptionMethodEnum.ZIP20
-): Readable => {
-  const zipArchive = password
-    ? archiver("zip-encrypted", {
-        encryptionMethod,
-        password,
-        zlib: { level: DEFAULT_ZLIB_LEVEL }
-      })
-    : archiver("zip", {
-        zlib: { level: DEFAULT_ZLIB_LEVEL }
-      });
-
-  Object.entries(data).forEach(([fileName, content]) => {
-    zipArchive.append(JSON.stringify(content), { name: fileName });
-  });
-  zipArchive.finalize();
-
-  return zipArchive;
-};
diff --git a/yarn.lock b/yarn.lock
@@ -549,6 +549,13 @@
   dependencies:
     defer-to-connect "^2.0.0"
 
+"@types/archiver@^3.1.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-3.1.0.tgz#0d5bd922ba5cf06e137cd6793db7942439b1805e"
+  integrity sha512-nTvHwgWONL+iXG+9CX+gnQ/tTOV+qucAjwpXqeUn4OCRMxP42T29FFP/7XaOo0EqqO3TlENhObeZEe7RUJAriw==
+  dependencies:
+    "@types/glob" "*"
+
 "@types/babel__core@^7.1.0":
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.2.tgz#608c74f55928033fce18b99b213c16be4b3d114f"
@@ -619,6 +626,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/events@*":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
+  integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
+
 "@types/express-serve-static-core@*":
   version "4.16.7"
   resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.16.7.tgz#50ba6f8a691c08a3dd9fa7fba25ef3133d298049"
@@ -636,6 +648,15 @@
     "@types/express-serve-static-core" "*"
     "@types/serve-static" "*"
 
+"@types/glob@*":
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
+  integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
+  dependencies:
+    "@types/events" "*"
+    "@types/minimatch" "*"
+    "@types/node" "*"
+
 "@types/http-cache-semantics@*":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz#9140779736aa2655635ee756e2467d787cfe8a2a"
@@ -695,6 +716,11 @@
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d"
   integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==
 
+"@types/minimatch@*":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
+  integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
+
 "@types/node-fetch@^2.5.6":
   version "2.5.7"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c"
@@ -723,6 +749,11 @@
   resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
   integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==
 
+"@types/randomstring@^1.1.6":
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/@types/randomstring/-/randomstring-1.1.6.tgz#45cdc060a6f043d610bcd46503a6887db2a209c3"
+  integrity sha512-XRIZIMTxjcUukqQcYBdpFWGbcRDyNBXrvTEtTYgFMIbBNUVt+9mCKsU+jUUDLeFO/RXopUgR5OLiBqbY18vSHQ==
+
 "@types/range-parser@*":
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c"