Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#172621882] small refactor: generate zip with user data #45

Merged
merged 5 commits into from
May 15, 2020
Merged

[#172621882] small refactor: generate zip with user data #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c71353e
refactor
May 15, 2020
fcc7e6b
Merge branch 'master' of https://github.com/pagopa/io-functions-admin…
May 15, 2020
78fbdb3
refactors
May 15, 2020
4e3a3f8
fix linter
May 15, 2020
528d921
fix naming
May 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 1 addition & 47 deletions ExtractUserDataActivity/__tests__/handler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,8 @@ import {
aMessageContent,
aRetrievedMessageWithoutContent,
aRetrievedNotification,
aRetrievedSenderService,
aRetrievedWebhookNotification
aRetrievedSenderService
} from "../../__mocks__/mocks";
import { AllUserData } from "../../utils/userData";
import { createCompressedStream } from "../../utils/zip";
import { NotificationModel } from "../notification"; // we use the local-defined model

const createMockIterator = <T>(a: ReadonlyArray<T>) => {
Expand Down Expand Up @@ -85,12 +82,6 @@ const blobServiceMock = ({

const aUserDataContainerName = "aUserDataContainerName" as NonEmptyString;

jest.mock("../../utils/zip", () => ({
createCompressedStream: jest.fn(() => ({
pipe: jest.fn()
}))
}));

describe("createExtractUserDataActivityHandler", () => {
beforeEach(() => {
jest.clearAllMocks();
Expand Down Expand Up @@ -125,36 +116,6 @@ describe("createExtractUserDataActivityHandler", () => {
);
});

it("should not export webhook notification data", async () => {
const notificationWebhookModelMock = ({
findNotificationsForMessage: jest.fn(() =>
createMockIterator([aRetrievedWebhookNotification])
)
} as any) as NotificationModel;

const handler = createExtractUserDataActivityHandler(
messageModelMock,
messageStatusModelMock,
notificationWebhookModelMock,
notificationStatusModelMock,
profileModelMock,
senderServiceModelMock,
blobServiceMock,
aUserDataContainerName
);
const input: ActivityInput = {
fiscalCode: aFiscalCode
};

await handler(contextMock, input);

// @ts-ignore as
const mockCall = createCompressedStream.mock.calls[0];
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is wrong and we must rewrite it. if you mock the blobService and call cb() immediately the code following createWriteStreamToBlockBlob won't never be executed

const allUserData: AllUserData = mockCall[0][`${aFiscalCode}.json`];

expect(allUserData.notifications[0].channels.WEBHOOK).toEqual({});
});

it("should query using correct data", async () => {
const handler = createExtractUserDataActivityHandler(
messageModelMock,
Expand Down Expand Up @@ -192,12 +153,5 @@ describe("createExtractUserDataActivityHandler", () => {
expect(
senderServiceModelMock.findSenderServicesForRecipient
).toHaveBeenCalledWith(aFiscalCode);

expect(createCompressedStream).toHaveBeenCalledWith(
{
[`${aFiscalCode}.json`]: expect.any(Object)
},
expect.any(String)
);
});
});
44 changes: 25 additions & 19 deletions ExtractUserDataActivity/handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* This activity extracts all the data about a user contained in our db.
*/

import * as archiver from "archiver";
import * as t from "io-ts";

import { sequenceS } from "fp-ts/lib/Apply";
Expand Down Expand Up @@ -49,7 +50,11 @@ import { readableReport } from "italia-ts-commons/lib/reporters";
import { FiscalCode, NonEmptyString } from "italia-ts-commons/lib/strings";
import { generateStrongPassword, StrongPassword } from "../utils/random";
import { AllUserData, MessageContentWithId } from "../utils/userData";
import { createCompressedStream } from "../utils/zip";
import {
DEFAULT_ZIP_ENCRYPTION_METHOD,
DEFAULT_ZLIB_LEVEL,
initArchiverZipEncryptedPlugin
} from "../utils/zip";
import { NotificationModel } from "./notification";

export const ArchiveInfo = t.interface({
Expand Down Expand Up @@ -352,15 +357,10 @@ export const createExtractUserDataActivityHandler = (
.reduce(
(queries, { id: notificationId }) => [
...queries,
...Object.values(NotificationChannelEnum).map(channel => {
switch (channel) {
case NotificationChannelEnum.EMAIL:
case NotificationChannelEnum.WEBHOOK:
return [notificationId, channel];
default:
assertNever(channel);
}
})
...Object.values(NotificationChannelEnum).map(channel => [
notificationId,
channel
])
],
[]
)
Expand Down Expand Up @@ -482,12 +482,15 @@ export const createExtractUserDataActivityHandler = (
}-${Date.now()}.zip` as NonEmptyString;
const fileName = `${data.profile.fiscalCode}.json` as NonEmptyString;

const readableZipStream = createCompressedStream(
{
[fileName]: data
},
password
);
initArchiverZipEncryptedPlugin.run();

const readableZipStream = archiver.create("zip-encrypted", {
encryptionMethod: DEFAULT_ZIP_ENCRYPTION_METHOD,
password,
zlib: { level: DEFAULT_ZLIB_LEVEL }
// following cast due to incomplete archive typings
// tslint:disable-next-line: no-any
} as any);

const writableBlobStream = blobService.createWriteStreamToBlockBlob(
userDataContainerName,
Expand All @@ -512,19 +515,22 @@ export const createExtractUserDataActivityHandler = (
}
);
readableZipStream.pipe(writableBlobStream);

readableZipStream.on("error", err =>
// tslint:disable-next-line: no-any
readableZipStream.on("error", (err: any) =>
cb(
ActivityResultArchiveGenerationFailure.encode({
kind: "ARCHIVE_GENERATION_FAILURE",
reason: err.message
})
)
);
readableZipStream.append(JSON.stringify(data), { name: fileName });
// TODO: handle this promise correctly
readableZipStream.finalize().catch();
}
)();

// the actual handler©
// the actual handler
return (context: Context, input: unknown) =>
fromEither(
ActivityInput.decode(input).mapLeft<ActivityResultFailure>(
Expand Down
2 changes: 2 additions & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
"@azure/arm-apimanagement": "^5.1.1",
"@azure/graph": "^4.0.1",
"@azure/ms-rest-nodeauth": "^2.0.5",
"@types/archiver": "^3.1.0",
"@types/randomstring": "^1.1.6",
"archiver": "^4.0.1",
"archiver-zip-encrypted": "^1.0.8",
"azure-storage": "^2.10.3",
Expand Down
7 changes: 0 additions & 7 deletions utils/__mocks__/zip.ts
View file
Open in desktop

This file was deleted.

27 changes: 0 additions & 27 deletions utils/__tests__/random.test.ts
View file
Open in desktop

This file was deleted.

50 changes: 8 additions & 42 deletions utils/random.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,58 +4,24 @@

import * as t from "io-ts";
import { readableReport } from "italia-ts-commons/lib/reporters";
import { NonEmptyString, PatternString } from "italia-ts-commons/lib/strings";
import { WithinRangeString } from "italia-ts-commons/lib/strings";
import * as randomstring from "randomstring";

const UPPERCASED_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWYZ";
const LOWERCASE_LETTERS = "abcdefghijklmnopqrstuvwyz";
const NUMBERS = "0123456789";
const SYMBOLS = "!£$%&()?+@=€";
const RANDOM_CHARSET =
"ABCDEFGHIJKLMNOPQRSTUVWYZabcdefghijklmnopqrstuvwyz0123456789!£$%&()?+@=€";

// at least 10 characters, at least one symbol one uppercase, one lowercase, one number
const STRONG_PASSWORD_PATTERN =
"(?=.{10,})(?=.*[!£$%&()?+@=€].*)(?=.*[a-z].*)(?=.*[A-Z].*)(?=.*[0-9].*)";

export const StrongPassword = t.intersection([
PatternString(STRONG_PASSWORD_PATTERN),
NonEmptyString
]);
export const StrongPassword = WithinRangeString(18, 19);
export type StrongPassword = t.TypeOf<typeof StrongPassword>;

const shuffleString = (str: string): string => {
const a = str.split("");
// tslint:disable-next-line: no-let
for (let i = a.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[a[i], a[j]] = [a[j], a[i]];
}
return a.join("");
};

/**
* Generates a randomic passwords with a high variety of characters
*/
export const generateStrongPassword = (): StrongPassword =>
StrongPassword.decode(
shuffleString(
// tslint:disable-next-line: restrict-plus-operands
randomstring.generate({
charset: UPPERCASED_LETTERS,
length: 5
}) +
randomstring.generate({
charset: LOWERCASE_LETTERS,
length: 5
}) +
randomstring.generate({
charset: NUMBERS,
length: 5
}) +
randomstring.generate({
charset: SYMBOLS,
length: 3
})
)
randomstring.generate({
charset: RANDOM_CHARSET,
length: 18
})
).getOrElseL(err => {
throw new Error(
`Failed generating strong password - ${readableReport(err)}`
Expand Down
45 changes: 16 additions & 29 deletions utils/zip.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,24 @@
import * as archiver from "archiver";
import * as achiverEncryptedFormat from "archiver-zip-encrypted";
import { NonEmptyString } from "italia-ts-commons/lib/strings";
import { Readable } from "stream";
archiver.registerFormat("zip-encrypted", achiverEncryptedFormat);

export const initArchiverZipEncryptedPlugin = {
called: false,
run(): void {
if (!initArchiverZipEncryptedPlugin.called) {
// tslint:disable-next-line: no-object-mutation
initArchiverZipEncryptedPlugin.called = true;
// note: only do it once per Node.js process/application, as duplicate registration will throw an error
archiver.registerFormat(
"zip-encrypted",
require("archiver-zip-encrypted")
);
}
}
};

export enum EncryptionMethodEnum {
ZIP20 = "zip20",
AES256 = "aes256"
}

export const DEFAULT_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20;
export const DEFAULT_ZIP_ENCRYPTION_METHOD = EncryptionMethodEnum.ZIP20;
export const DEFAULT_ZLIB_LEVEL = 8;

export const createCompressedStream = (
// tslint:disable-next-line: no-any
data: Record<string, any>,
password?: NonEmptyString,
encryptionMethod: EncryptionMethodEnum = EncryptionMethodEnum.ZIP20
): Readable => {
const zipArchive = password
? archiver("zip-encrypted", {
encryptionMethod,
password,
zlib: { level: DEFAULT_ZLIB_LEVEL }
})
: archiver("zip", {
zlib: { level: DEFAULT_ZLIB_LEVEL }
});

Object.entries(data).forEach(([fileName, content]) => {
zipArchive.append(JSON.stringify(content), { name: fileName });
});
zipArchive.finalize();
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a promise and must be awaited somewhere


return zipArchive;
};
31 changes: 31 additions & 0 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,13 @@
dependencies:
defer-to-connect "^2.0.0"

"@types/archiver@^3.1.0":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-3.1.0.tgz#0d5bd922ba5cf06e137cd6793db7942439b1805e"
integrity sha512-nTvHwgWONL+iXG+9CX+gnQ/tTOV+qucAjwpXqeUn4OCRMxP42T29FFP/7XaOo0EqqO3TlENhObeZEe7RUJAriw==
dependencies:
"@types/glob" "*"

"@types/babel__core@^7.1.0":
version "7.1.2"
resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.2.tgz#608c74f55928033fce18b99b213c16be4b3d114f"
Expand Down Expand Up @@ -619,6 +626,11 @@
dependencies:
"@types/node" "*"

"@types/events@*":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==

"@types/express-serve-static-core@*":
version "4.16.7"
resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.16.7.tgz#50ba6f8a691c08a3dd9fa7fba25ef3133d298049"
Expand All @@ -636,6 +648,15 @@
"@types/express-serve-static-core" "*"
"@types/serve-static" "*"

"@types/glob@*":
version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
dependencies:
"@types/events" "*"
"@types/minimatch" "*"
"@types/node" "*"

"@types/http-cache-semantics@*":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz#9140779736aa2655635ee756e2467d787cfe8a2a"
Expand Down Expand Up @@ -695,6 +716,11 @@
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d"
integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==

"@types/minimatch@*":
version "3.0.3"
resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==

"@types/node-fetch@^2.5.6":
version "2.5.7"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c"
Expand Down Expand Up @@ -723,6 +749,11 @@
resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==

"@types/randomstring@^1.1.6":
version "1.1.6"
resolved "https://registry.yarnpkg.com/@types/randomstring/-/randomstring-1.1.6.tgz#45cdc060a6f043d610bcd46503a6887db2a209c3"
integrity sha512-XRIZIMTxjcUukqQcYBdpFWGbcRDyNBXrvTEtTYgFMIbBNUVt+9mCKsU+jUUDLeFO/RXopUgR5OLiBqbY18vSHQ==

"@types/range-parser@*":
version "1.2.3"
resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.3.tgz#7ee330ba7caafb98090bece86a5ee44115904c2c"
Expand Down