[#IOPID-2503] add masking preprocessor for appinsights (#248)

pagopa · Nov 25, 2024 · 7b41c9d · 7b41c9d
1 parent 1a99e0e
commit 7b41c9d
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 33 deletions.
diff --git a/UserDataDeleteOrchestratorV2/handler.ts b/UserDataDeleteOrchestratorV2/handler.ts
@@ -662,6 +662,7 @@ export const createUserDataDeleteOrchestratorHandler = (
         "failed",
         E.toError(error),
         currentUserDataProcessing,
+        context,
         false
       );
 
@@ -698,6 +699,7 @@ export const createUserDataDeleteOrchestratorHandler = (
             "unhandled_failed_status",
             new Error(readableReport(err)),
             currentUserDataProcessing,
+            context,
             false
           );
           throw new Error(

diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts
@@ -180,7 +180,9 @@ export const handler = function*(
     trackUserDataDownloadException(
       "failed",
       E.toError(error),
-      currentUserDataProcessing
+      currentUserDataProcessing,
+      context,
+      false
     );
 
     context.log.error(`${logPrefix}|ERROR|${JSON.stringify(error)}`);
@@ -217,7 +219,9 @@ export const handler = function*(
         trackUserDataDownloadException(
           "unhandled_failed_status",
           new Error(readableReport(err)),
-          currentUserDataProcessing
+          currentUserDataProcessing,
+          context,
+          false
         );
 
         throw new Error(

diff --git a/utils/appinsights.ts b/utils/appinsights.ts
@@ -1,5 +1,6 @@
 import * as ai from "applicationinsights";
 import {
+  Data,
   EventTelemetry,
   ExceptionTelemetry
 } from "applicationinsights/out/Declarations/Contracts";
@@ -14,25 +15,69 @@ import { pipe } from "fp-ts/lib/function";
 // @see https://github.com/Azure/azure-functions-host/blob/master/src/WebJobs.Script/Config/ApplicationInsightsLoggerOptionsSetup.cs#L29
 const DEFAULT_SAMPLING_PERCENTAGE = 20;
 
+export const USER_DATA_PROCESSING_ID_KEY = "userDataProcessingId";
+const maskUserProcessingIdPreprocessor = (
+  envelope: ai.Contracts.Envelope,
+  _context?: {
+    readonly [name: string]: unknown;
+  }
+): boolean => {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data = (envelope.data as Data<any>).baseData;
+    const userDataProcessingId = data.properties[USER_DATA_PROCESSING_ID_KEY];
+    if (
+      userDataProcessingId !== undefined &&
+      typeof userDataProcessingId === "string"
+    ) {
+      const maskedUserDataProcessingId = userDataProcessingId.replace(
+        /^([A-Z]{6})(.{10})(-DOWNLOAD|-DELETE)$/,
+        "$1$3"
+      );
+      // eslint-disable-next-line functional/immutable-data
+      data.properties[USER_DATA_PROCESSING_ID_KEY] = maskedUserDataProcessingId;
+      // eslint-disable-next-line functional/immutable-data
+      envelope.tags[
+        ai.defaultClient.context.keys.operationId
+      ] = maskedUserDataProcessingId;
+      // eslint-disable-next-line functional/immutable-data
+      envelope.tags[
+        ai.defaultClient.context.keys.operationParentId
+      ] = maskedUserDataProcessingId;
+    }
+  } catch (e) {
+    // ignore errors caused by missing properties
+  }
+  // sending the event
+  return true;
+};
+
 // Avoid to initialize Application Insights more than once
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-export const initTelemetryClient = (env = process.env) =>
+export const initTelemetryClient = (
+  env = process.env
+): ai.TelemetryClient | undefined =>
   ai.defaultClient
     ? ai.defaultClient
     : pipe(
         env.APPINSIGHTS_INSTRUMENTATIONKEY,
         NonEmptyString.decode,
         E.fold(
           _ => undefined,
-          k =>
-            initAppInsights(k, {
+          instrumentationKey => {
+            initAppInsights(instrumentationKey, {
               disableAppInsights: env.APPINSIGHTS_DISABLE === "true",
               samplingPercentage: pipe(
                 env.APPINSIGHTS_SAMPLING_PERCENTAGE,
                 IntegerFromString.decode,
                 E.getOrElse(() => DEFAULT_SAMPLING_PERCENTAGE)
               )
-            })
+            });
+            ai.defaultClient.addTelemetryProcessor(
+              maskUserProcessingIdPreprocessor
+            );
+            return ai.defaultClient;
+          }
         )
       );
 

diff --git a/utils/appinsightsEvents.ts b/utils/appinsightsEvents.ts
@@ -1,7 +1,12 @@
 // eslint-disable sonarjs/no-duplicate-string
 
 import { UserDataProcessing } from "@pagopa/io-functions-commons/dist/src/models/user_data_processing";
-import { trackEvent, trackException } from "./appinsights";
+import { IOrchestrationFunctionContext } from "durable-functions/lib/src/iorchestrationfunctioncontext";
+import {
+  trackEvent,
+  trackException,
+  USER_DATA_PROCESSING_ID_KEY
+} from "./appinsights";
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export const trackUserDataDeleteEvent = (
@@ -12,7 +17,7 @@ export const trackUserDataDeleteEvent = (
     // eslint-disable-next-line sonarjs/no-duplicate-string
     name: `user.data.delete.${eventName}`,
     properties: {
-      userDataProcessingId: userDataProcessing.userDataProcessingId
+      [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId
     },
     tagOverrides: {
       "ai.operation.id": userDataProcessing.userDataProcessingId,
@@ -25,20 +30,25 @@ export const trackUserDataDeleteException = (
   eventName: string,
   exception: Error,
   userDataProcessing: UserDataProcessing,
+  context: IOrchestrationFunctionContext,
   isSampled: boolean = true
 ) =>
-  trackException({
-    exception,
-    properties: {
-      name: `user.data.delete.${eventName}`,
-      userDataProcessingId: userDataProcessing.userDataProcessingId
-    },
-    tagOverrides: {
-      "ai.operation.id": userDataProcessing.userDataProcessingId,
-      "ai.operation.parentId": userDataProcessing.userDataProcessingId,
-      samplingEnabled: String(isSampled)
-    }
-  });
+  // avoiding duplicate exceptions
+  context.df.isReplaying
+    ? void 0
+    : trackException({
+        exception,
+        properties: {
+          [USER_DATA_PROCESSING_ID_KEY]:
+            userDataProcessing.userDataProcessingId,
+          name: `user.data.delete.${eventName}`
+        },
+        tagOverrides: {
+          "ai.operation.id": userDataProcessing.userDataProcessingId,
+          "ai.operation.parentId": userDataProcessing.userDataProcessingId,
+          samplingEnabled: String(isSampled)
+        }
+      });
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export const trackUserDataDownloadEvent = (
@@ -49,7 +59,7 @@ export const trackUserDataDownloadEvent = (
     // eslint-disable-next-line sonarjs/no-duplicate-string
     name: `user.data.download.${eventName}`,
     properties: {
-      userDataProcessingId: userDataProcessing.userDataProcessingId
+      [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId
     },
     tagOverrides: {
       "ai.operation.id": userDataProcessing.userDataProcessingId,
@@ -61,16 +71,23 @@ export const trackUserDataDownloadEvent = (
 export const trackUserDataDownloadException = (
   eventName: string,
   exception: Error,
-  userDataProcessing: UserDataProcessing
+  userDataProcessing: UserDataProcessing,
+  context: IOrchestrationFunctionContext,
+  isSampled: boolean = true
 ) =>
-  trackException({
-    exception,
-    properties: {
-      name: `user.data.download.${eventName}`,
-      userDataProcessingId: userDataProcessing.userDataProcessingId
-    },
-    tagOverrides: {
-      "ai.operation.id": userDataProcessing.userDataProcessingId,
-      "ai.operation.parentId": userDataProcessing.userDataProcessingId
-    }
-  });
+  // avoiding duplicate exceptions
+  context.df.isReplaying
+    ? void 0
+    : trackException({
+        exception,
+        properties: {
+          [USER_DATA_PROCESSING_ID_KEY]:
+            userDataProcessing.userDataProcessingId,
+          name: `user.data.download.${eventName}`
+        },
+        tagOverrides: {
+          "ai.operation.id": userDataProcessing.userDataProcessingId,
+          "ai.operation.parentId": userDataProcessing.userDataProcessingId,
+          samplingEnabled: String(isSampled)
+        }
+      });