From 598dde6cc36d4cfb14b16973f4a3ae8a1feb3da5 Mon Sep 17 00:00:00 2001 From: arcogabbo Date: Mon, 25 Nov 2024 11:20:18 +0100 Subject: [PATCH 1/3] [#IOPID-2503] added masking preprocessor for ai --- UserDataDeleteOrchestratorV2/handler.ts | 2 + utils/appinsights.ts | 53 +++++++++++++++++++++++-- utils/appinsightsEvents.ts | 21 ++++++---- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/UserDataDeleteOrchestratorV2/handler.ts b/UserDataDeleteOrchestratorV2/handler.ts index e5847039..8695e9c5 100644 --- a/UserDataDeleteOrchestratorV2/handler.ts +++ b/UserDataDeleteOrchestratorV2/handler.ts @@ -662,6 +662,7 @@ export const createUserDataDeleteOrchestratorHandler = ( "failed", E.toError(error), currentUserDataProcessing, + context, false ); @@ -698,6 +699,7 @@ export const createUserDataDeleteOrchestratorHandler = ( "unhandled_failed_status", new Error(readableReport(err)), currentUserDataProcessing, + context, false ); throw new Error( diff --git a/utils/appinsights.ts b/utils/appinsights.ts index c916c322..264a9d37 100644 --- a/utils/appinsights.ts +++ b/utils/appinsights.ts @@ -1,5 +1,6 @@ import * as ai from "applicationinsights"; import { + Data, EventTelemetry, ExceptionTelemetry } from "applicationinsights/out/Declarations/Contracts"; @@ -14,9 +15,48 @@ import { pipe } from "fp-ts/lib/function"; // @see https://github.com/Azure/azure-functions-host/blob/master/src/WebJobs.Script/Config/ApplicationInsightsLoggerOptionsSetup.cs#L29 const DEFAULT_SAMPLING_PERCENTAGE = 20; +export const USER_DATA_PROCESSING_ID_KEY = "userDataProcessingId"; +const maskUserProcessingIdPreprocessor = ( + envelope: ai.Contracts.Envelope, + _context?: { + readonly [name: string]: unknown; + } +): boolean => { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data = (envelope.data as Data).baseData; + const userDataProcessingId = data.properties[USER_DATA_PROCESSING_ID_KEY]; + if ( + userDataProcessingId !== undefined && + typeof userDataProcessingId === "string" + ) { + const maskedUserDataProcessingId = userDataProcessingId.replace( + /^([A-Z]{6})(.{10})(-DOWNLOAD|-DELETE)$/, + "$1$3" + ); + // eslint-disable-next-line functional/immutable-data + data.properties[USER_DATA_PROCESSING_ID_KEY] = maskedUserDataProcessingId; + // eslint-disable-next-line functional/immutable-data + envelope.tags[ + ai.defaultClient.context.keys.operationId + ] = maskedUserDataProcessingId; + // eslint-disable-next-line functional/immutable-data + envelope.tags[ + ai.defaultClient.context.keys.operationParentId + ] = maskedUserDataProcessingId; + } + } catch (e) { + // ignore errors caused by missing properties + } + // sending the event + return true; +}; + // Avoid to initialize Application Insights more than once // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -export const initTelemetryClient = (env = process.env) => +export const initTelemetryClient = ( + env = process.env +): ai.TelemetryClient | undefined => ai.defaultClient ? ai.defaultClient : pipe( @@ -24,15 +64,20 @@ export const initTelemetryClient = (env = process.env) => NonEmptyString.decode, E.fold( _ => undefined, - k => - initAppInsights(k, { + instrumentationKey => { + initAppInsights(instrumentationKey, { disableAppInsights: env.APPINSIGHTS_DISABLE === "true", samplingPercentage: pipe( env.APPINSIGHTS_SAMPLING_PERCENTAGE, IntegerFromString.decode, E.getOrElse(() => DEFAULT_SAMPLING_PERCENTAGE) ) - }) + }); + ai.defaultClient.addTelemetryProcessor( + maskUserProcessingIdPreprocessor + ); + return ai.defaultClient; + } ) ); diff --git a/utils/appinsightsEvents.ts b/utils/appinsightsEvents.ts index a5e3a70e..2a2d9227 100644 --- a/utils/appinsightsEvents.ts +++ b/utils/appinsightsEvents.ts @@ -1,7 +1,12 @@ // eslint-disable sonarjs/no-duplicate-string import { UserDataProcessing } from "@pagopa/io-functions-commons/dist/src/models/user_data_processing"; -import { trackEvent, trackException } from "./appinsights"; +import { IOrchestrationFunctionContext } from "durable-functions/lib/src/iorchestrationfunctioncontext"; +import { + trackEvent, + trackException, + USER_DATA_PROCESSING_ID_KEY +} from "./appinsights"; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const trackUserDataDeleteEvent = ( @@ -12,7 +17,7 @@ export const trackUserDataDeleteEvent = ( // eslint-disable-next-line sonarjs/no-duplicate-string name: `user.data.delete.${eventName}`, properties: { - userDataProcessingId: userDataProcessing.userDataProcessingId + [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId }, tagOverrides: { "ai.operation.id": userDataProcessing.userDataProcessingId, @@ -25,13 +30,15 @@ export const trackUserDataDeleteException = ( eventName: string, exception: Error, userDataProcessing: UserDataProcessing, + context: IOrchestrationFunctionContext, isSampled: boolean = true ) => trackException({ exception, properties: { - name: `user.data.delete.${eventName}`, - userDataProcessingId: userDataProcessing.userDataProcessingId + [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId, + isReplay: context.df.isReplaying, + name: `user.data.delete.${eventName}` }, tagOverrides: { "ai.operation.id": userDataProcessing.userDataProcessingId, @@ -49,7 +56,7 @@ export const trackUserDataDownloadEvent = ( // eslint-disable-next-line sonarjs/no-duplicate-string name: `user.data.download.${eventName}`, properties: { - userDataProcessingId: userDataProcessing.userDataProcessingId + [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId }, tagOverrides: { "ai.operation.id": userDataProcessing.userDataProcessingId, @@ -66,8 +73,8 @@ export const trackUserDataDownloadException = ( trackException({ exception, properties: { - name: `user.data.download.${eventName}`, - userDataProcessingId: userDataProcessing.userDataProcessingId + [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId, + name: `user.data.download.${eventName}` }, tagOverrides: { "ai.operation.id": userDataProcessing.userDataProcessingId, From 1a257a8dc7ff8e0c065545d5a3ba8dc04688d42d Mon Sep 17 00:00:00 2001 From: arcogabbo Date: Mon, 25 Nov 2024 12:13:59 +0100 Subject: [PATCH 2/3] add info also to download procedure --- UserDataDownloadOrchestrator/handler.ts | 8 ++++++-- utils/appinsightsEvents.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/UserDataDownloadOrchestrator/handler.ts b/UserDataDownloadOrchestrator/handler.ts index 15cd6177..65a7de43 100644 --- a/UserDataDownloadOrchestrator/handler.ts +++ b/UserDataDownloadOrchestrator/handler.ts @@ -180,7 +180,9 @@ export const handler = function*( trackUserDataDownloadException( "failed", E.toError(error), - currentUserDataProcessing + currentUserDataProcessing, + context, + false ); context.log.error(`${logPrefix}|ERROR|${JSON.stringify(error)}`); @@ -217,7 +219,9 @@ export const handler = function*( trackUserDataDownloadException( "unhandled_failed_status", new Error(readableReport(err)), - currentUserDataProcessing + currentUserDataProcessing, + context, + false ); throw new Error( diff --git a/utils/appinsightsEvents.ts b/utils/appinsightsEvents.ts index 2a2d9227..f72a0dae 100644 --- a/utils/appinsightsEvents.ts +++ b/utils/appinsightsEvents.ts @@ -68,16 +68,20 @@ export const trackUserDataDownloadEvent = ( export const trackUserDataDownloadException = ( eventName: string, exception: Error, - userDataProcessing: UserDataProcessing + userDataProcessing: UserDataProcessing, + context: IOrchestrationFunctionContext, + isSampled: boolean = true ) => trackException({ exception, properties: { [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId, + isReplay: context.df.isReplaying, name: `user.data.download.${eventName}` }, tagOverrides: { "ai.operation.id": userDataProcessing.userDataProcessingId, - "ai.operation.parentId": userDataProcessing.userDataProcessingId + "ai.operation.parentId": userDataProcessing.userDataProcessingId, + samplingEnabled: String(isSampled) } }); From 0d6d0f3dcee05b5395d342040ab5bd8b0cc3db60 Mon Sep 17 00:00:00 2001 From: arcogabbo Date: Mon, 25 Nov 2024 16:52:08 +0100 Subject: [PATCH 3/3] fix: avoid sending multiple exceptions --- utils/appinsightsEvents.ts | 58 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/utils/appinsightsEvents.ts b/utils/appinsightsEvents.ts index f72a0dae..b334a5d4 100644 --- a/utils/appinsightsEvents.ts +++ b/utils/appinsightsEvents.ts @@ -33,19 +33,22 @@ export const trackUserDataDeleteException = ( context: IOrchestrationFunctionContext, isSampled: boolean = true ) => - trackException({ - exception, - properties: { - [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId, - isReplay: context.df.isReplaying, - name: `user.data.delete.${eventName}` - }, - tagOverrides: { - "ai.operation.id": userDataProcessing.userDataProcessingId, - "ai.operation.parentId": userDataProcessing.userDataProcessingId, - samplingEnabled: String(isSampled) - } - }); + // avoiding duplicate exceptions + context.df.isReplaying + ? void 0 + : trackException({ + exception, + properties: { + [USER_DATA_PROCESSING_ID_KEY]: + userDataProcessing.userDataProcessingId, + name: `user.data.delete.${eventName}` + }, + tagOverrides: { + "ai.operation.id": userDataProcessing.userDataProcessingId, + "ai.operation.parentId": userDataProcessing.userDataProcessingId, + samplingEnabled: String(isSampled) + } + }); // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const trackUserDataDownloadEvent = ( @@ -72,16 +75,19 @@ export const trackUserDataDownloadException = ( context: IOrchestrationFunctionContext, isSampled: boolean = true ) => - trackException({ - exception, - properties: { - [USER_DATA_PROCESSING_ID_KEY]: userDataProcessing.userDataProcessingId, - isReplay: context.df.isReplaying, - name: `user.data.download.${eventName}` - }, - tagOverrides: { - "ai.operation.id": userDataProcessing.userDataProcessingId, - "ai.operation.parentId": userDataProcessing.userDataProcessingId, - samplingEnabled: String(isSampled) - } - }); + // avoiding duplicate exceptions + context.df.isReplaying + ? void 0 + : trackException({ + exception, + properties: { + [USER_DATA_PROCESSING_ID_KEY]: + userDataProcessing.userDataProcessingId, + name: `user.data.download.${eventName}` + }, + tagOverrides: { + "ai.operation.id": userDataProcessing.userDataProcessingId, + "ai.operation.parentId": userDataProcessing.userDataProcessingId, + samplingEnabled: String(isSampled) + } + });