[PRDP-186] Implement AES encryption and PDV Tokenizer

.github/workflows/integration_test.yml

@@ -45,7 +45,7 @@ jobs:
           container_app_environment_name: ${{ vars.CONTAINER_APP_ENVIRONMENT_NAME }}
           resource_group_name: ${{ vars.CONTAINER_APP_ENVIRONMENT_RESOURCE_GROUP_NAME }} # RG of the runner
           pat_token: ${{ secrets.BOT_TOKEN_GITHUB }}
-          self_hosted_runner_image_tag: "v1.6.0"
+          # self_hosted_runner_image_tag: "v1.6.0"
 
   integration_test:
     needs: [ create_runner ]

README.md

@@ -68,21 +68,29 @@ On terminal type:
 then replace env variables with correct values
 (if there is NO default value, the variable HAS to be defined)
 
-| VARIABLE                          | USAGE                                                                            |      DEFAULT VALUE      |
-|-----------------------------------|----------------------------------------------------------------------------------|:-----------------------:|
-| `RECEIPTS_STORAGE_CONN_STRING`    | Connection string to the Receipt Storage                                         |                         |
-| `RECEIPT_QUEUE_TOPIC`             | Topic name of the Receipt Queue                                                  |                         |
-| `RECEIPT_QUEUE_MAX_RETRY`         | Number of retry to complete the generation process before being tagged as FAILED |           "5"           |
-| `RECEIPT_QUEUE_TOPIC_POISON`      | Topic name of the Receipt Poison Queue                                           |                         |
-| `BLOB_STORAGE_ACCOUNT_ENDPOINT`   | Endpoint to the Receipt Blob Storage                                             |                         |
-| `BLOB_STORAGE_CONTAINER_NAME`     | Container name of the Receipt container in the Blob Storage                      |                         |
-| `COSMOS_RECEIPTS_CONN_STRING`     | Connection string to the Receipt CosmosDB                                        |                         |
-| `COSMOS_RECEIPT_SERVICE_ENDPOINT` | Endpoint to the Receipt CosmosDB                                                 |                         |
-| `COSMOS_RECEIPT_KEY`              | Key to the Receipt CosmosDB                                                      |                         |
-| `COSMOS_RECEIPT_DB_NAME`          | Database name of the Receipt database in CosmosDB                                |                         |
-| `COSMOS_RECEIPT_CONTAINER_NAME`   | Container name of the Receipt container in CosmosDB                              |                         |
-| `PDF_ENGINE_ENDPOINT`             | Endpoint to the PDF engine                                                       |                         |
-| `OCP_APIM_SUBSCRIPTION_KEY`       | Auth key for Azure to access the PDF Engine                                      |                         |
+| VARIABLE                              | USAGE                                                                            |                     DEFAULT VALUE                      |
+|---------------------------------------|----------------------------------------------------------------------------------|:------------------------------------------------------:|
+| `RECEIPTS_STORAGE_CONN_STRING`        | Connection string to the Receipt Storage                                         |                                                        |
+| `RECEIPT_QUEUE_TOPIC`                 | Topic name of the Receipt Queue                                                  |                                                        |
+| `RECEIPT_QUEUE_MAX_RETRY`             | Number of retry to complete the generation process before being tagged as FAILED |                          "5"                           |
+| `RECEIPT_QUEUE_TOPIC_POISON`          | Topic name of the Receipt Poison Queue                                           |                                                        |
+| `BLOB_STORAGE_ACCOUNT_ENDPOINT`       | Endpoint to the Receipt Blob Storage                                             |                                                        |
+| `BLOB_STORAGE_CONTAINER_NAME`         | Container name of the Receipt container in the Blob Storage                      |                                                        |
+| `COSMOS_RECEIPTS_CONN_STRING`         | Connection string to the Receipt CosmosDB                                        |                                                        |
+| `COSMOS_RECEIPT_SERVICE_ENDPOINT`     | Endpoint to the Receipt CosmosDB                                                 |                                                        |
+| `COSMOS_RECEIPT_KEY`                  | Key to the Receipt CosmosDB                                                      |                                                        |
+| `COSMOS_RECEIPT_DB_NAME`              | Database name of the Receipt database in CosmosDB                                |                                                        |
+| `COSMOS_RECEIPT_CONTAINER_NAME`       | Container name of the Receipt container in CosmosDB                              |                                                        |
+| `PDF_ENGINE_ENDPOINT`                 | Endpoint to the PDF engine                                                       |                                                        |
+| `OCP_APIM_SUBSCRIPTION_KEY`           | Auth key for Azure to access the PDF Engine                                      |                                                        |
+| `PDV_TOKENIZER_BASE_PATH`             | PDV Tokenizer API base path                                                      | "https://api.uat.tokenizer.pdv.pagopa.it/tokenizer/v1" |
+| `PDV_TOKENIZER_SEARCH_TOKEN_ENDPOINT` | PDV Tokenizer API search token endpoint                                          |                    "/tokens/search"                    |
+| `PDV_TOKENIZER_FIND_PII_ENDPOINT`     | PDV Tokenizer API find pii endpoint                                              |                    "/tokens/%s/pii"                    |
+| `PDV_TOKENIZER_CREATE_TOKEN_ENDPOINT` | PDV Tokenizer API create token endpoint                                          |                       "/tokens"                        |
+| `PDV_TOKENIZER_SUBSCRIPTION_KEY`      | Tokenizer API azure ocp apim subscription key                                    |                                                        |
+| `TOKENIZER_APIM_HEADER_KEY`           | Tokenizer APIM header key                                                        |                       x-api-key                        |
+| `AES_SECRET_KEY`                      | AES encryption secret key                                                        |                                                        |
+| `AES_SALT`                            | AES encryption salt                                                              |                                                        |
 
 > to doc details about AZ fn config
 > see [here](https://stackoverflow.com/questions/62669672/azure-functions-what-is-the-purpose-of-having-host-json-and-local-settings-jso)

helm/values-dev.yaml

@@ -91,6 +91,7 @@ microservice-chart:
     BLOB_STORAGE_ACCOUNT_ENDPOINT: "https://pagopadweureceiptsfnsa.blob.core.windows.net/"
     BLOB_STORAGE_CONTAINER_NAME: "pagopa-d-weu-receipts-azure-blob-receipt-st-attach"
     WORKING_DIRECTORY_PATH: "/temp"
+    PDV_TOKENIZER_BASE_PATH: "https://api.uat.tokenizer.pdv.pagopa.it/tokenizer/v1"
     ENABLE_ECS_CONSOLE: "true"
     CONSOLE_LOG_THRESHOLD: "DEBUG"
     CONSOLE_LOG_PATTERN: "%d{HH:mm:ss.SSS}[%thread]%-5level%logger{36}-%msg%n"
@@ -113,6 +114,9 @@ microservice-chart:
     OCP_APIM_SUBSCRIPTION_KEY: "shared-apim-d-subscription-key"
     COSMOS_RECEIPT_KEY: "cosmos-receipt-pkey"
     OTEL_EXPORTER_OTLP_HEADERS: 'elastic-otl-secret-token'
+    PDV_TOKENIZER_SUBSCRIPTION_KEY: "tokenizer-api-key"
+    AES_SECRET_KEY: "aes-secret-key"
+    AES_SALT: "aes-salt"
   keyvault:
     name: "pagopa-d-receipts-kv"
     tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d"

helm/values-prod.yaml

@@ -91,6 +91,7 @@ microservice-chart:
     BLOB_STORAGE_ACCOUNT_ENDPOINT: "https://pagopapweureceiptsfnsa.blob.core.windows.net"
     BLOB_STORAGE_CONTAINER_NAME: "pagopa-p-weu-receipts-azure-blob-receipt-st-attach"
     WORKING_DIRECTORY_PATH: "/temp"
+    PDV_TOKENIZER_BASE_PATH: "https://api.tokenizer.pdv.pagopa.it"
     ENABLE_ECS_CONSOLE: "true"
     CONSOLE_LOG_THRESHOLD: "DEBUG"
     CONSOLE_LOG_PATTERN: "%d{HH:mm:ss.SSS}[%thread]%-5level%logger{36}-%msg%n"
@@ -113,6 +114,9 @@ microservice-chart:
     OCP_APIM_SUBSCRIPTION_KEY: "shared-apim-p-subscription-key"
     COSMOS_RECEIPT_KEY: "cosmos-receipt-pkey"
     OTEL_EXPORTER_OTLP_HEADERS: "elastic-otl-secret-token"
+    PDV_TOKENIZER_SUBSCRIPTION_KEY: "tokenizer-api-key"
+    AES_SECRET_KEY: "aes-secret-key"
+    AES_SALT: "aes-salt"
   keyvault:
     name: "pagopa-p-receipts-kv"
     tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d"

helm/values-uat.yaml

@@ -91,6 +91,7 @@ microservice-chart:
     BLOB_STORAGE_ACCOUNT_ENDPOINT: "https://pagopauweureceiptsfnsa.blob.core.windows.net"
     BLOB_STORAGE_CONTAINER_NAME: "pagopa-u-weu-receipts-azure-blob-receipt-st-attach"
     WORKING_DIRECTORY_PATH: "/temp"
+    PDV_TOKENIZER_BASE_PATH: "https://api.uat.tokenizer.pdv.pagopa.it/tokenizer/v1"
     ENABLE_ECS_CONSOLE: "true"
     CONSOLE_LOG_THRESHOLD: "DEBUG"
     CONSOLE_LOG_PATTERN: "%d{HH:mm:ss.SSS}[%thread]%-5level%logger{36}-%msg%n"
@@ -113,6 +114,9 @@ microservice-chart:
     OCP_APIM_SUBSCRIPTION_KEY: "shared-apim-u-subscription-key"
     COSMOS_RECEIPT_KEY: "cosmos-receipt-pkey"
     OTEL_EXPORTER_OTLP_HEADERS: "elastic-otl-secret-token"
+    PDV_TOKENIZER_SUBSCRIPTION_KEY: "tokenizer-api-key"
+    AES_SECRET_KEY: "aes-secret-key"
+    AES_SALT: "aes-salt"
   keyvault:
     name: "pagopa-u-receipts-kv"
     tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d"

integration-test/src/step_definitions/common.js

@@ -1,3 +1,6 @@
+const FISCAL_CODE = "AAAAAA00A00A000A";
+const TOKENIZED_FISCAL_CODE = "cd07268c-73e8-4df4-8305-a35085e32eff";
+
 function sleep(ms) {
 	return new Promise(resolve => setTimeout(resolve, ms));
 }
@@ -43,7 +46,7 @@ function createEventForPoisonQueue(id, attemptedPoisonRetry) {
 		"debtor": {
 			"fullName": "paGetPaymentName",
 			"entityUniqueIdentifierType": "G",
-			"entityUniqueIdentifierValue": "JHNDOE00A01F205N",
+			"entityUniqueIdentifierValue": FISCAL_CODE,
 			"streetName": "paGetPaymentStreet",
 			"civicNumber": "paGetPayment99",
 			"postalCode": "20155",
@@ -55,7 +58,7 @@ function createEventForPoisonQueue(id, attemptedPoisonRetry) {
 		"payer": {
 			"fullName": "name",
 			"entityUniqueIdentifierType": "G",
-			"entityUniqueIdentifierValue": "JHNDOE00A01F205S",
+			"entityUniqueIdentifierValue": FISCAL_CODE,
 			"streetName": "street",
 			"civicNumber": "civic",
 			"postalCode": "postal",
@@ -98,7 +101,7 @@ function createEventForPoisonQueue(id, attemptedPoisonRetry) {
 			"user": {
 				"fullName": "John Doe",
 				"type": "F",
-				"fiscalCode": "JHNDOE00A01F205N",
+				"fiscalCode": FISCAL_CODE,
 				"notificationEmail": "john.doe@mail.it",
 				"userId": "1234",
 				"userStatus": "11",
@@ -128,8 +131,8 @@ function createReceipt(id) {
 	{
 		"eventId": id,
 		"eventData": {
-			"payerFiscalCode": "JHNDOE00A01F205N",
-			"debtorFiscalCode": "JHNDOE00A01F205N",
+			"payerFiscalCode": TOKENIZED_FISCAL_CODE,
+			"debtorFiscalCode": TOKENIZED_FISCAL_CODE,
 			"amount": "200",
 			"cart": [
 				{

integration-test/src/step_definitions/receipt_pdf_generator_step.js

@@ -1,8 +1,8 @@
 const assert = require('assert');
 const { After, Given, When, Then, setDefaultTimeout } = require('@cucumber/cucumber');
 const { sleep, createEventForQueue, createEventForPoisonQueue } = require("./common");
-const { getDocumentByIdFromReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastoreByMessagePayload, deleteDocumentFromReceiptsDatastore, createDocumentInReceiptsDatastore, createDocumentInErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastore, getDocumentByMessagePayloadFromErrorReceiptsDatastore } = require("./receipts_datastore_client");
-const { putMessageOnPoisonQueue, putMessageOnReceiptQueue } = require("./reqeipt_queue_client");
+const { getDocumentByIdFromReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastoreByBizEventId, deleteDocumentFromReceiptsDatastore, createDocumentInReceiptsDatastore, createDocumentInErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastore, getDocumentByBizEventIdFromErrorReceiptsDatastore } = require("./receipts_datastore_client");
+const { putMessageOnPoisonQueue, putMessageOnReceiptQueue } = require("./receipts_queue_client");
 const { receiptPDFExist } = require("./receipts_blob_storage_client");
 
 // set timeout for Hooks function, it allows to wait for long task
@@ -78,14 +78,14 @@ Then('the blob storage has the PDF document', async function () {
 Given('a random biz event with id {string} enqueued on receipts poison queue with poison retry {string}', async function (id, value) {
     let attemptedPoisonRetry = (value === 'true');
     this.event = createEventForPoisonQueue(id, attemptedPoisonRetry);
-    await deleteDocumentFromErrorReceiptsDatastoreByMessagePayload(this.event);
+    await deleteDocumentFromErrorReceiptsDatastoreByBizEventId(id);
     await putMessageOnPoisonQueue(this.event);
 });
 
 When('the biz event has been properly stored on receipt-message-error datastore after {int} ms', async function (time) {
     // boundary time spent by azure function to process event
     await sleep(time);
-    this.responseToCheck = await getDocumentByMessagePayloadFromErrorReceiptsDatastore(this.event);
+    this.responseToCheck = await getDocumentByBizEventIdFromErrorReceiptsDatastore(this.event.id);
 });
 
 Then('the receipt-message-error datastore returns the error receipt', async function () {

integration-test/src/step_definitions/receipts_datastore_client.js

@@ -48,11 +48,11 @@ async function deleteDocumentFromReceiptsDatastore(id, partitionKey) {
 
 // receipts-message-error datastore
 
-async function getDocumentByMessagePayloadFromErrorReceiptsDatastore(messagePayload) {
+async function getDocumentByBizEventIdFromErrorReceiptsDatastore(bizEventId) {
     return await errorReceiptContainer.items
         .query({
-            query: "SELECT * from c WHERE c.messagePayload=@messagePayload",
-            parameters: [{ name: "@messagePayload", value: JSON.stringify(messagePayload) }]
+            query: "SELECT * from c WHERE c.bizEventId=@bizEventId",
+            parameters: [{ name: "@bizEventId", value: JSON.stringify(bizEventId) }]
         })
         .fetchNext();
 }
@@ -61,6 +61,7 @@ async function createDocumentInErrorReceiptsDatastore(id) {
     let event = createEventForPoisonQueue(id, true);
     let payload = {
         "messagePayload": JSON.stringify(event),
+        "bizEventId": id,
         "status": "REVIEWED",
         "id": id,
         "_rid": "Z9AJAJpW0pIhAAAAAAAAAA==",
@@ -86,14 +87,14 @@ async function deleteDocumentFromErrorReceiptsDatastore(id) {
     }
 }
 
-async function deleteDocumentFromErrorReceiptsDatastoreByMessagePayload(messagePayload) {
-    let documents = await getDocumentByMessagePayloadFromErrorReceiptsDatastore(messagePayload);
+async function deleteDocumentFromErrorReceiptsDatastoreByBizEventId(bizEventId) {
+    let documents = await getDocumentByBizEventIdFromErrorReceiptsDatastore(bizEventId);
 
     documents?.resources?.forEach((el) => {
         deleteDocumentFromErrorReceiptsDatastore(el.id);
     })
 }
 
 module.exports = {
-    getDocumentByIdFromReceiptsDatastore, deleteDocumentFromReceiptsDatastoreByEventId, createDocumentInReceiptsDatastore, deleteDocumentFromReceiptsDatastore, getDocumentByMessagePayloadFromErrorReceiptsDatastore, createDocumentInErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastoreByMessagePayload
+    getDocumentByIdFromReceiptsDatastore, deleteDocumentFromReceiptsDatastoreByEventId, createDocumentInReceiptsDatastore, deleteDocumentFromReceiptsDatastore, getDocumentByBizEventIdFromErrorReceiptsDatastore, createDocumentInErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastore, deleteDocumentFromErrorReceiptsDatastoreByBizEventId
 }

performance-test/src/modules/common.js

@@ -1,3 +1,5 @@
+const FISCAL_CODE = "AAAAAA00A00A000A";
+const TOKENIZED_FISCAL_CODE = "cd07268c-73e8-4df4-8305-a35085e32eff";
 
 export function randomString(length, charset) {
     let res = '';
@@ -41,7 +43,7 @@ export function createEvent(id) {
 		"debtor": {
 			"fullName": "paGetPaymentName",
 			"entityUniqueIdentifierType": "G",
-			"entityUniqueIdentifierValue": "JHNDOE00A01F205N",
+			"entityUniqueIdentifierValue": FISCAL_CODE,
 			"streetName": "paGetPaymentStreet",
 			"civicNumber": "paGetPayment99",
 			"postalCode": "20155",
@@ -53,7 +55,7 @@ export function createEvent(id) {
 		"payer": {
 			"fullName": "name",
 			"entityUniqueIdentifierType": "G",
-			"entityUniqueIdentifierValue": "JHNDOE00A01F205S",
+			"entityUniqueIdentifierValue": FISCAL_CODE,
 			"streetName": "street",
 			"civicNumber": "civic",
 			"postalCode": "postal",
@@ -96,7 +98,7 @@ export function createEvent(id) {
 			"user": {
 				"fullName": "John Doe",
 				"type": "F",
-				"fiscalCode": "JHNDOE00A01F205N",
+				"fiscalCode": FISCAL_CODE,
 				"notificationEmail": "john.doe@mail.it",
 				"userId": "1234",
 				"userStatus": "11",
@@ -126,8 +128,8 @@ export function createReceipt(id) {
 	{
 		"eventId": id,
 		"eventData": {
-			"payerFiscalCode": "GENERATOR_PERF_TEST",
-			"debtorFiscalCode": "GENERATOR_PERF_TEST",
+			"payerFiscalCode": TOKENIZED_FISCAL_CODE,
+			"debtorFiscalCode": TOKENIZED_FISCAL_CODE,
 			"amount": "200",
 			"cart": [
 				{

src/main/java/it/gov/pagopa/receipt/pdf/generator/ManageReceiptPoisonQueue.java

@@ -13,7 +13,9 @@
 import it.gov.pagopa.receipt.pdf.generator.entity.event.BizEvent;
 import it.gov.pagopa.receipt.pdf.generator.entity.receipt.ReceiptError;
 import it.gov.pagopa.receipt.pdf.generator.entity.receipt.enumeration.ReceiptErrorStatusType;
+import it.gov.pagopa.receipt.pdf.generator.exception.Aes256Exception;
 import it.gov.pagopa.receipt.pdf.generator.exception.UnableToQueueException;
+import it.gov.pagopa.receipt.pdf.generator.utils.Aes256Utils;
 import it.gov.pagopa.receipt.pdf.generator.utils.ObjectMapperUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,18 +94,30 @@ public void processManageReceiptPoisonQueue(
                  logger.error("[{}] error for the function called at {} when attempting" +
                                  "to requeue BizEvent wit id {}, saving to cosmos for review",
                          context.getFunctionName(), LocalDateTime.now(), bizEvent.getId(), e);
-                saveToDocument(context, errorMessage, documentdb);
+                saveToDocument(context, errorMessage, bizEvent.getId(), documentdb);
             }
         } else {
-            saveToDocument(context, errorMessage, documentdb);
+            saveToDocument(context, errorMessage, bizEvent != null ? bizEvent.getId() : null, documentdb);
         }
     }
 
-    private void saveToDocument(ExecutionContext context, String errorMessage,
+    private void saveToDocument(ExecutionContext context, String errorMessage, String bizEventId,
                                 OutputBinding<ReceiptError> documentdb) {
          logger.info("[{}] saving new entry to the retry error to review with payload {}",
                  context.getFunctionName(), errorMessage);
-        documentdb.setValue(ReceiptError.builder().messagePayload(errorMessage)
-                .status(ReceiptErrorStatusType.TO_REVIEW).build());
+
+         ReceiptError receiptError = ReceiptError.builder()
+                 .bizEventId(bizEventId)
+                 .status(ReceiptErrorStatusType.TO_REVIEW).build();
+
+        try {
+            String encodedEvent = Aes256Utils.encrypt(errorMessage);
+            receiptError.setMessagePayload(encodedEvent);
+
+        } catch (Aes256Exception e) {
+            receiptError.setMessageError(e.getMessage());
+        }
+
+        documentdb.setValue(receiptError);
     }
 }