Merge pull request #95 from NHSDigital/mm-mesh-1761-update-to-use-mes…

…h-python-client mesh-1761: use the mesh python client to abstract this from the raw http interface (somewhat)
NHSDigital · Dec 8, 2023 · ce44db3 · ce44db3
2 parents 7617cf0 + 66907f4
commit ce44db3
Show file tree

Hide file tree

Showing 17 changed files with 1,464 additions and 1,351 deletions.
diff --git a/.gitallowed b/.gitallowed
@@ -4,4 +4,8 @@
 mesh_sandbox/store/data/mailboxes.jsonl.*password.*
 "account": "123456789012"
 "accountId": "123456789012"
-123456789012:role
+123456789012:role
+
+mesh_client_aws_serverless/mesh_mailbox.py:[0-9]+:\s*password=self.params\[MeshMailbox.MAILBOX_PASSWORD\]
+tests/mesh_testing_common.py:[0-9]+:\s*-----BEGIN CERTIFICATE-----
+tests/mesh_testing_common.py:[0-9]+:\s*-----(BEGIN|END) PRIVATE KEY-----
diff --git a/.tool-versions b/.tool-versions
@@ -1,4 +1,3 @@
-docker-compose-v1 2.17.3
 python 3.9.12
 poetry 1.5.1
 terraform 1.4.5

diff --git a/CHANGE-LOG.md b/CHANGE-LOG.md
@@ -0,0 +1,13 @@
+Release Notes
+=============
+
+These are not all encompassing, but we will try and capture noteable differences here.
+
+----
+# 1.0
+### v1.0 release includes some significant changes, attempting to capture major differences here
+* migrated core API interactions to use the official [Python MESH Client](https://github.com/NHSDigital/mesh-client), which sends [application/vnd.mesh.v2+json](https://digital.nhs.uk/developer/api-catalogue/message-exchange-for-social-care-and-health-api)
+* as a result of the move to v2 MESH api features there will be some slight differences:
+  * message status headers value will be lowercase status: `accepted`, `acknowledged`, rather than capitalised `Accepted`, `Acknowledged` and so forth.
+  * mex header names are all lower case  ( though requests Response.headers is a CaseInsensitiveDict so this should not matter )
+  * mex-* metadata will be stored with in the s3 object metadata for a received object.
diff --git a/README.md b/README.md
@@ -1,54 +1,23 @@
-# mesh-client-aws-serverless
+# terraform-aws-mesh-client
 
-Common code for MESH AWS serverless client, built to spec and tested by NHS Digital Solutions Assurance, using the
+Common code for MESH AWS serverless client, built and tested by NHS England
 
-## Installation
+Release Notes
+------------
+see [CHANGE-LOG](CHANGE-LOG.md) for news on major changes
 
-Simply add the pre-built package to your python environment.
-
-The latest version can be obtained with the following curl command if your system has it present:
-
-```
-package_version=$(curl -SL https://github.com/NHSDigital/mesh-client-aws-serverless/releases/latest | grep -Po 'Release v\K(\d+.\d+.\d+)' | head -n1)
-```
-
-Or you can set a specific version:
-
-```
-package_version="0.0.1"
-```
-
-Alternatively the main page of this repo will display the latest version i.e. 0.2.3, and previous versions can be searched, which you can substitute in place of `${package_version}` in the below commands.
-
-### PIP
-
-```
-pip install https://github.com/NHSDigital/mesh-client-aws-serverless/releases/download/v${package_version}/mesh_client_aws_serverless-${package_version}-py3-none-any.whl
-```
-
-### requirements.txt
-
-```
-https://github.com/NHSDigital/mesh-client-aws-serverless/releases/download/v${package_version}/mesh_client_aws_serverless-${package_version}-py3-none-any.whl
-```
-
-### Poetry
-
-```
-poetry add https://github.com/NHSDigital/mesh-client-aws-serverless/releases/download/v${package_version}/mesh_client_aws_serverless-${package_version}-py3-none-any.whl
-```
 
 ## Usage
 
-# Mesh Lambdas
+# MESH Lambdas
 
-A terraform module to provide AWS infrastructure capable of sending and recieving Mesh messages
+A terraform module to provide AWS infrastructure capable of sending and receiving MESH messages
 
 ## Configuration
 
 Example configuration required to use this module:
 
-find the release you want from https://github.com/NHSDigital/terraform-aws-mesh-client/releases and substitute in the module version below ... e.g. ref=v0.2.1
+find the release you want from https://github.com/NHSDigital/terraform-aws-mesh-client/releases and substitute in the module version below ... e.g. ref=v1.0.1
 
 ```
 module "mesh" {
@@ -79,13 +48,13 @@ module "mesh" {
 }
 ```
 
-Release versions will be pushed to Github as git tags, with the format `v<major>.<minor>.<patch>` such as `v0.0.1`
+Release versions will be pushed to Github as git tags, with the format `v<major>.<minor>.<patch>` such as `v1.0.1`
 
 ## Tagging
 
 We do not tag any resources created by this module, to configure tags across all supported resources, use the provider level default tags
 
-Below is an example passing in Spines prefferred tags:
+Below is an example passing in Spines preferred tags:
 
 ```
 provider "aws" {

diff --git a/mesh_client_aws_serverless/mesh_common.py b/mesh_client_aws_serverless/mesh_common.py
@@ -2,6 +2,8 @@
 import json
 import os
 from collections import namedtuple
+from typing import Optional
+from urllib.parse import quote_plus
 
 from mypy_boto3_ssm.type_defs import ParameterTypeDef
 from nhs_aws_helpers import secrets_client, ssm_client, stepfunctions
@@ -23,6 +25,13 @@ def __init__(self, msg=None):
         self.msg = msg
 
 
+def nullsafe_quote(value: Optional[str]) -> str:
+    if not value:
+        return ""
+
+    return quote_plus(value, encoding="utf-8")
+
+
 class MeshCommon:
     """Common"""
 

diff --git a/mesh_client_aws_serverless/mesh_fetch_message_chunk_application.py b/mesh_client_aws_serverless/mesh_fetch_message_chunk_application.py
@@ -9,11 +9,38 @@
 from botocore.exceptions import ClientError
 from nhs_aws_helpers import s3_client
 from requests import Response
+from requests.structures import CaseInsensitiveDict
 from spine_aws_common import LambdaApplication
 
-from mesh_client_aws_serverless.mesh_common import MeshCommon
+from mesh_client_aws_serverless.mesh_common import MeshCommon, nullsafe_quote
 from mesh_client_aws_serverless.mesh_mailbox import MeshMailbox
 
+_METADATA_HEADERS = {
+    "mex-to",
+    "mex-from",
+    "mex-workflowid",
+    "mex-filename",
+    "mex-subject",
+    "mex-localid",
+    "mex-partnerid",
+    "mex-messagetype",
+    "mex-statuscode",
+    "mex-statusdescription",
+    "mex-statussuccess",
+}
+
+
+def metadata_from_headers(headers: CaseInsensitiveDict) -> dict[str, str]:
+    return {
+        mex: nullsafe_quote(headers.get(mex))
+        for mex in _METADATA_HEADERS
+        if mex in headers
+    }
+
+
+def get_content_type(response: Response) -> str:
+    return response.headers.get("Content-Type") or "application/octet-stream"
+
 
 class MeshFetchMessageChunkApplication(LambdaApplication):
     """
@@ -28,7 +55,6 @@ def __init__(self, additional_log_config=None, load_ssm_params=False):
         Init variables
         """
         super().__init__(additional_log_config, load_ssm_params)
-        self._mailbox: Optional[MeshMailbox] = None
         self.input = {}
         self.environment = os.environ.get("Environment", "default")
         self.chunk_size = os.environ.get("CHUNK_SIZE", MeshCommon.DEFAULT_CHUNK_SIZE)
@@ -60,12 +86,6 @@ def initialise(self):
         self.message_id = self.input["message_id"]
         self.response = self.event.raw_event
         self.log_object.internal_id = self.internal_id
-        self._setup_mailbox()
-
-    def _setup_mailbox(self):
-        self._mailbox = MeshMailbox(
-            self.log_object, self.input["dest_mailbox"], self.environment
-        )
 
     @property
     def http_response(self) -> Response:
@@ -74,11 +94,6 @@ def http_response(self) -> Response:
         ), "http_response not initialised, call start"
         return self._http_response
 
-    @property
-    def mailbox(self) -> MeshMailbox:
-        assert self._mailbox is not None, "mailbox not initialised, call initialise"
-        return self._mailbox
-
     def start(self):
         self.log_object.write_log(
             "MESHFETCH0001",
@@ -87,25 +102,28 @@ def start(self):
                 "message_id": self.message_id,
             },
         )
-        # get stream for this chunk
-        self._http_response = self.mailbox.get_chunk(
-            self.message_id, chunk_num=self.current_chunk
-        )
-        self.number_of_chunks = self._get_number_of_chunks()
-        self.http_response.raise_for_status()
-        self.chunked = (
-            self.http_response.status_code == HTTPStatus.PARTIAL_CONTENT.value
-        )
-        self._get_aws_bucket_and_key()
+        with MeshMailbox(
+            self.log_object, self.input["dest_mailbox"], self.environment
+        ) as mailbox:
+            # get stream for this chunk
+            self._http_response = mailbox.get_chunk(
+                self.message_id, chunk_num=self.current_chunk
+            )
+            self.number_of_chunks = self._get_number_of_chunks()
+            self.http_response.raise_for_status()
+            self.chunked = (
+                self.http_response.status_code == HTTPStatus.PARTIAL_CONTENT.value
+            )
+            self._get_aws_bucket_and_key(mailbox)
 
-        if self.http_response.headers.get("Mex-Messagetype") == "REPORT":
-            self._handle_report_message()
-        elif self.number_of_chunks == 1:
-            self._handle_single_chunk_message()
-        else:
-            self._handle_multiple_chunk_message()
+            if self.http_response.headers.get("Mex-Messagetype") == "REPORT":
+                self._handle_report_message(mailbox)
+            elif self.number_of_chunks == 1:
+                self._handle_single_chunk_message(mailbox)
+            else:
+                self._handle_multiple_chunk_message(mailbox)
 
-    def _handle_multiple_chunk_message(self):
+    def _handle_multiple_chunk_message(self, mailbox: MeshMailbox):
         self.log_object.write_log(
             "MESHFETCH0013", None, {"message_id": self.message_id}
         )
@@ -123,7 +141,7 @@ def _handle_multiple_chunk_message(self):
         last_chunk = self._is_last_chunk(self.current_chunk)
         if last_chunk:
             self._finish_multipart_upload()
-            self.mailbox.acknowledge_message(self.message_id)
+            mailbox.acknowledge_message(self.message_id)
             self.log_object.write_log(
                 "MESHFETCH0004", None, {"message_id": self.message_id}
             )
@@ -134,21 +152,26 @@ def _handle_multiple_chunk_message(self):
                 None,
                 {"chunk": self.current_chunk, "message_id": self.message_id},
             )
-        self._update_response_and_mailbox_cleanup(complete=last_chunk)
+        self._update_response(complete=last_chunk)
 
-    def _handle_single_chunk_message(self):
+    def _handle_single_chunk_message(self, mailbox: MeshMailbox):
         self.log_object.write_log(
             "MESHFETCH0011", None, {"message_id": self.message_id}
         )
         chunk_data = self.http_response.raw.read(decode_content=True)
-        self._upload_to_s3(chunk_data, s3_key=self.s3_key)
-        self.mailbox.acknowledge_message(self.message_id)
-        self._update_response_and_mailbox_cleanup(complete=True)
+        self._upload_to_s3(
+            chunk_data,
+            s3_key=self.s3_key,
+            content_type=get_content_type(self.http_response),
+            metadata=metadata_from_headers(self.http_response.headers),
+        )
+        mailbox.acknowledge_message(self.message_id)
+        self._update_response(complete=True)
         self.log_object.write_log(
             "MESHFETCH0012", None, {"message_id": self.message_id}
         )
 
-    def _handle_report_message(self):
+    def _handle_report_message(self, mailbox: MeshMailbox):
         self.log_object.write_log(
             "MESHFETCH0010",
             None,
@@ -160,9 +183,14 @@ def _handle_report_message(self):
         )
         buffer = json.dumps(dict(self.http_response.headers)).encode("utf-8")
         self.http_headers_bytes_read = len(buffer)
-        self._upload_to_s3(buffer, s3_key=self.s3_key)
-        self.mailbox.acknowledge_message(self.message_id)
-        self._update_response_and_mailbox_cleanup(complete=True)
+        self._upload_to_s3(
+            buffer,
+            s3_key=self.s3_key,
+            content_type="application/json",
+            metadata=metadata_from_headers(self.http_response.headers),
+        )
+        mailbox.acknowledge_message(self.message_id)
+        self._update_response(complete=True)
         self.log_object.write_log(
             "MESHFETCH0012", None, {"message_id": self.message_id}
         )
@@ -179,11 +207,9 @@ def _get_filename(self):
 
         return f"{self.message_id}.dat"
 
-    def _get_aws_bucket_and_key(self):
-        self.s3_bucket = self.mailbox.params["INBOUND_BUCKET"]
-        s3_folder = (
-            (self.mailbox.params.get("INBOUND_FOLDER", "") or "").strip().rstrip("/")
-        )
+    def _get_aws_bucket_and_key(self, mailbox: MeshMailbox):
+        self.s3_bucket = mailbox.params["INBOUND_BUCKET"]
+        s3_folder = (mailbox.params.get("INBOUND_FOLDER", "") or "").strip().rstrip("/")
         if len(s3_folder) > 0:
             s3_folder += "/"
         file_name = self._get_filename()
@@ -294,11 +320,21 @@ def _upload_part_to_s3(self, buffer):
         )
         return etag
 
-    def _upload_to_s3(self, buffer, s3_key):
+    def _upload_to_s3(
+        self,
+        buffer,
+        s3_key,
+        content_type: Optional[str] = None,
+        metadata: Optional[dict[str, str]] = None,
+    ):
+        metadata = metadata or {}
+        content_type = content_type or "application/octet-stream"
         self.s3_client.put_object(
             Bucket=self.s3_bucket,
             Key=s3_key,
             Body=buffer,
+            ContentType=content_type,
+            Metadata=metadata,
         )
         self.log_object.write_log(
             "MESHFETCH0002a",
@@ -326,6 +362,8 @@ def _create_multipart_upload(self):
             multipart_upload = self.s3_client.create_multipart_upload(
                 Bucket=self.s3_bucket,
                 Key=self.s3_key,
+                Metadata=metadata_from_headers(self.http_response.headers),
+                ContentType=get_content_type(self.http_response),
             )
             self.aws_upload_id = multipart_upload["UploadId"]
             self.log_object.write_log(
@@ -386,7 +424,7 @@ def _finish_multipart_upload(self):
             )
             raise e
 
-    def _update_response_and_mailbox_cleanup(self, complete: bool):
+    def _update_response(self, complete: bool):
         self.response.update({"statusCode": self.http_response.status_code})
         self.response["body"].update(
             {
@@ -399,7 +437,6 @@ def _update_response_and_mailbox_cleanup(self, complete: bool):
                 "file_name": self._get_filename(),
             }
         )
-        self.mailbox.clean_up()
 
     def _read_bytes_into_buffer(self):
         part_buffer = b""