From b0a8af6e6d27bca692c7e1f1efa3f6379ede1106 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Thu, 14 Jul 2022 14:33:44 -0700 Subject: [PATCH 1/2] feat: adding extra keys based on extra-config in input event --- .../snpp_level1a_generate_cmr/generate_cmr.py | 53 +++++++++++++------ setup.py | 2 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py b/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py index 205d3f57..b7b43a92 100644 --- a/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py +++ b/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py @@ -95,6 +95,12 @@ "required": [ "meta" ] + }, + "extra_config": { + "required": [], + "properties": { + "add_extra_keys": {"type": "boolean"} + } } }, "required": [] @@ -141,6 +147,37 @@ def __read_pds_metadata_file(self): self.__s3.target_key = self._pds_file_dict['key'] return self.__s3.read_small_txt_file() + def __is_adding_extra_keys(self): + if 'extra_config' not in self.__event['cma']: + return True + if 'add_extra_keys' not in self.__event['cma']['extra_config']: + return True + return self.__event['cma']['extra_config']['add_extra_keys'] + + def __generate_output_dict(self, echo_metadata_md5: str): + output_dict = { + + + + "checksumType": "md5", + "checksum": echo_metadata_md5, + "type": "metadata", + + "key": self.__s3.target_key, + "fileName": os.path.basename(self.__s3.target_key), + "bucket": self.__s3.target_bucket, + "size": int(self.__s3.get_size()), + } + if not self.__is_adding_extra_keys(): + return output_dict + output_dict = {**output_dict, **{ + "path": os.path.dirname(self.__s3.target_key), + "name": os.path.basename(self.__s3.target_key), + "source_bucket": self.__s3.target_bucket, + "url_path": f's3://{self.__s3.target_bucket}/{self.__s3.target_key}', + }} + return output_dict + def start(self): """ sample event @@ -371,21 +408,7 @@ def start(self): "granuleId": granule_id, "dataType": collection_name, "version": collection_version, - "files": self.__input_file_list + [{ - - "path": os.path.dirname(self.__s3.target_key), - "checksumType": "md5", - "checksum": echo_metadata_md5, - "type": "metadata", - - "key": self.__s3.target_key, - "name": os.path.basename(self.__s3.target_key), - "fileName": os.path.basename(self.__s3.target_key), - "bucket": self.__s3.target_bucket, - "source_bucket": self.__s3.target_bucket, - "url_path": f's3://{self.__s3.target_bucket}/{self.__s3.target_key}', - "size": int(self.__s3.get_size()), - }], + "files": self.__input_file_list + [self.__generate_output_dict(echo_metadata_md5)], # "files": self.__input_file_list, "sync_granule_duration": 20302, "createdAt": TimeUtils.get_current_unix_milli(), diff --git a/setup.py b/setup.py index 68156118..ad08c8a1 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="cumulus_lambda_functions", - version="1.5.14", + version="1.5.15", packages=find_packages(), install_requires=install_requires, tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage'], From 89425aa841228e81c8d4cb3052a207e9f31a1793 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Thu, 14 Jul 2022 14:34:58 -0700 Subject: [PATCH 2/2] chore: reformat code --- .../snpp_level1a_generate_cmr/generate_cmr.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py b/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py index b7b43a92..56ab5a04 100644 --- a/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py +++ b/cumulus_lambda_functions/snpp_level1a_generate_cmr/generate_cmr.py @@ -156,18 +156,15 @@ def __is_adding_extra_keys(self): def __generate_output_dict(self, echo_metadata_md5: str): output_dict = { + "checksumType": "md5", + "checksum": echo_metadata_md5, + "type": "metadata", - - - "checksumType": "md5", - "checksum": echo_metadata_md5, - "type": "metadata", - - "key": self.__s3.target_key, - "fileName": os.path.basename(self.__s3.target_key), - "bucket": self.__s3.target_bucket, - "size": int(self.__s3.get_size()), - } + "key": self.__s3.target_key, + "fileName": os.path.basename(self.__s3.target_key), + "bucket": self.__s3.target_bucket, + "size": int(self.__s3.get_size()), + } if not self.__is_adding_extra_keys(): return output_dict output_dict = {**output_dict, **{