From c9dd83fc33a39c8a8a1ab124f74f8792122891fe Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 28 Oct 2024 10:11:17 -0700 Subject: [PATCH 01/14] breaking: using latest uds-lib + update docker --- docker/stage-in-stage-out/dc-003-upload.yaml | 9 ++++++--- requirements.txt | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/stage-in-stage-out/dc-003-upload.yaml b/docker/stage-in-stage-out/dc-003-upload.yaml index 77d6653b..7f9a95ab 100644 --- a/docker/stage-in-stage-out/dc-003-upload.yaml +++ b/docker/stage-in-stage-out/dc-003-upload.yaml @@ -23,17 +23,20 @@ services: VERIFY_SSL: 'FALSE' DAPA_API: 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' - COLLECTION_ID: 'NEW_COLLECTION_EXAMPLE_L1B___9' + + PROJECT: 'NAME of the project if collection name in Granule STAC purely includes the name. It will be part of URN:NASA:UNITY:::' + VENUE: 'VENUE of the project if collection name in Granule STAC purely includes the name. It will be part of URN:NASA:UNITY:::' STAGING_BUCKET: 'uds-test-cumulus-staging' + DELETE_FILES: 'FALSE' - RESULT_PATH_PREFIX: 'stage_out/sample_dir (Optional:defaulted to stage_out if missing. NOTE: No need to begin or end with "/". )' + RESULT_PATH_PREFIX: 'stage_out/sample_dir (Optional:defaulted to stage_out if missing. NOTE: No need to begin or end with "/". It also accepts empty string now as default value)' GRANULES_SEARCH_DOMAIN: 'UNITY' GRANULES_UPLOAD_TYPE: 'UPLOAD_S3_BY_STAC_CATALOG' UPLOAD_DIR: '/etc/snpp_upload_test_1. or empty string' OUTPUT_DIRECTORY: '/some/directory' CATALOG_FILE: 'empty string or /path/to/stac/catalog file' - LOG_LEVEL: '20' + LOG_LEVEL: '20 NOTE: 10=DEBUG 20=info. 30=error' networks: - internal networks: diff --git a/requirements.txt b/requirements.txt index f7989aea..986ee5f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jsonschema==4.23.0 jsonschema-specifications==2023.12.1 lark==0.12.0 mangum==0.18.0 -mdps-ds-lib==0.5.1.dev1 +mdps-ds-lib==0.5.1.dev10000 pydantic==2.9.2 pydantic_core==2.23.4 pygeofilter==0.2.4 From d803e28cf348d65609a24049bb557c57f01c40dc Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 28 Oct 2024 12:08:17 -0700 Subject: [PATCH 02/14] feat: use latest uds-lib --- docker/stage-in-stage-out/dc-003-upload.yaml | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/stage-in-stage-out/dc-003-upload.yaml b/docker/stage-in-stage-out/dc-003-upload.yaml index 7f9a95ab..d02baca3 100644 --- a/docker/stage-in-stage-out/dc-003-upload.yaml +++ b/docker/stage-in-stage-out/dc-003-upload.yaml @@ -22,6 +22,7 @@ services: COGNITO_URL: 'ask U-CS. ex: https://cognito-idp.us-west-2.amazonaws.com' VERIFY_SSL: 'FALSE' + DRY_RUN: 'TRUE | FALSE . set to TRUE to confirm if all settings are correct bar aws credentials' DAPA_API: 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' PROJECT: 'NAME of the project if collection name in Granule STAC purely includes the name. It will be part of URN:NASA:UNITY:::' diff --git a/requirements.txt b/requirements.txt index 986ee5f1..c0cca764 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jsonschema==4.23.0 jsonschema-specifications==2023.12.1 lark==0.12.0 mangum==0.18.0 -mdps-ds-lib==0.5.1.dev10000 +mdps-ds-lib==0.5.1.dev010100 pydantic==2.9.2 pydantic_core==2.23.4 pygeofilter==0.2.4 From 7fe0fcec7ce71e5f6d57262674c58744cc1c4a78 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 14:29:01 -0600 Subject: [PATCH 03/14] chore: update log statement --- cumulus_lambda_functions/cumulus_wrapper/query_granules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py index 86eacefb..3720cf6f 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py @@ -150,6 +150,7 @@ def delete_entry(self, private_api_prefix: str, granule_id: str): """ {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} """ + LOGGER.debug(f'json query_result: {query_result}') if query_result['statusCode'] >= 500: LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}') return {'server_error': query_result} @@ -157,7 +158,6 @@ def delete_entry(self, private_api_prefix: str, granule_id: str): LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}') return {'client_error': query_result} query_result = json.loads(query_result['body']) - LOGGER.info(f'json query_result: {query_result}') """ { "detail": "Record deleted" From d73133c00df679b5e43b3a77aac7c24120812441 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 16:57:39 -0600 Subject: [PATCH 04/14] fix: adding facade delete endpoint --- .../uds_api/dapa/granules_dapa_query_es.py | 63 +++++++++++++++++++ .../uds_api/granules_api.py | 42 ++++++++++--- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py index 913c2f02..54663840 100644 --- a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py +++ b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py @@ -1,6 +1,10 @@ +import json +import os from copy import deepcopy +from mdps_ds_lib.lib.aws.aws_lambda import AwsLambda from pystac import Link +from starlette.datastructures import URL from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic from cumulus_lambda_functions.granules_to_es.granules_index_mapping import GranulesIndexMapping @@ -17,6 +21,7 @@ class GranulesDapaQueryEs: def __init__(self, collection_id, limit, offset, input_datetime, filter_input, pagination_link_obj: PaginationLinksGenerator, base_url): + self.__collection_cnm_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip() self.__pagination_link_obj = pagination_link_obj self.__input_datetime = input_datetime self.__collection_id = collection_id @@ -149,6 +154,64 @@ def get_single_granule(self, granule_id): self.__restructure_each_granule_result(each_granules_query_result_stripped) return each_granules_query_result_stripped + def delete_facade(self, current_url: URL): + actual_path = current_url.path + actual_path = actual_path if actual_path.endswith('/') else f'{actual_path}/' + actual_path = f'{actual_path}actual' + LOGGER.info(f'sanity_check') + + actual_event = { + 'resource': actual_path, + 'path': actual_path, + 'httpMethod': 'DELETE', + 'headers': { + 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': 'Bearer xxx', + 'Host': current_url.hostname, 'User-Agent': 'python-requests/2.28.2', + 'X-Amzn-Trace-Id': 'Root=1-64a66e90-6fa8b7a64449014639d4f5b4', 'X-Forwarded-For': '44.236.15.58', + 'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'}, + 'multiValueHeaders': { + 'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': ['Bearer xxx'], + 'Host': [current_url.hostname], 'User-Agent': ['python-requests/2.28.2'], + 'X-Amzn-Trace-Id': ['Root=1-64a66e90-6fa8b7a64449014639d4f5b4'], + 'X-Forwarded-For': ['127.0.0.1'], 'X-Forwarded-Port': ['443'], 'X-Forwarded-Proto': ['https'] + }, + 'queryStringParameters': {}, + 'multiValueQueryStringParameters': {}, + 'pathParameters': {}, + 'stageVariables': None, + 'requestContext': { + 'resourceId': '', + 'authorizer': {'principalId': '', 'integrationLatency': 0}, + 'resourcePath': actual_path, 'httpMethod': 'PUT', + 'extendedRequestId': '', 'requestTime': '', + 'path': actual_path, 'accountId': '', + 'protocol': 'HTTP/1.1', 'stage': '', 'domainPrefix': '', 'requestTimeEpoch': 0, + 'requestId': '', + 'identity': { + 'cognitoIdentityPoolId': None, 'accountId': None, 'cognitoIdentityId': None, 'caller': None, + 'sourceIp': '127.0.0.1', 'principalOrgId': None, 'accessKey': None, + 'cognitoAuthenticationType': None, + 'cognitoAuthenticationProvider': None, 'userArn': None, 'userAgent': 'python-requests/2.28.2', + 'user': None + }, + 'domainName': current_url.hostname, 'apiId': '' + }, + 'body': json.dumps({}), + 'isBase64Encoded': False + } + LOGGER.info(f'actual_event: {actual_event}') + response = AwsLambda().invoke_function( + function_name=self.__collection_cnm_lambda_name, + payload=actual_event, + ) + LOGGER.debug(f'async function started: {response}') + return { + 'statusCode': 202, + 'body': { + 'message': 'processing' + } + } + def start(self): try: granules_query_dsl = self.__generate_es_dsl() diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index 86e8dc23..2fa0e883 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -2,12 +2,10 @@ import os from typing import Union -from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 -from pystac import Item +from starlette.responses import Response, JSONResponse from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery -from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic from cumulus_lambda_functions.uds_api.dapa.daac_archive_crud import DaacArchiveCrud, DaacDeleteModel, DaacAddModel, \ DaacUpdateModel from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query_es import GranulesDapaQueryEs @@ -26,7 +24,6 @@ from fastapi import APIRouter, HTTPException, Request -from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query import GranulesDapaQuery from cumulus_lambda_functions.uds_api.dapa.pagination_links_generator import PaginationLinksGenerator from cumulus_lambda_functions.uds_api.web_service_constants import WebServiceConstants @@ -244,9 +241,9 @@ async def get_single_granule_dapa(request: Request, collection_id: str, granule_ raise HTTPException(status_code=500, detail=str(e)) return granules_result -@router.delete("/{collection_id}/items/{granule_id}") -@router.delete("/{collection_id}/items/{granule_id}/") -async def delete_single_granule_dapa(request: Request, collection_id: str, granule_id: str): +@router.delete("/{collection_id}/items/{granule_id}/actual") +@router.delete("/{collection_id}/items/{granule_id}/actual/") +async def delete_single_granule_dapa_actual(request: Request, collection_id: str, granule_id: str): authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ .get_instance(UDSAuthorizerFactory.cognito, es_url=os.getenv('ES_URL'), @@ -290,6 +287,37 @@ async def delete_single_granule_dapa(request: Request, collection_id: str, granu return {} +@router.delete("/{collection_id}/items/{granule_id}") +@router.delete("/{collection_id}/items/{granule_id}/") +async def delete_single_granule_dapa_facade(request: Request, collection_id: str, granule_id: str, response: Response, response_class=JSONResponse): + authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ + .get_instance(UDSAuthorizerFactory.cognito, + es_url=os.getenv('ES_URL'), + es_port=int(os.getenv('ES_PORT', '443')) + ) + auth_info = FastApiUtils.get_authorization_info(request) + collection_identifier = UdsCollections.decode_identifier(collection_id) + if not authorizer.is_authorized_for_collection(DBConstants.delete, collection_id, + auth_info['ldap_groups'], + collection_identifier.tenant, + collection_identifier.venue): + LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}') + raise HTTPException(status_code=403, detail=json.dumps({ + 'message': 'not authorized to execute this action' + })) + try: + LOGGER.debug(f'deleting granule: {granule_id}') + granules_dapa_query = GranulesDapaQueryEs(collection_id, -1, -1, None, None, None, '') + delete_prep_result = granules_dapa_query.delete_facade(request.url) + except Exception as e: + LOGGER.exception('failed during delete_single_granule_dapa') + raise HTTPException(status_code=500, detail=str(e)) + if delete_prep_result['statusCode'] < 300: + response.status_code = delete_prep_result['statusCode'] + return delete_prep_result['body'] + raise HTTPException(status_code=delete_prep_result['statusCode'], detail=delete_prep_result['body']) + + @router.put("/{collection_id}/archive/{granule_id}") @router.put("/{collection_id}/archive/{granule_id}/") async def archive_single_granule_dapa(request: Request, collection_id: str, granule_id: str): From 55812c509e4c7d91634a77a0eba73e11fe68bc68 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 17:14:49 -0600 Subject: [PATCH 05/14] fix: need to pass token to actual call as well --- .../uds_api/dapa/granules_dapa_query_es.py | 6 +++--- cumulus_lambda_functions/uds_api/granules_api.py | 2 +- tests/integration_tests/test_granules_deletion.py | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py index 54663840..62d025de 100644 --- a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py +++ b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py @@ -154,7 +154,7 @@ def get_single_granule(self, granule_id): self.__restructure_each_granule_result(each_granules_query_result_stripped) return each_granules_query_result_stripped - def delete_facade(self, current_url: URL): + def delete_facade(self, current_url: URL, bearer_token: str): actual_path = current_url.path actual_path = actual_path if actual_path.endswith('/') else f'{actual_path}/' actual_path = f'{actual_path}actual' @@ -165,12 +165,12 @@ def delete_facade(self, current_url: URL): 'path': actual_path, 'httpMethod': 'DELETE', 'headers': { - 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': 'Bearer xxx', + 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': bearer_token, 'Host': current_url.hostname, 'User-Agent': 'python-requests/2.28.2', 'X-Amzn-Trace-Id': 'Root=1-64a66e90-6fa8b7a64449014639d4f5b4', 'X-Forwarded-For': '44.236.15.58', 'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'}, 'multiValueHeaders': { - 'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': ['Bearer xxx'], + 'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': [bearer_token], 'Host': [current_url.hostname], 'User-Agent': ['python-requests/2.28.2'], 'X-Amzn-Trace-Id': ['Root=1-64a66e90-6fa8b7a64449014639d4f5b4'], 'X-Forwarded-For': ['127.0.0.1'], 'X-Forwarded-Port': ['443'], 'X-Forwarded-Proto': ['https'] diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index 2fa0e883..62091165 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -308,7 +308,7 @@ async def delete_single_granule_dapa_facade(request: Request, collection_id: str try: LOGGER.debug(f'deleting granule: {granule_id}') granules_dapa_query = GranulesDapaQueryEs(collection_id, -1, -1, None, None, None, '') - delete_prep_result = granules_dapa_query.delete_facade(request.url) + delete_prep_result = granules_dapa_query.delete_facade(request.url, request.headers.get('Authorization', '')) except Exception as e: LOGGER.exception('failed during delete_single_granule_dapa') raise HTTPException(status_code=500, detail=str(e)) diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index b05acd45..96660bea 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -1,6 +1,7 @@ import base64 import json import os +from time import sleep from unittest import TestCase import requests @@ -32,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'CCC-04' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'CCC-05' # 'uds_collection' # 'sbx_collection' self.collection_version = '08'.replace('.', '') # '2402011200' return @@ -79,10 +80,10 @@ def test_delete_all(self): query_result = requests.delete(url=post_url, headers=headers, ) - self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') + self.assertEqual(query_result.status_code, 202, f'wrong status code. {query_result.text}') response_json = json.loads(query_result.text) print(json.dumps(response_json, indent=4)) - + sleep(30) post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev query_result = requests.get(url=post_url, headers=headers,) self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') From a4d444630ed2d4ac8781845152d2b5f84747544f Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 17:38:31 -0600 Subject: [PATCH 06/14] chore: add log statement --- cumulus_lambda_functions/lib/uds_db/granules_db_index.py | 1 + tests/integration_tests/test_granules_deletion.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py index 24139081..44d18a12 100644 --- a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py +++ b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py @@ -210,6 +210,7 @@ def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ): if result is None: raise ValueError(f"no such granule: {doc_id}") for each_granule in result['hits']['hits']: + LOGGER.debug(f"deleting {each_granule['_id']} from each_granule['_index']") delete_result = self.__es.delete_by_query({ 'query': {'term': {'_id': each_granule['_id']}} }, each_granule['_index']) diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 96660bea..4a668610 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'CCC-05' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'CCC-07' # 'uds_collection' # 'sbx_collection' self.collection_version = '08'.replace('.', '') # '2402011200' return @@ -83,7 +83,7 @@ def test_delete_all(self): self.assertEqual(query_result.status_code, 202, f'wrong status code. {query_result.text}') response_json = json.loads(query_result.text) print(json.dumps(response_json, indent=4)) - sleep(30) + sleep(60) post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev query_result = requests.get(url=post_url, headers=headers,) self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') From fa32e31ebe29635025fbb7cf240b6f292b8158e4 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 17:48:39 -0600 Subject: [PATCH 07/14] fix: wrong logging statement --- cumulus_lambda_functions/lib/uds_db/granules_db_index.py | 2 +- tests/integration_tests/test_granules_deletion.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py index 44d18a12..c0f6925e 100644 --- a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py +++ b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py @@ -210,7 +210,7 @@ def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ): if result is None: raise ValueError(f"no such granule: {doc_id}") for each_granule in result['hits']['hits']: - LOGGER.debug(f"deleting {each_granule['_id']} from each_granule['_index']") + LOGGER.debug(f"deleting {each_granule['_id']} from {each_granule['_index']}") delete_result = self.__es.delete_by_query({ 'query': {'term': {'_id': each_granule['_id']}} }, each_granule['_index']) diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 4a668610..7f40e92d 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'CCC-07' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'CCC-08' # 'uds_collection' # 'sbx_collection' self.collection_version = '08'.replace('.', '') # '2402011200' return From 38f6952a0b8adad3fc0999196c0ddb9497ae50ed Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 18:41:46 -0600 Subject: [PATCH 08/14] fix: update dict key error --- cumulus_lambda_functions/cumulus_wrapper/query_granules.py | 4 ++-- tests/integration_tests/test_granules_deletion.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py index 3720cf6f..a9ab8ced 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py @@ -152,10 +152,10 @@ def delete_entry(self, private_api_prefix: str, granule_id: str): """ LOGGER.debug(f'json query_result: {query_result}') if query_result['statusCode'] >= 500: - LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}') + LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}') return {'server_error': query_result} if query_result['statusCode'] >= 400: - LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}') + LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}') return {'client_error': query_result} query_result = json.loads(query_result['body']) """ diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 7f40e92d..5316773c 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,8 +33,8 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'CCC-08' # 'uds_collection' # 'sbx_collection' - self.collection_version = '08'.replace('.', '') # '2402011200' + self.collection_name = 'AAA-05' # 'uds_collection' # 'sbx_collection' + # self.collection_version = '08'.replace('.', '') # '2402011200' return def test_01_setup_permissions(self): From e3c03c4650de0e9d5554dcdffb06d28ffb528d7d Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 17 Dec 2024 19:17:38 -0600 Subject: [PATCH 09/14] fix: why deleting twice? --- cumulus_lambda_functions/lib/uds_db/granules_db_index.py | 2 +- cumulus_lambda_functions/uds_api/granules_api.py | 2 +- tests/integration_tests/test_granules_deletion.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py index c0f6925e..9ace9ce9 100644 --- a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py +++ b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py @@ -212,7 +212,7 @@ def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ): for each_granule in result['hits']['hits']: LOGGER.debug(f"deleting {each_granule['_id']} from {each_granule['_index']}") delete_result = self.__es.delete_by_query({ - 'query': {'term': {'_id': each_granule['_id']}} + 'query': {'term': {'id': each_granule['_id']}} }, each_granule['_index']) LOGGER.debug(f'delete_result: {delete_result}') if delete_result is None: diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index 62091165..38828903 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -282,7 +282,7 @@ async def delete_single_granule_dapa_actual(request: Request, collection_id: str # delete_result = s3.delete_multiple(s3_urls=s3_urls) # LOGGER.debug(f'delete_result for {each_granule.id} - delete_result: {delete_result}') except Exception as e: - LOGGER.exception('failed during get_granules_dapa') + LOGGER.exception('failed during delete_single_granule_dapa_actual') raise HTTPException(status_code=500, detail=str(e)) return {} diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 5316773c..ff9291cc 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'AAA-05' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'AAA-07' # 'uds_collection' # 'sbx_collection' # self.collection_version = '08'.replace('.', '') # '2402011200' return @@ -75,15 +75,15 @@ def test_delete_all(self): asset_urls = [v['href'] for k, v in response_json['features'][0]['assets'].items()] print(asset_urls) - post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/' # MCP Dev + post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/actual' # MCP Dev print(post_url) query_result = requests.delete(url=post_url, headers=headers, ) - self.assertEqual(query_result.status_code, 202, f'wrong status code. {query_result.text}') + self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') response_json = json.loads(query_result.text) print(json.dumps(response_json, indent=4)) - sleep(60) + sleep(30) post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev query_result = requests.get(url=post_url, headers=headers,) self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') From decf6384cdd5d23be7ed64322fc8bd1d774bcf0a Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Wed, 18 Dec 2024 06:09:08 -0600 Subject: [PATCH 10/14] fix: disable cumulus part deletion to confirm es deletion is working as intended --- cumulus_lambda_functions/uds_api/granules_api.py | 4 ++-- tests/integration_tests/test_granules_deletion.py | 2 +- tests/integration_tests/test_stage_out_ingestion.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index 38828903..d54f869d 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -264,8 +264,8 @@ async def delete_single_granule_dapa_actual(request: Request, collection_id: str cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX') cumulus = GranulesQuery('https://na/dev', 'NA') cumulus.with_collection_id(collection_id) - cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID - LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') + # cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID + # LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') es_delete_result = GranulesDbIndex().delete_entry(collection_identifier.tenant, collection_identifier.venue, granule_id diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index ff9291cc..511e4d41 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'AAA-07' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'AAA-03' # 'uds_collection' # 'sbx_collection' # self.collection_version = '08'.replace('.', '') # '2402011200' return diff --git a/tests/integration_tests/test_stage_out_ingestion.py b/tests/integration_tests/test_stage_out_ingestion.py index f72b6e77..721c9cac 100644 --- a/tests/integration_tests/test_stage_out_ingestion.py +++ b/tests/integration_tests/test_stage_out_ingestion.py @@ -42,7 +42,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'CCC' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'DDD' # 'uds_collection' # 'sbx_collection' self.collection_version = '24.03.20.14.40'.replace('.', '') # '2402011200' return From a05298ffda3d6bc8e5cdc0b2bf38b7608b0eefee Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Wed, 18 Dec 2024 07:18:44 -0600 Subject: [PATCH 11/14] fix: cumulus deletion is causing ES deletes to persist --- cumulus_lambda_functions/uds_api/granules_api.py | 6 ++++-- tests/integration_tests/test_granules_deletion.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index d54f869d..ec4f7fd4 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -1,5 +1,6 @@ import json import os +from time import sleep from typing import Union from starlette.responses import Response, JSONResponse @@ -264,8 +265,9 @@ async def delete_single_granule_dapa_actual(request: Request, collection_id: str cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX') cumulus = GranulesQuery('https://na/dev', 'NA') cumulus.with_collection_id(collection_id) - # cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID - # LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') + cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID + LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') + sleep(3) # Testing if this helps es_delete_result = GranulesDbIndex().delete_entry(collection_identifier.tenant, collection_identifier.venue, granule_id diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 511e4d41..01c3a54a 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'AAA-03' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'DDD-01' # 'uds_collection' # 'sbx_collection' # self.collection_version = '08'.replace('.', '') # '2402011200' return @@ -83,7 +83,7 @@ def test_delete_all(self): self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') response_json = json.loads(query_result.text) print(json.dumps(response_json, indent=4)) - sleep(30) + sleep(10) post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev query_result = requests.get(url=post_url, headers=headers,) self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') From 69f8ef62534a1431991feabf12cac703fc3683f6 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Wed, 18 Dec 2024 07:46:17 -0600 Subject: [PATCH 12/14] fix: delete event is also inserting a reord to ES --- .../granules_to_es/granules_indexer.py | 3 + .../uds_api/granules_api.py | 59 +++++++++---------- .../test_granules_deletion.py | 2 +- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/cumulus_lambda_functions/granules_to_es/granules_indexer.py b/cumulus_lambda_functions/granules_to_es/granules_indexer.py index e9db5b7d..be2fe3de 100644 --- a/cumulus_lambda_functions/granules_to_es/granules_indexer.py +++ b/cumulus_lambda_functions/granules_to_es/granules_indexer.py @@ -73,6 +73,9 @@ def start(self): result = JsonValidator(self.CUMULUS_SCHEMA).validate(incoming_msg) if result is not None: raise ValueError(f'input json has CUMULUS validation errors: {result}') + if 'event' not in incoming_msg or incoming_msg['event'].upper() == 'DELETE': + LOGGER.debug(f'missing event or it is DELETE event. Not inserting to ES') + return self.__cumulus_record = incoming_msg['record'] if len(self.__cumulus_record['files']) < 1: # TODO ingest updating stage? diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index ec4f7fd4..9ea59e03 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -214,36 +214,36 @@ async def get_granules_dapa(request: Request, collection_id: str, limit: Union[i raise HTTPException(status_code=granules_result['statusCode'], detail=granules_result['body']) -@router.get("/{collection_id}/items/{granule_id}") -@router.get("/{collection_id}/items/{granule_id}/") -async def get_single_granule_dapa(request: Request, collection_id: str, granule_id: str): - authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ - .get_instance(UDSAuthorizerFactory.cognito, - es_url=os.getenv('ES_URL'), - es_port=int(os.getenv('ES_PORT', '443')) - ) - auth_info = FastApiUtils.get_authorization_info(request) - collection_identifier = UdsCollections.decode_identifier(collection_id) - if not authorizer.is_authorized_for_collection(DBConstants.read, collection_id, - auth_info['ldap_groups'], - collection_identifier.tenant, - collection_identifier.venue): - LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}') - raise HTTPException(status_code=403, detail=json.dumps({ - 'message': 'not authorized to execute this action' - })) - try: - api_base_prefix = FastApiUtils.get_api_base_prefix() - pg_link_generator = PaginationLinksGenerator(request) - granules_dapa_query = GranulesDapaQueryEs(collection_id, 1, None, None, filter, None, f'{pg_link_generator.base_url}/{api_base_prefix}') - granules_result = granules_dapa_query.get_single_granule(granule_id) - except Exception as e: - LOGGER.exception('failed during get_granules_dapa') - raise HTTPException(status_code=500, detail=str(e)) - return granules_result +# @router.get("/{collection_id}/items/{granule_id}") +# @router.get("/{collection_id}/items/{granule_id}/") +# async def get_single_granule_dapa(request: Request, collection_id: str, granule_id: str): +# authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ +# .get_instance(UDSAuthorizerFactory.cognito, +# es_url=os.getenv('ES_URL'), +# es_port=int(os.getenv('ES_PORT', '443')) +# ) +# auth_info = FastApiUtils.get_authorization_info(request) +# collection_identifier = UdsCollections.decode_identifier(collection_id) +# if not authorizer.is_authorized_for_collection(DBConstants.read, collection_id, +# auth_info['ldap_groups'], +# collection_identifier.tenant, +# collection_identifier.venue): +# LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}') +# raise HTTPException(status_code=403, detail=json.dumps({ +# 'message': 'not authorized to execute this action' +# })) +# try: +# api_base_prefix = FastApiUtils.get_api_base_prefix() +# pg_link_generator = PaginationLinksGenerator(request) +# granules_dapa_query = GranulesDapaQueryEs(collection_id, 1, None, None, filter, None, f'{pg_link_generator.base_url}/{api_base_prefix}') +# granules_result = granules_dapa_query.get_single_granule(granule_id) +# except Exception as e: +# LOGGER.exception('failed during get_granules_dapa') +# raise HTTPException(status_code=500, detail=str(e)) +# return granules_result -@router.delete("/{collection_id}/items/{granule_id}/actual") -@router.delete("/{collection_id}/items/{granule_id}/actual/") +@router.delete("/{collection_id}/items/{granule_id}") +@router.delete("/{collection_id}/items/{granule_id}/") async def delete_single_granule_dapa_actual(request: Request, collection_id: str, granule_id: str): authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ .get_instance(UDSAuthorizerFactory.cognito, @@ -267,7 +267,6 @@ async def delete_single_granule_dapa_actual(request: Request, collection_id: str cumulus.with_collection_id(collection_id) cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') - sleep(3) # Testing if this helps es_delete_result = GranulesDbIndex().delete_entry(collection_identifier.tenant, collection_identifier.venue, granule_id diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index 01c3a54a..fe0e7a1a 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -75,7 +75,7 @@ def test_delete_all(self): asset_urls = [v['href'] for k, v in response_json['features'][0]['assets'].items()] print(asset_urls) - post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/actual' # MCP Dev + post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/' # MCP Dev print(post_url) query_result = requests.delete(url=post_url, headers=headers, From 2472d7b52b036155704af555d39a4e915b7aeb4a Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Wed, 18 Dec 2024 07:55:22 -0600 Subject: [PATCH 13/14] fix: remove Makefile since this is only used locally --- ci.cd/Makefile | 64 ------------------- .../test_granules_deletion.py | 2 +- 2 files changed, 1 insertion(+), 65 deletions(-) diff --git a/ci.cd/Makefile b/ci.cd/Makefile index a2c54bce..e69de29b 100644 --- a/ci.cd/Makefile +++ b/ci.cd/Makefile @@ -1,64 +0,0 @@ -export IMAGE_PREFIX ?= cae-artifactory.jpl.nasa.gov:16001/gov/nasa/jpl/unity -export NAME ?= uds -export VERSION ?= latest - - -all: build_lambda upload_lambda update_lambda_function build_docker -local: build_lambda upload_lambda -build_docker: - docker build --no-cache -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile_download_granules.jpl . - -zip_docker: - docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz" - -build_lambda: - docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.9 ci.cd/create_s3_zip.sh - -build_lambda_public: - docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" python:3.9 ci.cd/create_s3_zip.sh - -upload_lambda: - aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/ - -move: - mv /Users/wphyo/Downloads/cumulus_lambda_functions-*.zip tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip - -upload_lambda_mcp_dev: - aws s3 cp tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip s3://uds-dev-cumulus-public/unity_cumulus_lambda/ -update_lambda_function_mcp_dev_6: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null -update_lambda_function_mcp_dev_7: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null -update_lambda_function_mcp_dev_auth_list: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_list --publish &>/dev/null -update_lambda_function_mcp_dev_auth_add: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_add --publish &>/dev/null -update_lambda_function_mcp_dev_auth_delete: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_delete --publish &>/dev/null - -update_lambda_function_mcp_dev_uds_api: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null - -update_lambda_function_mcp_sbx_uds_api: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-uds_api_1 --publish &>/dev/null - -mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_auth_list update_lambda_function_mcp_dev_auth_add update_lambda_function_mcp_dev_auth_delete - -mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api - -update_lambda_function_mcp_sbx_7: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null - -update_lambda_function_mcp_sbx_8: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_to_es --publish &>/dev/null - - -update_lambda_function_uds-sbx-cumulus-granules_cnm_response_writer: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_response_writer --publish &>/dev/null - -update_lambda_function_mcp_sbx_ingester: - aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_ingester --publish &>/dev/null - -mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7 update_lambda_function_mcp_sbx_8 - -mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py index fe0e7a1a..abf60f17 100644 --- a/tests/integration_tests/test_granules_deletion.py +++ b/tests/integration_tests/test_granules_deletion.py @@ -33,7 +33,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'DDD-01' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'DDD-03' # 'uds_collection' # 'sbx_collection' # self.collection_version = '08'.replace('.', '') # '2402011200' return From 6ef72646d47d25fb93f72adc974ac697d804ef00 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Wed, 18 Dec 2024 09:04:37 -0600 Subject: [PATCH 14/14] fix: pagination link cannot have int --- .../uds_api/dapa/granules_dapa_query_es.py | 3 ++- tests/integration_tests/test_uds_api.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py index 62d025de..1685f90f 100644 --- a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py +++ b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py @@ -227,7 +227,8 @@ def start(self): self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False) each_granules_query_result_stripped['links'].append(self_link) self.__restructure_each_granule_result(each_granules_query_result_stripped) - pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join(granules_query_result['hits']['hits'][-1]['sort']) + + pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join([k if isinstance(k, str) else str(k) for k in granules_query_result['hits']['hits'][-1]['sort']]) return { 'statusCode': 200, 'body': { diff --git a/tests/integration_tests/test_uds_api.py b/tests/integration_tests/test_uds_api.py index 4869162a..8ad16460 100644 --- a/tests/integration_tests/test_uds_api.py +++ b/tests/integration_tests/test_uds_api.py @@ -125,7 +125,7 @@ def test_collections_get(self): ) self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') query_result = json.loads(query_result.text) - print(query_result) + print(json.dumps(query_result, indent=4)) self.assertTrue('links' in query_result, 'links missing') links = {k['rel']: k for k in query_result['links']} self.assertTrue('next' in links, f'missing next in links: {links}') @@ -184,7 +184,8 @@ def test_collections_get_single_granule(self): return def test_granules_get(self): - post_url = f'{self.uds_url}collections/urn:nasa:unity:unity:dev:SBG-L2A_RFL___1/items/' # MCP Dev + # post_url = f'{self.uds_url}collections/urn:nasa:unity:unity:dev:SBG-L2A_RFL___1/items/' # MCP Dev + post_url = f'{self.uds_url}collections/urn:nasa:unity:asips:int:P1590011-T___1/items/' # MCP OPS headers = { 'Authorization': f'Bearer {self.bearer_token}', } @@ -192,9 +193,9 @@ def test_granules_get(self): query_result = requests.get(url=post_url, headers=headers, ) - self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') response_json = json.loads(query_result.text) print(json.dumps(response_json, indent=4)) + self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') links = {k['rel']: k['href'] for k in response_json['links'] if k['rel'] != 'root'} for k, v in links.items(): self.assertTrue(v.startswith(self.uds_url), f'missing stage: {self.stage} in {v} for {k}')