fix: Delete bug

ci.cd/Makefile

@@ -1,64 +0,0 @@
-export IMAGE_PREFIX ?= cae-artifactory.jpl.nasa.gov:16001/gov/nasa/jpl/unity
-export NAME ?= uds
-export VERSION ?= latest
-
-
-all: build_lambda upload_lambda update_lambda_function build_docker
-local: build_lambda upload_lambda
-build_docker:
-	docker build --no-cache -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile_download_granules.jpl .
-
-zip_docker:
-	docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz"
-
-build_lambda:
-	docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.9 ci.cd/create_s3_zip.sh
-
-build_lambda_public:
-	docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" python:3.9 ci.cd/create_s3_zip.sh
-
-upload_lambda:
-	aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/
-
-move:
-	 mv /Users/wphyo/Downloads/cumulus_lambda_functions-*.zip tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip
-
-upload_lambda_mcp_dev:
-	aws s3 cp tf-module/unity-cumulus/build/cumulus_lambda_functions_deployment.zip s3://uds-dev-cumulus-public/unity_cumulus_lambda/
-update_lambda_function_mcp_dev_6:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_s4pa_generate_cmr --publish &>/dev/null
-update_lambda_function_mcp_dev_7:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_list:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_list --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_add:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_add --publish &>/dev/null
-update_lambda_function_mcp_dev_auth_delete:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_auth_delete --publish &>/dev/null
-
-update_lambda_function_mcp_dev_uds_api:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-uds_api_1 --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_uds_api:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-uds_api_1 --publish &>/dev/null
-
-mcp_dev: upload_lambda_mcp_dev update_lambda_function_mcp_dev_auth_list update_lambda_function_mcp_dev_auth_add update_lambda_function_mcp_dev_auth_delete
-
-mcp_dev_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_dev_uds_api
-
-update_lambda_function_mcp_sbx_7:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-metadata_stac_generate_cmr --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_8:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_to_es --publish &>/dev/null
-
-
-update_lambda_function_uds-sbx-cumulus-granules_cnm_response_writer:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_response_writer --publish &>/dev/null
-
-update_lambda_function_mcp_sbx_ingester:
-	aws lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket uds-dev-cumulus-public --function-name arn:aws:lambda:us-west-2:237868187491:function:uds-sbx-cumulus-granules_cnm_ingester --publish &>/dev/null
-
-mcp_sbx: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_7 update_lambda_function_mcp_sbx_8
-
-mcp_sbx_fastapi: upload_lambda_mcp_dev update_lambda_function_mcp_sbx_uds_api

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -150,14 +150,14 @@ def delete_entry(self, private_api_prefix: str, granule_id: str):
             """
         {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False}
             """
+            LOGGER.debug(f'json query_result: {query_result}')
             if query_result['statusCode'] >= 500:
-                LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}')
+                LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}')
                 return {'server_error': query_result}
             if query_result['statusCode'] >= 400:
-                LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}')
+                LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}')
                 return {'client_error': query_result}
             query_result = json.loads(query_result['body'])
-            LOGGER.info(f'json query_result: {query_result}')
             """
             {
               "detail": "Record deleted"

cumulus_lambda_functions/granules_to_es/granules_indexer.py

@@ -73,6 +73,9 @@ def start(self):
         result = JsonValidator(self.CUMULUS_SCHEMA).validate(incoming_msg)
         if result is not None:
             raise ValueError(f'input json has CUMULUS validation errors: {result}')
+        if 'event' not in incoming_msg or incoming_msg['event'].upper() == 'DELETE':
+            LOGGER.debug(f'missing event or it is DELETE event. Not inserting to ES')
+            return
         self.__cumulus_record = incoming_msg['record']
         if len(self.__cumulus_record['files']) < 1:
             # TODO ingest updating stage?

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -210,8 +210,9 @@ def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ):
         if result is None:
             raise ValueError(f"no such granule: {doc_id}")
         for each_granule in result['hits']['hits']:
+            LOGGER.debug(f"deleting {each_granule['_id']} from {each_granule['_index']}")
             delete_result = self.__es.delete_by_query({
-                'query': {'term': {'_id': each_granule['_id']}}
+                'query': {'term': {'id': each_granule['_id']}}
             }, each_granule['_index'])
             LOGGER.debug(f'delete_result: {delete_result}')
             if delete_result is None:

cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py

@@ -1,6 +1,10 @@
+import json
+import os
 from copy import deepcopy
 
+from mdps_ds_lib.lib.aws.aws_lambda import AwsLambda
 from pystac import Link
+from starlette.datastructures import URL
 
 from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic
 from cumulus_lambda_functions.granules_to_es.granules_index_mapping import GranulesIndexMapping
@@ -17,6 +21,7 @@
 
 class GranulesDapaQueryEs:
     def __init__(self, collection_id, limit, offset, input_datetime, filter_input, pagination_link_obj: PaginationLinksGenerator, base_url):
+        self.__collection_cnm_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip()
         self.__pagination_link_obj = pagination_link_obj
         self.__input_datetime = input_datetime
         self.__collection_id = collection_id
@@ -149,6 +154,64 @@ def get_single_granule(self, granule_id):
         self.__restructure_each_granule_result(each_granules_query_result_stripped)
         return each_granules_query_result_stripped
 
+    def delete_facade(self, current_url: URL, bearer_token: str):
+        actual_path = current_url.path
+        actual_path = actual_path if actual_path.endswith('/') else f'{actual_path}/'
+        actual_path = f'{actual_path}actual'
+        LOGGER.info(f'sanity_check')
+
+        actual_event = {
+            'resource': actual_path,
+            'path': actual_path,
+            'httpMethod': 'DELETE',
+            'headers': {
+                'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': bearer_token,
+                'Host': current_url.hostname, 'User-Agent': 'python-requests/2.28.2',
+                'X-Amzn-Trace-Id': 'Root=1-64a66e90-6fa8b7a64449014639d4f5b4', 'X-Forwarded-For': '44.236.15.58',
+                'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'},
+            'multiValueHeaders': {
+                'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': [bearer_token],
+                'Host': [current_url.hostname], 'User-Agent': ['python-requests/2.28.2'],
+                'X-Amzn-Trace-Id': ['Root=1-64a66e90-6fa8b7a64449014639d4f5b4'],
+                'X-Forwarded-For': ['127.0.0.1'], 'X-Forwarded-Port': ['443'], 'X-Forwarded-Proto': ['https']
+            },
+            'queryStringParameters': {},
+            'multiValueQueryStringParameters': {},
+            'pathParameters': {},
+            'stageVariables': None,
+            'requestContext': {
+                'resourceId': '',
+                'authorizer': {'principalId': '', 'integrationLatency': 0},
+                'resourcePath': actual_path, 'httpMethod': 'PUT',
+                'extendedRequestId': '', 'requestTime': '',
+                'path': actual_path, 'accountId': '',
+                'protocol': 'HTTP/1.1', 'stage': '', 'domainPrefix': '', 'requestTimeEpoch': 0,
+                'requestId': '',
+                'identity': {
+                    'cognitoIdentityPoolId': None, 'accountId': None, 'cognitoIdentityId': None, 'caller': None,
+                    'sourceIp': '127.0.0.1', 'principalOrgId': None, 'accessKey': None,
+                    'cognitoAuthenticationType': None,
+                    'cognitoAuthenticationProvider': None, 'userArn': None, 'userAgent': 'python-requests/2.28.2',
+                    'user': None
+                },
+                'domainName': current_url.hostname, 'apiId': ''
+            },
+            'body': json.dumps({}),
+            'isBase64Encoded': False
+        }
+        LOGGER.info(f'actual_event: {actual_event}')
+        response = AwsLambda().invoke_function(
+            function_name=self.__collection_cnm_lambda_name,
+            payload=actual_event,
+        )
+        LOGGER.debug(f'async function started: {response}')
+        return {
+            'statusCode': 202,
+            'body': {
+                'message': 'processing'
+            }
+        }
+
     def start(self):
         try:
             granules_query_dsl = self.__generate_es_dsl()
@@ -164,7 +227,8 @@ def start(self):
                 self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False)
                 each_granules_query_result_stripped['links'].append(self_link)
                 self.__restructure_each_granule_result(each_granules_query_result_stripped)
-            pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join(granules_query_result['hits']['hits'][-1]['sort'])
+
+            pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join([k if isinstance(k, str) else str(k) for k in granules_query_result['hits']['hits'][-1]['sort']])
             return {
                 'statusCode': 200,
                 'body': {

cumulus_lambda_functions/uds_api/granules_api.py

@@ -1,13 +1,12 @@
 import json
 import os
+from time import sleep
 from typing import Union
 
-from mdps_ds_lib.lib.aws.aws_s3 import AwsS3
-from pystac import Item
+from starlette.responses import Response, JSONResponse
 
 from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery
 
-from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic
 from cumulus_lambda_functions.uds_api.dapa.daac_archive_crud import DaacArchiveCrud, DaacDeleteModel, DaacAddModel, \
     DaacUpdateModel
 from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query_es import GranulesDapaQueryEs
@@ -26,7 +25,6 @@
 
 from fastapi import APIRouter, HTTPException, Request
 
-from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query import GranulesDapaQuery
 from cumulus_lambda_functions.uds_api.dapa.pagination_links_generator import PaginationLinksGenerator
 from cumulus_lambda_functions.uds_api.web_service_constants import WebServiceConstants
 
@@ -246,7 +244,7 @@ async def get_single_granule_dapa(request: Request, collection_id: str, granule_
 
 @router.delete("/{collection_id}/items/{granule_id}")
 @router.delete("/{collection_id}/items/{granule_id}/")
-async def delete_single_granule_dapa(request: Request, collection_id: str, granule_id: str):
+async def delete_single_granule_dapa_actual(request: Request, collection_id: str, granule_id: str):
     authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
         .get_instance(UDSAuthorizerFactory.cognito,
                       es_url=os.getenv('ES_URL'),
@@ -285,11 +283,42 @@ async def delete_single_granule_dapa(request: Request, collection_id: str, granu
         #     delete_result = s3.delete_multiple(s3_urls=s3_urls)
         #     LOGGER.debug(f'delete_result for {each_granule.id} - delete_result: {delete_result}')
     except Exception as e:
-        LOGGER.exception('failed during get_granules_dapa')
+        LOGGER.exception('failed during delete_single_granule_dapa_actual')
         raise HTTPException(status_code=500, detail=str(e))
     return {}
 
 
+# @router.delete("/{collection_id}/items/{granule_id}")
+# @router.delete("/{collection_id}/items/{granule_id}/")
+# async def delete_single_granule_dapa_facade(request: Request, collection_id: str, granule_id: str, response: Response, response_class=JSONResponse):
+#     authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
+#         .get_instance(UDSAuthorizerFactory.cognito,
+#                       es_url=os.getenv('ES_URL'),
+#                       es_port=int(os.getenv('ES_PORT', '443'))
+#                       )
+#     auth_info = FastApiUtils.get_authorization_info(request)
+#     collection_identifier = UdsCollections.decode_identifier(collection_id)
+#     if not authorizer.is_authorized_for_collection(DBConstants.delete, collection_id,
+#                                                    auth_info['ldap_groups'],
+#                                                    collection_identifier.tenant,
+#                                                    collection_identifier.venue):
+#         LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}')
+#         raise HTTPException(status_code=403, detail=json.dumps({
+#             'message': 'not authorized to execute this action'
+#         }))
+#     try:
+#         LOGGER.debug(f'deleting granule: {granule_id}')
+#         granules_dapa_query = GranulesDapaQueryEs(collection_id, -1, -1, None, None, None, '')
+#         delete_prep_result = granules_dapa_query.delete_facade(request.url, request.headers.get('Authorization', ''))
+#     except Exception as e:
+#         LOGGER.exception('failed during delete_single_granule_dapa')
+#         raise HTTPException(status_code=500, detail=str(e))
+#     if delete_prep_result['statusCode'] < 300:
+#         response.status_code = delete_prep_result['statusCode']
+#         return delete_prep_result['body']
+#     raise HTTPException(status_code=delete_prep_result['statusCode'], detail=delete_prep_result['body'])
+
+
 @router.put("/{collection_id}/archive/{granule_id}")
 @router.put("/{collection_id}/archive/{granule_id}/")
 async def archive_single_granule_dapa(request: Request, collection_id: str, granule_id: str):

tests/integration_tests/test_granules_deletion.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import os
+from time import sleep
 from unittest import TestCase
 
 import requests
@@ -32,8 +33,8 @@ def setUp(self) -> None:
 
         self.tenant = 'UDS_LOCAL_TEST_3'  # 'uds_local_test'  # 'uds_sandbox'
         self.tenant_venue = 'DEV'  # 'DEV1'  # 'dev'
-        self.collection_name = 'CCC-04'  # 'uds_collection'  # 'sbx_collection'
-        self.collection_version = '08'.replace('.', '')  # '2402011200'
+        self.collection_name = 'DDD-03'  # 'uds_collection'  # 'sbx_collection'
+        # self.collection_version = '08'.replace('.', '')  # '2402011200'
         return
 
     def test_01_setup_permissions(self):
@@ -82,7 +83,7 @@ def test_delete_all(self):
         self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')
         response_json = json.loads(query_result.text)
         print(json.dumps(response_json, indent=4))
-
+        sleep(10)
         post_url = f'{self.uds_url}collections/{collection_id}/items/'  # MCP Dev
         query_result = requests.get(url=post_url, headers=headers,)
         self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')

tests/integration_tests/test_stage_out_ingestion.py

@@ -42,7 +42,7 @@ def setUp(self) -> None:
 
         self.tenant = 'UDS_LOCAL_TEST_3'  # 'uds_local_test'  # 'uds_sandbox'
         self.tenant_venue = 'DEV'  # 'DEV1'  # 'dev'
-        self.collection_name = 'CCC'  # 'uds_collection'  # 'sbx_collection'
+        self.collection_name = 'DDD'  # 'uds_collection'  # 'sbx_collection'
         self.collection_version = '24.03.20.14.40'.replace('.', '')  # '2402011200'
         return