Merge decf638 into 75f029d

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -150,14 +150,14 @@ def delete_entry(self, private_api_prefix: str, granule_id: str):
             """
         {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False}
             """
+            LOGGER.debug(f'json query_result: {query_result}')
             if query_result['statusCode'] >= 500:
-                LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}')
+                LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}')
                 return {'server_error': query_result}
             if query_result['statusCode'] >= 400:
-                LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}')
+                LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}')
                 return {'client_error': query_result}
             query_result = json.loads(query_result['body'])
-            LOGGER.info(f'json query_result: {query_result}')
             """
             {
               "detail": "Record deleted"

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -210,8 +210,9 @@ def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ):
         if result is None:
             raise ValueError(f"no such granule: {doc_id}")
         for each_granule in result['hits']['hits']:
+            LOGGER.debug(f"deleting {each_granule['_id']} from {each_granule['_index']}")
             delete_result = self.__es.delete_by_query({
-                'query': {'term': {'_id': each_granule['_id']}}
+                'query': {'term': {'id': each_granule['_id']}}
             }, each_granule['_index'])
             LOGGER.debug(f'delete_result: {delete_result}')
             if delete_result is None:

cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py

@@ -1,6 +1,10 @@
+import json
+import os
 from copy import deepcopy
 
+from mdps_ds_lib.lib.aws.aws_lambda import AwsLambda
 from pystac import Link
+from starlette.datastructures import URL
 
 from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic
 from cumulus_lambda_functions.granules_to_es.granules_index_mapping import GranulesIndexMapping
@@ -17,6 +21,7 @@
 
 class GranulesDapaQueryEs:
     def __init__(self, collection_id, limit, offset, input_datetime, filter_input, pagination_link_obj: PaginationLinksGenerator, base_url):
+        self.__collection_cnm_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip()
         self.__pagination_link_obj = pagination_link_obj
         self.__input_datetime = input_datetime
         self.__collection_id = collection_id
@@ -149,6 +154,64 @@ def get_single_granule(self, granule_id):
         self.__restructure_each_granule_result(each_granules_query_result_stripped)
         return each_granules_query_result_stripped
 
+    def delete_facade(self, current_url: URL, bearer_token: str):
+        actual_path = current_url.path
+        actual_path = actual_path if actual_path.endswith('/') else f'{actual_path}/'
+        actual_path = f'{actual_path}actual'
+        LOGGER.info(f'sanity_check')
+
+        actual_event = {
+            'resource': actual_path,
+            'path': actual_path,
+            'httpMethod': 'DELETE',
+            'headers': {
+                'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Authorization': bearer_token,
+                'Host': current_url.hostname, 'User-Agent': 'python-requests/2.28.2',
+                'X-Amzn-Trace-Id': 'Root=1-64a66e90-6fa8b7a64449014639d4f5b4', 'X-Forwarded-For': '44.236.15.58',
+                'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'},
+            'multiValueHeaders': {
+                'Accept': ['*/*'], 'Accept-Encoding': ['gzip, deflate'], 'Authorization': [bearer_token],
+                'Host': [current_url.hostname], 'User-Agent': ['python-requests/2.28.2'],
+                'X-Amzn-Trace-Id': ['Root=1-64a66e90-6fa8b7a64449014639d4f5b4'],
+                'X-Forwarded-For': ['127.0.0.1'], 'X-Forwarded-Port': ['443'], 'X-Forwarded-Proto': ['https']
+            },
+            'queryStringParameters': {},
+            'multiValueQueryStringParameters': {},
+            'pathParameters': {},
+            'stageVariables': None,
+            'requestContext': {
+                'resourceId': '',
+                'authorizer': {'principalId': '', 'integrationLatency': 0},
+                'resourcePath': actual_path, 'httpMethod': 'PUT',
+                'extendedRequestId': '', 'requestTime': '',
+                'path': actual_path, 'accountId': '',
+                'protocol': 'HTTP/1.1', 'stage': '', 'domainPrefix': '', 'requestTimeEpoch': 0,
+                'requestId': '',
+                'identity': {
+                    'cognitoIdentityPoolId': None, 'accountId': None, 'cognitoIdentityId': None, 'caller': None,
+                    'sourceIp': '127.0.0.1', 'principalOrgId': None, 'accessKey': None,
+                    'cognitoAuthenticationType': None,
+                    'cognitoAuthenticationProvider': None, 'userArn': None, 'userAgent': 'python-requests/2.28.2',
+                    'user': None
+                },
+                'domainName': current_url.hostname, 'apiId': ''
+            },
+            'body': json.dumps({}),
+            'isBase64Encoded': False
+        }
+        LOGGER.info(f'actual_event: {actual_event}')
+        response = AwsLambda().invoke_function(
+            function_name=self.__collection_cnm_lambda_name,
+            payload=actual_event,
+        )
+        LOGGER.debug(f'async function started: {response}')
+        return {
+            'statusCode': 202,
+            'body': {
+                'message': 'processing'
+            }
+        }
+
     def start(self):
         try:
             granules_query_dsl = self.__generate_es_dsl()

cumulus_lambda_functions/uds_api/granules_api.py

@@ -2,12 +2,10 @@
 import os
 from typing import Union
 
-from mdps_ds_lib.lib.aws.aws_s3 import AwsS3
-from pystac import Item
+from starlette.responses import Response, JSONResponse
 
 from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery
 
-from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic
 from cumulus_lambda_functions.uds_api.dapa.daac_archive_crud import DaacArchiveCrud, DaacDeleteModel, DaacAddModel, \
     DaacUpdateModel
 from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query_es import GranulesDapaQueryEs
@@ -26,7 +24,6 @@
 
 from fastapi import APIRouter, HTTPException, Request
 
-from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query import GranulesDapaQuery
 from cumulus_lambda_functions.uds_api.dapa.pagination_links_generator import PaginationLinksGenerator
 from cumulus_lambda_functions.uds_api.web_service_constants import WebServiceConstants
 
@@ -244,9 +241,9 @@ async def get_single_granule_dapa(request: Request, collection_id: str, granule_
         raise HTTPException(status_code=500, detail=str(e))
     return granules_result
 
-@router.delete("/{collection_id}/items/{granule_id}")
-@router.delete("/{collection_id}/items/{granule_id}/")
-async def delete_single_granule_dapa(request: Request, collection_id: str, granule_id: str):
+@router.delete("/{collection_id}/items/{granule_id}/actual")
+@router.delete("/{collection_id}/items/{granule_id}/actual/")
+async def delete_single_granule_dapa_actual(request: Request, collection_id: str, granule_id: str):
     authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
         .get_instance(UDSAuthorizerFactory.cognito,
                       es_url=os.getenv('ES_URL'),
@@ -267,8 +264,8 @@ async def delete_single_granule_dapa(request: Request, collection_id: str, granu
         cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX')
         cumulus = GranulesQuery('https://na/dev', 'NA')
         cumulus.with_collection_id(collection_id)
-        cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id)  # TODO not sure it is correct granule ID
-        LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}')
+        # cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id)  # TODO not sure it is correct granule ID
+        # LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}')
         es_delete_result = GranulesDbIndex().delete_entry(collection_identifier.tenant,
                                                           collection_identifier.venue,
                                                           granule_id
@@ -285,11 +282,42 @@ async def delete_single_granule_dapa(request: Request, collection_id: str, granu
         #     delete_result = s3.delete_multiple(s3_urls=s3_urls)
         #     LOGGER.debug(f'delete_result for {each_granule.id} - delete_result: {delete_result}')
     except Exception as e:
-        LOGGER.exception('failed during get_granules_dapa')
+        LOGGER.exception('failed during delete_single_granule_dapa_actual')
         raise HTTPException(status_code=500, detail=str(e))
     return {}
 
 
+@router.delete("/{collection_id}/items/{granule_id}")
+@router.delete("/{collection_id}/items/{granule_id}/")
+async def delete_single_granule_dapa_facade(request: Request, collection_id: str, granule_id: str, response: Response, response_class=JSONResponse):
+    authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
+        .get_instance(UDSAuthorizerFactory.cognito,
+                      es_url=os.getenv('ES_URL'),
+                      es_port=int(os.getenv('ES_PORT', '443'))
+                      )
+    auth_info = FastApiUtils.get_authorization_info(request)
+    collection_identifier = UdsCollections.decode_identifier(collection_id)
+    if not authorizer.is_authorized_for_collection(DBConstants.delete, collection_id,
+                                                   auth_info['ldap_groups'],
+                                                   collection_identifier.tenant,
+                                                   collection_identifier.venue):
+        LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}')
+        raise HTTPException(status_code=403, detail=json.dumps({
+            'message': 'not authorized to execute this action'
+        }))
+    try:
+        LOGGER.debug(f'deleting granule: {granule_id}')
+        granules_dapa_query = GranulesDapaQueryEs(collection_id, -1, -1, None, None, None, '')
+        delete_prep_result = granules_dapa_query.delete_facade(request.url, request.headers.get('Authorization', ''))
+    except Exception as e:
+        LOGGER.exception('failed during delete_single_granule_dapa')
+        raise HTTPException(status_code=500, detail=str(e))
+    if delete_prep_result['statusCode'] < 300:
+        response.status_code = delete_prep_result['statusCode']
+        return delete_prep_result['body']
+    raise HTTPException(status_code=delete_prep_result['statusCode'], detail=delete_prep_result['body'])
+
+
 @router.put("/{collection_id}/archive/{granule_id}")
 @router.put("/{collection_id}/archive/{granule_id}/")
 async def archive_single_granule_dapa(request: Request, collection_id: str, granule_id: str):

tests/integration_tests/test_granules_deletion.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import os
+from time import sleep
 from unittest import TestCase
 
 import requests
@@ -32,8 +33,8 @@ def setUp(self) -> None:
 
         self.tenant = 'UDS_LOCAL_TEST_3'  # 'uds_local_test'  # 'uds_sandbox'
         self.tenant_venue = 'DEV'  # 'DEV1'  # 'dev'
-        self.collection_name = 'CCC-04'  # 'uds_collection'  # 'sbx_collection'
-        self.collection_version = '08'.replace('.', '')  # '2402011200'
+        self.collection_name = 'AAA-03'  # 'uds_collection'  # 'sbx_collection'
+        # self.collection_version = '08'.replace('.', '')  # '2402011200'
         return
 
     def test_01_setup_permissions(self):
@@ -74,15 +75,15 @@ def test_delete_all(self):
 
         asset_urls = [v['href'] for k, v in response_json['features'][0]['assets'].items()]
         print(asset_urls)
-        post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/'  # MCP Dev
+        post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/actual'  # MCP Dev
         print(post_url)
         query_result = requests.delete(url=post_url,
                                        headers=headers,
                                        )
         self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')
         response_json = json.loads(query_result.text)
         print(json.dumps(response_json, indent=4))
-
+        sleep(30)
         post_url = f'{self.uds_url}collections/{collection_id}/items/'  # MCP Dev
         query_result = requests.get(url=post_url, headers=headers,)
         self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')

tests/integration_tests/test_stage_out_ingestion.py

@@ -42,7 +42,7 @@ def setUp(self) -> None:
 
         self.tenant = 'UDS_LOCAL_TEST_3'  # 'uds_local_test'  # 'uds_sandbox'
         self.tenant_venue = 'DEV'  # 'DEV1'  # 'dev'
-        self.collection_name = 'CCC'  # 'uds_collection'  # 'sbx_collection'
+        self.collection_name = 'DDD'  # 'uds_collection'  # 'sbx_collection'
         self.collection_version = '24.03.20.14.40'.replace('.', '')  # '2402011200'
         return