From b0801de6578a6c3a643b8b4b233a0c8bc2905a60 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:23:36 -0700 Subject: [PATCH 1/6] feat: add AUDIT log level for upload --- .../upload_arbitrary_files_as_granules.py | 1 + .../stage_in_out/upload_granules_abstract.py | 15 + .../upload_granules_by_complete_catalog_s3.py | 2 +- tests/integration_tests/.env.tpl | 6 + tests/integration_tests/__init__.py | 0 tests/integration_tests/test_docker_entry.py | 375 ++++ .../integration_tests/test_docker_stage_in.py | 1928 +++++++++++++++++ .../test_docker_stage_out.py | 1334 ++++++++++++ 8 files changed, 3660 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/.env.tpl create mode 100644 tests/integration_tests/__init__.py create mode 100644 tests/integration_tests/test_docker_entry.py create mode 100644 tests/integration_tests/test_docker_stage_in.py create mode 100644 tests/integration_tests/test_docker_stage_out.py diff --git a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py index 9a53a75..a545ce4 100644 --- a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py +++ b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py @@ -77,6 +77,7 @@ def execute_job(self, job_obj, lock) -> bool: sample_stac_item = self.generate_sample_stac(job_obj) updating_assets = {} try: + LOGGER.audit(f'uploading auxiliary file: {job_obj}') s3_url = self.__s3.upload(job_obj, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{sample_stac_item.id}', self.__delete_files) updating_assets[os.path.basename(s3_url)] = s3_url uploading_current_granule_stac = f'{s3_url}.stac.json' diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 66543e5..5d803d0 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -1,3 +1,18 @@ +import logging + +# Define the new log level AUDIT +AUDIT_LEVEL = 60 +logging.addLevelName(AUDIT_LEVEL, "AUDIT") + + +# Define a custom method for logging at AUDIT level +def audit(self, message, *args, **kwargs): + if self.isEnabledFor(AUDIT_LEVEL): + self._log(AUDIT_LEVEL, message, args, **kwargs) + + +logging.Logger.audit = audit + import os from abc import ABC, abstractmethod diff --git a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py index 409a41b..2f7d8e7 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py @@ -64,7 +64,7 @@ def execute_job(self, each_child, lock) -> bool: uploading_current_granule_stac = None for asset_type, asset_hrefs in current_assets.items(): for each_asset_href in asset_hrefs: - LOGGER.debug(f'uploading {asset_type}, {each_asset_href}') + LOGGER.audit(f'uploading {asset_type}: {each_asset_href}') s3_url = self.__s3.upload(each_asset_href, self.__staging_bucket, f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', self.__delete_files) diff --git a/tests/integration_tests/.env.tpl b/tests/integration_tests/.env.tpl new file mode 100644 index 0000000..5bb2ddf --- /dev/null +++ b/tests/integration_tests/.env.tpl @@ -0,0 +1,6 @@ +USERNAME=base64-encoded-str +PASSWORD=base64-encoded-str +CLIENT_ID=7a1fglm2d54eoggj13lccivp25 +COGNITO_URL=https://cognito-idp.us-west-2.amazonaws.com +UNITY_URL=https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev +UNITY_STAGE=sbx-uds-dapa diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py new file mode 100644 index 0000000..df328b3 --- /dev/null +++ b/tests/integration_tests/test_docker_entry.py @@ -0,0 +1,375 @@ +import logging + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import os +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from cumulus_lambda_functions.docker_entrypoint.__main__ import choose_process +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerEntry(TestCase): + def test_01_search_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '4000' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + search_result_str = choose_process() + search_result = json.loads(search_result_str) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4000, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4000, f'wrong length. not unique') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + self.assertEqual(sorted(json.dumps(FileUtils.read_json(os.environ['OUTPUT_FILE']))), sorted(search_result_str), f'not identical result') + return + + def test_01_search_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A___1' + os.environ['LIMITS'] = '100' + os.environ['DATE_FROM'] = '2016-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2016-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + print(search_result) + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 20, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result),20, f'wrong length. not unique') + return + + def test_01_search_part_03(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '-1' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + # self.assertTrue(isinstance(search_result, list), f'search_result is not list: {search_result}') + self.assertEqual(len(item_collections.items), 4381, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 4381, f'wrong length. not unique') + return + + def test_01_search_part_04(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'L0_SNPP_ATMS_SCIENCE___1' + os.environ['LIMITS'] = '347' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2022-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 347, f'wrong length') + search_result = set([k.id for k in item_collections.items]) + self.assertEqual(len(search_result), 347, f'wrong length. not unique') + return + + def test_01_search_part_05(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['COLLECTION_ID'] = 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030' + os.environ['LIMITS'] = '37' + os.environ['DATE_FROM'] = '1990-01-14T08:00:00Z' + os.environ['DATE_TO'] = '2024-01-14T11:59:59Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 37, f'wrong length') + return + + def test_01_1_search_cmr_part_01(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '2120' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 2120, f'wrong length') + return + + def test_01_1_search_cmr_part_02(self): + """ + :return: + """ + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['COLLECTION_ID'] = 'C1996881146-POCLOUD' # 'C1666605425-PODAAC' # C1996881146-POCLOUD + os.environ['LIMITS'] = '23' + os.environ['DATE_FROM'] = '2002-06-01T12:06:00.000Z' + os.environ['DATE_TO'] = '2011-10-04T06:51:45.000Z' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['GRANULES_SEARCH_DOMAIN'] = 'CMR' + os.environ['CMR_BASE_URL'] = 'https://cmr.earthdata.nasa.gov' + if len(argv) > 1: + argv.pop(-1) + argv.append('SEARCH') + search_result = choose_process() + search_result = json.loads(search_result) + self.assertTrue('type' in search_result, f'missing type in search_result') + item_collections = ItemCollection.from_dict(search_result) + self.assertEqual(len(item_collections.items), 23, f'wrong length') + return + + def test_04_catalog(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['UPLOADED_FILES_JSON'] = json.dumps(upload_result) + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertEqual('registered', catalog_result, 'wrong status') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_04_catalog_from_file(self): + upload_result = [{'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9', 'assets': {'metadata': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas'}, 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc'}}}] + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + # Example result: {'cataloging_request_status': 'registered', 'status_result': {'cataloged': False, 'missing_granules': ['NEW_COLLECTION_EXAMPLE_L1B___9:test_file01'], 'registered_granules': []}} + return + + def test_04_catalog_from_file_item_collection(self): + upload_result = {'type': 'FeatureCollection', 'features': [ + {'type': 'Feature', 'stac_version': '1.0.0', 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', 'end_datetime': '2016-01-31T19:59:59.991043Z', + 'created': '2016-02-01T02:45:59.639000Z', 'updated': '2022-03-23T15:48:21.578000Z', + 'datetime': '1970-01-01T00:00:00Z'}, 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [], 'assets': { + 'data': {'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, 'bbox': [0.0, 0.0, 0.0, 0.0], 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + os.environ['DELAY_SECOND'] = '35' + os.environ['REPEAT_TIMES'] = '3' + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], 'registered', f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + # TODO update this after it is deployed to MCP Test + status_result = catalog_result['status_result'] + + self.assertTrue('cataloged' in status_result, f'missing cataloged') + self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return + + def test_04_catalog_from_file_item_collection_large(self): + upload_result = FileUtils.read_json('./stage-out-results.json') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ[Constants.CLIENT_ID] = '71g0c73jl77gsqhtlfg2ht388c' # MCP Dev + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['GRANULES_CATALOG_TYPE'] = 'UNITY' + os.environ['CHUNK_SIZE'] = '250' + # os.environ['DELAY_SECOND'] = '5' + # os.environ['REPEAT_TIMES'] = '3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('CATALOG') + with tempfile.TemporaryDirectory() as tmp_dir_name: + input_file_path = os.path.join(tmp_dir_name, 'uploaded_files.json') + FileUtils.write_json(input_file_path, upload_result) + os.environ['UPLOADED_FILES_JSON'] = input_file_path + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + catalog_result_str = choose_process() + catalog_result = json.loads(catalog_result_str) + self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') + self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') + + catalog_result = catalog_result[0] + self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') + self.assertTrue('status_result' in catalog_result, f'missing status_result') + self.assertEqual(catalog_result['cataloging_request_status'], {'message': 'processing'}, f'mismatched cataloging_request_status value') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + + status_result = catalog_result['status_result'] + # TODO disabling this as we are not waiting for them to be registered. + # self.assertTrue('cataloged' in status_result, f'missing cataloged') + # self.assertTrue('missing_granules' in status_result, f'missing missing_granules') + # self.assertTrue('registered_granules' in status_result, f'missing registered_granules') + # self.assertTrue(isinstance(status_result['cataloged'], bool), f'cataloged is not boolean: {status_result["cataloged"]}') + return diff --git a/tests/integration_tests/test_docker_stage_in.py b/tests/integration_tests/test_docker_stage_in.py new file mode 100644 index 0000000..6336640 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_in.py @@ -0,0 +1,1928 @@ +import os + +from mdps_ds_lib.stage_in_out.download_granules_factory import DownloadGranulesFactory + +os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' +os.environ['no_proxy'] = '*' +import logging + +import requests + +logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +import math +from unittest.mock import patch, MagicMock +import json +import tempfile +from glob import glob +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageIn(TestCase): + def test_02_download(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.16", "properties": {"start_datetime": "2016-01-14T11:24:00Z", "end_datetime": "2016-01-14T11:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.917000Z", "datetime": "2022-08-15T06:26:19.027000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.16.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.17", "properties": {"start_datetime": "2016-01-14T11:30:00Z", "end_datetime": "2016-01-14T11:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.907000Z", "datetime": "2022-08-15T06:26:19.042000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.17.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.10", "properties": {"start_datetime": "2016-01-14T10:48:00Z", "end_datetime": "2016-01-14T10:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.446000Z", "datetime": "2022-08-15T06:26:18.730000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.10.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.14", "properties": {"start_datetime": "2016-01-14T11:12:00Z", "end_datetime": "2016-01-14T11:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.354000Z", "datetime": "2022-08-15T06:26:17.758000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.14.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.12", "properties": {"start_datetime": "2016-01-14T11:00:00Z", "end_datetime": "2016-01-14T11:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:25.344000Z", "datetime": "2022-08-15T06:26:17.938000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.09", "properties": {"start_datetime": "2016-01-14T10:42:00Z", "end_datetime": "2016-01-14T10:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:24.910000Z", "datetime": "2022-08-15T06:26:20.688000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.09.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.20", "properties": {"start_datetime": "2016-01-14T11:48:00Z", "end_datetime": "2016-01-14T11:54:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.929000Z", "datetime": "2022-08-15T06:26:19.091000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.20.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.15", "properties": {"start_datetime": "2016-01-14T11:18:00Z", "end_datetime": "2016-01-14T11:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.732000Z", "datetime": "2022-08-15T06:26:19.282000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.15.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.07", "properties": {"start_datetime": "2016-01-14T10:30:00Z", "end_datetime": "2016-01-14T10:36:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.371000Z", "datetime": "2022-08-15T06:26:19.047000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.07.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.19", "properties": {"start_datetime": "2016-01-14T11:42:00Z", "end_datetime": "2016-01-14T11:48:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:23.268000Z", "datetime": "2022-08-15T06:26:18.576000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.19.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.03", "properties": {"start_datetime": "2016-01-14T10:06:00Z", "end_datetime": "2016-01-14T10:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.930000Z", "datetime": "2022-08-15T06:26:17.714000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.03.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.11", "properties": {"start_datetime": "2016-01-14T10:54:00Z", "end_datetime": "2016-01-14T11:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.863000Z", "datetime": "2022-08-15T06:26:17.648000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.11.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.05", "properties": {"start_datetime": "2016-01-14T10:18:00Z", "end_datetime": "2016-01-14T10:24:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.649000Z", "datetime": "2022-08-15T06:26:18.060000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.05.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.13", "properties": {"start_datetime": "2016-01-14T11:06:00Z", "end_datetime": "2016-01-14T11:12:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.277000Z", "datetime": "2022-08-15T06:26:18.090000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.13.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.02", "properties": {"start_datetime": "2016-01-14T10:00:00Z", "end_datetime": "2016-01-14T10:06:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:22.169000Z", "datetime": "2022-08-15T06:26:17.466000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.02.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_from_url(self): + os.environ['STAC_JSON'] = 'https://cmr.earthdata.nasa.gov/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_num=1&page_size=1' + granule_json = requests.get(os.environ['STAC_JSON']) + granule_json = json.loads(granule_json.text) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + # os.environ['STAC_JSON'] = json.dumps(granule_json) + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data,metadata' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(download_result) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * len(os.environ['DOWNLOADING_KEYS'].split(',')) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match: {glob(os.path.join(tmp_dir_name, "*"))} v. {download_result["features"]}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac__from_file(self): + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020601090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-01T21:00:00.000Z", "start_datetime": "2002-06-01T21:00:00.000Z", "end_datetime": "2002-06-02T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020602090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106835-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106835-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-02T21:00:00.000Z", "start_datetime": "2002-06-02T21:00:00.000Z", "end_datetime": "2002-06-03T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106890-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106890-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-03T21:00:00.000Z", "start_datetime": "2002-06-03T21:00:00.000Z", "end_datetime": "2002-06-04T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020604090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106962-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106962-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-06-04T21:00:00.000Z", "start_datetime": "2002-06-04T21:00:00.000Z", "end_datetime": "2002-06-05T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"metadata": {"href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.xml", "type": "application/xml"}, "opendap": {"title": "OPeNDAP request URL", "href": "https://opendap.earthdata.nasa.gov/providers/POCLOUD/collections/GHRSST%20Level%204%20MUR%20Global%20Foundation%20Sea%20Surface%20Temperature%20Analysis%20(v4.1)/granules/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1"}, "data": {"title": "Download 20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc", "href": "https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/MUR-JPL-L4-GLOB-v4.1/20020605090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc"}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2028106862-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2028106862-POCLOUD.umm_json"}]}]}' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'metadata' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['metadata']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['metadata']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_502__from_file(self): + granule_json = '''{ + "type": "FeatureCollection", + "stac_version": "1.0.0", + "numberMatched": 242, + "numberReturned": 10, + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&page_size=10&temporal%5B%5D=2021-02-01T00%3A00%3A00Z%2C2021-02-02T00%3A00%3A00Z&page_num=1" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "next", + "body": { + "collection_concept_id": "C2011289787-GES_DISC", + "page_num": "2", + "page_size": "10", + "temporal": [ + "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + ], + "temporal[]": "2021-02-01T00:00:00Z,2021-02-02T00:00:00Z" + }, + "method": "POST", + "merge": true, + "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac" + } + ], + "context": { + "returned": 10, + "limit": 1000000, + "matched": 242 + }, + "features": [ + { + "properties": { + "datetime": "2021-01-31T23:54:00.000Z", + "start_datetime": "2021-01-31T23:54:00.000Z", + "end_datetime": "2021-02-01T00:00:00.000Z" + }, + "bbox": [ + -164.22, + -59.19, + -127.42, + -34.4 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/031/SNDR.SS1330.CHIRP.20210131T2354.m06.g240.L1_J1.std.v02_48.G.200408101645.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -127.42, + -53.49 + ], + [ + -140.59, + -34.4 + ], + [ + -164.22, + -38.4 + ], + [ + -161.84, + -59.19 + ], + [ + -127.42, + -53.49 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327589-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327589-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:00:00.000Z", + "start_datetime": "2021-02-01T00:00:00.000Z", + "end_datetime": "2021-02-01T00:06:00.000Z" + }, + "bbox": [ + -167.61, + -38.28, + -140.62, + -14.06 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0000.m06.g001.L1_J1.std.v02_48.G.200408101657.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -140.62, + -34.28 + ], + [ + -147.98, + -14.06 + ], + [ + -167.61, + -17.44 + ], + [ + -164.31, + -38.28 + ], + [ + -140.62, + -34.28 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327600-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327600-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:06:00.000Z", + "start_datetime": "2021-02-01T00:06:00.000Z", + "end_datetime": "2021-02-01T00:12:00.000Z" + }, + "bbox": [ + -171.93, + -17.31, + -148.0, + 6.74 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0006.m06.g002.L1_J1.std.v02_48.G.200408101655.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -148.0, + -13.93 + ], + [ + -153.06, + 6.74 + ], + [ + -171.93, + 3.48 + ], + [ + -167.7, + -17.31 + ], + [ + -148.0, + -13.93 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327624-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327624-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:12:00.000Z", + "start_datetime": "2021-02-01T00:12:00.000Z", + "end_datetime": "2021-02-01T00:18:00.000Z" + }, + "bbox": [ + -177.83, + 3.6, + -153.07, + 27.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0012.m06.g003.L1_J1.std.v02_48.G.200408101705.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -153.07, + 6.87 + ], + [ + -156.97, + 27.71 + ], + [ + -177.83, + 24.15 + ], + [ + -172.02, + 3.6 + ], + [ + -153.07, + 6.87 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327611-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327611-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:18:00.000Z", + "start_datetime": "2021-02-01T00:18:00.000Z", + "end_datetime": "2021-02-01T00:24:00.000Z" + }, + "bbox": [ + 172.75, + 24.27, + -156.97, + 48.71 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0018.m06.g004.L1_J1.std.v02_48.G.200408101716.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -156.97, + 27.84 + ], + [ + -160.07, + 48.71 + ], + [ + 172.75, + 44.19 + ], + [ + -177.94, + 24.27 + ], + [ + -156.97, + 27.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327811-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327811-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:24:00.000Z", + "start_datetime": "2021-02-01T00:24:00.000Z", + "end_datetime": "2021-02-01T00:30:00.000Z" + }, + "bbox": [ + 152.97, + 44.29, + -160.05, + 69.67 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0024.m06.g005.L1_J1.std.v02_48.G.200408101749.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -160.05, + 48.84 + ], + [ + -161.92, + 69.67 + ], + [ + 152.97, + 62.28 + ], + [ + 172.6, + 44.29 + ], + [ + -160.05, + 48.84 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327763-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327763-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:30:00.000Z", + "start_datetime": "2021-02-01T00:30:00.000Z", + "end_datetime": "2021-02-01T00:36:00.000Z" + }, + "bbox": [ + -180.0, + 62.35, + 180.0, + 90.0 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0030.m06.g006.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -161.83, + 69.79 + ], + [ + -45.22, + 88.83 + ], + [ + 101.33, + 71.69 + ], + [ + 152.69, + 62.35 + ], + [ + -161.83, + 69.79 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327782-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327782-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:36:00.000Z", + "start_datetime": "2021-02-01T00:36:00.000Z", + "end_datetime": "2021-02-01T00:42:00.000Z" + }, + "bbox": [ + -41.63, + 61.83, + 100.91, + 89.26695841076085 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0036.m06.g007.L1_J1.std.v02_48.G.200408101747.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -41.63, + 88.73 + ], + [ + 6.42, + 68.41 + ], + [ + 50.76, + 61.83 + ], + [ + 100.91, + 71.65 + ], + [ + -41.63, + 88.73 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327806-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327806-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:42:00.000Z", + "start_datetime": "2021-02-01T00:42:00.000Z", + "end_datetime": "2021-02-01T00:48:00.000Z" + }, + "bbox": [ + 4.45, + 43.65, + 50.62, + 68.30202151196218 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0042.m06.g008.L1_J1.std.v02_48.G.200408101807.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 6.29, + 68.29 + ], + [ + 4.45, + 47.46 + ], + [ + 31.46, + 43.65 + ], + [ + 50.62, + 61.71 + ], + [ + 6.29, + 68.29 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327773-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327773-GES_DISC.umm_json" + } + ] + }, + { + "properties": { + "datetime": "2021-02-01T00:48:00.000Z", + "start_datetime": "2021-02-01T00:48:00.000Z", + "end_datetime": "2021-02-01T00:54:00.000Z" + }, + "bbox": [ + 1.29, + 23.6, + 31.41, + 47.33 + ], + "assets": { + "metadata": { + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.xml", + "type": "application/xml" + }, + "opendap": { + "title": "The OPENDAP location for the granule. (GET DATA : OPENDAP DATA)", + "href": "https://sounder.gesdisc.eosdis.nasa.gov/opendap/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "type": "application/x-netcdf" + }, + "data": { + "title": "Download SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc", + "href": "https://data.gesdisc.earthdata.nasa.gov/data/CHIRP/SNDR13CHRP1.2/2021/032/SNDR.SS1330.CHIRP.20210201T0048.m06.g009.L1_J1.std.v02_48.G.200408101736.nc" + } + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 4.35, + 47.33 + ], + [ + 1.29, + 26.46 + ], + [ + 22.18, + 23.6 + ], + [ + 31.41, + 43.52 + ], + [ + 4.35, + 47.33 + ] + ] + ] + }, + "stac_extensions": [], + "id": "G2031327751-GES_DISC", + "stac_version": "1.0.0", + "collection": "C2011289787-GES_DISC", + "links": [ + { + "rel": "self", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.stac" + }, + { + "rel": "parent", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "collection", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac" + }, + { + "rel": "root", + "href": "https://cmr.earthdata.nasa.gov:443/search/" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.json" + }, + { + "rel": "via", + "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2031327751-GES_DISC.umm_json" + } + ] + } + ] + }''' + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + os.environ['DOWNLOADING_KEYS'] = 'data' + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__daac_error(self): # TODO update this later + granule_json = '{"type": "FeatureCollection", "stac_version": "1.0.0", "numberMatched": 3413, "numberReturned": 23, "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C1996881146-POCLOUD&page_size=23&temporal%5B%5D=2002-06-01T12%3A06%3A00.000Z%2C2011-10-04T06%3A51%3A45.000Z&page_num=1"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "next", "body": {"collection_concept_id": "C1996881146-POCLOUD", "page_num": "2", "page_size": "23", "temporal": ["2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"], "temporal[]": "2002-06-01T12:06:00.000Z,2011-10-04T06:51:45.000Z"}, "method": "POST", "merge": true, "href": "https://cmr.earthdata.nasa.gov:443/search/granules.stac"}], "context": {"returned": 23, "limit": 1000000, "matched": 3413}, "features": [{"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}, {"properties": {"datetime": "2002-05-31T21:00:00.000Z", "start_datetime": "2002-05-31T21:00:00.000Z", "end_datetime": "2002-06-01T21:00:00.000Z"}, "bbox": [-180.0, -90.0, 180.0, 90.0], "assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}, "type": "Feature", "geometry": {"type": "Polygon", "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}, "stac_extensions": [], "id": "G2030963432-POCLOUD", "stac_version": "1.0.0", "collection": "C1996881146-POCLOUD", "links": [{"rel": "self", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.stac"}, {"rel": "parent", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "collection", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/C1996881146-POCLOUD.stac"}, {"rel": "root", "href": "https://cmr.earthdata.nasa.gov:443/search/"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.json"}, {"rel": "via", "href": "https://cmr.earthdata.nasa.gov:443/search/concepts/G2030963432-POCLOUD.umm_json"}]}]}' + # granule_json = [{"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601161248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00414.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601172624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00415.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601190536-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00416.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601204344-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00417.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/152/20020601222152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00418.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602000000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00419.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602013912-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00420.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602031720-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00421.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602045528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00422.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602063440-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00423.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602081248-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00424.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602095056-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00425.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602112904-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00426.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602130816-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00427.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602144624-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00428.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602162432-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00429.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602180240-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00430.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602194152-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00431.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602212000-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00432.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/153/20020602225808-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00433.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603003616-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00434.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603021528-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00435.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}, {"assets": {"data": {"href": "https://podaac-tools.jpl.nasa.gov/drive/files/allData/ghrsst/data/GDS2/L2P/AMSRE/REMSS/v7/2002/154/20020603035336-REMSS-L2P_GHRSST-SSTsubskin-AMSRE-l2b_v07a_r00436.dat-v02.0-fv01.0.nc", "title": "The HTTP location for the granule."}}}] + granule_json = json.loads(granule_json) + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + # TODO this is downloading a login page HTML + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + print(glob(os.path.join(tmp_dir_name, '*'))) + self.assertEqual(len(download_result['features']) + 2, len(glob(os.path.join(tmp_dir_name, '*'))), + f'downloaded file does not match') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_dapa_url(self): + dapa_url = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000/items?limit=37&offset=0&datetime=1990-01-14T08:00:00Z/2024-01-14T11:59:59Z' + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['API_PREFIX'] = 'sbx-uds-2-dapa' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '71g0c73jl77gsqhtlfg2ht388c' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['STAC_AUTH_TYPE'] = 'UNITY' + + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['STAC_JSON'] = dapa_url + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + print(download_result) + expected_downloaded_feature = [{'type': 'Feature', 'stac_version': '1.0.0', + 'id': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000:test_file05', + 'properties': {'tag': '#sample', 'c_data1': [1, 10, 100, 1000], + 'c_data2': [False, True, True, False, True], + 'c_data3': ['Bellman Ford'], + 'datetime': '2024-01-31T22:34:06.556000Z', + 'start_datetime': '2016-01-31T18:00:00.009000Z', + 'end_datetime': '2016-01-31T19:59:59.991000Z', + 'created': '1970-01-01T00:00:00Z', + 'updated': '2024-01-31T22:34:49.583000Z', + 'status': 'completed', 'provider': 'unity'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, + 'links': [{'rel': 'collection', 'href': '.'}], 'assets': { + 'test_file05.cmr.xml': {'href': './test_file05.cmr.xml', 'title': 'test_file05.cmr.xml', + 'description': 'size=1768;checksumType=md5;checksum=29d1b69df5587d7ee0a945250adfd16f;', + 'roles': ['metadata']}, + 'test_file05.nc.stac.json': {'href': './test_file05.nc.stac.json', + 'title': 'test_file05.nc.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.nc.cas': {'href': './test_file05.nc.cas', 'title': 'test_file05.nc.cas', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['metadata']}, + 'test_file05.data.stac.json': {'href': './test_file05.data.stac.json', + 'title': 'test_file05.data.stac.json', + 'description': 'size=-1;checksumType=md5;checksum=unknown;', + 'roles': ['data']}}, 'bbox': [-180.0, -90.0, 180.0, 90.0], + 'stac_extensions': [], + 'collection': 'URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2401310000'}] + + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['test_file05.data.stac.json']['href'] for k in download_result]) + for each_granule in zip(expected_downloaded_feature, download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['test_file05.data.stac.json']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file(self): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + print(downloaded_file_hrefs) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + print() + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large_github_data_file(self): + granule_json = ''' + { + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "title": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc", + "description": "SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc" + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + } + ''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + with tempfile.TemporaryDirectory() as tmp_dir_name: + # tmp_dir_name = '/tmp/unity-ds' + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 40 * 2**20, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + @patch('requests.get') + def test_02_download__from_file_with_http(self, mock_requests): + granule_json = '{"numberMatched": {"total_size": 5}, "numberReturned": 6, "stac_version": "1.0.0", "type": "FeatureCollection", ' \ + '"links": [{"rel": "self", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev/sbx-uds-2-dapa/collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/?limit=10"}, {"rel": "root", "href": "https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev"}], ' \ + '"features": [' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:21:04.234000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:21:47.477000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.cmr.xml", "title": "test_file01.cmr.xml", "description": "size=1768;checksumType=md5;checksum=4d1935f25f3b508ca1e1a0368eeda10c;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.stac.json", "title": "test_file01.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc.cas", "title": "test_file01.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file01/test_file01.nc", "title": "test_file01.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:44:56.784000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:45:40.118000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.stac.json", "title": "test_file02.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.cmr.xml", "title": "test_file02.cmr.xml", "description": "size=1768;checksumType=md5;checksum=88b82e1824d51713d0bc897d970f3b0a;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc.cas", "title": "test_file02.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file02/test_file02.nc", "title": "test_file02.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:01.078000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:54:42.272000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.cmr.xml", "title": "test_file03.cmr.xml", "description": "size=1768;checksumType=md5;checksum=cd84e6a6138b3aad77d013ca4fb3ded4;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.stac.json", "title": "test_file03.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc.cas", "title": "test_file03.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file03/test_file03.nc", "title": "test_file03.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:54:33.221000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:55:12.198000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.cmr.xml", "title": "test_file04.cmr.xml", "description": "size=1768;checksumType=md5;checksum=47574084df6d14bbe9df60a2d40617ef;"}, "metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.stac.json", "title": "test_file04.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc.cas", "title": "test_file04.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file04/test_file04.nc", "title": "test_file04.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"}, ' \ + '{"type": "Feature", "stac_version": "1.0.0", "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05", "properties": {"tag": "#sample", "c_data1": [1, 10, 100, 1000], "c_data2": [false, true, true, false, true], "c_data3": ["Bellman Ford"], "datetime": "2023-12-04T18:58:31.381000Z", "start_datetime": "2016-01-31T18:00:00.009000Z", "end_datetime": "2016-01-31T19:59:59.991000Z", "created": "1970-01-01T00:00:00Z", "updated": "2023-12-04T18:58:42.027000Z", "status": "completed", "provider": "unity"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"metadata__stac": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.stac.json", "title": "test_file05.nc.stac.json", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "metadata__cmr": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.cmr.xml", "title": "test_file05.cmr.xml", "description": "size=1768;checksumType=md5;checksum=03e639becc6c74ad5128ccd438fc35ae;"}, "metadata__cas": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc.cas", "title": "test_file05.nc.cas", "description": "size=-1;checksumType=md5;checksum=unknown;"}, "data": {"href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030:test_file05/test_file05.nc", "title": "test_file05.nc", "description": "size=-1;checksumType=md5;checksum=unknown;"}}, "bbox": [-180.0, -90.0, 180.0, 90.0], "stac_extensions": [], "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030"} ' \ + ']}' + # granule_json = json.loads(granule_json) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.return_value.content.decode.return_value = granule_json + + # mock_response.content = granule_json.encode() + # mock_response.content.return_value = granule_json.encode() + # mock_response.json.return_value = json.loads(granule_json) + # specify the return value of the get() method + mock_requests.return_value.get.return_value = mock_response + mock_requests.return_value.content.decode.return_value = granule_json + + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'S3' + os.environ['STAC_JSON'] = 'https://example.com/get_feature_collection' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) * 4 + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + granule_json = json.loads(granule_json) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_file_large(self): + granule_json = FileUtils.read_json('./stage-in.json') + # granule_json['features'] = granule_json['features'][0:5] + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ[Constants.EDL_USERNAME] = '/unity/uds/user/wphyo/edl_username' + os.environ[Constants.EDL_PASSWORD] = '/unity/uds/user/wphyo/edl_dwssap' + os.environ[Constants.EDL_PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.EDL_BASE_URL] = 'urs.earthdata.nasa.gov' + os.environ['STAC_JSON'] = json.dumps(granule_json) + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'DAAC' + # os.environ['PARALLEL_COUNT'] = '5' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + print(tmp_dir_name) + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + print(len(download_result['features'])) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match') + error_file = os.path.join(downloading_dir, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http(self): + granule_json = '{"numberMatched": 20, "numberReturned": 20, "stac_version": "1.0.0", "type": "FeatureCollection", "links": [{"rel": "self", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}, {"rel": "root", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com"}, {"rel": "next", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100"}, {"rel": "prev", "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0"}], "features": [{"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", "properties": {"start_datetime": "2016-01-14T09:54:00Z", "end_datetime": "2016-01-14T10:00:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:39.830000Z", "datetime": "2022-08-15T06:26:37.029000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", "properties": {"start_datetime": "2016-01-14T10:36:00Z", "end_datetime": "2016-01-14T10:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.078000Z", "datetime": "2022-08-15T06:26:19.333000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", "properties": {"start_datetime": "2016-01-14T10:24:00Z", "end_datetime": "2016-01-14T10:30:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.068000Z", "datetime": "2022-08-15T06:26:18.641000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", "properties": {"start_datetime": "2016-01-14T11:36:00Z", "end_datetime": "2016-01-14T11:42:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.060000Z", "datetime": "2022-08-15T06:26:19.698000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}, {"type": "Feature", "stac_version": "1.0.0", "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", "properties": {"start_datetime": "2016-01-14T10:12:00Z", "end_datetime": "2016-01-14T10:18:00Z", "created": "2020-12-14T13:50:00Z", "updated": "2022-08-15T06:26:26.050000Z", "datetime": "2022-08-15T06:26:19.491000Z"}, "geometry": {"type": "Point", "coordinates": [0.0, 0.0]}, "links": [{"rel": "collection", "href": "."}], "assets": {"data": {"href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc"}, "metadata__data": {"href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas"}, "metadata__cmr": {"href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml"}}, "bbox": [0.0, 0.0, 0.0, 0.0], "stac_extensions": [], "collection": "SNDR_SNPP_ATMS_L1A___1"}]}' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_KEYS'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([k['assets']['data']['href'] for k in download_result]) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = os.path.basename(each_granule[0]['assets']['data']['href']) + self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, + f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_02_download__from_http_with_role(self): + granule_json = '''{ + "numberMatched": 20, + "numberReturned": 20, + "stac_version": "1.0.0", + "type": "FeatureCollection", + "links": [ + { + "rel": "self", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + }, + { + "rel": "root", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com" + }, + { + "rel": "next", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=100" + }, + { + "rel": "prev", + "href": "https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test/am-uds-dapa/collections/SNDR_SNPP_ATMS_L1A___1/items?datetime=2016-01-14T08:00:00Z/2016-01-14T11:59:59Z&limit=100&offset=0" + } + ], + "features": [ + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.01", + "properties": { + "start_datetime": "2016-01-14T09:54:00Z", + "end_datetime": "2016-01-14T10:00:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:39.830000Z", + "datetime": "2022-08-15T06:26:37.029000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/README.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.01.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.08", + "properties": { + "start_datetime": "2016-01-14T10:36:00Z", + "end_datetime": "2016-01-14T10:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.078000Z", + "datetime": "2022-08-15T06:26:19.333000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CHANGELOG.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.08.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.06", + "properties": { + "start_datetime": "2016-01-14T10:24:00Z", + "end_datetime": "2016-01-14T10:30:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.068000Z", + "datetime": "2022-08-15T06:26:18.641000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CODE_OF_CONDUCT.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.06.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.18", + "properties": { + "start_datetime": "2016-01-14T11:36:00Z", + "end_datetime": "2016-01-14T11:42:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.060000Z", + "datetime": "2022-08-15T06:26:19.698000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/CONTRIBUTING.md", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.18.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + }, + { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.04", + "properties": { + "start_datetime": "2016-01-14T10:12:00Z", + "end_datetime": "2016-01-14T10:18:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:26.050000Z", + "datetime": "2022-08-15T06:26:19.491000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc": { + "href": "https://raw.githubusercontent.com/unity-sds/unity-data-services/develop/LICENSE", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc", + "roles": [ + "data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.nc.cas", + "roles": [ + "metadata__data" + ] + }, + "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.04.cmr.xml", + "roles": [ + "metadata__cmr" + ] + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + ] + }''' + granule_json = json.loads(granule_json) + if len(argv) > 1: + argv.pop(-1) + argv.append('DOWNLOAD') + os.environ['GRANULES_DOWNLOAD_TYPE'] = 'HTTP' + os.environ['PARALLEL_COUNT'] = '3' + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + granule_json_file = os.path.join(tmp_dir_name, 'input_file.json') + downloading_dir = os.path.join(tmp_dir_name, 'downloading_dir') + FileUtils.mk_dir_p(downloading_dir) + FileUtils.write_json(granule_json_file, granule_json) + os.environ['STAC_JSON'] = granule_json_file + os.environ['DOWNLOAD_DIR'] = downloading_dir + os.environ['DOWNLOADING_ROLES'] = 'data' + download_result_str = DownloadGranulesFactory().get_class(os.getenv('GRANULES_DOWNLOAD_TYPE', 'MISSING_GRANULES_DOWNLOAD_TYPE')).download() + download_result = json.loads(download_result_str) + self.assertTrue('features' in download_result, f'missing features in download_result') + self.assertEqual(len(download_result['features']) + 1, len(glob(os.path.join(downloading_dir, '*'))), + f'downloaded file does not match: {download_result["features"]} v. {glob(os.path.join(downloading_dir, "*"))}') + error_file = os.path.join(tmp_dir_name, 'error.log') + if FileUtils.file_exist(error_file): + self.assertTrue(False, f'some downloads failed. error.log exists. {FileUtils.read_json(error_file)}') + download_result = download_result['features'] + self.assertTrue('assets' in download_result[0], f'no assets in download_result: {download_result}') + downloaded_file_hrefs = set([list(k['assets'].values())[0]['href'] for k in download_result]) + # print(granule_json['features'], download_result) + print(FileUtils.read_json(f'{downloading_dir}/downloaded_feature_collection.json')) + for each_granule in zip(granule_json['features'], download_result): + remote_filename = [k['href'] for k in each_granule[0]['assets'].values() if 'data' in k['roles']] + remote_filename = os.path.basename(remote_filename[0]) + # self.assertTrue(os.path.join('.', remote_filename) in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(remote_filename in downloaded_file_hrefs, f'mismatched: {remote_filename}') + self.assertTrue(FileUtils.get_size(os.path.join(downloading_dir, remote_filename)) > 0, f'empty file: {remote_filename}') + # self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py new file mode 100644 index 0000000..a1b92b5 --- /dev/null +++ b/tests/integration_tests/test_docker_stage_out.py @@ -0,0 +1,1334 @@ +import logging +logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + +from datetime import datetime + +from mdps_ds_lib.stage_in_out.upload_granules_by_complete_catalog_s3 import \ + UploadGranulesByCompleteCatalogS3 + +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from mdps_ds_lib.stage_in_out.upoad_granules_factory import UploadGranulesFactory + + +import json +import os +import tempfile +from sys import argv +from unittest import TestCase + +from pystac import Item, Asset, Catalog, Link, ItemCollection + +from mdps_ds_lib.lib.constants import Constants + +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from mdps_ds_lib.lib.utils.file_utils import FileUtils + + +class TestDockerStageOut(TestCase): + def setUp(self) -> None: + super().setUp() + self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' + self.tenant_venue = 'DEV' # 'DEV1' # 'dev' + self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' + self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + + def not_in_used_test_03_upload(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'S3' + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = tmp_dir_name + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata' in upload_result['assets'], 'missing assets#metadata') + self.assertTrue('href' in upload_result['assets']['metadata'], 'missing assets#metadata#href') + self.assertTrue( + upload_result['assets']['metadata']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def not_in_used_test_03_upload_catalog(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '6ir9qveln397i0inh9pmsabq1' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + os.environ['DAPA_API'] = 'https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test' + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-test-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + os.environ['GRANULES_UPLOAD_TYPE'] = 'CATALOG_S3' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + with open(os.path.join(tmp_dir_name, 'test_file01.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id='NA', + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + collection='NA', + assets={ + 'data': Asset(os.path.join(tmp_dir_name, 'test_file01.nc'), title='main data'), + 'metadata__cas': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.cas'), title='metadata cas'), + 'metadata__stac': Asset(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), title='metadata stac'), + }) + with open(os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + catalog.add_link(Link('child', os.path.join(tmp_dir_name, 'test_file01.nc.stac.json'), 'application/json')) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + print(upload_result) + self.assertEqual(1, len(upload_result), 'wrong length of upload_result features') + upload_result = upload_result[0] + self.assertTrue('assets' in upload_result, 'missing assets') + self.assertTrue('metadata__cas' in upload_result['assets'], 'missing assets#metadata__cas') + self.assertTrue('href' in upload_result['assets']['metadata__cas'], 'missing assets#metadata__cas#href') + self.assertTrue( + upload_result['assets']['metadata__cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue('data' in upload_result['assets'], 'missing assets#data') + self.assertTrue('href' in upload_result['assets']['data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets']['data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/')) + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + return + + def test_03_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_02_upload_complete_catalog(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{UploadGranulesByCompleteCatalogS3.DEFAULT_RESULT_PATH_PREFIX}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_03_upload_auxiliary_files(self): + temp_collection_id = f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:{self.collection_name}___{self.collection_version}' + os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_AUXILIARY_FILE_AS_GRANULE' + os.environ['COLLECTION_ID'] = temp_collection_id + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['PARALLEL_COUNT'] = '1' + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + real_base_dir = os.path.join(tmp_dir_name, 'auxiliary_base') + FileUtils.mk_dir_p(real_base_dir) + os.environ['BASE_DIRECTORY'] = real_base_dir + + with open(os.path.join(tmp_dir_name, 'excluding_file.json'), 'w') as ff: + ff.write('{"message": "excluding file"}') + with open(os.path.join(real_base_dir, 'test_file_0.json'), 'w') as ff: + ff.write('{"message": "some file at root"}') + sub_folders = [ + os.path.join(real_base_dir, 'son'), + os.path.join(real_base_dir, 'daughter'), + os.path.join(real_base_dir, os.path.join('son', 'grandson')), + os.path.join(real_base_dir, os.path.join('son', 'granddaughter')), + os.path.join(real_base_dir, os.path.join('daughter', 'grandson')), + os.path.join(real_base_dir, os.path.join('daughter', 'granddaughter')), + ] + for i, each_sub_folder in enumerate(sub_folders): + FileUtils.mk_dir_p(each_sub_folder) + with open(os.path.join(each_sub_folder, f'test_file_{i}.json'), 'w') as ff: + ff.write(json.dumps({"message": f"some file at {each_sub_folder}"})) + FileUtils.mk_dir_p(os.path.join(real_base_dir, 'nephew')) # should not throw error for empty folders + FileUtils.mk_dir_p(os.path.join(real_base_dir, os.path.join('nephew', 'grandson'))) # should not throw error for empty folders + total_files = len(sub_folders) + 1 + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + self.assertTrue(f'{result_key}.stac.json' in upload_result['assets'], f'missing assets#metadata asset: test_file_0.json') + self.assertTrue('href' in upload_result['assets'][f'{result_key}.stac.json'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'{result_key}.stac.json']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/'), f"wrong HREF (no S3?): upload_result['assets'][f'{result_key}.stac.json']['href']") + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_upload_complete_catalog_invalid_bucket(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('An error occurred (NoSuchBucket)'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return + + def test_03_upload_complete_catalog_missing_data(self): + os.environ['VERIFY_SSL'] = 'FALSE' + + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'invalid_bucket' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data1']), + f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'{filename}.nc.stac.json': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result_str = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result_str) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), 0, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), total_files, f'wrong length: {failed_feature_collection}') + + upload_result = failed_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('properties' in upload_result, 'missing properties') + self.assertTrue('upload_error' in upload_result['properties'], 'missing upload_error') + self.assertTrue('missing "data" in assets'in upload_result['properties']['upload_error'], f"wrong upload_error: {upload_result['properties']['upload_error']}") + + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + self.assertTrue(result_key.startswith('test_file'), f'worng asset key: {result_key}') + result_key_prefix = result_key.split('.')[0] + + self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') + self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') + self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') + self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + return From 00d6bded6edaeb42a0d554103e00302296781da2 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 14 Oct 2024 10:31:23 -0700 Subject: [PATCH 2/6] chore: update outdated tests --- tests/integration_tests/test_docker_entry.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/test_docker_entry.py b/tests/integration_tests/test_docker_entry.py index df328b3..c5b396d 100644 --- a/tests/integration_tests/test_docker_entry.py +++ b/tests/integration_tests/test_docker_entry.py @@ -1,5 +1,8 @@ import logging +from mdps_ds_lib.stage_in_out.catalog_granules_factory import CatalogGranulesFactory +from mdps_ds_lib.stage_in_out.search_granules_factory import SearchGranulesFactory + logging.basicConfig(level=10, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") import math @@ -42,7 +45,7 @@ def test_01_search_part_01(self): argv.append('SEARCH') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - search_result_str = choose_process() + search_result_str = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result_str) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -73,7 +76,7 @@ def test_01_search_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() print(search_result) search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') @@ -103,7 +106,7 @@ def test_01_search_part_03(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -132,7 +135,7 @@ def test_01_search_part_04(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -161,7 +164,7 @@ def test_01_search_part_05(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -188,7 +191,7 @@ def test_01_1_search_cmr_part_01(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -215,7 +218,8 @@ def test_01_1_search_cmr_part_02(self): if len(argv) > 1: argv.pop(-1) argv.append('SEARCH') - search_result = choose_process() + search_result = SearchGranulesFactory().get_class(os.getenv('GRANULES_SEARCH_DOMAIN', 'MISSING_GRANULES_SEARCH_DOMAIN')).search() + search_result = json.loads(search_result) self.assertTrue('type' in search_result, f'missing type in search_result') item_collections = ItemCollection.from_dict(search_result) @@ -239,7 +243,7 @@ def test_04_catalog(self): argv.append('CATALOG') with tempfile.TemporaryDirectory() as tmp_dir_name: os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertEqual('registered', catalog_result, 'wrong status') self.assertTrue(FileUtils.file_exist(os.environ['OUTPUT_FILE']), f'missing output file') @@ -266,7 +270,7 @@ def test_04_catalog_from_file(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -317,7 +321,7 @@ def test_04_catalog_from_file_item_collection(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue('cataloging_request_status' in catalog_result, f'missing cataloging_request_status') self.assertTrue('status_result' in catalog_result, f'missing status_result') @@ -355,7 +359,7 @@ def test_04_catalog_from_file_item_collection_large(self): FileUtils.write_json(input_file_path, upload_result) os.environ['UPLOADED_FILES_JSON'] = input_file_path os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') - catalog_result_str = choose_process() + catalog_result_str = CatalogGranulesFactory().get_class(os.getenv('GRANULES_CATALOG_TYPE', 'MISSING_GRANULES_CATALOG_TYPE')).catalog() catalog_result = json.loads(catalog_result_str) self.assertTrue(isinstance(catalog_result, list), f'catalog_result is not list. {catalog_result}') self.assertEqual(len(catalog_result), math.ceil(len(upload_result['features']) / 250), f'mismatched catalog_result count') From a51314b0f34e70a66368f45d04bcc582c76fda57 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 21 Oct 2024 12:12:14 -0700 Subject: [PATCH 3/6] fix: allow empty str as RESULT_PATH_PREFIX & replace w/ default val --- mdps_ds_lib/stage_in_out/upload_granules_abstract.py | 2 ++ tests/integration_tests/test_docker_stage_out.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index 5d803d0..ceedbee 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -47,6 +47,8 @@ def _set_props_from_env(self): self._collection_id = os.environ.get(self.COLLECTION_ID_KEY) self._staging_bucket = os.environ.get(self.STAGING_BUCKET_KEY) self._result_path_prefix = os.environ.get(self.RESULT_PATH_PREFIX, self.DEFAULT_RESULT_PATH_PREFIX) + if self._result_path_prefix is None or self._result_path_prefix.strip() == '': + self._result_path_prefix = self.DEFAULT_RESULT_PATH_PREFIX self._result_path_prefix = self._result_path_prefix[:-1] if self._result_path_prefix.endswith('/') else self._result_path_prefix self._result_path_prefix = self._result_path_prefix[1:] if self._result_path_prefix.startswith('/') else self._result_path_prefix diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index a1b92b5..7ee0298 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -1,4 +1,5 @@ import logging + logging.basicConfig(level=20, format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") from datetime import datetime @@ -30,7 +31,7 @@ def setUp(self) -> None: self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' self.collection_name = 'UDS_UNIT_COLLECTION' # 'uds_collection' # 'sbx_collection' - self.collection_version = '24.08.29.09.00'.replace('.', '') # '2402011200' + self.collection_version = '24.10.21.12.00'.replace('.', '') # '2402011200' def not_in_used_test_03_upload(self): os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' @@ -791,7 +792,7 @@ def test_03_03_upload_auxiliary_files(self): os.environ['COLLECTION_ID'] = temp_collection_id os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['VERIFY_SSL'] = 'FALSE' - os.environ['RESULT_PATH_PREFIX'] = 'stage_out' + os.environ['RESULT_PATH_PREFIX'] = '' os.environ['PARALLEL_COUNT'] = '1' if len(argv) > 1: @@ -882,7 +883,7 @@ def test_03_03_upload_auxiliary_files(self): """ s3 = AwsS3() s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], - f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + f"stage_out/successful_features_{starting_time}", )] s3_keys = sorted(s3_keys) print(f's3_keys: {s3_keys}') From ea976abca9ae325fc2149ab0610744bcfb69085f Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Mon, 21 Oct 2024 15:07:58 -0700 Subject: [PATCH 4/6] fix: re-use keys for input stac item during stage out --- .../lib/cumulus_stac/granules_catalog.py | 12 +- .../upload_granules_by_complete_catalog_s3.py | 20 +- .../test_docker_stage_out.py | 229 ++++++++++++++++++ .../lib/cumulus_stac/test_granules_catalog.py | 126 ++++++++-- 4 files changed, 353 insertions(+), 34 deletions(-) diff --git a/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py b/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py index ae37281..be645ec 100644 --- a/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py +++ b/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py @@ -46,23 +46,23 @@ def extract_assets_href(self, granules_stac: Item, dir_name: str = '') -> dict: self_dir = os.path.dirname(granules_stac.self_href) except: self_dir = None - assets = defaultdict(list) + assets = defaultdict(dict) for k, v in granules_stac.get_assets().items(): href = v.href if v.roles is None or len(v.roles) < 1: LOGGER.warning(f'asset do not have roles: {v}') continue - k = v.roles[0] + role_key = v.roles[0] if not FileUtils.is_relative_path(href): - assets[k].append(href) + assets[role_key][k] = href continue if dir_name is not None and len(dir_name) > 0: - assets[k].append(os.path.join(dir_name, href)) + assets[role_key][k] = os.path.join(dir_name, href) continue if self_dir is not None and len(self_dir) > 0: - assets[k].append(os.path.join(self_dir, href)) + assets[role_key][k] = os.path.join(self_dir, href) continue - assets[k].append(href) + assets[role_key][k] = href return assets def update_assets_href(self, granules_stac: Item, new_assets: dict): diff --git a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py index 2f7d8e7..bd3ab5a 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py @@ -5,7 +5,7 @@ from mdps_ds_lib.lib.utils.file_utils import FileUtils -from pystac import ItemCollection +from pystac import ItemCollection, Item from mdps_ds_lib.lib.cumulus_stac.granules_catalog import GranulesCatalog from mdps_ds_lib.lib.processing_jobs.job_executor_abstract import JobExecutorAbstract @@ -48,10 +48,10 @@ def validate_job(self, job_obj): # return def execute_job(self, each_child, lock) -> bool: - current_granule_stac = self.__gc.get_granules_item(each_child) + current_granule_stac: Item = self.__gc.get_granules_item(each_child) try: current_granules_dir = os.path.dirname(each_child) - current_assets = self.__gc.extract_assets_href(current_granule_stac, current_granules_dir) + current_assets = self.__gc.extract_assets_href(current_granule_stac, current_granules_dir) # returns defaultdict(list) if 'data' not in current_assets: # this is still ok .coz extract_assets_href is {'data': [url1, url2], ...} LOGGER.warning(f'skipping {each_child}. no data in {current_assets}') current_granule_stac.properties['upload_error'] = f'missing "data" in assets' @@ -63,14 +63,14 @@ def execute_job(self, each_child, lock) -> bool: updating_assets = {} uploading_current_granule_stac = None for asset_type, asset_hrefs in current_assets.items(): - for each_asset_href in asset_hrefs: - LOGGER.audit(f'uploading {asset_type}: {each_asset_href}') - s3_url = self.__s3.upload(each_asset_href, self.__staging_bucket, - f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', - self.__delete_files) - if each_asset_href == each_child: + for asset_name, asset_href in asset_hrefs.items(): + LOGGER.audit(f'uploading type={asset_type}, name={asset_name}, href={asset_href}') + s3_url = self.__s3.upload(asset_href, self.__staging_bucket, + f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', + self.__delete_files) + if asset_href == each_child: uploading_current_granule_stac = s3_url - updating_assets[os.path.basename(s3_url)] = s3_url + updating_assets[asset_name] = s3_url self.__gc.update_assets_href(current_granule_stac, updating_assets) current_granule_stac.id = current_granule_id current_granule_stac.collection_id = self.__collection_id diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index 7ee0298..ab31914 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -548,6 +548,7 @@ def test_03_upload_complete_catalog(self): 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} """ s3 = AwsS3() + print(f'starting_time: {starting_time}') s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", )] @@ -560,6 +561,234 @@ def test_03_upload_complete_catalog(self): self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') return + def test_03_upload_complete_catalog_role_as_key(self): + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' + os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' + + os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' + # os.environ['GRANULES_UPLOAD_TYPE'] = 'UPLOAD_S3_BY_STAC_CATALOG' + # defaulted to this value + + if len(argv) > 1: + argv.pop(-1) + argv.append('UPLOAD') + + starting_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M') + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['OUTPUT_FILE'] = os.path.join(tmp_dir_name, 'some_output', 'output.json') + os.environ['UPLOAD_DIR'] = '' # not needed + os.environ['OUTPUT_DIRECTORY'] = os.path.join(tmp_dir_name, 'output_dir') + FileUtils.mk_dir_p(os.environ.get('OUTPUT_DIRECTORY')) + os.environ['CATALOG_FILE'] = os.path.join(tmp_dir_name, 'catalog.json') + total_files = 10 + # os.environ['PARALLEL_COUNT'] = str(total_files) + granules_dir = os.path.join(tmp_dir_name, 'some_granules') + FileUtils.mk_dir_p(granules_dir) + catalog = Catalog( + id='NA', + description='NA') + catalog.set_self_href(os.environ['CATALOG_FILE']) + + for i in range(1, total_files+1): + filename = f'test_file{i:02d}' + with open(os.path.join(granules_dir, f'{filename}.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(granules_dir, f'{filename}.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + + ''') + stac_item = Item(id=filename, + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, 0.0, 0.0, 0.0], + datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z" + }, + href=os.path.join('some_granules', f'{filename}.nc.stac.json'), + collection='NA', + assets={ + f'data': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), + f'metadata1': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), + f'metadata2': Asset(os.path.join('.', f'{filename}.nc.stac.json'), title='test_file01.nc.stac.json', roles=['metadata']), + }) + with open(os.path.join(granules_dir, f'{filename}.nc.stac.json'), 'w') as ff: + ff.write(json.dumps(stac_item.to_dict(False, False))) + catalog.add_link(Link('item', os.path.join('some_granules', f'{filename}.nc.stac.json'), 'application/json')) + print(json.dumps(catalog.to_dict(False, False))) + with open(os.environ['CATALOG_FILE'], 'w') as ff: + ff.write(json.dumps(catalog.to_dict(False, False))) + + upload_result = UploadGranulesFactory().get_class(os.getenv('GRANULES_UPLOAD_TYPE', UploadGranulesFactory.UPLOAD_S3_BY_STAC_CATALOG)).upload() + upload_result = json.loads(upload_result) + print(upload_result) + """ + {'type': 'Catalog', 'id': 'NA', 'stac_version': '1.0.0', 'description': 'NA', 'links': [{'rel': 'root', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/catalog.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/successful_features.json', 'type': 'application/json'}, {'rel': 'item', 'href': '/var/folders/33/xhq97d6s0dq78wg4h2smw23m0000gq/T/tmprew515jo/failed_features.json', 'type': 'application/json'}]} + """ + self.assertTrue('type' in upload_result, 'missing type') + self.assertEqual(upload_result['type'], 'Catalog', 'missing type') + upload_result = Catalog.from_dict(upload_result) + child_links = [k.href for k in upload_result.get_links(rel='item')] + self.assertEqual(len(child_links), 2, f'wrong length: {child_links}') + self.assertTrue(FileUtils.file_exist(child_links[0]), f'missing file: {child_links[0]}') + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[0])) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + + self.assertTrue(FileUtils.file_exist(child_links[1]), f'missing file: {child_links[1]}') + failed_feature_collection = ItemCollection.from_dict(FileUtils.read_json(child_links[1])) + failed_feature_collection = list(failed_feature_collection.items) + self.assertEqual(len(failed_feature_collection), 0, f'wrong length: {failed_feature_collection}') + + upload_result = successful_feature_collection[0].to_dict(False, False) + print(f'example feature: {upload_result}') + self.assertTrue('assets' in upload_result, 'missing assets') + result_key = [k for k in upload_result['assets'].keys()][0] + print(f'result_key: {result_key}') + self.assertEqual(result_key, 'data', f'worng asset key: {result_key}') + self.assertTrue(f'metadata1' in upload_result['assets'], f'missing assets#metadata asset: metadata1') + self.assertTrue('href' in upload_result['assets'][f'metadata1'], 'missing assets#metadata__cas#href') + self.assertTrue(upload_result['assets'][f'metadata1']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(f'data' in upload_result['assets'], f'missing assets#data: data') + self.assertTrue('href' in upload_result['assets'][f'data'], 'missing assets#data#href') + self.assertTrue(upload_result['assets'][f'data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + """ + Example output: + { + 'type': 'FeatureCollection', + 'features': [{ + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'NEW_COLLECTION_EXAMPLE_L1B___9:test_file01', + 'properties': {'start_datetime': '2016-01-31T18:00:00.009057Z', + 'end_datetime': '2016-01-31T19:59:59.991043Z', 'created': '2016-02-01T02:45:59.639000Z', + 'updated': '2022-03-23T15:48:21.578000Z', 'datetime': '1970-01-01T00:00:00Z'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [], + 'assets': {'data': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc', + 'title': 'main data'}, 'metadata__cas': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.cas', + 'title': 'metadata cas'}, 'metadata__stac': { + 'href': 's3://uds-test-cumulus-staging/NEW_COLLECTION_EXAMPLE_L1B___9/NEW_COLLECTION_EXAMPLE_L1B___9:test_file01/test_file01.nc.stac.json', + 'title': 'metadata stac'}}, + 'bbox': [0.0, 0.0, 0.0, 0.0], + 'stac_extensions': [], + 'collection': 'NEW_COLLECTION_EXAMPLE_L1B___9'}]} + """ + s3 = AwsS3() + s3_keys = [k for k in s3.get_child_s3_files(os.environ['STAGING_BUCKET'], + f"{os.environ['RESULT_PATH_PREFIX']}/successful_features_{starting_time}", + )] + s3_keys = sorted(s3_keys) + print(f's3_keys: {s3_keys}') + self.assertTrue(len(s3_keys) > 0, f'empty files in S3') + local_file = s3.set_s3_url(f's3://{os.environ["STAGING_BUCKET"]}/{s3_keys[-1][0]}').download(tmp_dir_name) + successful_feature_collection = ItemCollection.from_dict(FileUtils.read_json(local_file)) + successful_feature_collection = list(successful_feature_collection.items) + self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') + return + + def test_03_02_upload_complete_catalog(self): os.environ['VERIFY_SSL'] = 'FALSE' os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' diff --git a/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py b/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py index 0e57f60..58e457e 100644 --- a/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py +++ b/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py @@ -208,9 +208,10 @@ def test_extract_assets_href(self): pystac_catalog = gc.get_granules_item(granules_catalog_path) self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') assets = gc.extract_assets_href(pystac_catalog) - expected_assets = {'data': ['s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc'], - 'metadata__data': ['s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml']} + expected_assets = {'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'} + } self.assertEqual(assets, expected_assets, 'wrong assets') return @@ -276,9 +277,9 @@ def test_extract_assets_relative_href_01(self): self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') assets = gc.extract_assets_href(pystac_catalog) expected_assets = { - 'data': ['./SNDR.SNPP.ATMS.L1A.nominal2.12.nc'], - 'metadata__data': ['SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml']} + 'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': './SNDR.SNPP.ATMS.L1A.nominal2.12.nc'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': 'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'}} self.assertEqual(assets, expected_assets, 'wrong assets') return @@ -350,9 +351,10 @@ def test_extract_assets_relative_href_02(self): self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') assets = gc.extract_assets_href(pystac_catalog, '/some/temp/directory/../hehe') expected_assets = { - 'data': ['/some/temp/directory/../hehe/./SNDR.SNPP.ATMS.L1A.nominal2.12.nc', 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc'], - 'metadata__data': ['/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml']} + 'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': '/some/temp/directory/../hehe/./SNDR.SNPP.ATMS.L1A.nominal2.12.nc', + 'SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.1.nc'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': '/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'}} self.assertEqual(assets, expected_assets, 'wrong assets') return @@ -428,9 +430,10 @@ def test_extract_assets_relative_href_03(self): self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') assets = gc.extract_assets_href(pystac_catalog) expected_assets = { - 'data': ['/some/temp/directory/../hehe/./SNDR.SNPP.ATMS.L1A.nominal2.12.nc'], - 'metadata__data': ['/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas', '/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.2.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml']} + 'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': '/some/temp/directory/../hehe/./SNDR.SNPP.ATMS.L1A.nominal2.12.nc'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': '/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas', + 'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.2.cas': '/some/temp/directory/../hehe/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.2.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'}} self.assertEqual(assets, expected_assets, 'wrong assets') return @@ -496,9 +499,9 @@ def test_update_assets_href(self): self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') assets = gc.extract_assets_href(pystac_catalog) expected_assets = { - 'data': ['s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc'], - 'metadata__data': ['s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'] + 'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'} } self.assertEqual(assets, expected_assets, 'wrong assets') updating_assets = { @@ -509,9 +512,96 @@ def test_update_assets_href(self): } updating_assets_result = { - 'data': ['file:///absolute/file/some/file/data'], - 'metadata__data': ['s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'], - 'metadata__cmr': ['s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'] + 'data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc': 'file:///absolute/file/some/file/data'}, + 'metadata__data': {'SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas'}, + 'metadata__cmr': {'SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml'} + } + gc.update_assets_href(pystac_catalog, updating_assets) + updated_assets = gc.extract_assets_href(pystac_catalog) + self.assertEqual(updated_assets, updating_assets_result, 'wrong updated assets') + + return + + def test_update_assets_href_02(self): + sample_granules = { + "type": "Feature", + "stac_version": "1.0.0", + "id": "SNDR.SNPP.ATMS.L1A.nominal2.12", + "properties": { + "start_datetime": "2016-01-14T11:00:00Z", + "end_datetime": "2016-01-14T11:06:00Z", + "created": "2020-12-14T13:50:00Z", + "updated": "2022-08-15T06:26:25.344000Z", + "datetime": "2022-08-15T06:26:17.938000Z" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "data1": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc", + "roles": ["data"], + }, + "metadata1": { + "href": "s3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas", + "roles": ["metadata"], + }, + "metadata2": { + "href": "s3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", + "title": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", + "description": "SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml", + "roles": ["metadata"], + } + }, + "bbox": [ + 0.0, + 0.0, + 0.0, + 0.0 + ], + "stac_extensions": [], + "collection": "SNDR_SNPP_ATMS_L1A___1" + } + with tempfile.TemporaryDirectory() as tmp_dir_name: + granules_catalog_path = os.path.join(tmp_dir_name, 'sample_granules.json') + FileUtils.write_json(granules_catalog_path, sample_granules) + gc = GranulesCatalog() + pystac_catalog = gc.get_granules_item(granules_catalog_path) + self.assertEqual(pystac_catalog.id, 'SNDR.SNPP.ATMS.L1A.nominal2.12') + assets = gc.extract_assets_href(pystac_catalog) + expected_assets = { + 'data': {'data1': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc'}, + 'metadata': {'metadata1': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas', + 'metadata2': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml' + }, + } + self.assertEqual(assets, expected_assets, 'wrong assets') + updating_assets = { + 'data1': 'file:///absolute/file/some/file/data', + 'metadata1': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas', + 'other.name': '/absolute/file/some/file/metadata__extra', + 'metadata2': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml' + } + + updating_assets_result = { + 'data': {'data1': 'file:///absolute/file/some/file/data'}, + 'metadata': {'metadata1': 's3://uds-test-cumulus-protected/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.nc.cas', + 'metadata2': 's3://uds-test-cumulus-private/SNDR_SNPP_ATMS_L1A___1/SNDR.SNPP.ATMS.L1A.nominal2.12.cmr.xml' + }, } gc.update_assets_href(pystac_catalog, updating_assets) updated_assets = gc.extract_assets_href(pystac_catalog) From 047c74c31dacb92b3f6a74ae55a6fe3888bc9df8 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 22 Oct 2024 10:28:16 -0700 Subject: [PATCH 5/6] BREAKING CHANGE : not using collection id from ENV --- .../lib/cumulus_stac/granules_catalog.py | 16 +++++++++++++ .../upload_arbitrary_files_as_granules.py | 2 ++ .../stage_in_out/upload_granules_abstract.py | 10 ++++++-- .../upload_granules_by_complete_catalog_s3.py | 22 ++++++++++------- .../test_docker_stage_out.py | 22 +++++++++-------- .../lib/cumulus_stac/test_granules_catalog.py | 24 +++++++++++++++++++ 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py b/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py index be645ec..7b55d24 100644 --- a/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py +++ b/mdps_ds_lib/lib/cumulus_stac/granules_catalog.py @@ -9,6 +9,22 @@ class GranulesCatalog: + @staticmethod + def get_unity_formatted_collection_id(current_collection_id: str, project_venue_set: tuple): + if current_collection_id == '' or current_collection_id is None: + raise ValueError(f'NULL or EMPTY collection_id: {current_collection_id}') + collection_identifier_parts = current_collection_id.split(':') + if len(collection_identifier_parts) >= 6: + LOGGER.debug(f'current_collection_id is assumed to be in UNITY format: {current_collection_id}') + return current_collection_id + + LOGGER.info(f'current_collection_id is not UNITY formatted ID: {current_collection_id}') + if project_venue_set[0] is None or project_venue_set[1] is None: + raise ValueError(f'missing project or venue in ENV which is needed due to current_collection_id not UNITY format: {project_venue_set}') + new_collection = f'URN:NASA:UNITY:{project_venue_set[0]}:{project_venue_set[1]}:{current_collection_id}' + LOGGER.info(f'UNITY formatted ID: {new_collection}') + return new_collection + def update_catalog(self, catalog_file_path: str, file_paths: list, rel_name: str = 'item'): if not FileUtils.file_exist(catalog_file_path): raise ValueError(f'missing file: {catalog_file_path}') diff --git a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py index a545ce4..9b159be 100644 --- a/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py +++ b/mdps_ds_lib/stage_in_out/upload_arbitrary_files_as_granules.py @@ -111,6 +111,8 @@ def upload(self, **kwargs) -> str: :return: """ self._set_props_from_env() + if self._collection_id is None: + raise ValueError(f'missing COLLECTION ID in ENV') output_dir = os.environ.get(self.OUTPUT_DIRECTORY) if not FileUtils.dir_exist(output_dir): raise ValueError(f'OUTPUT_DIRECTORY: {output_dir} does not exist') diff --git a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py index ceedbee..3980405 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_abstract.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_abstract.py @@ -23,7 +23,9 @@ class UploadGranulesAbstract(ABC): RESULT_PATH_PREFIX = 'RESULT_PATH_PREFIX' # s3 prefix DEFAULT_RESULT_PATH_PREFIX = 'stage_out' # default s3 prefix OUTPUT_DIRECTORY = 'OUTPUT_DIRECTORY' # To store successful & failed features json - COLLECTION_ID_KEY = 'COLLECTION_ID' # Need this + COLLECTION_ID_KEY = 'COLLECTION_ID' # Need this only for arbitrary upload + PROJECT_KEY = 'PROJECT' # Need this only for process stageout + VENUE_KEY = 'VENUE' # Need this only for process stageout STAGING_BUCKET_KEY = 'STAGING_BUCKET' # S3 Bucket VERIFY_SSL_KEY = 'VERIFY_SSL' DELETE_FILES_KEY = 'DELETE_FILES' @@ -31,6 +33,8 @@ class UploadGranulesAbstract(ABC): def __init__(self) -> None: super().__init__() self._collection_id = '' + self._project = '' + self._venue = '' self._staging_bucket = '' self._result_path_prefix = '' self._parallel_count = int(os.environ.get(Constants.PARALLEL_COUNT, '-1')) @@ -40,11 +44,13 @@ def __init__(self) -> None: self._delete_files = False def _set_props_from_env(self): - missing_keys = [k for k in [self.COLLECTION_ID_KEY, self.STAGING_BUCKET_KEY] if k not in os.environ] + missing_keys = [k for k in [self.STAGING_BUCKET_KEY] if k not in os.environ] if len(missing_keys) > 0: raise ValueError(f'missing environment keys: {missing_keys}') self._collection_id = os.environ.get(self.COLLECTION_ID_KEY) + self._project = os.environ.get(self.PROJECT_KEY) + self._venue = os.environ.get(self.VENUE_KEY) self._staging_bucket = os.environ.get(self.STAGING_BUCKET_KEY) self._result_path_prefix = os.environ.get(self.RESULT_PATH_PREFIX, self.DEFAULT_RESULT_PATH_PREFIX) if self._result_path_prefix is None or self._result_path_prefix.strip() == '': diff --git a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py index bd3ab5a..8f9bf20 100644 --- a/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py +++ b/mdps_ds_lib/stage_in_out/upload_granules_by_complete_catalog_s3.py @@ -22,9 +22,9 @@ class UploadItemExecutor(JobExecutorAbstract): - def __init__(self, result_list, error_list, collection_id, staging_bucket, retry_wait_time_sec, retry_times, delete_files: bool) -> None: + def __init__(self, result_list, error_list, project_venue_set, staging_bucket, retry_wait_time_sec, retry_times, delete_files: bool) -> None: super().__init__() - self.__collection_id = collection_id + self.__project_venue_set = project_venue_set self.__staging_bucket = staging_bucket self.__delete_files = delete_files @@ -49,7 +49,10 @@ def validate_job(self, job_obj): def execute_job(self, each_child, lock) -> bool: current_granule_stac: Item = self.__gc.get_granules_item(each_child) + current_collection_id = current_granule_stac.collection_id.strip() try: + current_collection_id = GranulesCatalog.get_unity_formatted_collection_id(current_collection_id, self.__project_venue_set) + LOGGER.debug(f'reformatted current_collection_id: {current_collection_id}') current_granules_dir = os.path.dirname(each_child) current_assets = self.__gc.extract_assets_href(current_granule_stac, current_granules_dir) # returns defaultdict(list) if 'data' not in current_assets: # this is still ok .coz extract_assets_href is {'data': [url1, url2], ...} @@ -66,18 +69,18 @@ def execute_job(self, each_child, lock) -> bool: for asset_name, asset_href in asset_hrefs.items(): LOGGER.audit(f'uploading type={asset_type}, name={asset_name}, href={asset_href}') s3_url = self.__s3.upload(asset_href, self.__staging_bucket, - f'{self.__collection_id}/{self.__collection_id}:{current_granule_id}', - self.__delete_files) + f'{current_collection_id}/{current_collection_id}:{current_granule_id}', + self.__delete_files) if asset_href == each_child: uploading_current_granule_stac = s3_url updating_assets[asset_name] = s3_url self.__gc.update_assets_href(current_granule_stac, updating_assets) current_granule_stac.id = current_granule_id - current_granule_stac.collection_id = self.__collection_id + current_granule_stac.collection_id = current_collection_id if uploading_current_granule_stac is not None: # upload metadata file again self.__s3.set_s3_url(uploading_current_granule_stac) self.__s3.upload_bytes(json.dumps(current_granule_stac.to_dict(False, False)).encode()) - current_granule_stac.id = f'{self.__collection_id}:{current_granule_id}' + current_granule_stac.id = f'{current_collection_id}:{current_granule_id}' self.__result_list.put(current_granule_stac.to_dict(False, False)) except Exception as e: current_granule_stac.properties['upload_error'] = str(e) @@ -110,10 +113,11 @@ def upload(self, **kwargs) -> str: for each_child in child_links: job_manager_props.memory_job_dict[each_child] = each_child + project_venue_set = (self._project, self._venue) # https://www.infoworld.com/article/3542595/6-python-libraries-for-parallel-processing.html multithread_processor_props = MultiThreadProcessorProps(self._parallel_count) multithread_processor_props.job_manager = JobManagerMemory(job_manager_props) - multithread_processor_props.job_executor = UploadItemExecutor(local_items, error_list, self._collection_id, self._staging_bucket, self._retry_wait_time_sec, self._retry_times, self._delete_files) + multithread_processor_props.job_executor = UploadItemExecutor(local_items, error_list, project_venue_set, self._staging_bucket, self._retry_wait_time_sec, self._retry_times, self._delete_files) multithread_processor = MultiThreadProcessor(multithread_processor_props) multithread_processor.start() @@ -130,12 +134,12 @@ def upload(self, **kwargs) -> str: failed_item_collections = ItemCollection(items=errors) successful_features_file = os.path.join(output_dir, 'successful_features.json') - - failed_features_file = os.path.join(output_dir, 'failed_features.json') LOGGER.debug(f'writing results: {successful_features_file} && {failed_features_file}') FileUtils.write_json(successful_features_file, successful_item_collections.to_dict(False)) FileUtils.write_json(failed_features_file, failed_item_collections.to_dict(False)) + if len(failed_item_collections.items) > 0: + LOGGER.fatal(f'One or more Failures: {failed_item_collections.to_dict(False)}') s3_url = self.__s3.upload(successful_features_file, self._staging_bucket, self._result_path_prefix, s3_name=f'successful_features_{TimeUtils.get_current_time()}.json', diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index ab31914..5050e0b 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -336,7 +336,8 @@ def not_in_used_test_03_upload_catalog(self): def test_03_upload_complete_catalog(self): os.environ['VERIFY_SSL'] = 'FALSE' os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' - os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['PROJECT'] = 'LOCAL' + os.environ['VENUE'] = 'UNIT_TEST' os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' @@ -520,10 +521,10 @@ def test_03_upload_complete_catalog(self): result_key_prefix = result_key.split('.')[0] self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) """ Example output: { @@ -564,7 +565,8 @@ def test_03_upload_complete_catalog(self): def test_03_upload_complete_catalog_role_as_key(self): os.environ['VERIFY_SSL'] = 'FALSE' os.environ['RESULT_PATH_PREFIX'] = 'integration_test/stage_out' - os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['PROJECT'] = 'LOCAL' + os.environ['VENUE'] = 'UNIT_TEST' os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' @@ -748,10 +750,10 @@ def test_03_upload_complete_catalog_role_as_key(self): self.assertEqual(result_key, 'data', f'worng asset key: {result_key}') self.assertTrue(f'metadata1' in upload_result['assets'], f'missing assets#metadata asset: metadata1') self.assertTrue('href' in upload_result['assets'][f'metadata1'], 'missing assets#metadata__cas#href') - self.assertTrue(upload_result['assets'][f'metadata1']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'metadata1']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) self.assertTrue(f'data' in upload_result['assets'], f'missing assets#data: data') self.assertTrue('href' in upload_result['assets'][f'data'], 'missing assets#data#href') - self.assertTrue(upload_result['assets'][f'data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'data']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) """ Example output: { @@ -788,10 +790,10 @@ def test_03_upload_complete_catalog_role_as_key(self): self.assertEqual(len(successful_feature_collection), total_files, f'wrong length: {successful_feature_collection}') return - def test_03_02_upload_complete_catalog(self): os.environ['VERIFY_SSL'] = 'FALSE' - os.environ['COLLECTION_ID'] = 'NEW_COLLECTION_EXAMPLE_L1B___9' + os.environ['PROJECT'] = 'LOCAL' + os.environ['VENUE'] = 'UNIT_TEST' os.environ['STAGING_BUCKET'] = 'uds-sbx-cumulus-staging' os.environ['GRANULES_SEARCH_DOMAIN'] = 'UNITY' @@ -975,10 +977,10 @@ def test_03_02_upload_complete_catalog(self): result_key_prefix = result_key.split('.')[0] self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/{os.environ["COLLECTION_ID"]}/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) """ Example output: { diff --git a/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py b/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py index 58e457e..ef4c3c4 100644 --- a/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py +++ b/tests/mdps_ds_lib/lib/cumulus_stac/test_granules_catalog.py @@ -912,3 +912,27 @@ def test_manual_validdate_stac(self): validation_result = stac_item.validate() return + def test_get_unity_formatted_collection_id(self): + with self.assertRaises(ValueError) as context: + GranulesCatalog.get_unity_formatted_collection_id(None, (None, None)) + self.assertTrue(str(context.exception).startswith('NULL or EMPTY collection_id')) + with self.assertRaises(ValueError) as context: + GranulesCatalog.get_unity_formatted_collection_id('', (None, None)) + self.assertTrue(str(context.exception).startswith('NULL or EMPTY collection_id')) + with self.assertRaises(ValueError) as context: + GranulesCatalog.get_unity_formatted_collection_id('NA', (None, None)) + self.assertTrue(str(context.exception).startswith('missing project or venue')) + with self.assertRaises(ValueError) as context: + GranulesCatalog.get_unity_formatted_collection_id('NA', (None, 'DEV')) + self.assertTrue(str(context.exception).startswith('missing project or venue')) + with self.assertRaises(ValueError) as context: + GranulesCatalog.get_unity_formatted_collection_id('NA', ('LOCAL', None)) + self.assertTrue(str(context.exception).startswith('missing project or venue')) + + result = GranulesCatalog.get_unity_formatted_collection_id('NA', ('LOCAL', 'DEV')) + self.assertEqual(result, 'URN:NASA:UNITY:LOCAL:DEV:NA', f'wrong collection id output') + + result = GranulesCatalog.get_unity_formatted_collection_id('URN:JPL:IDS:LOCAL1:DEV2:A:B:C:D:E:F:G', ('LOCAL', 'DEV')) + self.assertEqual(result, 'URN:JPL:IDS:LOCAL1:DEV2:A:B:C:D:E:F:G', f'wrong collection id output') + + return From 978470f3dce74dfcf6e6ad0e8fde323f036445e9 Mon Sep 17 00:00:00 2001 From: Wai Phyo Date: Tue, 22 Oct 2024 11:02:57 -0700 Subject: [PATCH 6/6] chore: add test to see if it works with multiple collections --- tests/integration_tests/test_docker_stage_out.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/test_docker_stage_out.py b/tests/integration_tests/test_docker_stage_out.py index 5050e0b..d23e1f2 100644 --- a/tests/integration_tests/test_docker_stage_out.py +++ b/tests/integration_tests/test_docker_stage_out.py @@ -479,7 +479,7 @@ def test_03_upload_complete_catalog(self): "datetime": "2022-03-23T15:48:19.079000Z" }, href=os.path.join('some_granules', f'{filename}.nc.stac.json'), - collection='NA', + collection=f'NA_{i:02d}', assets={ f'{filename}.nc': Asset(os.path.join('.', f'{filename}.nc'), title='test_file01.nc', roles=['data']), f'{filename}.nc.cas': Asset(os.path.join('.', f'{filename}.nc.cas'), title='test_file01.nc.cas', roles=['metadata']), @@ -521,10 +521,10 @@ def test_03_upload_complete_catalog(self): result_key_prefix = result_key.split('.')[0] self.assertTrue(f'{result_key_prefix}.nc.cas' in upload_result['assets'], f'missing assets#metadata asset: {result_key_prefix}.nc.cas') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc.cas'], 'missing assets#metadata__cas#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc.cas']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA_0')) self.assertTrue(f'{result_key_prefix}.nc' in upload_result['assets'], f'missing assets#data: {result_key_prefix}.nc') self.assertTrue('href' in upload_result['assets'][f'{result_key_prefix}.nc'], 'missing assets#data#href') - self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA/')) + self.assertTrue(upload_result['assets'][f'{result_key_prefix}.nc']['href'].startswith(f's3://{os.environ["STAGING_BUCKET"]}/URN:NASA:UNITY:{os.environ["PROJECT"]}:{os.environ["VENUE"]}:NA_0')) """ Example output: {