diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 6047ddb1e..7740de9ed 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,4 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 129ff5f16..99489d507 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index cd3cc6bf1..c6af17100 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -27,8 +27,8 @@ jobs: deploy_ref: refs/heads/main job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge - default_max_vcpus: 1200 - expanded_max_vcpus: 2400 + default_max_vcpus: 1500 + expanded_max_vcpus: 3000 required_surplus: 2000 security_environment: EDC ami_id: image_id_ecs_amz2 @@ -50,8 +50,8 @@ jobs: job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge - default_max_vcpus: 1200 - expanded_max_vcpus: 2400 + default_max_vcpus: 1500 + expanded_max_vcpus: 3000 required_surplus: 2000 security_environment: EDC ami_id: image_id_ecs_amz2 @@ -107,6 +107,6 @@ jobs: call-bump-version-workflow: if: github.ref == 'refs/heads/main' needs: deploy - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index 06b1a3406..339d5c678 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -47,8 +47,9 @@ jobs: default_application_status: APPROVED cost_profile: DEFAULT job_files: >- + job_spec/ARIA_AUTORIFT.yml job_spec/ARIA_RAIDER.yml - job_spec/INSAR_ISCE_TEST.yml + job_spec/INSAR_ISCE.yml instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge default_max_vcpus: 640 expanded_max_vcpus: 640 @@ -78,7 +79,7 @@ jobs: - environment: hyp3-lavas-test domain: hyp3-lavas-test.asf.alaska.edu - template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 + template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 image_tag: test product_lifetime_in_days: 14 default_credits_per_user: 0 diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index f5db008ba..73c44e088 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -26,8 +26,8 @@ jobs: job_spec/AUTORIFT_ITS_LIVE.yml job_spec/S1_CORRECTION_ITS_LIVE.yml instance_types: r6id.xlarge,r6idn.xlarge,r5dn.xlarge,r5d.xlarge - default_max_vcpus: 2000 - expanded_max_vcpus: 2000 + default_max_vcpus: 2000 # Max: 10,406 + expanded_max_vcpus: 2000 # Max: 10,406 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id @@ -42,11 +42,12 @@ jobs: default_application_status: APPROVED cost_profile: DEFAULT job_files: >- + job_spec/ARIA_AUTORIFT.yml job_spec/ARIA_RAIDER.yml job_spec/INSAR_ISCE.yml - instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge - default_max_vcpus: 0 - expanded_max_vcpus: 0 + instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge + default_max_vcpus: 4000 # Max: 13000 + expanded_max_vcpus: 4000 # Max: 13000 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id @@ -61,11 +62,12 @@ jobs: default_application_status: APPROVED cost_profile: DEFAULT job_files: >- + job_spec/ARIA_AUTORIFT.yml job_spec/ARIA_RAIDER.yml - job_spec/INSAR_ISCE_TEST.yml + job_spec/INSAR_ISCE.yml instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge - default_max_vcpus: 1000 - expanded_max_vcpus: 1000 + default_max_vcpus: 1000 # Max: 10316 + expanded_max_vcpus: 1000 # Max: 10316 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id @@ -79,10 +81,13 @@ jobs: default_credits_per_user: 0 default_application_status: APPROVED cost_profile: DEFAULT - job_files: job_spec/INSAR_ISCE.yml - instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge - default_max_vcpus: 1600 - expanded_max_vcpus: 1600 + job_files: >- + job_spec/ARIA_AUTORIFT.yml + job_spec/ARIA_RAIDER.yml + job_spec/INSAR_ISCE.yml + instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge + default_max_vcpus: 1600 # Max 1652 + expanded_max_vcpus: 1600 # Max 1652 required_surplus: 0 security_environment: JPL-public ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id @@ -228,7 +233,7 @@ jobs: - environment: hyp3-lavas domain: hyp3-lavas.asf.alaska.edu - template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 + template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 image_tag: latest product_lifetime_in_days: 14 default_credits_per_user: 0 diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 3ba21604e..f89f3e3bd 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index fa6ffb607..8311cb7b1 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,7 +7,7 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 permissions: pull-requests: write with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d0cbe17ed..06194fe08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 with: release_prefix: HyP3 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 36567e935..cb29f7574 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -87,4 +87,4 @@ jobs: snyk iac test --severity-threshold=high call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index bff66176b..7b6489fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [7.7.0] + +### Added +- `ARIA_AUTORIFT.yml` job spec for Solid Earth offset tracking in the ARIA JPL deployments + +### Changed +- Increased throughput for `hyp3-a19-jpl` (0 -> 4,000 vCPUs) to support continued processing of ARIA GUNW products. +- The `hyp3-a19-jpl` and `hyp3-nisar-jpl` deployments now use the `m6id[n]` instance families to reduce the high number of spot interruptions seen with wth `c6id` instance family. +- Increased available vCPUs for DAAC deployments. + +### Removed +- The `INSAR_ISCE_TEST.yml` job spec, which only differed from `INSAR_ISCE.yml` in support of different instance families, has been removed now that all ARIA JPL deployments are using the same instance families again. + + ## [7.6.0] ### Changed diff --git a/job_spec/ARIA_AUTORIFT.yml b/job_spec/ARIA_AUTORIFT.yml new file mode 100644 index 000000000..ad3bc6e0b --- /dev/null +++ b/job_spec/ARIA_AUTORIFT.yml @@ -0,0 +1,65 @@ +AUTORIFT: + required_parameters: + - granules + parameters: + granules: + default: '""' + api_schema: + type: array + minItems: 2 + maxItems: 2 + example: + - S2B_MSIL1C_20200105T152259_N0208_R039_T13CES_20200105T181230 + - S2B_MSIL1C_20200315T152259_N0209_R039_T13CES_20200315T181115 + items: + anyOf: + - description: The name of the Sentinel-1 SLC granule to process + type: string + pattern: "^S1[AB]_IW_SLC__1S[SD][VH]" + minLength: 67 + maxLength: 67 + example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8 + - description: The name of the Sentinel-2 granule to process (ESA naming convention) + type: string + pattern: "^S2[AB]_MSIL1C_" + minLength: 60 + maxLength: 60 + example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912 + - description: The name of the Landsat 4, 5, 7, 8 or 9 Collection 2 granule to process + type: string + pattern: "^L([CO]0[89]|E07|T0[45])_L1" + minLength: 40 + maxLength: 40 + example: LC08_L1GT_118112_20210107_20210107_02_T2 + bucket_prefix: + default: '""' + parameter_file: + api_schema: + description: Shapefile for determining the correct search parameters by geographic location. Path to shapefile must be understood by GDAL. + type: string + default: '/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_solidearth_0120m.shp' + cost_profiles: + DEFAULT: + cost: 1.0 + validators: [] + tasks: + - name: '' + image: ghcr.io/asfhyp3/hyp3-autorift + command: + - --bucket + - '!Ref Bucket' + - --bucket-prefix + - Ref::bucket_prefix + - --parameter-file + - Ref::parameter_file + - --naming-scheme + - ITS_LIVE_OD + - Ref::granules + timeout: 10800 + vcpu: 1 + memory: 31500 + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD + - ESA_USERNAME + - ESA_PASSWORD diff --git a/job_spec/INSAR_ISCE.yml b/job_spec/INSAR_ISCE.yml index 69ea74edc..8715ca330 100644 --- a/job_spec/INSAR_ISCE.yml +++ b/job_spec/INSAR_ISCE.yml @@ -97,7 +97,7 @@ INSAR_ISCE: image: ghcr.io/access-cloud-based-insar/dockerizedtopsapp command: - ++omp-num-threads - - '4' + - '2' # 2 for the m instance family; 4 for the c - --bucket - '!Ref Bucket' - --bucket-prefix diff --git a/job_spec/INSAR_ISCE_TEST.yml b/job_spec/INSAR_ISCE_TEST.yml deleted file mode 100644 index 8715ca330..000000000 --- a/job_spec/INSAR_ISCE_TEST.yml +++ /dev/null @@ -1,156 +0,0 @@ -INSAR_ISCE: - required_parameters: - - granules - - secondary_granules - - frame_id - parameters: - granules: - default: '""' - api_schema: - type: array - minItems: 1 - items: - description: The name of the Sentinel-1 SLC granules to use as reference scenes for InSAR processing - type: string - pattern: "^S1[AB]_IW_SLC__1S[SD]V" - minLength: 67 - maxLength: 67 - example: S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 - secondary_granules: - default: '""' - api_schema: - type: array - minItems: 1 - items: - description: The name of the Sentinel-1 SLC granules to use as secondary scenes for InSAR processing - type: string - pattern: "^S1[AB]_IW_SLC__1S[SD]V" - minLength: 67 - maxLength: 67 - example: S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 - frame_id: - api_schema: - description: Subset GUNW products to this frame. Standard products require frame id to be specified. When set to `-1`, no subsetting applied. If no frame specified (i.e. `-1`), then output GUNW is not a standard ARIA-S1-GUNW. - type: integer - minimum: -1 - compute_solid_earth_tide: - api_schema: - default: true - type: boolean - description: Whether to compute a solid earth tide correction layer for ARIA-S1-GUNW products; standard ARIA products must include this layer. - esd_coherence_threshold: - api_schema: - description: Coherence threshold value for determining which points to include when performing ESD. Must be any value between 0 and 1 (inclusive), or -1 for no ESD correction. Standard ARIA products do NOT include ESD. - default: -1.0 - type: number - minimum: -1.0 - maximum: 1.0 - estimate_ionosphere_delay: - api_schema: - description: Whether to apply ionosphere correction to the ARIA-S1-GUNW as an additional layer; standard ARIA products include the ionsphere correction to be turned on and include multiple ionosphere correction layers. - default: true - type: boolean - goldstein_filter_power: - api_schema: - description: The filter power of the adaptive goldsetin filter; 0 means no filter applied. Standard product requires strength to be set to 0.5. - default: 0.5 - type: number - minimum: 0 - output_resolution: - api_schema: - default: 90 - description: Desired output resolution in meters of GUNW product; standard ARIA products requires resolution to be set to 90 m. - type: integer - enum: - - 30 - - 90 - unfiltered_coherence: - api_schema: - default: true - type: boolean - description: Whether to add unfiltered_coherence layer to ARIA-S1-GUNW products; standard ARIA products must include this layer. - dense_offsets: - api_schema: - default: false - type: boolean - description: Whether to include 2 extra layers (azimuth and range pixel offsets) that measure via patch cross-correlation the change between reference and secondary in radar coordinates; standard ARIA products do NOT include this layer. - weather_model: - api_schema: - description: Weather model used to generate tropospheric delay estimations. - default: None - type: string - enum: - - None - - ERA5 - - ERA5T - - GMAO - - HRES - - HRRR - bucket_prefix: - default: '""' - cost_profiles: - DEFAULT: - cost: 1.0 - validators: [] - tasks: - - name: '' - image: ghcr.io/access-cloud-based-insar/dockerizedtopsapp - command: - - ++omp-num-threads - - '2' # 2 for the m instance family; 4 for the c - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --reference-scenes - - Ref::granules - - --secondary-scenes - - Ref::secondary_granules - - --esd-coherence-threshold - - Ref::esd_coherence_threshold - - --estimate-ionosphere-delay - - Ref::estimate_ionosphere_delay - - --frame-id - - Ref::frame_id - - --compute-solid-earth-tide - - Ref::compute_solid_earth_tide - - --output-resolution - - Ref::output_resolution - - --dense-offsets - - Ref::dense_offsets - - --goldstein-filter-power - - Ref::goldstein_filter_power - - --unfiltered-coherence - - Ref::unfiltered_coherence - timeout: 21600 - vcpu: 1 - memory: 7500 - secrets: - - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD - - ESA_USERNAME - - ESA_PASSWORD - - name: TROPOSPHERE - image: ghcr.io/dbekaert/raider - command: - - ++process - - calcDelaysGUNW - - --bucket - - '!Ref Bucket' - - --bucket-prefix - - Ref::bucket_prefix - - --weather-model - - Ref::weather_model - timeout: 10800 - vcpu: 1 - memory: 7500 - secrets: - - EARTHDATA_USERNAME - - EARTHDATA_PASSWORD - - ESA_USERNAME - - ESA_PASSWORD - - RAIDER_ECMWF_ERA5_UID - - RAIDER_ECMWF_ERA5_API_KEY - - RAIDER_HRES_EMAIL - - RAIDER_HRES_API_KEY - - RAIDER_HRES_URL diff --git a/requirements-all.txt b/requirements-all.txt index cbf68fcd2..9ffec8c6c 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,9 +5,9 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.34.131 +boto3==1.34.136 jinja2==3.1.4 -moto[dynamodb]==5.0.9 +moto[dynamodb]==5.0.10 pytest==8.2.2 PyYAML==6.0.1 responses==0.25.3 @@ -15,6 +15,6 @@ flake8==7.1.0 flake8-import-order==0.18.2 flake8-blind-except==0.2.1 flake8-builtins==2.5.0 -setuptools==70.1.0 +setuptools==70.2.0 openapi-spec-validator==0.7.1 -cfn-lint==1.3.4 +cfn-lint==1.4.2 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index b0b6f1c78..1b1b393df 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.34.131 +boto3==1.34.136 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index b7f1ab6c0..ce83bb4b5 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.34.131 +boto3==1.34.136 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index c8a36e361..9453043fe 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.34.131 +boto3==1.34.136 ./lib/lambda_logging/