diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index d1bfc1ba7..d8f240e63 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -66,7 +66,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index d4479499a..0872503ea 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -102,7 +102,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 024c27fdd..7eb29e145 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -259,7 +259,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml index 95eea8fde..b073dd11f 100644 --- a/.github/workflows/deploy-multi-burst-sandbox.yml +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -43,7 +43,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 0fa2dc434..aa5180bca 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -6,7 +6,7 @@ jobs: flake8: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -23,7 +23,7 @@ jobs: matrix: security_environment: [ASF, EDC, JPL, JPL-public] steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -37,7 +37,7 @@ jobs: openapi-spec-validator: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: python-version: 3.9 @@ -50,7 +50,7 @@ jobs: statelint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 @@ -70,7 +70,7 @@ jobs: snyk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: snyk/actions/setup@0.4.0 - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e66b82cb..d7824a78c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.1 + - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 18de461ca..f8e6ace63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.0.0] + +### Changed +- All failed jobs now have a `processing_times` value of `null`. + +### Fixed +- Resolve a regression introduced by the previous release (v8.0.0) in which a processing step could report a negative processing time if the underlying AWS Batch job had a failed attempt that did not include a `StartedAt` field. Fixes +- Upgrade from Flask v2.2.5 to v3.0.3. Fixes +- Specify our custom JSON encoder by subclassing `flask.json.provider.JSONProvider`. See + ## [8.0.0] ### Added diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index ebee991a0..2fc6042af 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -409,8 +409,9 @@ components: processing_times: description: > List of run times for the job's processing steps in the order that they were executed. - An empty list represents a failure to calculate processing times. + This field is null for failed jobs and non-null for successful jobs. type: array + nullable: true items: oneOf: - type: array @@ -422,10 +423,10 @@ components: processing_time_in_seconds: description: > - Run time in seconds for a processing step's final attempt (regardless of whether it succeeded). - A value of zero indicates that there were no attempts. + Run time in seconds for a processing step's final attempt. type: number minimum: 0 + exclusiveMinimum: true example: 50 securitySchemes: diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index e3c820d05..e0d69998f 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -6,6 +6,7 @@ import yaml from flask import abort, g, jsonify, make_response, redirect, render_template, request +from flask.json.provider import JSONProvider from flask_cors import CORS from openapi_core import OpenAPI from openapi_core.contrib.flask.decorators import FlaskOpenAPIViewDecorator @@ -87,13 +88,24 @@ def default(self, o): if isinstance(o, datetime.date): return o.isoformat() + if isinstance(o, Decimal): if o == int(o): return int(o) return float(o) + + # Raises a TypeError json.JSONEncoder.default(self, o) +class CustomJSONProvider(JSONProvider): + def dumps(self, o): + return json.dumps(o, cls=CustomEncoder) + + def loads(self, s): + return json.loads(s) + + class ErrorHandler(FlaskOpenAPIErrorsHandler): def __init__(self): super().__init__() @@ -104,7 +116,7 @@ def __call__(self, errors): return handlers.problem_format(error['status'], error['title']) -app.json_encoder = CustomEncoder +app.json = CustomJSONProvider(app) openapi = FlaskOpenAPIViewDecorator( api_spec, diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index bb5456e13..2b0950499 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -1,24 +1,11 @@ -import json from typing import Union -def get_time_from_attempts(attempts: list[dict]) -> float: - if len(attempts) == 0: - return 0 - attempts.sort(key=lambda attempt: attempt['StoppedAt']) - final_attempt = attempts[-1] - return (final_attempt['StoppedAt'] - final_attempt['StartedAt']) / 1000 - - def get_time_from_result(result: Union[list, dict]) -> Union[list, float]: if isinstance(result, list): return [get_time_from_result(item) for item in result] - if 'start' in result: - attempts = [{'StartedAt': start, 'StoppedAt': stop} for start, stop in zip(result['start'], result['stop'])] - return get_time_from_attempts(attempts) - - return get_time_from_attempts(json.loads(result['Cause'])['Attempts']) + return (result['StoppedAt'] - result['StartedAt']) / 1000 def lambda_handler(event, _) -> list[Union[list, float]]: diff --git a/apps/render_cf.py b/apps/render_cf.py index 37f248602..9688782ee 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -103,8 +103,8 @@ def get_batch_submit_job_state(job_spec: dict, step: dict, filter_batch_params=F }, }, 'ResultSelector': { - 'start.$': '$.Attempts[*].StartedAt', - 'stop.$': '$.Attempts[*].StoppedAt', + 'StartedAt.$': '$.StartedAt', + 'StoppedAt.$': '$.StoppedAt', }, 'Retry': [ { diff --git a/apps/step-function.json.j2 b/apps/step-function.json.j2 index 567f1cef4..02279c91a 100644 --- a/apps/step-function.json.j2 +++ b/apps/step-function.json.j2 @@ -4,7 +4,6 @@ "SET_DEFAULT_RESULTS": { "Type": "Pass", "Result": { - "processing_times": [], "get_files": { "logs": [], "expiration_time": null @@ -208,7 +207,7 @@ "status_code": "FAILED", "logs.$": "$.results.get_files.logs", "expiration_time.$": "$.results.get_files.expiration_time", - "processing_times.$": "$.results.processing_times" + "processing_times": null }, "Retry": [ { diff --git a/requirements-all.txt b/requirements-all.txt index 2935a6425..bf08b02e1 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,9 +5,9 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.35.44 +boto3==1.35.58 jinja2==3.1.4 -moto[dynamodb]==5.0.18 +moto[dynamodb]==5.0.20 pytest==8.3.3 PyYAML==6.0.2 responses==0.25.3 @@ -15,6 +15,6 @@ flake8==7.1.1 flake8-import-order==0.18.2 flake8-blind-except==0.2.1 flake8-builtins==2.5.0 -setuptools==75.2.0 +setuptools==75.4.0 openapi-spec-validator==0.7.1 -cfn-lint==1.18.1 +cfn-lint==1.19.0 diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt index fd4e8e1c0..f8f4d6b66 100644 --- a/requirements-apps-api.txt +++ b/requirements-apps-api.txt @@ -1,11 +1,11 @@ -flask==2.2.5 +flask==3.0.3 Flask-Cors==5.0.0 jsonschema==4.23.0 openapi-core==0.19.4 prance==23.6.21.0 PyJWT==2.9.0 requests==2.32.3 -serverless_wsgi==3.0.4 +serverless_wsgi==3.0.5 shapely==2.0.6 strict-rfc3339==0.7 ./lib/dynamo/ diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index a54ad61db..c430ea9e0 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.35.44 +boto3==1.35.58 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index d5e084a91..5c3450830 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.35.44 +boto3==1.35.58 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index 0b09d31f4..84e70f46e 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.35.44 +boto3==1.35.58 ./lib/lambda_logging/ diff --git a/tests/test_api/test_patch_user.py b/tests/test_api/test_patch_user.py index a3086a63d..7185d77e8 100644 --- a/tests/test_api/test_patch_user.py +++ b/tests/test_api/test_patch_user.py @@ -18,7 +18,7 @@ def test_patch_new_user(client, tables): assert response.json == { 'user_id': 'foo', 'application_status': APPLICATION_PENDING, - 'remaining_credits': Decimal(0), + 'remaining_credits': 0, 'job_names': [], 'use_case': 'I want data.', } @@ -45,7 +45,7 @@ def test_patch_pending_user(client, tables): assert response.status_code == HTTPStatus.OK assert response.json == { 'user_id': 'foo', - 'remaining_credits': Decimal(0), + 'remaining_credits': 0, 'application_status': APPLICATION_PENDING, 'use_case': 'New use case.', 'job_names': [], @@ -92,7 +92,7 @@ def test_patch_user_access_code(client, tables): assert response.json == { 'user_id': 'foo', 'application_status': APPLICATION_APPROVED, - 'remaining_credits': Decimal(25), + 'remaining_credits': 25, 'job_names': [], 'use_case': 'I want data.', 'access_code': '123', diff --git a/tests/test_check_processing_time.py b/tests/test_check_processing_time.py index 9ad6d1e90..8cc262e78 100644 --- a/tests/test_check_processing_time.py +++ b/tests/test_check_processing_time.py @@ -1,99 +1,25 @@ import check_processing_time -def test_single_attempt(): - attempts = [{'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 2800}] - assert check_processing_time.get_time_from_attempts(attempts) == 2.3 - - -def test_multiple_attempts(): - attempts = [ - {'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 1000}, - {'Container': {}, 'StartedAt': 3000, 'StatusReason': '', 'StoppedAt': 8700} - ] - assert check_processing_time.get_time_from_attempts(attempts) == 5.7 - - -def test_unsorted_attempts(): - # I'm not sure if the lambda would ever be given unsorted attempts, but it seems worth testing that it doesn't - # depend on the list to be sorted. - attempts = [ - {'Container': {}, 'StartedAt': 3000, 'StatusReason': '', 'StoppedAt': 8700}, - {'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 1000} - ] - assert check_processing_time.get_time_from_attempts(attempts) == 5.7 - - -def test_missing_start_time(): - # There are some cases in which at least one of the attempts may not have a StartedAt time. - # https://github.com/ASFHyP3/hyp3/issues/936 - attempts = [ - {'Container': {}, 'StartedAt': 500, 'StatusReason': '', 'StoppedAt': 1000}, - {'Container': {}, 'StatusReason': '', 'StoppedAt': 8700}, - {'Container': {}, 'StartedAt': 12000, 'StatusReason': '', 'StoppedAt': 15200} - ] - assert check_processing_time.get_time_from_attempts(attempts) == 3.2 - - -def test_no_attempts(): - assert check_processing_time.get_time_from_attempts([]) == 0 - - -def test_get_time_from_result(): - result = { - 'start': [500, 3000], - 'stop': [1000, 8700], - } - assert check_processing_time.get_time_from_result(result) == 5.7 - - -def test_get_time_from_result_list(): - result = [ - { - 'start': [500, 3000], - 'stop': [1000, 8900], - }, - { - 'start': [500, 4000], - 'stop': [3000, 4200], - }, - ] - assert check_processing_time.get_time_from_result(result) == [5.9, 0.2] - - -def test_get_time_from_result_failed(): - result = { - 'Error': 'States.TaskFailed', - 'Cause': '{"Attempts": [' - '{"Container": {}, "StartedAt": 500, "StatusReason": "", "StoppedAt": 1000}, ' - '{"Container": {}, "StartedAt": 1500, "StatusReason": "", "StoppedAt": 2000}, ' - '{"Container": {}, "StartedAt": 3000, "StatusReason": "", "StoppedAt": 9400}]}' - } - assert check_processing_time.get_time_from_result(result) == 6.4 - - def test_lambda_handler(): event = { 'processing_results': { 'step_0': { - 'start': [500, 3000], - 'stop': [1000, 8700], + 'StartedAt': 3000, + 'StoppedAt': 8700, }, 'step_1': { - 'Error': 'States.TaskFailed', - 'Cause': '{"Attempts": [' - '{"Container": {}, "StartedAt": 500, "StatusReason": "", "StoppedAt": 1000}, ' - '{"Container": {}, "StartedAt": 1500, "StatusReason": "", "StoppedAt": 2000}, ' - '{"Container": {}, "StartedAt": 3000, "StatusReason": "", "StoppedAt": 9400}]}' + 'StartedAt': 3000, + 'StoppedAt': 9400, }, 'step_2': [ { - 'start': [500, 3000], - 'stop': [1000, 8900], + 'StartedAt': 3000, + 'StoppedAt': 8900, }, { - 'start': [500, 4000], - 'stop': [3000, 4200], + 'StartedAt': 4000, + 'StoppedAt': 4200, }, ] }