Merge pull request #2493 from ASFHyP3/develop

Release v9.0.0

.github/workflows/deploy-daac.yml

@@ -66,7 +66,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-enterprise-test.yml

@@ -102,7 +102,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-enterprise.yml

@@ -259,7 +259,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-multi-burst-sandbox.yml

@@ -43,7 +43,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/static-analysis.yml

@@ -6,7 +6,7 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -23,7 +23,7 @@ jobs:
       matrix:
         security_environment: [ASF, EDC, JPL, JPL-public]
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -37,7 +37,7 @@ jobs:
   openapi-spec-validator:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -50,7 +50,7 @@ jobs:
   statelint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
@@ -70,7 +70,7 @@ jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
       - uses: snyk/actions/setup@0.4.0
       - uses: actions/setup-python@v5
         with:

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4.2.1
+      - uses: actions/checkout@v4.2.2
 
       - uses: actions/setup-python@v5
         with:

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [9.0.0]
+
+### Changed
+- All failed jobs now have a `processing_times` value of `null`.
+
+### Fixed
+- Resolve a regression introduced by the previous release (v8.0.0) in which a processing step could report a negative processing time if the underlying AWS Batch job had a failed attempt that did not include a `StartedAt` field. Fixes <https://github.com/ASFHyP3/hyp3/issues/2485>
+- Upgrade from Flask v2.2.5 to v3.0.3. Fixes <https://github.com/ASFHyP3/hyp3/issues/2491>
+- Specify our custom JSON encoder by subclassing `flask.json.provider.JSONProvider`. See <https://github.com/pallets/flask/pull/4692>
+
 ## [8.0.0]
 
 ### Added

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -409,8 +409,9 @@ components:
     processing_times:
       description: >
         List of run times for the job's processing steps in the order that they were executed.
-        An empty list represents a failure to calculate processing times.
+        This field is null for failed jobs and non-null for successful jobs.
       type: array
+      nullable: true
       items:
         oneOf:
           - type: array
@@ -422,10 +423,10 @@ components:
 
     processing_time_in_seconds:
       description: >
-        Run time in seconds for a processing step's final attempt (regardless of whether it succeeded).
-        A value of zero indicates that there were no attempts.
+        Run time in seconds for a processing step's final attempt.
       type: number
       minimum: 0
+      exclusiveMinimum: true
       example: 50
 
   securitySchemes:

apps/api/src/hyp3_api/routes.py

@@ -6,6 +6,7 @@
 
 import yaml
 from flask import abort, g, jsonify, make_response, redirect, render_template, request
+from flask.json.provider import JSONProvider
 from flask_cors import CORS
 from openapi_core import OpenAPI
 from openapi_core.contrib.flask.decorators import FlaskOpenAPIViewDecorator
@@ -87,13 +88,24 @@ def default(self, o):
 
         if isinstance(o, datetime.date):
             return o.isoformat()
+
         if isinstance(o, Decimal):
             if o == int(o):
                 return int(o)
             return float(o)
+
+        # Raises a TypeError
         json.JSONEncoder.default(self, o)
 
 
+class CustomJSONProvider(JSONProvider):
+    def dumps(self, o):
+        return json.dumps(o, cls=CustomEncoder)
+
+    def loads(self, s):
+        return json.loads(s)
+
+
 class ErrorHandler(FlaskOpenAPIErrorsHandler):
     def __init__(self):
         super().__init__()
@@ -104,7 +116,7 @@ def __call__(self, errors):
         return handlers.problem_format(error['status'], error['title'])
 
 
-app.json_encoder = CustomEncoder
+app.json = CustomJSONProvider(app)
 
 openapi = FlaskOpenAPIViewDecorator(
     api_spec,

apps/check-processing-time/src/check_processing_time.py

@@ -1,24 +1,11 @@
-import json
 from typing import Union
 
 
-def get_time_from_attempts(attempts: list[dict]) -> float:
-    if len(attempts) == 0:
-        return 0
-    attempts.sort(key=lambda attempt: attempt['StoppedAt'])
-    final_attempt = attempts[-1]
-    return (final_attempt['StoppedAt'] - final_attempt['StartedAt']) / 1000
-
-
 def get_time_from_result(result: Union[list, dict]) -> Union[list, float]:
     if isinstance(result, list):
         return [get_time_from_result(item) for item in result]
 
-    if 'start' in result:
-        attempts = [{'StartedAt': start, 'StoppedAt': stop} for start, stop in zip(result['start'], result['stop'])]
-        return get_time_from_attempts(attempts)
-
-    return get_time_from_attempts(json.loads(result['Cause'])['Attempts'])
+    return (result['StoppedAt'] - result['StartedAt']) / 1000
 
 
 def lambda_handler(event, _) -> list[Union[list, float]]:

apps/render_cf.py

@@ -103,8 +103,8 @@ def get_batch_submit_job_state(job_spec: dict, step: dict, filter_batch_params=F
             },
         },
         'ResultSelector': {
-            'start.$': '$.Attempts[*].StartedAt',
-            'stop.$': '$.Attempts[*].StoppedAt',
+            'StartedAt.$': '$.StartedAt',
+            'StoppedAt.$': '$.StoppedAt',
         },
         'Retry': [
             {

apps/step-function.json.j2

@@ -4,7 +4,6 @@
     "SET_DEFAULT_RESULTS": {
       "Type": "Pass",
       "Result": {
-        "processing_times": [],
         "get_files": {
           "logs": [],
           "expiration_time": null
@@ -208,7 +207,7 @@
         "status_code": "FAILED",
         "logs.$": "$.results.get_files.logs",
         "expiration_time.$": "$.results.get_files.expiration_time",
-        "processing_times.$": "$.results.processing_times"
+        "processing_times": null
       },
       "Retry": [
         {

requirements-all.txt

@@ -5,16 +5,16 @@
 -r requirements-apps-start-execution-worker.txt
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
-boto3==1.35.44
+boto3==1.35.58
 jinja2==3.1.4
-moto[dynamodb]==5.0.18
+moto[dynamodb]==5.0.20
 pytest==8.3.3
 PyYAML==6.0.2
 responses==0.25.3
 flake8==7.1.1
 flake8-import-order==0.18.2
 flake8-blind-except==0.2.1
 flake8-builtins==2.5.0
-setuptools==75.2.0
+setuptools==75.4.0
 openapi-spec-validator==0.7.1
-cfn-lint==1.18.1
+cfn-lint==1.19.0

requirements-apps-api.txt

@@ -1,11 +1,11 @@
-flask==2.2.5
+flask==3.0.3
 Flask-Cors==5.0.0
 jsonschema==4.23.0
 openapi-core==0.19.4
 prance==23.6.21.0
 PyJWT==2.9.0
 requests==2.32.3
-serverless_wsgi==3.0.4
+serverless_wsgi==3.0.5
 shapely==2.0.6
 strict-rfc3339==0.7
 ./lib/dynamo/

requirements-apps-disable-private-dns.txt

@@ -1 +1 @@
-boto3==1.35.44
+boto3==1.35.58

requirements-apps-start-execution-manager.txt

@@ -1,3 +1,3 @@
-boto3==1.35.44
+boto3==1.35.58
 ./lib/dynamo/
 ./lib/lambda_logging/

requirements-apps-start-execution-worker.txt

@@ -1,2 +1,2 @@
-boto3==1.35.44
+boto3==1.35.58
 ./lib/lambda_logging/

tests/test_api/test_patch_user.py

@@ -18,7 +18,7 @@ def test_patch_new_user(client, tables):
     assert response.json == {
         'user_id': 'foo',
         'application_status': APPLICATION_PENDING,
-        'remaining_credits': Decimal(0),
+        'remaining_credits': 0,
         'job_names': [],
         'use_case': 'I want data.',
     }
@@ -45,7 +45,7 @@ def test_patch_pending_user(client, tables):
     assert response.status_code == HTTPStatus.OK
     assert response.json == {
         'user_id': 'foo',
-        'remaining_credits': Decimal(0),
+        'remaining_credits': 0,
         'application_status': APPLICATION_PENDING,
         'use_case': 'New use case.',
         'job_names': [],
@@ -92,7 +92,7 @@ def test_patch_user_access_code(client, tables):
     assert response.json == {
         'user_id': 'foo',
         'application_status': APPLICATION_APPROVED,
-        'remaining_credits': Decimal(25),
+        'remaining_credits': 25,
         'job_names': [],
         'use_case': 'I want data.',
         'access_code': '123',