Merge pull request #2294 from ASFHyP3/develop

Release v7.3.0

.github/workflows/deploy-daac.yml

@@ -62,7 +62,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-enterprise-test.yml

@@ -82,7 +82,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-enterprise.yml

@@ -37,16 +37,16 @@ jobs:
             domain: hyp3-a19-jpl.asf.alaska.edu
             template_bucket: cf-templates-v4pvone059de-us-west-2
             image_tag: latest
-            product_lifetime_in_days: 180
+            product_lifetime_in_days: 14
             default_credits_per_user: 0
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
               job_spec/ARIA_RAIDER.yml
               job_spec/INSAR_ISCE.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 4000
-            expanded_max_vcpus: 4000
+            default_max_vcpus: 0
+            expanded_max_vcpus: 0
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -229,7 +229,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/deploy-whitelisting-sandbox.yml

@@ -41,7 +41,7 @@ jobs:
       url: https://${{ matrix.domain }}
 
     steps:
-      - uses: actions/checkout@v4.1.2
+      - uses: actions/checkout@v4.1.6
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/static-analysis.yml

@@ -6,7 +6,7 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -23,7 +23,7 @@ jobs:
       matrix:
         security_environment: [ASF, EDC, JPL, JPL-public]
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -37,7 +37,7 @@ jobs:
   openapi-spec-validator:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
@@ -50,7 +50,7 @@ jobs:
   statelint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
@@ -70,7 +70,7 @@ jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
       - uses: snyk/actions/setup@0.4.0
       - uses: actions/setup-python@v5
         with:

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4.1.5
+      - uses: actions/checkout@v4.1.6
 
       - uses: actions/setup-python@v5
         with:

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [7.3.0]
+
+This release adds support for access codes. If a user specifies an active access code when they apply for HyP3 access, they will be granted automatic approval without the need for a HyP3 operator to review their application.
+
+If you operate a HyP3 deployment, you can create a new access code by adding an item to the `AccessCodesTable` DynamoDB table for your deployment, with any string for the `access_code` attribute and an ISO-formatted UTC timestamp for the `start_date` and `end_date` attributes, e.g. `2024-06-01T00:00:00+00:00` and `2024-06-02T00:00:00+00:00` for an access code that becomes active on June 1, 2024 and expires on June 2, 2024.
+
+### Added
+- The `PATCH /user` endpoint now includes an optional `access_code` parameter and returns a `403` response if given an invalid or inactive access code.
+
+### Changed
+- Turn off hyp3 ACCESS spend by zeroing the max VCPUs in the associated deployment.
+- Reduce product lifetime in hyp3 ACCESS deployment to 14 days.
+
 ## [7.2.1]
 
 ### Fixed

apps/api/api-cf.yml.j2

@@ -6,6 +6,9 @@ Parameters:
   UsersTable:
     Type: String
 
+  AccessCodesTable:
+    Type: String
+
   AuthPublicKey:
     Type: String
 
@@ -171,6 +174,10 @@ Resources:
               - dynamodb:PutItem
               - dynamodb:UpdateItem
             Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${UsersTable}*"
+          - Effect: Allow
+            Action:
+              - dynamodb:GetItem
+            Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${AccessCodesTable}*"
 
   Lambda:
     Type: AWS::Lambda::Function
@@ -179,6 +186,7 @@ Resources:
         Variables:
           JOBS_TABLE_NAME: !Ref JobsTable
           USERS_TABLE_NAME: !Ref UsersTable
+          ACCESS_CODES_TABLE_NAME: !Ref AccessCodesTable
           AUTH_PUBLIC_KEY: !Ref AuthPublicKey
           AUTH_ALGORITHM: !Ref AuthAlgorithm
           DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -197,6 +197,8 @@ components:
       properties:
         use_case:
           $ref: "#/components/schemas/use_case"
+        access_code:
+          $ref: "#/components/schemas/access_code"
 
     user:
       description: Information about a user
@@ -315,6 +317,11 @@ components:
       type: string
       example: I want to process data.
 
+    access_code:
+      description: Grants automatic user approval while the code remains active.
+      type: string
+      example: 123
+
     user_id:
       description: Username from Earthdata Login.
       type: string

apps/api/src/hyp3_api/handlers.py

@@ -4,7 +4,7 @@
 from flask import abort, jsonify, request
 
 import dynamo
-from dynamo.exceptions import InsufficientCreditsError, UnexpectedApplicationStatusError
+from dynamo.exceptions import AccessCodeError, InsufficientCreditsError, UnexpectedApplicationStatusError
 from hyp3_api import util
 from hyp3_api.validation import GranuleValidationError, validate_jobs
 
@@ -65,6 +65,8 @@ def patch_user(body: dict, user: str, edl_access_token: str) -> dict:
     print(body)
     try:
         user_record = dynamo.user.update_user(user, edl_access_token, body)
+    except AccessCodeError as e:
+        abort(problem_format(403, str(e)))
     except UnexpectedApplicationStatusError as e:
         abort(problem_format(403, str(e)))
     return _user_response(user_record)

apps/main-cf.yml.j2

@@ -120,6 +120,7 @@ Resources:
       Parameters:
         JobsTable: !Ref JobsTable
         UsersTable: !Ref UsersTable
+        AccessCodesTable: !Ref AccessCodesTable
         AuthPublicKey: !Ref AuthPublicKey
         AuthAlgorithm: !Ref AuthAlgorithm
         DefaultCreditsPerUser: !Ref DefaultCreditsPerUser
@@ -356,6 +357,17 @@ Resources:
         - AttributeName: user_id
           KeyType: HASH
 
+  AccessCodesTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      BillingMode: PAY_PER_REQUEST
+      AttributeDefinitions:
+        - AttributeName: access_code
+          AttributeType: S
+      KeySchema:
+        - AttributeName: access_code
+          KeyType: HASH
+
   {% if security_environment == 'EDC' %}
   DisablePrivateDNS:
     Type: AWS::CloudFormation::Stack

lib/dynamo/dynamo/exceptions.py

@@ -5,6 +5,10 @@ class DatabaseConditionException(Exception):
     """Raised when a DynamoDB condition expression check fails."""
 
 
+class AccessCodeError(Exception):
+    """Raised when a user application includes an invalid or expired access code."""
+
+
 class InsufficientCreditsError(Exception):
     """Raised when trying to submit jobs whose total cost exceeds the user's remaining credits."""
 

lib/dynamo/dynamo/jobs.py

@@ -1,5 +1,4 @@
 import json
-from datetime import datetime, timezone
 from decimal import Decimal
 from os import environ
 from pathlib import Path
@@ -17,7 +16,7 @@
     RejectedApplicationError,
 )
 from dynamo.user import APPLICATION_APPROVED, APPLICATION_NOT_STARTED, APPLICATION_PENDING, APPLICATION_REJECTED
-from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time, get_request_time_expression
+from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, current_utc_time, get_request_time_expression
 
 costs_file = Path(__file__).parent / 'costs.json'
 COSTS = convert_floats_to_decimals(json.loads(costs_file.read_text()))
@@ -32,7 +31,7 @@
 
 def put_jobs(user_id: str, jobs: List[dict], dry_run=False) -> List[dict]:
     table = DYNAMODB_RESOURCE.Table(environ['JOBS_TABLE_NAME'])
-    request_time = format_time(datetime.now(timezone.utc))
+    request_time = current_utc_time()
 
     user_record = dynamo.user.get_or_create_user(user_id)
 

lib/dynamo/dynamo/user.py

@@ -6,8 +6,13 @@
 import botocore.exceptions
 import requests
 
+import dynamo.util
 from dynamo.exceptions import (
-    ApprovedApplicationError, DatabaseConditionException, InvalidApplicationStatusError, RejectedApplicationError
+    AccessCodeError,
+    ApprovedApplicationError,
+    DatabaseConditionException,
+    InvalidApplicationStatusError,
+    RejectedApplicationError,
 )
 from dynamo.util import DYNAMODB_RESOURCE
 
@@ -21,26 +26,44 @@ def update_user(user_id: str, edl_access_token: str, body: dict) -> dict:
     user = get_or_create_user(user_id)
     application_status = user['application_status']
     if application_status in (APPLICATION_NOT_STARTED, APPLICATION_PENDING):
+        access_code = body.get('access_code')
+        if access_code:
+            _validate_access_code(access_code)
+            updated_application_status = APPLICATION_APPROVED
+            access_code_expression = ', access_code = :access_code'
+            access_code_value = {':access_code': access_code}
+        else:
+            updated_application_status = APPLICATION_PENDING
+            access_code_expression = ''
+            access_code_value = {}
         edl_profile = _get_edl_profile(user_id, edl_access_token)
         users_table = DYNAMODB_RESOURCE.Table(environ['USERS_TABLE_NAME'])
         try:
             user = users_table.update_item(
                 Key={'user_id': user_id},
-                UpdateExpression='SET #edl_profile = :edl_profile, use_case = :use_case, application_status = :pending',
+                UpdateExpression=(
+                    'SET #edl_profile = :edl_profile,'
+                    '    use_case = :use_case,'
+                    '    application_status = :updated_application_status'
+                    f'{access_code_expression}'
+                ),
                 ConditionExpression='application_status IN (:not_started, :pending)',
                 ExpressionAttributeNames={'#edl_profile': '_edl_profile'},
                 ExpressionAttributeValues={
                     ':edl_profile': edl_profile,
                     ':use_case': body['use_case'],
                     ':not_started': APPLICATION_NOT_STARTED,
-                    ':pending': APPLICATION_PENDING
+                    ':pending': APPLICATION_PENDING,
+                    ':updated_application_status': updated_application_status,
+                    **access_code_value
                 },
                 ReturnValues='ALL_NEW',
             )['Attributes']
         except botocore.exceptions.ClientError as e:
             if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
                 raise DatabaseConditionException(f'Failed to update record for user {user_id}')
             raise
+        user = _reset_credits_if_needed(user=user, current_month=_get_current_month(), users_table=users_table)
         return user
     if application_status == APPLICATION_REJECTED:
         raise RejectedApplicationError(user_id)
@@ -49,6 +72,21 @@ def update_user(user_id: str, edl_access_token: str, body: dict) -> dict:
     raise InvalidApplicationStatusError(user_id, application_status)
 
 
+def _validate_access_code(access_code: str) -> None:
+    access_codes_table = DYNAMODB_RESOURCE.Table(environ['ACCESS_CODES_TABLE_NAME'])
+    item = access_codes_table.get_item(Key={'access_code': access_code}).get('Item')
+
+    if item is None:
+        raise AccessCodeError(f'{access_code} is not a valid access code')
+
+    now = dynamo.util.current_utc_time()
+    if now < item['start_date']:
+        raise AccessCodeError(f'Access code {access_code} will become active on {item["start_date"]}')
+
+    if now >= item['end_date']:
+        raise AccessCodeError(f'Access code {access_code} expired on {item["end_date"]}')
+
+
 def _get_edl_profile(user_id: str, edl_access_token: str) -> dict:
     url = f'https://urs.earthdata.nasa.gov/api/users/{user_id}'
     response = requests.get(url, headers={'Authorization': f'Bearer {edl_access_token}'})
@@ -57,21 +95,13 @@ def _get_edl_profile(user_id: str, edl_access_token: str) -> dict:
 
 
 def get_or_create_user(user_id: str) -> dict:
-    current_month = _get_current_month()
-    default_credits = Decimal(os.environ['DEFAULT_CREDITS_PER_USER'])
-
     users_table = DYNAMODB_RESOURCE.Table(environ['USERS_TABLE_NAME'])
     user = users_table.get_item(Key={'user_id': user_id}).get('Item')
 
     if user is None:
         user = _create_user(user_id, users_table)
 
-    return _reset_credits_if_needed(
-        user=user,
-        default_credits=default_credits,
-        current_month=current_month,
-        users_table=users_table,
-    )
+    return _reset_credits_if_needed(user=user, current_month=_get_current_month(), users_table=users_table)
 
 
 def _get_current_month() -> str:
@@ -93,7 +123,7 @@ def _create_user(user_id: str, users_table) -> dict:
     return user
 
 
-def _reset_credits_if_needed(user: dict, default_credits: Decimal, current_month: str, users_table) -> dict:
+def _reset_credits_if_needed(user: dict, current_month: str, users_table) -> dict:
     if (
             user['application_status'] == APPLICATION_APPROVED
             and user.get('_month_of_last_credit_reset', '0') < current_month  # noqa: W503
@@ -112,7 +142,7 @@ def _reset_credits_if_needed(user: dict, default_credits: Decimal, current_month
                 ExpressionAttributeNames={'#month_of_last_credit_reset': '_month_of_last_credit_reset'},
                 ExpressionAttributeValues={
                     ':approved': APPLICATION_APPROVED,
-                    ':credits': user.get('credits_per_month', default_credits),
+                    ':credits': user.get('credits_per_month', Decimal(os.environ['DEFAULT_CREDITS_PER_USER'])),
                     ':current_month': current_month,
                     ':number': 'N',
                 },