diff --git a/.github/workflows/fetch-and-ingest-branch.yaml b/.github/workflows/fetch-and-ingest-branch.yaml index ce834c4f..61e0c8c5 100644 --- a/.github/workflows/fetch-and-ingest-branch.yaml +++ b/.github/workflows/fetch-and-ingest-branch.yaml @@ -8,45 +8,42 @@ on: description: 'Specific container image to use for build (will override the default of "nextstrain build")' required: false -env: - NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }} jobs: - fetch-and-ingest: + set_config_overrides: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: nextstrain/.github/actions/setup-nextstrain-cli@master - with: - cli-version: ">=7.1.0" - python-version: "3.10" - - - name: install-pyyaml - run: python3 -m pip install pyyaml - - - name: run_pipeline + - id: upload_config run: | + # Create JSON string for the nested upload config GITHUB_BRANCH=${GITHUB_REF#refs/heads/} - ./phylogenetic/bin/set-branch-ingest-config \ - --config-yaml ingest/config/optional.yaml \ - --s3-dst s3://nextstrain-data/files/workflows/monkeypox/branch/"${GITHUB_BRANCH}" \ - > ingest/config/optional-branch.yaml + S3_DST="s3://nextstrain-data/files/workflows/monkeypox/branch/${GITHUB_BRANCH}" + UPLOAD_CONFIG=$(jq -cn --arg S3_DST "$S3_DST" '{"s3": {"dst": $S3_DST }}') - nextstrain build \ - --aws-batch \ - --detach \ - --no-download \ - --cpus 32 \ - --memory 64gib \ - --env AWS_DEFAULT_REGION \ - --env AWS_ACCESS_KEY_ID \ - --env AWS_SECRET_ACCESS_KEY \ - --env GITHUB_RUN_ID \ - ingest \ - --configfiles config/config.yaml config/optional-branch.yaml \ - --config trigger_rebuild=False - env: - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_RUN_ID: ${{ github.run_id }} + echo "upload_config=$UPLOAD_CONFIG" >> "$GITHUB_OUTPUT" + outputs: + upload_config: ${{ steps.upload_config.outputs.upload_config }} + + fetch-and-ingest: + needs: [set_config_overrides] + permissions: + id-token: write + uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master + secrets: inherit + with: + runtime: aws-batch + env: | + NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }} + UPLOAD_CONFIG: ${{ needs.set_config_overrides.outputs.upload_config }} + run: | + nextstrain build \ + --aws-batch \ + --detach \ + --no-download \ + --cpus 32 \ + --memory 64gib \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + ingest \ + --configfiles config/config.yaml config/optional.yaml \ + --config trigger_rebuild=False send_slack_notifications=False upload="$UPLOAD_CONFIG" diff --git a/.github/workflows/fetch-and-ingest.yaml b/.github/workflows/fetch-and-ingest.yaml index 2b5856aa..c2a8e27b 100644 --- a/.github/workflows/fetch-and-ingest.yaml +++ b/.github/workflows/fetch-and-ingest.yaml @@ -49,7 +49,6 @@ jobs: --no-download \ --cpus 32 \ --memory 64gib \ - --env AWS_DEFAULT_REGION \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ --env GITHUB_RUN_ID \ diff --git a/.github/workflows/rebuild-hmpxv1-big.yaml b/.github/workflows/rebuild-hmpxv1-big.yaml index b65aa363..947186ce 100644 --- a/.github/workflows/rebuild-hmpxv1-big.yaml +++ b/.github/workflows/rebuild-hmpxv1-big.yaml @@ -39,7 +39,6 @@ jobs: --no-download \ --cpus 8 \ --memory 68gib \ - --env AWS_DEFAULT_REGION \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ --env GITHUB_RUN_ID \ diff --git a/.github/workflows/rebuild-hmpxv1.yaml b/.github/workflows/rebuild-hmpxv1.yaml index 4cf2b68d..a2d6e199 100644 --- a/.github/workflows/rebuild-hmpxv1.yaml +++ b/.github/workflows/rebuild-hmpxv1.yaml @@ -39,7 +39,6 @@ jobs: --no-download \ --cpus 8 \ --memory 32gib \ - --env AWS_DEFAULT_REGION \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ --env GITHUB_RUN_ID \ diff --git a/.github/workflows/rebuild-mpxv.yaml b/.github/workflows/rebuild-mpxv.yaml index 1522e77b..53bbfcb5 100644 --- a/.github/workflows/rebuild-mpxv.yaml +++ b/.github/workflows/rebuild-mpxv.yaml @@ -39,7 +39,6 @@ jobs: --no-download \ --cpus 8 \ --memory 32gib \ - --env AWS_DEFAULT_REGION \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ --env GITHUB_RUN_ID \ diff --git a/ingest/README.md b/ingest/README.md index a94ff53b..cc88f291 100644 --- a/ingest/README.md +++ b/ingest/README.md @@ -70,7 +70,6 @@ The complete ingest pipeline with AWS S3 uploads and Slack notifications uses th #### Required -- `AWS_DEFAULT_REGION` - `AWS_ACCESS_KEY_ID` - `AWS_SECRET_ACCESS_KEY` - `SLACK_TOKEN` diff --git a/phylogenetic/bin/set-branch-ingest-config b/phylogenetic/bin/set-branch-ingest-config deleted file mode 100755 index d1a2d627..00000000 --- a/phylogenetic/bin/set-branch-ingest-config +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -""" -Edits the config YAML file and outputs the new config to stdout. -""" -import argparse -import yaml - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument("--config-yaml", default="ingest/config/optional.yaml", - help="Path to config YAML file to edit.") - parser.add_argument("--s3-dst", required=True, - help="The S3 destination to add to the config YAML file") - parser.add_argument("--send-slack-notifications", action="store_true", - help="Set the `send_slack_notifications` config to True") - - args = parser.parse_args() - - with open(args.config_yaml, 'r') as config_fh: - config = yaml.safe_load(config_fh) - - config['upload']['s3']['dst'] = args.s3_dst - config['send_slack_notifications'] = args.send_slack_notifications - - print(yaml.dump(config, default_flow_style=False, sort_keys=False))