Merge pull request #218 from nextstrain/remove-set-branch-config

GitHub Action workflows clean up
nextstrain · Oct 31, 2023 · 9ff822f · 9ff822f
2 parents f432dde + 0f0ba05
commit 9ff822f
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 69 deletions.
diff --git a/.github/workflows/fetch-and-ingest-branch.yaml b/.github/workflows/fetch-and-ingest-branch.yaml
@@ -8,45 +8,42 @@ on:
         description: 'Specific container image to use for build (will override the default of "nextstrain build")'
         required: false
 
-env:
-  NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
 
 jobs:
-  fetch-and-ingest:
+  set_config_overrides:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-        with:
-          cli-version: ">=7.1.0"
-          python-version: "3.10"
-
-      - name: install-pyyaml
-        run: python3 -m pip install pyyaml
-
-      - name: run_pipeline
+      - id: upload_config
         run: |
+          # Create JSON string for the nested upload config
           GITHUB_BRANCH=${GITHUB_REF#refs/heads/}
-          ./phylogenetic/bin/set-branch-ingest-config \
-            --config-yaml ingest/config/optional.yaml \
-            --s3-dst s3://nextstrain-data/files/workflows/monkeypox/branch/"${GITHUB_BRANCH}" \
-            > ingest/config/optional-branch.yaml
+          S3_DST="s3://nextstrain-data/files/workflows/monkeypox/branch/${GITHUB_BRANCH}"
+          UPLOAD_CONFIG=$(jq -cn --arg S3_DST "$S3_DST" '{"s3": {"dst": $S3_DST }}')
 
-          nextstrain build \
-            --aws-batch \
-            --detach \
-            --no-download \
-            --cpus 32 \
-            --memory 64gib \
-            --env AWS_DEFAULT_REGION \
-            --env AWS_ACCESS_KEY_ID \
-            --env AWS_SECRET_ACCESS_KEY \
-            --env GITHUB_RUN_ID \
-            ingest \
-              --configfiles config/config.yaml config/optional-branch.yaml \
-              --config trigger_rebuild=False
-        env:
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
+          echo "upload_config=$UPLOAD_CONFIG" >> "$GITHUB_OUTPUT"
+    outputs:
+      upload_config: ${{ steps.upload_config.outputs.upload_config }}
+
+  fetch-and-ingest:
+    needs: [set_config_overrides]
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      env: |
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
+        UPLOAD_CONFIG: ${{ needs.set_config_overrides.outputs.upload_config }}
+      run: |
+        nextstrain build \
+          --aws-batch \
+          --detach \
+          --no-download \
+          --cpus 32 \
+          --memory 64gib \
+          --env AWS_ACCESS_KEY_ID \
+          --env AWS_SECRET_ACCESS_KEY \
+          ingest \
+            --configfiles config/config.yaml config/optional.yaml \
+            --config trigger_rebuild=False send_slack_notifications=False upload="$UPLOAD_CONFIG"
diff --git a/.github/workflows/fetch-and-ingest.yaml b/.github/workflows/fetch-and-ingest.yaml
@@ -49,7 +49,6 @@ jobs:
           --no-download \
           --cpus 32 \
           --memory 64gib \
-          --env AWS_DEFAULT_REGION \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           --env GITHUB_RUN_ID \

diff --git a/.github/workflows/rebuild-hmpxv1-big.yaml b/.github/workflows/rebuild-hmpxv1-big.yaml
@@ -39,7 +39,6 @@ jobs:
           --no-download \
           --cpus 8 \
           --memory 68gib \
-          --env AWS_DEFAULT_REGION \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           --env GITHUB_RUN_ID \

diff --git a/.github/workflows/rebuild-hmpxv1.yaml b/.github/workflows/rebuild-hmpxv1.yaml
@@ -39,7 +39,6 @@ jobs:
           --no-download \
           --cpus 8 \
           --memory 32gib \
-          --env AWS_DEFAULT_REGION \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           --env GITHUB_RUN_ID \

diff --git a/.github/workflows/rebuild-mpxv.yaml b/.github/workflows/rebuild-mpxv.yaml
@@ -39,7 +39,6 @@ jobs:
           --no-download \
           --cpus 8 \
           --memory 32gib \
-          --env AWS_DEFAULT_REGION \
           --env AWS_ACCESS_KEY_ID \
           --env AWS_SECRET_ACCESS_KEY \
           --env GITHUB_RUN_ID \

diff --git a/ingest/README.md b/ingest/README.md
@@ -70,7 +70,6 @@ The complete ingest pipeline with AWS S3 uploads and Slack notifications uses th
 
 #### Required
 
-- `AWS_DEFAULT_REGION`
 - `AWS_ACCESS_KEY_ID`
 - `AWS_SECRET_ACCESS_KEY`
 - `SLACK_TOKEN`

diff --git a/phylogenetic/bin/set-branch-ingest-config b/phylogenetic/bin/set-branch-ingest-config