Clean up workflows

.github/workflows/ci.yaml

@@ -1,8 +1,11 @@
 name: CI
 
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   test-cram:

.github/workflows/run-models.yaml

@@ -17,60 +17,64 @@ on:
         required: false
 
 jobs:
-  run_models:
+  set_config_overrides:
     runs-on: ubuntu-latest
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
-      DATA_PROVENANCE: ${{ github.event.inputs.data_provenance || github.event.client_payload.data_provenance }}
-      GITHUB_RUN_ID: ${{ github.run_id }}
-      PAT_GITHUB_DISPATCH: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH }}
-      SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-      SLACK_CHANNELS: ${{ github.event.inputs.slack_channel || 'nextstrain-counts-updates' }}
-      TRIAL_NAME: ${{ github.event.inputs.trial_name }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
+      - id: s3_dst
+        run: |
+          S3_DST=s3://nextstrain-data/files/workflows/forecasts-ncov
 
-    - name: install-pyyaml
-      run: python3 -m pip install pyyaml
+          if [[ -n "$TRIAL_NAME" ]]; then
+            S3_DST+=/trial/"$TRIAL_NAME"
+          fi
 
-    - name: run_pipeline
-      run: |
-        ./ingest/bin/write-envdir env.d \
-          AWS_DEFAULT_REGION \
-          GITHUB_RUN_ID \
-          SLACK_TOKEN \
-          SLACK_CHANNELS \
-          PAT_GITHUB_DISPATCH \
+          echo "s3_dst=$S3_DST" >> "$GITHUB_OUTPUT"
+        env:
+          TRIAL_NAME: ${{ inputs.trial_name }}
+
+      - id: data_provenances
+        run: |
+          DATA_PROVENANCES='["gisaid", "open"]'
 
-        # Need to edit the optional file with a script because we need to edit nested values
-        if [[ "$TRIAL_NAME" ]]; then
-          ./bin/set-trial-optional-config \
-            --config-yaml config/optional.yaml \
-            --s3-trial-prefix "$TRIAL_NAME" > optional.yaml
-        else
-          cat config/optional.yaml > optional.yaml
-        fi
+          if [[ -n "$INPUT_DATA_PROVENANCE" ]]; then
+            DATA_PROVENANCES="$INPUT_DATA_PROVENANCE"
+          fi
 
-        declare -a override_config
-        if [[ "$DATA_PROVENANCE" ]]; then
-          override_config=(--config data_provenances="$DATA_PROVENANCE")
-        fi
+          echo "data_provenances=$DATA_PROVENANCES" >> "$GITHUB_OUTPUT"
+        env:
+          INPUT_DATA_PROVENANCE: ${{ inputs.data_provenance }}
+    outputs:
+      s3_dst: ${{ steps.s3_dst.outputs.s3_dst }}
+      data_provenances: ${{ steps.data_provenances.outputs.data_provenances }}
 
+  run_models:
+    needs: [set_config_overrides]
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      run: |
         nextstrain build \
           --aws-batch \
           --detach \
           --no-download \
           --image nextstrain/base:build-20230720T001758Z \
           --cpus 8 \
           --memory 16GiB \
-          --exec env \
+          --env AWS_DEFAULT_REGION \
+          --env AWS_ACCESS_KEY_ID \
+          --env AWS_SECRET_ACCESS_KEY \
+          --env GITHUB_RUN_ID \
+          --env SLACK_TOKEN \
+          --env SLACK_CHANNELS \
           . \
-            envdir env.d snakemake \
-              --configfile config/config.yaml optional.yaml \
-              --printshellcmds \
-              --cores 8 \
-              --keep-going \
-              "${override_config[@]}"
+            --configfile config/config.yaml config/optional.yaml \
+            --config s3_dst="$S3_DST" data_provenances="$DATA_PROVENANCES" \
+            --keep-going
+      env: |
+        DATA_PROVENANCES: ${{ needs.set_config_overrides.outputs.data_provenances }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        SLACK_CHANNELS: ${{ github.event.inputs.slack_channel || vars.SLACK_CHANNELS }}
+        S3_DST: ${{ needs.set_config_overrides.outputs.s3_dst }}

.github/workflows/update-ncov-case-counts.yaml

@@ -46,7 +46,7 @@ jobs:
           S3_DST+=/trial/"$TRIAL_NAME"
         fi
 
-        ./ingest/bin/upload-to-s3 global_case_counts.tsv "$S3_DST"/global.tsv.gz $CLOUDFRONT_DOMAIN
+        ./ingest/vendored/upload-to-s3 global_case_counts.tsv "$S3_DST"/global.tsv.gz $CLOUDFRONT_DOMAIN
       env:
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -55,4 +55,4 @@ jobs:
 
     - name: notify_pipeline_failed
       if: ${{ failure() }}
-      run: ./ingest/bin/notify-on-job-fail
+      run: ./ingest/vendored/notify-on-job-fail "Case counts ingest" "nextstrain/forecasts-ncov"

Snakefile

@@ -69,7 +69,7 @@ def _get_all_input(w):
             model=models_to_run,
             date=run_date
         ))
-        if config.get("upload"):
+        if config.get("s3_dst"):
             all_input.extend(expand(
                 [
                     "results/{data_provenance}/{variant_classification}/{geo_resolution}/{model}/{date}_results_s3_upload.done",
@@ -95,5 +95,5 @@ include: "workflow/snakemake_rules/models.smk"
 if config.get("send_slack_notifications"):
     include: "workflow/snakemake_rules/slack_notifications.smk"
 
-if config.get("upload"):
+if config.get("s3_dst"):
     include: "workflow/snakemake_rules/upload.smk"

bin/nextstrain-remote-upload-with-slack-notification

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 base="$(realpath "$(dirname "$0")/..")"
-ingest_bin="$base/ingest/bin"
+ingest_vendored="$base/ingest/vendored"
 
 main() {
     local quiet=0
@@ -31,7 +31,7 @@ main() {
     # the basename of the local file is used in combination with the remote URL in the final remote destination URL.
     local dst="$remote_url$(basename "$file_to_upload")"
 
-    if ! "$ingest_bin"/notify-slack "Updated $dst available."; then
+    if ! "$ingest_vendored"/notify-slack "Updated $dst available."; then
         echo "Notifying Slack failed, but exiting with success anyway."
     fi
 

config/optional.yaml

@@ -1,12 +1,6 @@
 # Optional configs used by the Nextstrain team
-# Upload params for each data provenance
-upload:
-  gisaid:
-    # AWS S3 Bucket with prefix
-    s3_dst: "s3://nextstrain-data/files/workflows/forecasts-ncov/gisaid"
-  open:
-    # AWS S3 Bucket with prefix
-    s3_dst: "s3://nextstrain-data/files/workflows/forecasts-ncov/open"
+
+s3_dst: "s3://nextstrain-data/files/workflows/forecasts-ncov"
 
 # Send Slack notifications
 send_slack_notifications: True