ingest.yaml

name: Ingest

defaults:
  run:
    # This is the same as GitHub Action's `bash` keyword as of 20 June 2023:
    # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
    #
    # Completely spelling it out here so that GitHub can't change it out from under us
    # and we don't have to refer to the docs to know the expected behavior.
    shell: bash --noprofile --norc -eo pipefail {0}

on:
  workflow_call:
    inputs:
      image:
        description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
        required: false
        type: string

  workflow_dispatch:
    inputs:
      image:
        description: 'Specific container image to use for ingest workflow (will override the default of "nextstrain build")'
        required: false
        type: string
      trial_name:
        description: |
          Trial name for outputs.
          If not set, outputs will overwrite files at s3://nextstrain-data/files/workflows/dengue/
          If set, outputs will be uploaded to s3://nextstrain-data/files/workflows/dengue/trials/<trial_name>/
        required: false
        type: string

jobs:
  set_config_overrides:
    runs-on: ubuntu-latest
    steps:
      - id: config
        name: Set config overrides
        env:
          TRIAL_NAME: ${{ inputs.trial_name }}
        run: |
          config=""
          if [[ "$TRIAL_NAME" ]]; then
            config+="--config"
            config+=" s3_dst='s3://nextstrain-data/files/workflows/dengue/trials/"$TRIAL_NAME"'"
          fi

          echo "config=$config" >> "$GITHUB_OUTPUT"
    outputs:
      config_overrides: ${{ steps.config.outputs.config }}

  ingest:
    needs: [set_config_overrides]
    permissions:
      id-token: write
    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
    secrets: inherit
    with:
      # Starting with the default docker runtime
      # We can migrate to AWS Batch when/if we need to for more resources or if
      # the job runs longer than the GH Action limit of 6 hours.
      runtime: docker
      env: |
        NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
        CONFIG_OVERRIDES: ${{ needs.set_config_overrides.outputs.config_overrides }}
      run: |
        nextstrain build \
          ingest \
            upload_all \
            --configfile build-configs/nextstrain-automation/config.yaml \
            $CONFIG_OVERRIDES
      # Specifying artifact name to differentiate ingest build outputs from
      # the phylogenetic build outputs
      artifact-name: ingest-build-output
      artifact-paths: |
        ingest/results/
        ingest/benchmarks/
        ingest/logs/
        ingest/.snakemake/log/