deploy.yml

name: Deploy dags to aws

on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      environment:
        type: environment
        description: The environment to deploy to.

jobs:
  detect-environments:
    runs-on: ubuntu-latest
    outputs:
      environments: ${{ steps.environments.outputs.result }}
    steps:
      - uses: actions/github-script@v6
        id: environments
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          result-encoding: json
          script: |
            if (context.payload?.inputs?.environment) return [context.payload?.inputs?.environment];
            const {data: {environments}} =
              await github.request(`GET /repos/${process.env.GITHUB_REPOSITORY}/environments`);
            return environments.map(e => e.name)  

  test:
    runs-on: ubuntu-latest
    env:
      ENVIRONMENT: development
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - run: |
          pip install -r requirements/dev-requirements.txt

      - run: |
          make dags/config.json
        
      - run: |
          make test

  deploy:
    runs-on: ubuntu-latest
    needs: [test, detect-environments]
    strategy:
      matrix:
        environment: ${{ fromJSON(needs.detect-environments.outputs.environments) }}
    environment: ${{ matrix.environment }}
    env:
      AWS_S3_BUCKET: ${{ secrets.DAGS_S3_BUCKET }}
      ENVIRONMENT: ${{ matrix.environment }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - run: |
          pip install -r requirements/requirements.txt
        
      - run: |
          make dags/config.json
      
      - run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip -q awscliv2.zip
          sudo ./aws/install --update
          sudo apt-get update
          sudo apt-get install -y rsync

      - uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.DEPLOY_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DEPLOY_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2

      - run: |
          aws s3 sync ./dags s3://$AWS_S3_BUCKET/dags \
            --delete \
            --exclude '.git/*' \
            --exclude '*/__pycache__/*'