cicd.yml

name: CI/CD

on:
  pull_request:  # any pull request
  push:
    branches:
      - master

defaults:
  run:
    shell: bash

env:
  AWS_DEFAULT_OUTPUT: json
  AWS_DEFAULT_REGION: us-east-1
  AWS_MAX_ATTEMPTS: 20  # retry attempts for AWS API calls
  AWS_RETRY_MODE: adaptive  # defaults to "legacy"; this handles more errors
  NODE_VERSION: '16'
  PYTEST_ADDOPTS: --color=yes
  RUNWAY_TEST_NAMESPACE: gh-${{ github.run_id }}
  PIPENV_IGNORE_VIRTUALENVS: '1'


jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      infra-test: ${{ steps.filter.outputs.infrastructure-test }}
      infra-test-alt: ${{ steps.filter.outputs.infrastructure-test-alt }}
    steps:
      - uses: actions/checkout@v4  # not needed for pull_request
        if: |
          github.event_name == 'push'
      - uses: dorny/paths-filter@v2  # cspell:ignore dorny
        id: filter
        with:
          filters: |
            infrastructure-test:
              - 'infrastructure/blueprints/admin_user.py'
              - 'infrastructure/blueprints/cfngin_bucket.py'
              - 'infrastructure/blueprints/prevent_privilege_escalation.py'
              - 'infrastructure/blueprints/test_runner_boundary.py'
              - 'infrastructure/blueprints/test_runner_user.py'
              - 'infrastructure/test/common/**'
            infrastructure-test-alt:
              - 'infrastructure/blueprints/admin_role.py'
              - 'infrastructure/blueprints/cfngin_bucket.py'
              - 'infrastructure/blueprints/prevent_privilege_escalation.py'
              - 'infrastructure/blueprints/test_runner_boundary.py'
              - 'infrastructure/test-alt/common/**'
  deploy-test-infrastructure:
    name: Deploy Test Infrastructure
    environment: test
    concurrency: test-infrastructure
    needs: changes
    if: |
      github.repository == 'onicagroup/runway' &&
      (needs.changes.outputs.infra-test == 'true' || needs.changes.outputs.infra-test-alt == 'true')
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.9
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: runner.os != 'Windows' && steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.DEPLOY_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DEPLOY_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
      - run: make deploy test
        working-directory: infrastructure
      - run: make deploy test-alt
        working-directory: infrastructure
  lint-python:
    name: Lint Python
    strategy:
      matrix:
        python-version: [3.9, '3.10', '3.11']
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    env:
      # populating AWS creds with fake values
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: runner.os != 'Windows' && steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - name: Install Node Dependencies
        run: make npm-ci
      - name: Run Linters
        run: make lint
  pre-commit:
    name: pre-commit
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.9]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: runner.os != 'Windows' && steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - uses: pre-commit/action@v3.0.0
  test-functional:
    name: Functional Tests
    needs: deploy-test-infrastructure
    # will fail if run from forks
    if: |
      always() &&
      github.repository == 'onicagroup/runway' &&
      github.actor != 'dependabot[bot]' &&
      (needs.deploy-test-infrastructure.result == 'success' || needs.deploy-test-infrastructure.result == 'skipped')
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.9
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: runner.os != 'Windows' && steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - name: Install Ubuntu Dependencies
        run: |
          sudo apt update -y
          sudo apt install -y default-libmysqlclient-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl pkg-config
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.TEST_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.TEST_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
      - name: Run Tests
        run: make test-functional
  test-python:
    name: Test Python
    strategy:
      matrix:
        python-version: [3.9, '3.10', '3.11']
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    env:
      # populating AWS creds with fake values
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
    steps:
      - name: Checkout Repo (complete)
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - name: Install Node Dependencies
        run: make npm-install
      - name: Configure Pagefile  # avoid MemoryError during tests
        if: runner.os == 'Windows'
        uses: al-cheb/configure-pagefile-action@v1.3  # cspell:ignore cheb
        with:
          minimum-size: 16GB
          maximum-size: 16GB
          disk-root: 'C:'
      - name: Run Integration & Unit Tests
        # assertions assume linux so some fail when run on windows
        run: make test cov-xml
      - name: Upload to Codecov
        uses: codecov/codecov-action@v3.1.4
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          fail_ci_if_error: true
  build-pypi:
    name: Build PyPi 📦
    needs:
      - pre-commit
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repo (complete)
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: 3.9
      # Remove apt repos that are known to break from time to time
      # See https://github.com/actions/virtual-environments/issues/323
      - name: Remove broken apt repos (ubuntu)
        run: |
          for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done
      - name: Install Dependencies (ubuntu)
        run: sudo apt-get update && sudo apt-get install sed -y
      - uses: Gr1N/setup-poetry@v8
      - uses: actions/cache@v3
        id: cache
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      - name: Ensure Cache Is Healthy
        if: runner.os != 'Windows' && steps.cache.outputs.cache-hit == 'true'
        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
      - run: poetry install --extras docs -vv
      - name: Run Build
        run: make build
      - name: Upload Distribution Artifact
        uses: actions/upload-artifact@v3
        with:
          name: pypi-dist
          path: dist