gradle.yml

name: Airbyte CI

on:
  #ability to start task manually in Web UI
  workflow_dispatch:
  schedule:
    - cron: "0 */1 * * *"
  push:
    branches-ignore:
      - "gitbook/v1"

jobs:
  # COMMON TASKS
  ensure-images-exist:
    name: "Ensure all required Docker images exist on Dockerhub"
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Check images exist
        run: ./tools/bin/check_images_exist.sh all
  # The output of this job is used to trigger the following builds.
  changes:
    name: "Detect Modified Files"
    # The filtering action does not deal with well scheduled events so skip to avoid errors.
    # See https://github.com/dorny/paths-filter/issues/100 for more info.
    # This is okay this workflow is only scheduled on master, where we want to build everything
    # so filtering is not required. Use always() in each start block to force the start task.
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      build: ${{ steps.filter.outputs.build }}
      cli: ${{ steps.filter.outputs.cli }}
      connectors: ${{ steps.filter.outputs.connectors }}
      db: ${{ steps.filter.outputs.db }}
      frontend: ${{ steps.filter.outputs.frontend }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          # Note, the following glob expression within a filters are ORs.
          filters: |
            backend:
              - 'airbyte-!(cdk|integrations|webapp|webapp-e2e-tests)/**'
            build:
              - '.github/**'
              - 'buildSrc/**'
              - 'tools/**'
              - '*.gradle'
            cli:
              - 'airbyte-api/**'
              - 'octavia-cli/**'
            connectors:
              - 'airbyte-cdk/**'
              - 'airbyte-protocol/**'
              - 'airbyte-integrations/**'
            db:
              - 'airbyte-db/**'
            frontend:
              - 'airbyte-webapp/**'
              - 'airbyte-webapp-e2e-tests/**'
  find_valid_pat:
    name: "Find a PAT with room for actions"
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      pat: ${{ steps.variables.outputs.pat }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Check PAT rate limits
        id: variables
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.AIRBYTEIO_PAT }} \
            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
            ${{ secrets.SUPERTOPHER_PAT }} \
            ${{ secrets.DAVINCHIA_PAT }}

  # Uncomment to debug.
  #  changes-output:
  #    name: "Debug Change Detection Logic"
  #    needs: changes
  #    runs-on: ubuntu-latest
  #    steps:
  #      - uses: actions/checkout@v2
  #      - run: |
  #          echo '${{ toJSON(needs) }}'

  ## BUILDS
  octavia-cli-build:
    needs: changes
    runs-on: ubuntu-latest
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
    name: "Octavia CLI: Build"
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-python@v2
        with:
          python-version: "3.9"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace

      - name: Ensure no file change
        run: ./tools/bin/check_for_file_changes

      - name: Build
        run: |
          SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan

      - name: Build Platform Docker Images
        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan

      - name: Run integration tests
        run: ./tools/bin/integration_tests_octavia.sh

  # Connectors Base
  # In case of self-hosted EC2 errors, remove this block.
  start-connectors-base-build-runner:
    name: "Connectors Base: Start Build EC2 Runner"
    needs:
      - changes
      - find_valid_pat
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: |
      needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ needs.find_valid_pat.outputs.pat }}

  build-connectors-base:
    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
    needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
    name: "Connectors Base: Build"
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-node@v1
        with:
          node-version: "16.13.0"

      - uses: actions/setup-python@v2
        with:
          python-version: "3.9"

      - name: Install Pyenv
        run: python3 -m pip install virtualenv==16.7.9 --user

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Generate Template scaffold
        run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan

      - name: Format
        run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace

      - name: Build
        run: SUB_BUILD=CONNECTORS_BASE ./gradlew build -x :airbyte-workers:test --scan

      - name: Ensure no file change
        run: git --no-pager diff && test -z "$(git --no-pager diff)"

  # In case of self-hosted EC2 errors, remove this block.
  stop-connectors-base-build-runner:
    name: "Connectors Base: Stop Build EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-connectors-base-build-runner # required to get output from the start-runner job
      - build-connectors-base # required to wait when the main job is done
      - find_valid_pat
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-connectors-base-build-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
          label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}

  ## Frontend Test
  # In case of self-hosted EC2 errors, remove this block.
  start-frontend-runner:
    name: "Frontend: Start EC2 Runner"
    needs:
      - changes
      - find_valid_pat
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: |
      needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/master'
      || (always() && needs.changes.outputs.backend == 'true')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
  frontend-build:
    name: "Frontend: Build"
    needs: start-frontend-runner
    runs-on: ${{ needs.start-frontend-runner.outputs.label }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-node@v1
        with:
          node-version: "16.13.0"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Build :airbyte-webapp
        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
  frontend-test:
    name: "Frontend: Run End-to-End Tests"
    needs:
      - start-frontend-runner # required to have runner started
    runs-on: ${{ needs.start-frontend-runner.outputs.label }} # run the job on the newly created runner
    timeout-minutes: 120
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-node@v1
        with:
          node-version: "16.13.0"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Build Platform Docker Images
        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan

      - name: Run End-to-End Frontend Tests
        env:
          CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }}
        run: ./tools/bin/e2e_test.sh
  # In case of self-hosted EC2 errors, remove this block.
  stop-frontend-runner:
    name: "Frontend: Stop Runner"
    timeout-minutes: 10
    needs:
      - start-frontend-runner # required to get output from the start-runner job
      - frontend-test # required to wait when the e2e-test job is done
      - frontend-build # required to wait when then build job is done
      - find_valid_pat
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-frontend-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
          label: ${{ needs.start-frontend-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }}

  ## FOLLOWING BUILDS ARE ALL PLATFORM BUILDS.

  # Main Platform
  # In case of self-hosted EC2 errors, remove this block.
  start-platform-build-runner:
    name: "Platform: Start Build EC2 Runner"
    needs:
      - changes
      - find_valid_pat
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
  platform-build:
    name: "Platform: Build"
    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
    needs: start-platform-build-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-node@v1
        with:
          node-version: "16.13.0"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace

      - name: Ensure no file change
        run: git --no-pager diff && test -z "$(git --no-pager diff)"

      - name: Build
        run: |
          SUB_BUILD=PLATFORM ./gradlew build javadoc --scan

      - name: Integration test
        run: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest

      - name: Slow integration test
        if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
        run: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest

      - name: Test if Seed spec is updated
        run: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"

      # todo (cgardens) - scope by platform.
      - name: Check documentation
        if: success() && github.ref == 'refs/heads/master'
        run: ./tools/site/link_checker.sh check_docs

      # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
      #      - name: Get Docker Space
      #        run: docker run --rm busybox df -h

      - name: Run End-to-End Acceptance Tests
        run: ./tools/bin/acceptance_test.sh

      - name: Automatic Migration Acceptance Test
        run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i

  # In case of self-hosted EC2 errors, remove this block.
  stop-platform-build-runner:
    name: "Platform: Stop Build EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-platform-build-runner # required to get output from the start-runner job
      - platform-build # required to wait when the main job is done
      - find_valid_pat
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-platform-build-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
          label: ${{ needs.start-platform-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}

  ## Kube Acceptance Tests (with scheduler v2 - both temporal changes and container orchestrator)
  # Docker acceptance tests run as part of the build job.
  # In case of self-hosted EC2 errors, remove this block.
  start-kube-acceptance-test-runner-v2:
    name: "Platform: Start Scheduler V2 Kube Acceptance Test Runner"
    needs:
      - changes
      - find_valid_pat
    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
          ec2-image-id: ami-0c1a9bc22624339d8
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
  kube-acceptance-test-v2:
    name: "Platform: Acceptance Tests (Kube v2)"
    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
    needs: start-kube-acceptance-test-runner-v2 # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }} # run the job on the newly created runner
    environment: more-secrets
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v1
        with:
          java-version: "17"

      - uses: actions/setup-node@v1
        with:
          node-version: "16.13.0"

      - name: Fix EC-2 Runner
        run: |
          mkdir -p /home/runner

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Create cluster config file
        run: |
          cat > /tmp/kind-config.yaml <<EOF
          kind: Cluster
          apiVersion: kind.x-k8s.io/v1alpha4
          nodes:
          - role: control-plane
          - role: worker
          EOF

      - name: KIND Kubernetes Cluster Setup
        uses: helm/kind-action@v1.2.0
        with:
          node_image: kindest/node:v1.21.2
          config: /tmp/kind-config.yaml
        # In case of self-hosted EC2 errors, remove this env block.
        env:
          USER: root
          HOME: /home/runner
          CHANGE_MINIKUBE_NONE_USER: true

      - name: Build Platform Docker Images
        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan

      - name: Run Kubernetes End-to-End Acceptance Tests
        env:
          USER: root
          HOME: /home/runner
          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
        run: |
          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh

      - uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: Kubernetes Logs
          path: /tmp/kubernetes_logs/*
  # In case of self-hosted EC2 errors, remove this block.
  stop-kube-acceptance-test-runner-v2:
    name: "Platform: Stop Kube Acceptance Test EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-kube-acceptance-test-runner-v2 # required to get output from the start-runner job
      - kube-acceptance-test-v2 # required to wait when the main job is done
      - find_valid_pat
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-kube-acceptance-test-runner-v2.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ needs.find_valid_pat.outputs.pat }}
          label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }}
          ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }}

  notify-failure-slack-channel:
    name: "Notify Slack Channel on Build Failures"
    runs-on: ubuntu-latest
    needs:
      - build-connectors-base
      - frontend-build
      - octavia-cli-build
      - platform-build
      - kube-acceptance-test-v2
    if: ${{ failure() && github.ref == 'refs/heads/master' }}
    steps:
      - name: Publish to OSS Build Failure Slack Channel
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
            {\"type\":\"divider\"}]}

  notify-failure-slack-channel-fixed-broken-build:
    name: "Notify Slack Channel on Build Fixes"
    runs-on: ubuntu-latest
    needs:
      - build-connectors-base
      - frontend-build
      - octavia-cli-build
      - platform-build
      - kube-acceptance-test-v2
    if: success()
    steps:
      - name: Get Previous Workflow Status
        uses: Mercymeilya/last-workflow-status@v0.3
        id: last_status
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
      # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
      - name: Publish Build Fixed Message to OSS Build Failure Slack Channel
        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
            {\"type\":\"divider\"}]}