go.yaml

# Copyright (C) 2021 ScyllaDB

name: Go

on:
  push:
    # Restrict the branches to only those we want to promote from.
    branches:
    - 'master'
    - 'v[0-9]+.[0-9]+'
    # Act only on temporary tags, the final ones are created by manually promoting
    # an existing candidate image, after it goes through QA. We need to be carefull
    # to avoid overwritting those, building from the same sources may not result
    # in the same image content that was tested, e.g. if a base image changed in the meantime.
    tags:
    - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
    - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
    - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+'
  schedule:
  - cron:  '0 23 * * *' # daily at 11pm

env:
  go_version: '1.20'
  GOPATH: ${{ github.workspace }}/go
  git_repo_path: ${{ github.workspace }}/go/src/github.com/scylladb/scylla-operator
  image_repo_ref: docker.io/scylladb/scylla-operator
  retention_days: 90

defaults:
  run:
    shell: bash
    working-directory: "./go/src/github.com/scylladb/scylla-operator"

jobs:
  verify:
    name: Verify
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Install tools
      run: |
        set -x
        go install github.com/mikefarah/yq/v4@v4.11.2
    - name: Verify
      run: make verify --warn-undefined-variables

  verify-deps:
    name: Verify dependencies
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Verify dependencies
      run: make verify-deps --warn-undefined-variables

  verify-vendorability:
    name: Verify vendorability
    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    env:
      github_self_ref: 'github.com/${{ github.repository }}@${{ github.sha }}'
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Override repository ref on pull requests
      if: ${{ github.event_name == 'pull_request' }}
      run: |
        echo "Initial github_self_ref=${github_self_ref}"
        echo 'github_self_ref=github.com/${{ github.event.pull_request.head.repo.full_name }}@${{ github.event.pull_request.head.sha }}' | tee -a ${GITHUB_ENV}
    - name: Verify vendorability
      run: |
        set -x
        pushd "$( mktemp -d )"
        
        cat > ./main.go <<EOF
        package main
        import _ "github.com/scylladb/scylla-operator/pkg/scheme"
        func main() {}
        EOF
        
        go mod init foo.com
        go mod edit -replace "github.com/scylladb/scylla-operator=${github_self_ref}"
        cat go.mod
        
        go mod tidy
        cat go.{mod,sum}
        
        go mod vendor
        
        go build ./...

  build-and-test:
    name: Build and test
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0
    - name: Setup git tags
      if: ${{ startsWith(github.ref, 'refs/tags/') }}
      run: ./hack/ci-detect-tags.sh
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Build
      run: make --warn-undefined-variables
    - name: Make sure new unit tests aren't flaky
      if: ${{ github.event_name == 'pull_request' }}
      run: |
        echo "GO_TEST_COUNT=-count=3" | tee -a ${GITHUB_ENV}
    - name: Test
      run: make test --warn-undefined-variables

  images:
    name: Build images
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Setup git tags
      if: ${{ startsWith(github.ref, 'refs/tags/') }}
      run: ./hack/ci-detect-tags.sh
    - name: Install podman
      run: |
        set -euExo pipefail
        shopt -s inherit_errexit
        
        sudo apt-get update
        sudo apt-get -y install podman
    - name: Build image
      run: |
        set -x
        # Avoid podman permission error on Ubuntu 20.04 by using it as root, although it shouldn't be needed.
        # Use --format=docker to support SHELL instruction in the Dockerfile. (SHELL didn't make it to OCI spec.)
        sudo podman --cgroup-manager=cgroupfs build --format=docker --squash --network=none -f ./Dockerfile -t '${{ env.image_repo_ref }}:ci' .
        sudo podman images '${{ env.image_repo_ref }}:ci'
        sudo podman save '${{ env.image_repo_ref }}:ci' | lz4 - ~/operatorimage.tar.lz4
    - name: Upload image artifact
      uses: actions/upload-artifact@v3
      with:
        name: operatorimage.tar.lz4
        path: ~/operatorimage.tar.lz4
        if-no-files-found: error
        retention-days: ${{ env.retention_days }}

  charts:
    name: Build Helm charts
    runs-on: ubuntu-22.04
    timeout-minutes: 15
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Setup git tags
      if: ${{ startsWith(github.ref, 'refs/tags/') }}
      run: ./hack/ci-detect-tags.sh
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Build Helm charts
      run: make helm-build
    - name: Upload charts artifacts
      uses: actions/upload-artifact@v3
      with:
        name: helm-charts.zip
        path: ${{ env.git_repo_path }}/helm/build/*
        if-no-files-found: error
        retention-days: ${{ env.retention_days }}

  test-e2e-parallel:
    name: Test e2e parallel
    runs-on: ubuntu-22.04
    needs: images
    timeout-minutes: 120
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Run e2e
      uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/run-e2e
      with:
        repositoryPath: ${{ env.git_repo_path }}
        jobSuffix: "parallel"
        suite: "scylla-operator/conformance/parallel"
        extraArgs: "--parallelism=8"

  test-e2e-serial:
    name: Test e2e serial
    runs-on: ubuntu-22.04
    needs: images
    timeout-minutes: 90
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Run e2e
      uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/run-e2e
      with:
        repositoryPath: ${{ env.git_repo_path }}
        jobSuffix: "serial"
        suite: "scylla-operator/conformance/serial"

  test-e2e-parallel-alpha:
    name: Test e2e parallel (with alpha features)
    runs-on: ubuntu-22.04
    needs: images
    timeout-minutes: 120
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Run e2e
      uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/run-e2e
      with:
        repositoryPath: ${{ env.git_repo_path }}
        jobSuffix: "parallel-alpha"
        suite: "scylla-operator/conformance/parallel"
        featureGates: "AllAlpha=true,AllBeta=true"
        extraArgs: "--parallelism=8"

  test-e2e-serial-alpha:
    name: Test e2e serial (with alpha features)
    runs-on: ubuntu-22.04
    needs: images
    timeout-minutes: 90
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        fetch-depth: 0  # also fetch tags
    - name: Run e2e
      uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/run-e2e
      with:
        repositoryPath: ${{ env.git_repo_path }}
        jobSuffix: "serial-alpha"
        suite: "scylla-operator/conformance/serial"
        featureGates: "AllAlpha=true,AllBeta=true"

  # TODO: Add upgrade-e2e - use the same image sha from images step

  # Dummy step for different promotion jobs to depend on
  success:
    name: All tests successfull
    runs-on: ubuntu-22.04
    needs:
    - verify
    - verify-deps
    - build-and-test
    - images
    - charts
    - test-e2e-parallel
    - test-e2e-serial
    - test-e2e-parallel-alpha
    - test-e2e-serial-alpha
    # TODO: Depend on upgrade-e2e when available
    timeout-minutes: 15
    steps:
    - name: Always succeed
      working-directory: .
      run: true

  promote:
    name: Promote artifacts
    runs-on: ubuntu-22.04
    needs: [success]
    timeout-minutes: 15
    if: ${{ github.event_name != 'pull_request' }}
    steps:
    - uses: actions/checkout@v3
      with:
        path: ${{ env.git_repo_path }}
        # Helm Chart version needs to be semantic, we need tags in checked out repo to determine latest one.
        fetch-depth: 0
    - name: Setup git tags
      if: ${{ startsWith(github.ref, 'refs/tags/') }}
      run: ./hack/ci-detect-tags.sh
    - name: Setup go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ env.go_version }}
    - name: Determine promotion tag
      if: ${{ github.event_name != 'schedule' }}
      run: |
        source ./hack/lib/tag-from-gh-ref.sh
        CI_IMAGE_TAG=$( tag_from_gh_ref "${GITHUB_REF}" )
        echo "CI_IMAGE_TAG=${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV}
    - name: Determine promotion tag for scheduled job
      if: ${{ github.event_name == 'schedule' }}
      run: |
        CI_IMAGE_TAG=nightly
        echo "CI_IMAGE_TAG=${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV}
        echo "HELM_CHART_VERSION_SUFFIX=-${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV}
    - uses: actions/download-artifact@v3
      with:
        name: operatorimage.tar.lz4
        path: ~/
    - name: Load image
      run: |
        set -x
        unlz4 ~/operatorimage.tar.lz4 - | docker load
        # docker looses the registry part on save/load
        docker tag "$( echo "${image_repo_ref}:ci" | sed -E -e 's~[^/]+/(.*)~\1~' )" "${image_repo_ref}:ci"
        docker images '${{ env.image_repo_ref }}:ci'
    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}
    - name: Promote image
      run: |
        set -x
        docker tag '${{ env.image_repo_ref }}:ci' '${{ env.image_repo_ref }}:${{ env.CI_IMAGE_TAG }}'
        docker push '${{ env.image_repo_ref }}:${{ env.CI_IMAGE_TAG }}'
    - name: Authenticate to GCP
      uses: google-github-actions/auth@v1
      with:
        credentials_json: ${{ secrets.GCP_SA_KEY }}
    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v1
    - name: Publish Helm Chart
      env:
        HELM_CHANNEL: latest
        HELM_APP_VERSION: ${{ env.CI_IMAGE_TAG }}
      run: make helm-publish

  failure-notifications:
    name: Failure notifications
    runs-on: ubuntu-22.04
    needs:
    - success
    - promote
    timeout-minutes: 15
    if: ${{ failure() && github.event_name != 'pull_request' }}
    steps:
    - name: Report failures to Slack
      if: ${{ always() }}
      working-directory: .
      run: |
        # We have to avoid printing the secret to logs.
        set +x
        curl -X POST -H 'Content-type: application/json' --data @<( cat <<-EOF
        {
          "text": ":warning: CI workflow \"${{ github.workflow }}\" triggered on \"${{ github.event_name }}\" event from ${{ github.ref }} (${{ github.sha }}) failed!\n:fire_extinguisher: See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for details.:fire:"
        }
        EOF
        ) '${{ secrets.SLACK_WEBHOOK_URL }}'