Deploy APL #2779
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy APL | |
| on: | |
| workflow_call: | |
| inputs: | |
| kubernetes_versions: | |
| description: 'Kubernetes version' | |
| type: string | |
| default: "['1.31']" | |
| install_profile: | |
| description: 'APL installation profile' | |
| type: string | |
| default: minimal-with-team | |
| domain_zone: | |
| description: 'Select Domain Zone' | |
| type: string | |
| default: DNS-Integration | |
| kms: | |
| description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?' | |
| type: string | |
| default: age | |
| certificate: | |
| description: 'Select certificate issuer' | |
| type: string | |
| default: letsencrypt_production | |
| is_pre_installed: | |
| description: Fake if Otomi is pre-installed by Installer | |
| type: string | |
| default: 'false' | |
| workflow_dispatch: | |
| inputs: | |
| kubernetes_versions: | |
| description: 'Kubernetes version' | |
| type: choice | |
| options: | |
| - "['1.29']" | |
| - "['1.30']" | |
| - "['1.31']" | |
| default: "['1.31']" | |
| install_profile: | |
| description: APL installation profile | |
| default: minimal-with-team | |
| type: choice | |
| options: | |
| - minimal | |
| - minimal-with-team | |
| - full | |
| - upgrade | |
| - no-apl | |
| domain_zone: | |
| type: choice | |
| description: 'Select Domain Zone' | |
| options: | |
| - Zone-1 | |
| - Zone-2 | |
| - Random | |
| - DNS-Integration | |
| kms: | |
| type: choice | |
| description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)? | |
| options: | |
| - age | |
| - no_kms | |
| default: age | |
| certificate: | |
| type: choice | |
| description: Select certificate issuer | |
| options: | |
| - gen_custom_ca | |
| - letsencrypt_staging | |
| - letsencrypt_production | |
| default: letsencrypt_production | |
| is_pre_installed: | |
| type: choice | |
| description: Fake if Otomi is pre-installed by Installer | |
| options: | |
| - 'true' | |
| - 'false' | |
| default: 'false' | |
| env: | |
| CACHE_REGISTRY: ghcr.io | |
| CACHE_REPO: linode/apl-core | |
| REPO: linode/apl-core | |
| GIT_USER: svcAPLBot | |
| CHECK_CONTEXT: continuous-integration/integration-test | |
| COMMIT_ID: '${{ github.event.pull_request.head.sha || github.sha }}' | |
| BOT_EMAIL: ${{ vars.BOT_EMAIL }} | |
| BOT_USERNAME: ${{ vars.BOT_USERNAME }} | |
| DEV_DOMAINS: ${{ secrets.DEV_DOMAINS }} | |
| jobs: | |
| preprocess-input: | |
| name: Preprocess input variables | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Print user input | |
| run: | | |
| echo 'ref: ${{ github.event.pull_request.head.ref || github.ref }}' | |
| echo 'install_profile: ${{ inputs.install_profile }}' | |
| echo 'kubernetes_versions: ${{ inputs.kubernetes_versions }}' | |
| echo 'kms: ${{ inputs.kms }}' | |
| echo 'domain_zone: ${{ inputs.domain_zone }}' | |
| echo 'certificate: ${{ inputs.certificate }}' | |
| echo 'is_pre_installed: ${{ inputs.is_pre_installed }}' | |
| preprocess-linode-input: | |
| needs: preprocess-input | |
| name: Preprocess input variables for linode | |
| runs-on: ubuntu-latest | |
| outputs: | |
| kubernetes_versions: ${{ steps.k8s-versions.outputs.versions }} | |
| steps: | |
| - name: Install the Linode CLI | |
| uses: linode/action-linode-cli@v1 | |
| with: | |
| token: ${{ secrets.LINODE_TOKEN }} | |
| - name: Check if cluster is running | |
| run: | | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;; | |
| "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;; | |
| "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;; | |
| "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;; | |
| esac | |
| [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl | |
| if [[ $(linode-cli lke clusters-list --json | jq --arg name "$LINODE_CLUSTER_NAME" '[.[] | select(.label == $name)] | length > 0') == "true" ]]; then | |
| echo "An LKE cluster with the same name ($LINODE_CLUSTER_NAME) already exists." | |
| echo "Visit https://cloud.linode.com/kubernetes/clusters to delete your cluster" | |
| echo "Exiting workflow..." | |
| exit 1 | |
| fi | |
| - id: k8s-versions | |
| name: Process k8s version input | |
| run: | | |
| if [ -z '${{ inputs.kubernetes_versions }}' ]; then | |
| echo "Kubernetes versions not specified, determine Linode supported versions" | |
| versions=`linode-cli lke versions-list --json | jq -ce '.[] | .id'` | |
| else | |
| versions='${{ inputs.kubernetes_versions }}' | |
| fi | |
| echo $versions | |
| echo "versions=$versions" >> $GITHUB_OUTPUT | |
| run-integration-test-linode: | |
| name: Run integration test on linode cluster | |
| needs: preprocess-linode-input | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kubernetes_versions: ${{ fromJSON(needs.preprocess-linode-input.outputs.kubernetes_versions) }} | |
| max-parallel: 5 | |
| steps: | |
| - name: Install the Linode CLI | |
| uses: linode/action-linode-cli@v1 | |
| with: | |
| token: ${{ secrets.LINODE_TOKEN }} | |
| - name: Set k8s cluster name | |
| run: | | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;; | |
| "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;; | |
| "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;; | |
| "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;; | |
| esac | |
| [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl | |
| echo LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME >> $GITHUB_ENV | |
| - name: Determine exact k8s version | |
| run: | | |
| echo LINODE_K8S_VERSION=$(linode-cli lke versions-list --json | jq -ce --arg version "$(echo ${{ matrix.kubernetes_versions }} | sed -E 's/^([0-9]+\.[0-9])$/\10/')" '.[] | select(.id | tostring | startswith($version)) | .id') >> $GITHUB_ENV | |
| - name: Determine domain name to use for scheduled integration test | |
| env: | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| if: ${{ inputs.domain_zone == 'DNS-Integration' && inputs.install_profile != 'no-apl'}} | |
| run: | | |
| RAND=$(openssl rand -hex 4) | |
| DOMAIN="integration-${RAND}.${EDGEDNS_ZONE}" | |
| echo "::add-mask::$DOMAIN" | |
| echo DOMAIN=$DOMAIN >> $GITHUB_ENV | |
| - name: Determine domain name | |
| if: ${{ inputs.domain_zone != 'DNS-Integration' && inputs.install_profile != 'no-apl' }} | |
| env: | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| run: | | |
| # Mapping of domain_zone to domain names | |
| case "${{ inputs.domain_zone }}" in | |
| "Zone-1") DOMAIN=$(jq -r '."${{ github.actor }}"[0]' <<< ${{ env.DEV_DOMAINS }}) ;; | |
| "Zone-2") DOMAIN=$(jq -r '."${{ github.actor }}"[1]' <<< ${{ env.DEV_DOMAINS }}) ;; | |
| "Random") DOMAIN="$(openssl rand -hex 4)$(date +"%d%m%y").${EDGEDNS_ZONE}" ;; | |
| esac | |
| echo "::add-mask::$DOMAIN" | |
| echo DOMAIN=$DOMAIN >> $GITHUB_ENV | |
| - name: Create k8s cluster for testing | |
| run: | | |
| linode-cli lke cluster-create \ | |
| --label ${{ env.LINODE_CLUSTER_NAME }} \ | |
| --region nl-ams \ | |
| --k8s_version ${{ env.LINODE_K8S_VERSION }} \ | |
| --control_plane.high_availability true \ | |
| --node_pools.type g6-dedicated-8 --node_pools.count 3 \ | |
| --node_pools.autoscaler.enabled true \ | |
| --node_pools.autoscaler.max 3 \ | |
| --node_pools.autoscaler.min 3 \ | |
| --tags testing \ | |
| --tags delete_me_tonight \ | |
| --no-defaults | |
| - name: Retrieve cluster id | |
| run: echo "LINODE_CLUSTER_ID=$(linode-cli lke clusters-list --json | jq -ce '.[] | select(.label | startswith("${{ env.LINODE_CLUSTER_NAME }}")) | .id')" >> $GITHUB_ENV | |
| - name: Wait for cluster to be ready | |
| run: | | |
| echo "Waiting for the cluster to be active..." | |
| while :; do | |
| rawOutput=$(linode-cli lke pools-list ${{ env.LINODE_CLUSTER_ID }} --json) | |
| allReady=$(echo "$rawOutput" | jq -r 'map(.nodes | .status == "ready") | all') | |
| echo "All nodes ready: $allReady" | |
| if [ "$allReady" == "true" ]; then | |
| echo "Cluster is ready" | |
| break | |
| fi | |
| sleep 30 | |
| done | |
| - name: Save kubectl config with auth token and Get kubectl environment and create docker secret | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| run: | | |
| # Get the kubeconfig from linode-cli | |
| kubeconfig=$(linode-cli lke kubeconfig-view ${{ env.LINODE_CLUSTER_ID }} --text | sed 1d | base64 --decode) | |
| # Save the kubeconfig to a file | |
| kubeconfigDir="$HOME/.kube" | |
| kubeconfigPath="$HOME/.kube/config" | |
| mkdir -p "$kubeconfigDir" # Create the directory if it doesn't exist | |
| echo "$kubeconfig" > "$kubeconfigPath" | |
| echo "Kubeconfig saved to $kubeconfigPath" | |
| # Set the kubectl context to use the new kubeconfig | |
| export KUBECONFIG="$kubeconfigPath" | |
| contextName=$(kubectl config get-contexts -o name | head -n 1) | |
| kubectl config use-context "$contextName" | |
| echo "Kubectl context set to linode" | |
| echo LINODE_CLUSTER_CONTEXT=`kubectl config current-context` >> $GITHUB_ENV | |
| - name: Create image pull secret on test cluster | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| run: | | |
| kubectl create secret docker-registry reg-otomi-github \ | |
| --docker-server=${{ env.CACHE_REGISTRY }} \ | |
| --docker-username=${{ env.BOT_USERNAME }} \ | |
| --docker-password='${{ secrets.BOT_PULL_TOKEN }}' | |
| - name: Checkout | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| uses: actions/checkout@v4 | |
| - name: Prepare APL chart | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| run: | | |
| ref=${{ github.event.pull_request.head.ref || github.ref }} | |
| tag=${ref##*/} | |
| sed --in-place "s/APP_VERSION_PLACEHOLDER/$tag/g" chart/apl/Chart.yaml | |
| sed --in-place "s/CONTEXT_PLACEHOLDER/${{ env.LINODE_CLUSTER_CONTEXT }}/g" tests/integration/${{ inputs.install_profile }}.yaml | |
| sed --in-place "s/CLUSTER_NAME_PLACEHOLDER/aplinstall${{ env.LINODE_CLUSTER_ID }}/g" tests/integration/${{ inputs.install_profile }}.yaml | |
| sed --in-place "s/OTOMI_VERSION_PLACEHOLDER/${GITHUB_REF##*/}/g" tests/integration/${{ inputs.install_profile }}.yaml | |
| touch values-container-registry.yaml | |
| # If a pipeline installs APL from the semver tag then pull container image from DockerHub | |
| [[ ${GITHUB_REF##*/} =~ ^v[0-9].+$ ]] && exit 0 | |
| # Pull image from cache registry | |
| cat << EOF > values-container-registry.yaml | |
| imageName: "${{ env.CACHE_REGISTRY }}/${{ env.CACHE_REPO }}" | |
| imagePullSecretNames: | |
| - reg-otomi-github | |
| EOF | |
| - name: APL install | |
| if: ${{ inputs.install_profile != 'no-apl' }} | |
| env: | |
| LETSENCRYPT_STAGING: ${{ secrets.LETSENCRYPT_STAGING }} | |
| LETSENCRYPT_PRODUCTION: ${{ secrets.LETSENCRYPT_PRODUCTION }} | |
| EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }} | |
| EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }} | |
| EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }} | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }} | |
| run: | | |
| touch values.yaml | |
| adminPassword="$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)" | |
| [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml | |
| [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml | |
| [[ '${{ inputs.kms }}' == 'age' ]] && kms="--set kms.sops.provider=age" | |
| if [[ '${{ inputs.is_pre_installed }}' == 'true' ]]; then | |
| cat <<EOF >> values.yaml | |
| otomi: | |
| isPreInstalled: true | |
| EOF | |
| fi | |
| if [[ '${{ inputs.kms }}' == 'age' ]]; then | |
| cat <<EOF >> values.yaml | |
| kms: | |
| sops: | |
| provider: age | |
| EOF | |
| fi | |
| install_args="otomi chart/apl --wait --wait-for-jobs --timeout 90m0s \ | |
| --values tests/integration/${{ inputs.install_profile }}.yaml \ | |
| --values values-container-registry.yaml \ | |
| --values values.yaml \ | |
| --set cluster.provider=linode \ | |
| --set dns.domainFilters[0]=${{ env.DOMAIN }} \ | |
| --set dns.provider.akamai.clientSecret=${EDGEDNS_CLIENT_SECRET} \ | |
| --set dns.provider.akamai.host=${EDGEDNS_HOST} \ | |
| --set dns.provider.akamai.accessToken=${EDGEDNS_ACCESS_TOKEN} \ | |
| --set dns.provider.akamai.clientToken=${EDGEDNS_CLIENT_TOKEN} \ | |
| --set otomi.hasExternalDNS=true \ | |
| --set cluster.domainSuffix=${{ env.DOMAIN }} \ | |
| --set otomi.adminPassword=$adminPassword \ | |
| $kms" | |
| helm install $install_args & | |
| HELM_PID=$! | |
| sleep 120 | |
| # While helm is installing we can crete the wildcard dns record | |
| while true; do | |
| PUB_IP=$(kubectl get svc ingress-nginx-platform-controller -n ingress -ojson | jq '.status.loadBalancer.ingress[0].ip' -r) | |
| if [[ -n "$PUB_IP" ]]; then | |
| echo "::add-mask::$PUB_IP" | |
| echo PUB_IP=$PUB_IP >> $GITHUB_ENV | |
| break | |
| else | |
| echo "Waiting for ingress-nginx-platform-controller IP..." | |
| sleep 5 | |
| fi | |
| done | |
| pip3 install edgegrid-python requests | |
| python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP || \ | |
| (echo "Will try to recreate it" && \ | |
| python3 bin/edgedns_A_record.py delete "*.${DOMAIN}" && \ | |
| python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP) | |
| wait $HELM_PID | |
| - name: Gather k8s events on failure | |
| if: failure() | |
| run: | | |
| kubectl get events --sort-by='.lastTimestamp' -A | |
| - name: Gather k8s pods on failure | |
| if: failure() | |
| run: | | |
| kubectl get pods -A -o wide | |
| - name: Gather APL logs on failure | |
| if: failure() | |
| run: | | |
| kubectl logs jobs/otomi-apl --tail 150 | |
| - name: Gather otomi-e2e logs on failure | |
| if: failure() | |
| run: | | |
| kubectl logs -n maintenance -l app.kubernetes.io/instance=job-e2e --tail 15000 | |
| - name: Remove the test cluster | |
| if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }} | |
| run: | | |
| linode-cli lke cluster-delete ${{ env.LINODE_CLUSTER_ID }} | |
| - name: Delete Domain | |
| if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }} | |
| env: | |
| EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }} | |
| EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }} | |
| EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }} | |
| EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }} | |
| EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }} | |
| run: | | |
| python3 bin/edgedns_A_record.py delete "*.${DOMAIN}" | |
| - name: Slack Notification | |
| if: always() | |
| uses: rtCamp/action-slack-notify@v2 | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| SLACK_CHANNEL: github-ci | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_ICON: https://github.com/redkubes.png?size=48 | |
| SLACK_TITLE: Scheduled integration tests | |
| SLACK_USERNAME: RedKubesBot |