feat(cicd) azure aks as a target in cicd (#1880) (#1945)

* feat(cicd) login to Azure ACR container registry Signed-off-by: Bart Smykla <bartek@smykla.com> * feat(cicd) add workflow parameters It will let us start the workflows via CircleCI's API, which is very helpful when developing and debugging tests. Also added to e2e-aks workflow triggers, to run the workflow periodically (every week), which I forgot to do before. Signed-off-by: Bart Smykla <bartek@smykla.com> (cherry picked from commit b92fb94) Co-authored-by: Bart Smykla <bartek@smykla.com>
kumahq · May 5, 2021 · c608a9f · c608a9f
1 parent 420d308
commit c608a9f
Show file tree

Hide file tree

Showing 2 changed files with 338 additions and 14 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,6 +6,26 @@ orbs:
   aws-eks: circleci/aws-eks@1.0.3
   kubernetes: circleci/kubernetes@0.11.2
   slack: circleci/slack@4.1
+  azure-cli: circleci/azure-cli@1.1.0
+  azure-acr: circleci/azure-acr@0.2.0
+  azure-aks: circleci/azure-aks@0.2.1
+
+parameters:
+  run_workflow_clean_eks:
+    default: true
+    type: boolean
+
+  run_workflow_clean_aks:
+    default: true
+    type: boolean
+
+  run_workflow_e2e_eks:
+    default: true
+    type: boolean
+
+  run_workflow_e2e_aks:
+    default: true
+    type: boolean
 
 commands:
   early_return_for_forked_pull_requests:
@@ -71,15 +91,6 @@ reusable:
         branches:
           ignore: master, /^release-.*/, gh-pages
 
-parameters:
-  run_workflow_eks_e2e:
-    default: true
-    type: boolean
-
-  run_workflow_clean_eks:
-    default: true
-    type: boolean
-
 executors:
   golang:
     docker:
@@ -223,9 +234,18 @@ jobs:
         # prefer the exact match
         - vm-executor-go.mod-{{ .Branch }}-{{ checksum "go.sum" }}
 
+    - run:
+        name: "Install Go"
+        command: |
+          # see https://golang.org/doc/install#tarball
+          curl -L https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz | tar -xz -C $HOME
+
     - run:
         name: "Download Go modules"
-        command: go mod download
+        command: |
+          export PATH="$HOME/go/bin:$PATH"
+
+          go mod download
 
     # since execution of go commands might change contents of "go.sum",
     #  we have to save cache immediately
@@ -236,7 +256,10 @@ jobs:
 
     - run:
         name: "Install all development tools"
-        command: make dev/tools
+        command: |
+          export PATH="$HOME/go/bin:$PATH"
+
+          make dev/tools
 
     - run:
         name: "Setup Helm"
@@ -250,6 +273,7 @@ jobs:
     - run:
         name: "Run E2E tests"
         command: |
+          export PATH="$HOME/go/bin:$PATH"
           export KUMA_UNIVERSAL_IMAGE="${ECR_REGISTRY}/kuma-universal:${TAG}";
           export E2E_PKG_LIST="./test/e2e/helm";
           export API_VERSION="<< parameters.api_version >>"
@@ -287,6 +311,260 @@ jobs:
         event: pass
         template: basic_success_1
 
+  aks-acr-clean-registry:
+    executor: azure-cli/default
+    steps:
+    - azure-cli/install
+
+    - azure-cli/login-with-service-principal
+
+    - run:
+        name: "Remove all test images from repository"
+        command: |
+          PURGE_CMD="acr purge \
+            --filter 'kumactl:.*' \
+            --filter 'kuma-cp:.*' \
+            --filter 'kuma-dp:.*' \
+            --filter 'kuma-init:.*' \
+            --filter 'kuma-universal:.*' \
+            --untagged \
+            --ago 1s"
+
+          az acr run \
+             --cmd "${PURGE_CMD}" \
+             --registry "${AZURE_ACR_REGISTRY}" \
+             /dev/null
+
+  aks-delete-clusters:
+    executor: azure-cli/default
+    steps:
+    - azure-cli/install
+
+    - azure-cli/login-with-service-principal
+
+    - run:
+        name: "Remove all test images from repository"
+        command: |
+          for name in $(az aks list --resource-group=${AZURE_RESOURCE_GROUP} | jq -r '.[].name'); do
+            az aks delete --resource-group=${AZURE_RESOURCE_GROUP} --name ${name} --yes &
+          done
+
+          wait
+
+  aks-e2e:
+    executor: vm
+    environment:
+      GOPATH: /home/circleci/.go-kuma-go
+    parameters:
+      prefix:
+        description: |
+          Prefix for the cluster names
+        type: string
+        default: kuma-cicd-e2e-test
+      api_version:
+        description: |
+          XDS API version
+        type: string
+        default: v3
+      node_count:
+        description: |
+          Amount of nodes for cluster's node pool
+        type: integer
+        default: 1
+      node_vm_size:
+        description: |
+          Size of Virtual Machines to create as Kubernetes nodes
+        type: string
+        default: "Standard_DS2_v2"
+      kubernetes_version:
+        description: |
+          Version of Kubernetes to use for creating the cluster, such as "1.11.8" or "1.12.6".
+        type: string
+        default: "1.18.17"
+    steps:
+    - early_return_for_forked_pull_requests
+
+    - checkout
+
+    - run:
+        name: "Setup Environment"
+        command: |
+          echo "export PATH=$HOME/.local/bin:$PATH" >> $BASH_ENV
+          echo "export PATH=$HOME/go/bin:$PATH" >> $BASH_ENV
+          echo "export PATH=$HOME/bin:$PATH" >> $BASH_ENV
+          echo "export TAG=${CIRCLE_SHA1:0:7}" >> $BASH_ENV
+          echo "export CLUSTER_1=<< parameters.prefix >>-${CIRCLE_SHA1:0:7}-1" >> $BASH_ENV
+          echo "export CLUSTER_2=<< parameters.prefix >>-${CIRCLE_SHA1:0:7}-2" >> $BASH_ENV
+          echo "export AZURE_ACR_REGISTRY_FULLNAME=${AZURE_ACR_REGISTRY}.azurecr.io" >> $BASH_ENV
+
+    - azure-cli/install
+
+    - azure-cli/login-with-service-principal
+
+    - run:
+        name: "Install AKS Preview Azure CLI Extension"
+        command: |
+          az extension add --name aks-preview
+
+    - kubernetes/install
+
+    # Mount files from the upstream jobs
+    - attach_workspace:
+        at: build
+
+    - run:
+        name: "Load Docker images from TAR archives"
+        command: make docker/load
+
+    - azure-acr/acr-login:
+        registry-name: ${AZURE_ACR_REGISTRY}
+
+    - run:
+        name: "Push images to ECR & Create clusters"
+        command: |
+          for image in kuma-cp kuma-dp kuma-init kumactl kuma-universal; do
+            make docker/tag/${image} \
+              BINTRAY_REGISTRY="${AZURE_ACR_REGISTRY_FULLNAME}" \
+              KUMA_VERSION="${TAG}" &
+          done
+
+          wait
+
+          for image in kuma-cp kuma-dp kuma-init kumactl kuma-universal; do
+            docker push "${AZURE_ACR_REGISTRY_FULLNAME}"/${image}:"${TAG}" &
+          done
+
+          clusters="${CLUSTER_1} ${CLUSTER_2}"
+
+          for name in ${clusters}; do
+            az aks create \
+              --name "${name}" \
+              --resource-group "${AZURE_RESOURCE_GROUP}" \
+              --location "${AZURE_LOCATION}" \
+              --subscription ${AZURE_SUBSCRIPTION} \
+              --node-count << parameters.node_count >> \
+              --node-vm-size << parameters.node_vm_size >> \
+              --kubernetes-version << parameters.kubernetes_version >> \
+              --enable-addons monitoring \
+              --generate-ssh-keys \
+              --network-plugin azure \
+              --enable-managed-identity \
+              --assign-identity "${AZURE_IDENTITY_CONTROL_PLANE}" \
+              --assign-kubelet-identity "${AZURE_IDENTITY_KUBELET}" \
+              &
+          done
+
+          FAIL="0"
+          for job in $(jobs -p); do
+            wait "${job}" || FAIL=$((FAIL + 1))
+          done
+
+          exit ${FAIL}
+
+    - azure-aks/update-kubeconfig-with-credentials:
+        cluster-name: "${CLUSTER_1}"
+        resource-group: "${AZURE_RESOURCE_GROUP}"
+        subscription: "${AZURE_SUBSCRIPTION}"
+        file: ~/.kube/kind-kuma-1-config
+
+    - azure-aks/update-kubeconfig-with-credentials:
+        cluster-name: "${CLUSTER_2}"
+        resource-group: "${AZURE_RESOURCE_GROUP}"
+        subscription: "${AZURE_SUBSCRIPTION}"
+        file: ~/.kube/kind-kuma-2-config
+
+    - restore_cache:
+        keys:
+        # prefer the exact match
+        - vm-executor-go.mod-{{ .Branch }}-{{ checksum "go.sum" }}
+
+    - run:
+        name: "Install Go"
+        command: |
+          # see https://golang.org/doc/install#tarball
+          curl -L https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz | tar -xz -C $HOME
+
+    - run:
+        name: "Download Go modules"
+        command: |
+          export PATH="$HOME/go/bin:$PATH"
+
+          go mod download
+
+    # since execution of go commands might change contents of "go.sum",
+    #  we have to save cache immediately
+    - save_cache:
+        key: vm-executor-go.mod-{{ .Branch }}-{{ checksum "go.sum" }}
+        paths:
+        - "/home/circleci/.go-kuma-go"
+
+    - run:
+        name: "Install all development tools"
+        command: |
+          export PATH="$HOME/go/bin:$PATH"
+
+          make dev/tools
+
+    - run:
+        name: "Setup Helm"
+        command: helm repo add kuma https://kumahq.github.io/charts
+
+    # We need kumactl to be built to run tests, and to not build it again,
+    #  we can use the one already build by earlier jobs
+    - attach_workspace:
+        at: build
+
+    - run:
+        name: "Run E2E tests"
+        command: |
+          export PATH="$HOME/go/bin:$PATH"
+          export KUMA_UNIVERSAL_IMAGE="${AZURE_ACR_REGISTRY_FULLNAME}/kuma-universal:${TAG}";
+          export E2E_PKG_LIST="./test/e2e/helm";
+          export API_VERSION="<< parameters.api_version >>"
+
+          make test/e2e/test \
+            KUMA_GLOBAL_IMAGE_REGISTRY="${AZURE_ACR_REGISTRY_FULLNAME}" \
+            KUMA_GLOBAL_IMAGE_TAG="${TAG}" \
+            KUMA_CNI_CONF_NAME="10-azure.conflist" \
+            KUMA_UNIVERSAL_IMAGE="${KUMA_UNIVERSAL_IMAGE}" \
+            KUMA_DEFAULT_RETRIES=30 \
+            KUMA_DEFAULT_TIMEOUT="3s" \
+            KUMA_USE_LOAD_BALANCER=true
+
+    - run:
+        name: "Cleanup"
+        when: always
+        command: |
+          for image in kumactl kuma-cp kuma-dp kuma-init kuma-universal; do
+            az acr repository delete \
+              --subscription "${AZURE_SUBSCRIPTION}" \
+              --name "${AZURE_ACR_REGISTRY}" \
+              --image "${image}:${TAG}" \
+              --yes \
+              &
+          done
+
+          clusters="${CLUSTER_1} ${CLUSTER_2}"
+
+          for name in ${clusters}; do
+            az aks delete \
+              --subscription "${AZURE_SUBSCRIPTION}" \
+              --name "${name}" \
+              --resource-group "${AZURE_RESOURCE_GROUP}" \
+              --yes \
+              &
+          done
+
+          wait || true
+
+    - slack/notify:
+        event: fail
+        template: basic_fail_1
+
+    - slack/notify:
+        event: pass
+        template: basic_success_1
+
   eks-delete-clusters:
     executor: aws-eks/python3
     steps:
@@ -947,8 +1225,21 @@ workflows:
     - eks-ecr-clean-registry
     - eks-delete-clusters
 
+  clean-aks:
+    when: << pipeline.parameters.run_workflow_clean_aks >>
+    triggers:
+    - schedule:
+        # run cleanup everyday at 02:00 - it's the last resort for cleanup of
+        # AKS resources if anything went wrong and because of any reason
+        # they were not cleaned immediately after tests
+        cron: "0 2 * * *"
+        <<: *master_only_workflow_filters
+    jobs:
+    - aks-acr-clean-registry
+    - aks-delete-clusters
+
   e2e-eks:
-    when: << pipeline.parameters.run_workflow_eks_e2e >>
+    when: << pipeline.parameters.run_workflow_e2e_eks >>
     triggers:
     - schedule:
         # run every Monday at 00:00
@@ -971,6 +1262,30 @@ workflows:
         requires:
         - images
 
+  e2e-aks:
+    when: << pipeline.parameters.run_workflow_e2e_aks >>
+    triggers:
+    - schedule:
+        # run every Monday at 00:00
+        cron: "0 0 * * 1"
+        <<: *master_only_workflow_filters
+    jobs:
+    - go_cache: *master_only_workflow_filters
+    - build:
+        <<: *master_only_workflow_filters
+        requires:
+        - go_cache
+    - images:
+        <<: *master_only_workflow_filters
+        requires:
+        - build
+        docker_registry: ${AZURE_ACR_REGISTRY}.azurecr.io
+    - aks-e2e:
+        <<: *master_only_workflow_filters
+        name: test/e2e/aks
+        requires:
+        - images
+
   kuma-master:
     jobs:
       - dev_golang: *master_workflow_filters