Add e2e github actions

Signed-off-by: Anna Jung (VMware) <antheaj@vmware.com>

.github/workflows/e2e_test.yaml

@@ -0,0 +1,122 @@
+name: E2E Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  e2e:
+    name: AWS
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        juju: [2.9/stable]
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Install juju
+      run: |
+        sudo snap install juju --classic --channel ${{ matrix.juju }}
+        sudo snap install juju-wait --classic
+
+    - name: Bootstrap onto AWS
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: |
+        set -eux
+        juju autoload-credentials --client aws
+        juju bootstrap aws/us-east-1 uk8saws --debug --show-log --config test-mode=true --model-default test-mode=true
+        juju deploy ubuntu --constraints 'instance-type=t3a.xlarge root-disk=80G'
+        juju wait -vw
+        juju status --format yaml
+
+    - name: Copy manifests to AWS instance
+      run: |
+        set -eux
+        juju scp -- -r $(pwd)/ ubuntu/0:~/
+
+    - name: Install Docker
+      run: juju ssh ubuntu/0 ./manifests/tests/e2e/setup/install_docker.sh
+
+    - name: Install Python
+      run: juju ssh ubuntu/0 ./manifests/tests/e2e/setup/install_python.sh
+
+    - name: Install Kubectl
+      run: juju ssh ubuntu/0 ./manifests/tests/e2e/setup/install_kubectl.sh
+
+    - name: Install Kustomize
+      run: juju ssh ubuntu/0 ./manifests/tests/gh-actions/install_kustomize.sh
+
+    - name: Install KinD
+      run: juju ssh ubuntu/0 ./manifests/tests/gh-actions/install_kind.sh
+
+    - name: Create a KinD Cluster
+      run: juju ssh ubuntu/0 kind create cluster --config ./manifests/tests/gh-actions/kind-cluster.yaml
+
+    - name: Deploy Kubeflow
+      run: juju ssh ubuntu/0 ./manifests/tests/e2e/setup/install_kubeflow.sh
+      timeout-minutes: 45
+
+    - name: Test Kubeflow with mnist
+      run: |
+        juju ssh ubuntu/0 <<EOF
+          cd manifests/tests/e2e
+          ./runner.sh
+        EOF
+      timeout-minutes: 20
+
+    - name: Generate KinD logs
+      env:
+        JOB_INDEX: ${{ strategy.job-index }}
+      run: |
+        set -eux
+        juju ssh ubuntu/0 ./manifests/tests/e2e/setup/generate_kind_logs.sh
+        juju scp ubuntu/0:~/kind-logs-$JOB_INDEX.tar.gz .
+      if: failure()
+
+    - name: Upload KinD logs tarball
+      uses: actions/upload-artifact@v2
+      with:
+        name: kind-logs
+        path: kind-logs-${{ strategy.job-index }}.tar.gz
+      if: failure()
+
+    - name: Generate kubectl describe logs
+      env:
+        JOB_INDEX: ${{ strategy.job-index }}
+      run: |
+        set -eux
+        juju ssh ubuntu/0 ./manifests/tests/e2e/setup/generate_describe_logs.sh
+        juju scp ubuntu/0:~/describe.tar.gz kubectl-describe-$JOB_INDEX.tar.gz
+      if: failure()
+
+    - name: Upload kubectl describe tarball
+      uses: actions/upload-artifact@v2
+      with:
+        name: kubectl-describe
+        path: kubectl-describe-${{ strategy.job-index }}.tar.gz
+      if: failure()
+
+    - name: Generate pod logs
+      env:
+        JOB_INDEX: ${{ strategy.job-index }}
+      run: |
+        set -eux
+        juju ssh ubuntu/0 ./manifests/tests/e2e/setup/generate_pod_logs.sh
+        juju scp ubuntu/0:~/stdout.tar.gz kubectl-stdout-$JOB_INDEX.tar.gz
+      if: failure()
+
+    - name: Upload pod logs tarball
+      uses: actions/upload-artifact@v2
+      with:
+        name: kubectl-stdout
+        path: kubectl-stdout-${{ strategy.job-index }}.tar.gz
+      if: failure()
+
+    - name: Destroy controller
+      run: juju destroy-controller -y uk8saws --destroy-all-models --destroy-storage
+      if: always()

tests/e2e/README.md

@@ -13,7 +13,7 @@ This test is using the following Kubeflow CRDs:
 The heart of this test is the `mnist.py` python script, which applies and waits
 for the CRDs to complete. The python scripts are all expecting that
 1. `kubectl` is configured with access to a Kubeflow cluster
-2. `kustomize` 3.2.0 is available
+2. `kustomize` 5.0.0 is available
 3. The KFP backend is proxied to localhost
 
 While the `mnist.py` is used for running the test, it is advised to use the

tests/e2e/mnist.py

@@ -1,8 +1,7 @@
-"""E2E Kubeflow test that tesst Pipelines, Katib, TFJobs and KServe.
+"""
+E2E Kubeflow test that test Pipelines, Katib, TFJobs and KServe.
 
-Requires:
-pip install kfp==1.8.4
-pip install kubeflow-katib==0.12.0
+Requires: pip install -r ./requirements.txt
 """
 import kfp
 import kfp.dsl as dsl

tests/e2e/requirements.txt

@@ -1,3 +1,5 @@
-kubernetes==21.7.0
-kfp==1.8.4
-kubeflow-katib==0.12.0
+kubernetes==19.15.0
+kfp==1.8.19
+kubeflow-katib==0.14.0
+protobuf==3.19.5
+PyYAML==5.4.1

tests/e2e/runner.sh

@@ -1,17 +1,20 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-echo "Installing necessary RBAC."""
+echo "Installing required python packages."
+pip3 install -r requirements.txt
+
+echo "Installing necessary RBAC."
 kubectl apply -f yamls
 
 echo "Setting up port-forward..."
 ./hack/proxy_istio.sh
 ./hack/proxy_pipelines.sh
 
-echo "Running the tests."""
+echo "Running the tests."
 python3 mnist.py
 
-echo "Cleaning up opened processes."""
+echo "Cleaning up opened processes."
 ./hack/cleanup_proxies.sh
 
 echo "Leaving the cluster as is for further inspection."

tests/e2e/setup/generate_describe_logs.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eux
+
+kubectl describe pods -A
+
+mkdir describe-${JOB_INDEX}
+
+for RESOURCE in $(kubectl api-resources -o name | sort); do
+  kubectl describe $RESOURCE -A > "describe-${JOB_INDEX}/$RESOURCE.describe" || true
+done
+
+tar -cvzf describe.tar.gz describe-${JOB_INDEX}/*.describe

tests/e2e/setup/generate_kind_logs.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -eux
+
+kind export logs ./logs
+tar -czvf kind-logs-${JOB_INDEX}.tar.gz ./logs

tests/e2e/setup/generate_pod_logs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eux
+
+kubectl get pods -A
+
+mkdir stdout-${JOB_INDEX}
+
+for NAMESPACE in kubeflow-user-example-com kubeflow; do
+    for POD in $(kubectl get pods -n $NAMESPACE -o custom-columns=:metadata.name --no-headers); do
+        for CONTAINER in $(kubectl get pods -n $NAMESPACE -o jsonpath="{.spec.containers[*].name}" $POD); do
+          kubectl logs -n $NAMESPACE --timestamps $POD -c $CONTAINER > stdout-${JOB_INDEX}/$NAMESPACE-$POD-$CONTAINER.log || true
+        done
+    done
+done
+
+tar -cvzf stdout.tar.gz stdout-${JOB_INDEX}/*.log

tests/e2e/setup/install_docker.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -eux
+sudo apt update
+
+# Install prereq packages
+sudo apt install -y apt-transport-https ca-certificates curl software-properties-common tar
+
+# Add GPG key for the official Docker repository
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
+# Add the Docker repository APT sources
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable"
+
+# Update to install from Docker repo
+apt-cache policy docker-ce
+
+# Install Docker
+sudo apt install -y docker-ce
+
+# Verify Docker is running
+sudo systemctl status docker
+
+# Add user to the docker group
+sudo usermod -a -G docker ubuntu

tests/e2e/setup/install_kubectl.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eux
+
+# Download kubectl
+sudo curl -L "https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl
+sudo chmod +x /usr/local/bin/kubectl
+kubectl version --short --client

tests/e2e/setup/install_kubeflow.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eux
+
+# Install kubeflow using raw manifests
+cd manifests
+while ! kustomize build example | awk '!/well-defined/' | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done
+
+# Wait for pods to be ready
+kubectl -n kubeflow wait --for=condition=Ready pods --all --timeout=1200s

tests/e2e/setup/install_python.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eux
+
+# Install python
+sudo apt install python3
+python3 --version
+
+# Install pip
+sudo apt install -y python3-pip