Skip to content

Commit

Permalink
E2E test to cover scenario of Kubeapps cluster not part of clusters l…
Browse files Browse the repository at this point in the history
…ist (#5573)

### Description of the change

This PR adds an E2E test in CI to avoid regression on #5566 and hence to
avoid that #4564 happens again.

### Benefits

Kubeapps is usable even if Kubeapps cluster is not among the clusters
configured.

### Possible drawbacks

N/A

### Applicable issues

- fixes #4563

Signed-off-by: Rafa Castelblanque <rcastelblanq@vmware.com>
  • Loading branch information
castelblanque authored Nov 3, 2022
1 parent 7cf1b15 commit 317934a
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 19 deletions.
1 change: 1 addition & 0 deletions .github/workflows/kubeapps-general.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ jobs:
tests_group:
- main
- multicluster
- multicluster-nokubeapps
- carvel
- operator
env:
Expand Down
9 changes: 5 additions & 4 deletions cmd/kubeapps-apis/plugins/helm/packages/v1alpha1/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path"
"strings"

appRepov1 "github.com/vmware-tanzu/kubeapps/cmd/apprepository-controller/pkg/apis/apprepository/v1alpha1"
"github.com/vmware-tanzu/kubeapps/cmd/kubeapps-apis/core"
corev1 "github.com/vmware-tanzu/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
Expand Down Expand Up @@ -39,11 +44,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
log "k8s.io/klog/v2"
"net/url"
"os"
"path"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
"strings"
)

type helmActionConfigGetter func(ctx context.Context, pkgContext *corev1.Context) (*action.Configuration, error)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright 2022 the Kubeapps contributors.
// SPDX-License-Identifier: Apache-2.0

const { test, expect } = require("@playwright/test");
const { KubeappsLogin } = require("../utils/kubeapps-login");
const utils = require("../utils/util-functions");

test("Deploys package in the only additional cluster while Kubeapps cluster is not available", async ({ page }) => {
test.setTimeout(120000);

// Log in
const k = new KubeappsLogin(page);
await k.doLogin("kubeapps-operator@example.com", "password", process.env.ADMIN_TOKEN);

// Check and select cluster using ui
await page.click(".kubeapps-dropdown .kubeapps-nav-link");
await page.selectOption('select[name="clusters"]', "second-cluster");
// Check that there is only one cluster
const clustersLength = await page.locator('select[name="clusters"] option').count()
expect(clustersLength).toEqual(1)
await page.click('cds-button:has-text("Change Context")');

// Select package to deploy
await page.click('a.nav-link:has-text("Catalog")');
await page.locator("input#search").fill("apache");
await page.waitForTimeout(3000);
await page.click('a:has-text("foo apache chart for CI")');
await page.click('cds-button:has-text("Deploy") >> nth=0');

// Deploy package
const releaseNameLocator = page.locator("#releaseName");
await releaseNameLocator.waitFor();
await expect(releaseNameLocator).toHaveText("");
const releaseName = utils.getRandomName("test-05-release");
console.log(`Creating release "${releaseName}"`);
await releaseNameLocator.fill(releaseName);
await page.locator('cds-button:has-text("Deploy")').click();

// Assertions
await page.waitForSelector("css=.application-status-pie-chart-number >> text=1", {
timeout: utils.getDeploymentTimeout(),
});
await page.waitForSelector("css=.application-status-pie-chart-title >> text=Ready", {
timeout: utils.getDeploymentTimeout(),
});

// Clean up
await page.locator('cds-button:has-text("Delete")').click();
await page.locator('cds-modal-actions button:has-text("Delete")').click();
});
27 changes: 26 additions & 1 deletion script/chart-museum.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,26 @@ pullBitnamiChart() {
curl -LO "${CHART_URL}"
}

# Delete chart from ChartsMuseum
# Arguments:
# $1: Chart name
# $2: Chart version
deleteChart() {
if [ -z "$1" ]; then
echo "No chart name supplied"
exit 1
fi
if [ -z "$2" ]; then
echo "No chart version supplied"
exit 1
fi

local CHART_NAME=$1
local CHART_VERSION=$2

curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" -X DELETE http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}
}

# Push a local chart to ChartsMuseum
# Arguments:
# $1: Chart name
Expand Down Expand Up @@ -69,7 +89,7 @@ pushChartToChartMuseum() {
CHART_EXISTS=$(curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" "http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}" | jq -r 'any([ .error] ; . > 0)')
if [ "$CHART_EXISTS" == "true" ]; then
echo ">> Chart ${CHART_NAME} v${CHART_VERSION} already exists: deleting"
curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" -X DELETE "http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}"
deleteChart ${CHART_NAME} ${CHART_VERSION}
fi

echo ">> Uploading chart from file ${CHART_FILE}"
Expand Down Expand Up @@ -148,6 +168,11 @@ if (($# > 0)); then
pushChartToChartMuseum $2 $3 $4
;;


deleteChart)
deleteChart $2 $3
;;

*)
;;
esac
Expand Down
105 changes: 91 additions & 14 deletions script/e2e-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,13 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null && pwd)"
ALL_TESTS="all"
MAIN_TESTS="main"
MULTICLUSTER_TESTS="multicluster"
MULTICLUSTER_NOKUBEAPPS_TESTS="multicluster-nokubeapps"
CARVEL_TESTS="carvel"
FLUX_TESTS="flux"
OPERATOR_TESTS="operator"
SUPPORTED_TESTS_GROUPS=("${ALL_TESTS}" "${MAIN_TESTS}" "${MULTICLUSTER_TESTS}" "${CARVEL_TESTS}" "${FLUX_TESTS}" "${OPERATOR_TESTS}")
SUPPORTED_TESTS_GROUPS=("${ALL_TESTS}" "${MAIN_TESTS}" "${MULTICLUSTER_TESTS}" "${CARVEL_TESTS}" "${FLUX_TESTS}" "${OPERATOR_TESTS}" "${MULTICLUSTER_NOKUBEAPPS_TESTS}")
INTEGRATION_HOST=kubeapps-ci.kubeapps
INTEGRATION_ENTRYPOINT="http://${INTEGRATION_HOST}"

# Params
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV:-"false"}
Expand Down Expand Up @@ -296,8 +299,8 @@ installOrUpgradeKubeapps() {
# See https://stackoverflow.com/a/36296000 for "${arr[@]+"${arr[@]}"}" notation.
cmd=(helm upgrade --install kubeapps-ci --namespace kubeapps "${chartSource}"
"${img_flags[@]}"
"${@:2}"
"${multiclusterFlags[@]+"${multiclusterFlags[@]}"}"
"${@:2}"
--set frontend.replicaCount=1
--set dashboard.replicaCount=1
--set kubeappsapis.replicaCount=2
Expand Down Expand Up @@ -379,7 +382,7 @@ img_flags=(
additional_flags_file=$(generateAdditionalValuesFile)

if [ "$USE_MULTICLUSTER_OIDC_ENV" = true ]; then
multiclusterFlags=(
basicAuthFlags=(
"--values" "${additional_flags_file}"
"--set" "authProxy.enabled=true"
"--set" "authProxy.provider=oidc"
Expand All @@ -389,17 +392,20 @@ if [ "$USE_MULTICLUSTER_OIDC_ENV" = true ]; then
"--set" "authProxy.extraFlags[0]=\"--oidc-issuer-url=https://${DEX_IP}:32000\""
"--set" "authProxy.extraFlags[1]=\"--scope=openid email groups audience:server:client_id:second-cluster audience:server:client_id:third-cluster\""
"--set" "authProxy.extraFlags[2]=\"--ssl-insecure-skip-verify=true\""
"--set" "authProxy.extraFlags[3]=\"--redirect-url=http://kubeapps-ci.kubeapps/oauth2/callback\""
"--set" "authProxy.extraFlags[3]=\"--redirect-url=${INTEGRATION_ENTRYPOINT}/oauth2/callback\""
"--set" "authProxy.extraFlags[4]=\"--cookie-secure=false\""
"--set" "authProxy.extraFlags[5]=\"--cookie-domain=kubeapps-ci.kubeapps\""
"--set" "authProxy.extraFlags[6]=\"--whitelist-domain=kubeapps-ci.kubeapps\""
"--set" "authProxy.extraFlags[5]=\"--cookie-domain=${INTEGRATION_HOST}\""
"--set" "authProxy.extraFlags[6]=\"--whitelist-domain=${INTEGRATION_HOST}\""
"--set" "authProxy.extraFlags[7]=\"--set-authorization-header=true\""
)
multiclusterFlags=(
"--set" "clusters[0].name=default"
"--set" "clusters[1].name=second-cluster"
"--set" "clusters[1].apiServiceURL=https://${ADDITIONAL_CLUSTER_IP}:6443"
"--set" "clusters[1].insecure=true"
"--set" "clusters[1].serviceToken=$(kubectl --context=kind-kubeapps-ci-additional --kubeconfig="${HOME}/.kube/kind-config-kubeapps-ci-additional" get secret kubeapps-namespace-discovery -o go-template='{{.data.token | base64decode}}')"
)
multiclusterFlags+=("${basicAuthFlags[@]+"${basicAuthFlags[@]}"}")
fi

helm repo add bitnami https://charts.bitnami.com/bitnami
Expand Down Expand Up @@ -477,16 +483,17 @@ done

# Browser tests
cd "${ROOT_DIR}/integration"
info "Using E2E runner image '${IMG_PREFIX}integration-tests${IMG_MODIFIER}:${IMG_DEV_TAG}'"
kubectl create deployment e2e-runner --image "${IMG_PREFIX}integration-tests${IMG_MODIFIER}:${IMG_DEV_TAG}"
k8s_wait_for_deployment default e2e-runner
pod=$(kubectl get po -l app=e2e-runner -o custom-columns=:metadata.name --no-headers)
## Copy config and latest tests
for f in *.js; do
kubectl cp "./${f}" "${pod}:/app/"
kubectl cp "./${f}" "default/${pod}:/app/"
done

kubectl cp ./tests "${pod}:/app/"
info "Copied tests to e2e-runner pod ${pod}"
kubectl cp ./tests "default/${pod}:/app/"
info "Copied tests to e2e-runner pod default/${pod}"

## Create admin user
kubectl create serviceaccount kubeapps-operator -n kubeapps
Expand Down Expand Up @@ -542,7 +549,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${MAIN_TESTS}"
DOCKER_PASSWORD=${DOCKER_PASSWORD} \
DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
Expand Down Expand Up @@ -573,7 +580,7 @@ if [[ -z "${GKE_BRANCH-}" && ("${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GRO
DOCKER_PASSWORD=${DOCKER_PASSWORD} \
DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
Expand Down Expand Up @@ -611,7 +618,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${CARVEL_TESTS}
test_command="
CI_TIMEOUT_MINUTES=20 \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
Expand Down Expand Up @@ -650,7 +657,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${FLUX_TESTS}"
test_command="
CI_TIMEOUT_MINUTES=20 \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
Expand Down Expand Up @@ -699,7 +706,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${OPERATOR_TEST
test_command="
CI_TIMEOUT_MINUTES=20 \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
Expand All @@ -717,5 +724,75 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${OPERATOR_TEST
fi
fi

############################################################
######## Multi-cluster without Kubeapps tests group ########
############################################################
if [[ -z "${GKE_BRANCH-}" && ("${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${MULTICLUSTER_NOKUBEAPPS_TESTS}") ]]; then
sectionStartTime=$(date +%s)
info "Running multi-cluster (without Kubeapps cluster) integration tests..."

info "Updating Kubeapps to exclude Kubeapps cluster from the list of clusters"

# Update Kubeapps
kubeappsChartPath="${ROOT_DIR}/chart/kubeapps"
info "Installing Kubeapps from ${kubeappsChartPath}..."
kubectl -n kubeapps delete secret localhost-tls || true

# See https://stackoverflow.com/a/36296000 for "${arr[@]+"${arr[@]}"}" notation.
cmd=(helm upgrade --install kubeapps-ci --namespace kubeapps "${kubeappsChartPath}"
"${img_flags[@]}"
"${basicAuthFlags[@]+"${basicAuthFlags[@]}"}"
--set clusters[0].name=second-cluster
--set clusters[0].apiServiceURL=https://${ADDITIONAL_CLUSTER_IP}:6443
--set clusters[0].insecure=true
--set clusters[0].serviceToken=$(kubectl --context=kind-kubeapps-ci-additional --kubeconfig=${HOME}/.kube/kind-config-kubeapps-ci-additional get secret kubeapps-namespace-discovery -o go-template='{{.data.token | base64decode}}')
--set frontend.replicaCount=1
--set dashboard.replicaCount=1
--set kubeappsapis.replicaCount=2
--set postgresql.architecture=standalone
--set postgresql.primary.persistence.enabled=false
--set postgresql.auth.password=password
--set redis.auth.password=password
--set apprepository.initialRepos[0].name=bitnami
--set apprepository.initialRepos[0].url=http://chartmuseum.chart-museum.svc.cluster.local:8080
--set apprepository.initialRepos[0].basicAuth.user=admin
--set apprepository.initialRepos[0].basicAuth.password=password
--set apprepository.globalReposNamespaceSuffix=-repos-global
--set global.postgresql.auth.postgresPassword=password
--wait)

echo "${cmd[@]}"
"${cmd[@]}"

info "Waiting for updated Kubeapps components to be ready..."
k8s_wait_for_deployment kubeapps kubeapps-ci

test_command="
CI_TIMEOUT_MINUTES=40 \
DOCKER_USERNAME=${DOCKER_USERNAME} \
DOCKER_PASSWORD=${DOCKER_PASSWORD} \
DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
ADMIN_TOKEN=${admin_token} \
VIEW_TOKEN=${view_token} \
EDIT_TOKEN=${edit_token} \
yarn test \"tests/multicluster-nokubeapps/\"
"
info "${test_command}"

if ! kubectl exec -it "$pod" -- /bin/sh -c "${test_command}"; then
## Integration tests failed, get report screenshot
warn "PODS status on failure"
kubectl cp "${pod}:/app/reports" ./reports
exit 1
fi
info "Multi-cluster integration tests succeeded!!"

sectionEndTime=$(date +%s)
info "Multi-cluster tests execution time: $(formattedElapsedTime sectionEndTime-sectionStartTime)"
fi

info "Integration tests succeeded!"
info "Total execution time: $(elapsedTimeSince "$startTime")"

0 comments on commit 317934a

Please sign in to comment.