E2E test to cover scenario of Kubeapps cluster not part of clusters list

.github/workflows/kubeapps-general.yaml

@@ -411,6 +411,7 @@ jobs:
         tests_group:
           - main
           - multicluster
+          - multicluster-nokubeapps
           - carvel
           - operator
     env:

cmd/kubeapps-apis/plugins/helm/packages/v1alpha1/server.go

@@ -8,6 +8,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+
 	appRepov1 "github.com/vmware-tanzu/kubeapps/cmd/apprepository-controller/pkg/apis/apprepository/v1alpha1"
 	"github.com/vmware-tanzu/kubeapps/cmd/kubeapps-apis/core"
 	corev1 "github.com/vmware-tanzu/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
@@ -39,11 +44,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	log "k8s.io/klog/v2"
-	"net/url"
-	"os"
-	"path"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
-	"strings"
 )
 
 type helmActionConfigGetter func(ctx context.Context, pkgContext *corev1.Context) (*action.Configuration, error)

integration/tests/multicluster-nokubeapps/11-multicluster-nokubeapps-cluster.spec.js

@@ -0,0 +1,50 @@
+// Copyright 2022 the Kubeapps contributors.
+// SPDX-License-Identifier: Apache-2.0
+
+const { test, expect } = require("@playwright/test");
+const { KubeappsLogin } = require("../utils/kubeapps-login");
+const utils = require("../utils/util-functions");
+
+test("Deploys package in the only additional cluster while Kubeapps cluster is not available", async ({ page }) => {
+  test.setTimeout(120000);
+
+  // Log in
+  const k = new KubeappsLogin(page);
+  await k.doLogin("kubeapps-operator@example.com", "password", process.env.ADMIN_TOKEN);
+
+  // Check and select cluster using ui
+  await page.click(".kubeapps-dropdown .kubeapps-nav-link");
+  await page.selectOption('select[name="clusters"]', "second-cluster");
+  // Check that there is only one cluster
+  const clustersLength = await page.locator('select[name="clusters"] option').count()
+  expect(clustersLength).toEqual(1)
+  await page.click('cds-button:has-text("Change Context")');
+
+  // Select package to deploy
+  await page.click('a.nav-link:has-text("Catalog")');
+  await page.locator("input#search").fill("apache");
+  await page.waitForTimeout(3000);
+  await page.click('a:has-text("foo apache chart for CI")');
+  await page.click('cds-button:has-text("Deploy") >> nth=0');
+
+  // Deploy package
+  const releaseNameLocator = page.locator("#releaseName");
+  await releaseNameLocator.waitFor();
+  await expect(releaseNameLocator).toHaveText("");
+  const releaseName = utils.getRandomName("test-05-release");
+  console.log(`Creating release "${releaseName}"`);
+  await releaseNameLocator.fill(releaseName);
+  await page.locator('cds-button:has-text("Deploy")').click();
+
+  // Assertions
+  await page.waitForSelector("css=.application-status-pie-chart-number >> text=1", {
+    timeout: utils.getDeploymentTimeout(),
+  });
+  await page.waitForSelector("css=.application-status-pie-chart-title >> text=Ready", {
+    timeout: utils.getDeploymentTimeout(),
+  });
+
+  // Clean up
+  await page.locator('cds-button:has-text("Delete")').click();
+  await page.locator('cds-modal-actions button:has-text("Delete")').click();
+});

script/chart-museum.sh

@@ -37,6 +37,26 @@ pullBitnamiChart() {
   curl -LO "${CHART_URL}"
 }
 
+# Delete chart from ChartsMuseum
+# Arguments:
+#   $1: Chart name
+#   $2: Chart version
+deleteChart() {
+  if [ -z "$1" ]; then
+    echo "No chart name supplied"
+    exit 1
+  fi
+  if [ -z "$2" ]; then
+    echo "No chart version supplied"
+    exit 1
+  fi
+
+  local CHART_NAME=$1
+  local CHART_VERSION=$2
+
+  curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" -X DELETE http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}
+}
+
 # Push a local chart to ChartsMuseum
 # Arguments:
 #   $1: Chart name
@@ -69,7 +89,7 @@ pushChartToChartMuseum() {
   CHART_EXISTS=$(curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" "http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}" | jq -r 'any([ .error] ; . > 0)')
   if [ "$CHART_EXISTS" == "true" ]; then
     echo ">> Chart ${CHART_NAME} v${CHART_VERSION} already exists: deleting"
-    curl -Lk -u "${CHARTMUSEUM_USER}:${CHARTMUSEUM_PWD}" -H "Host: ${CHARTMUSEUM_HOSTNAME}" -X DELETE "http://${CHARTMUSEUM_IP}/api/charts/${CHART_NAME}/${CHART_VERSION}"
+    deleteChart ${CHART_NAME} ${CHART_VERSION}
   fi
 
   echo ">> Uploading chart from file ${CHART_FILE}"
@@ -148,6 +168,11 @@ if (($# > 0)); then
       pushChartToChartMuseum $2 $3 $4
       ;;
 
+
+    deleteChart)
+      deleteChart $2 $3
+      ;;
+
     *)
       ;;
   esac

script/e2e-test.sh

@@ -14,10 +14,13 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null && pwd)"
 ALL_TESTS="all"
 MAIN_TESTS="main"
 MULTICLUSTER_TESTS="multicluster"
+MULTICLUSTER_NOKUBEAPPS_TESTS="multicluster-nokubeapps"
 CARVEL_TESTS="carvel"
 FLUX_TESTS="flux"
 OPERATOR_TESTS="operator"
-SUPPORTED_TESTS_GROUPS=("${ALL_TESTS}" "${MAIN_TESTS}" "${MULTICLUSTER_TESTS}" "${CARVEL_TESTS}" "${FLUX_TESTS}" "${OPERATOR_TESTS}")
+SUPPORTED_TESTS_GROUPS=("${ALL_TESTS}" "${MAIN_TESTS}" "${MULTICLUSTER_TESTS}" "${CARVEL_TESTS}" "${FLUX_TESTS}" "${OPERATOR_TESTS}" "${MULTICLUSTER_NOKUBEAPPS_TESTS}")
+INTEGRATION_HOST=kubeapps-ci.kubeapps
+INTEGRATION_ENTRYPOINT="http://${INTEGRATION_HOST}"
 
 # Params
 USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV:-"false"}
@@ -296,8 +299,8 @@ installOrUpgradeKubeapps() {
   # See https://stackoverflow.com/a/36296000 for "${arr[@]+"${arr[@]}"}" notation.
   cmd=(helm upgrade --install kubeapps-ci --namespace kubeapps "${chartSource}"
     "${img_flags[@]}"
-    "${@:2}"
     "${multiclusterFlags[@]+"${multiclusterFlags[@]}"}"
+    "${@:2}"
     --set frontend.replicaCount=1
     --set dashboard.replicaCount=1
     --set kubeappsapis.replicaCount=2
@@ -379,7 +382,7 @@ img_flags=(
 additional_flags_file=$(generateAdditionalValuesFile)
 
 if [ "$USE_MULTICLUSTER_OIDC_ENV" = true ]; then
-  multiclusterFlags=(
+  basicAuthFlags=(
     "--values" "${additional_flags_file}"
     "--set" "authProxy.enabled=true"
     "--set" "authProxy.provider=oidc"
@@ -389,17 +392,20 @@ if [ "$USE_MULTICLUSTER_OIDC_ENV" = true ]; then
     "--set" "authProxy.extraFlags[0]=\"--oidc-issuer-url=https://${DEX_IP}:32000\""
     "--set" "authProxy.extraFlags[1]=\"--scope=openid email groups audience:server:client_id:second-cluster audience:server:client_id:third-cluster\""
     "--set" "authProxy.extraFlags[2]=\"--ssl-insecure-skip-verify=true\""
-    "--set" "authProxy.extraFlags[3]=\"--redirect-url=http://kubeapps-ci.kubeapps/oauth2/callback\""
+    "--set" "authProxy.extraFlags[3]=\"--redirect-url=${INTEGRATION_ENTRYPOINT}/oauth2/callback\""
     "--set" "authProxy.extraFlags[4]=\"--cookie-secure=false\""
-    "--set" "authProxy.extraFlags[5]=\"--cookie-domain=kubeapps-ci.kubeapps\""
-    "--set" "authProxy.extraFlags[6]=\"--whitelist-domain=kubeapps-ci.kubeapps\""
+    "--set" "authProxy.extraFlags[5]=\"--cookie-domain=${INTEGRATION_HOST}\""
+    "--set" "authProxy.extraFlags[6]=\"--whitelist-domain=${INTEGRATION_HOST}\""
     "--set" "authProxy.extraFlags[7]=\"--set-authorization-header=true\""
+  )
+  multiclusterFlags=(
     "--set" "clusters[0].name=default"
     "--set" "clusters[1].name=second-cluster"
     "--set" "clusters[1].apiServiceURL=https://${ADDITIONAL_CLUSTER_IP}:6443"
     "--set" "clusters[1].insecure=true"
     "--set" "clusters[1].serviceToken=$(kubectl --context=kind-kubeapps-ci-additional --kubeconfig="${HOME}/.kube/kind-config-kubeapps-ci-additional" get secret kubeapps-namespace-discovery -o go-template='{{.data.token | base64decode}}')"
   )
+  multiclusterFlags+=("${basicAuthFlags[@]+"${basicAuthFlags[@]}"}")
 fi
 
 helm repo add bitnami https://charts.bitnami.com/bitnami
@@ -477,16 +483,17 @@ done
 
 # Browser tests
 cd "${ROOT_DIR}/integration"
+info "Using E2E runner image '${IMG_PREFIX}integration-tests${IMG_MODIFIER}:${IMG_DEV_TAG}'"
 kubectl create deployment e2e-runner --image "${IMG_PREFIX}integration-tests${IMG_MODIFIER}:${IMG_DEV_TAG}"
 k8s_wait_for_deployment default e2e-runner
 pod=$(kubectl get po -l app=e2e-runner -o custom-columns=:metadata.name --no-headers)
 ## Copy config and latest tests
 for f in *.js; do
-  kubectl cp "./${f}" "${pod}:/app/"
+  kubectl cp "./${f}" "default/${pod}:/app/"
 done
 
-kubectl cp ./tests "${pod}:/app/"
-info "Copied tests to e2e-runner pod ${pod}"
+kubectl cp ./tests "default/${pod}:/app/"
+info "Copied tests to e2e-runner pod default/${pod}"
 
 ## Create admin user
 kubectl create serviceaccount kubeapps-operator -n kubeapps
@@ -542,7 +549,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${MAIN_TESTS}"
     DOCKER_PASSWORD=${DOCKER_PASSWORD} \
     DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
     TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
-    INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
+    INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
     USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
     ADMIN_TOKEN=${admin_token} \
     VIEW_TOKEN=${view_token} \
@@ -573,7 +580,7 @@ if [[ -z "${GKE_BRANCH-}" && ("${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GRO
     DOCKER_PASSWORD=${DOCKER_PASSWORD} \
     DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
     TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
-    INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
+    INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
     USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
     ADMIN_TOKEN=${admin_token} \
     VIEW_TOKEN=${view_token} \
@@ -611,7 +618,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${CARVEL_TESTS}
   test_command="
     CI_TIMEOUT_MINUTES=20 \
     TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
-    INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
+    INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
     USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
     ADMIN_TOKEN=${admin_token} \
     VIEW_TOKEN=${view_token} \
@@ -650,7 +657,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${FLUX_TESTS}"
   test_command="
     CI_TIMEOUT_MINUTES=20 \
     TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
-    INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
+    INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
     USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
     ADMIN_TOKEN=${admin_token} \
     VIEW_TOKEN=${view_token} \
@@ -699,7 +706,7 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${OPERATOR_TEST
     test_command="
       CI_TIMEOUT_MINUTES=20 \
       TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
-      INTEGRATION_ENTRYPOINT=http://kubeapps-ci.kubeapps \
+      INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
       USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
       ADMIN_TOKEN=${admin_token} \
       VIEW_TOKEN=${view_token} \
@@ -717,5 +724,75 @@ if [[ "${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${OPERATOR_TEST
   fi
 fi
 
+############################################################
+######## Multi-cluster without Kubeapps tests group ########
+############################################################
+if [[ -z "${GKE_BRANCH-}" && ("${TESTS_GROUP}" == "${ALL_TESTS}" || "${TESTS_GROUP}" == "${MULTICLUSTER_NOKUBEAPPS_TESTS}") ]]; then
+  sectionStartTime=$(date +%s)
+  info "Running multi-cluster (without Kubeapps cluster) integration tests..."
+
+  info "Updating Kubeapps to exclude Kubeapps cluster from the list of clusters"
+
+  # Update Kubeapps
+  kubeappsChartPath="${ROOT_DIR}/chart/kubeapps"
+  info "Installing Kubeapps from ${kubeappsChartPath}..."
+  kubectl -n kubeapps delete secret localhost-tls || true
+
+  # See https://stackoverflow.com/a/36296000 for "${arr[@]+"${arr[@]}"}" notation.
+  cmd=(helm upgrade --install kubeapps-ci --namespace kubeapps "${kubeappsChartPath}"
+    "${img_flags[@]}"
+    "${basicAuthFlags[@]+"${basicAuthFlags[@]}"}"
+    --set clusters[0].name=second-cluster
+    --set clusters[0].apiServiceURL=https://${ADDITIONAL_CLUSTER_IP}:6443
+    --set clusters[0].insecure=true
+    --set clusters[0].serviceToken=$(kubectl --context=kind-kubeapps-ci-additional --kubeconfig=${HOME}/.kube/kind-config-kubeapps-ci-additional get secret kubeapps-namespace-discovery -o go-template='{{.data.token | base64decode}}')
+    --set frontend.replicaCount=1
+    --set dashboard.replicaCount=1
+    --set kubeappsapis.replicaCount=2
+    --set postgresql.architecture=standalone
+    --set postgresql.primary.persistence.enabled=false
+    --set postgresql.auth.password=password
+    --set redis.auth.password=password
+    --set apprepository.initialRepos[0].name=bitnami
+    --set apprepository.initialRepos[0].url=http://chartmuseum.chart-museum.svc.cluster.local:8080
+    --set apprepository.initialRepos[0].basicAuth.user=admin
+    --set apprepository.initialRepos[0].basicAuth.password=password
+    --set apprepository.globalReposNamespaceSuffix=-repos-global
+    --set global.postgresql.auth.postgresPassword=password
+    --wait)
+
+  echo "${cmd[@]}"
+  "${cmd[@]}"
+
+  info "Waiting for updated Kubeapps components to be ready..."
+  k8s_wait_for_deployment kubeapps kubeapps-ci
+
+  test_command="
+    CI_TIMEOUT_MINUTES=40 \
+    DOCKER_USERNAME=${DOCKER_USERNAME} \
+    DOCKER_PASSWORD=${DOCKER_PASSWORD} \
+    DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL} \
+    TEST_TIMEOUT_MINUTES=${TEST_TIMEOUT_MINUTES} \
+    INTEGRATION_ENTRYPOINT=${INTEGRATION_ENTRYPOINT} \
+    USE_MULTICLUSTER_OIDC_ENV=${USE_MULTICLUSTER_OIDC_ENV} \
+    ADMIN_TOKEN=${admin_token} \
+    VIEW_TOKEN=${view_token} \
+    EDIT_TOKEN=${edit_token} \
+    yarn test \"tests/multicluster-nokubeapps/\"
+    "
+  info "${test_command}"
+
+  if ! kubectl exec -it "$pod" -- /bin/sh -c "${test_command}"; then
+    ## Integration tests failed, get report screenshot
+    warn "PODS status on failure"
+    kubectl cp "${pod}:/app/reports" ./reports
+    exit 1
+  fi
+  info "Multi-cluster integration tests succeeded!!"
+
+  sectionEndTime=$(date +%s)
+  info "Multi-cluster tests execution time: $(formattedElapsedTime sectionEndTime-sectionStartTime)"
+fi
+
 info "Integration tests succeeded!"
 info "Total execution time: $(elapsedTimeSince "$startTime")"