Add Kuttl (declarative E2E test) implem (#283)

* Add kuttl test scenario scaleUP (WIP)

* Test scale up with Kuttl

* Add crds in helm directory (casskop & multicasskop)

* Delete old beta crds

* Update doc

* Fix kuttl ScaleUp tests

* Fix helm install CRD override

* Add the necessary config for test parallelisation (random namespaces) & fix scaleup icarus assert incorrect matching

* Add ScaleDown step

* Rename test case folder

* Add Kuttl to circleci build

* Fix indent

* Fix makefile

* Update Makefile

* Update CircleCI kuttl integration

* Update config.yml

* Update config.yml

* Debug cci

* Fix config

* Add kuttl install circleci

* Install brew

* Install brew 2

* Fix brew

* fix brew

* Fix kuttl install

* Update config.yml

* Fix kuttl install in docker image

* Update Dockerfile

* Using binary kuttl for installation

* Using binary kuttl for installation2

* Update Dockerfile

* Update Dockerfile

* use cci for kuttl

* Update config.yml

* Update config.yml

* fix arg

* Add helm installation

* Add doc on "How to run kuttl tests" in developer guide

* Remove useless namespace option in doc

* Remove useless namespace option in doc (multi casskop)

* Fix backup & restore doc example

* Update and add spec of Backup & restore documentation

* Remove old helm 2 docs refs

* set cassandraCluster (with uppercase) in CRDs and definitions

* update CRD

* Fix tests

* Log secret validation error

* Update test with new typo

* FIX undo some wrong typo changes

* Fix test

* Applied review asked changes

* Rename test

* Update with merge of #282

* Remove unused topology

* Undo change post merge (distant repo not yet updated)

* Fix test

* Revert "Merge branch 'helm-3-install-crds' into bdd-testing"

This reverts commit bb84620, reversing
changes made to ba64023.

* Remove event collect

Co-authored-by: Cyril Scetbon <cscetbon@gmail.com>

Makefile

@@ -403,18 +403,28 @@ endif
 docker-e2e-test-fix:
 	docker run --env GO111MODULE=on --rm -v $(PWD):$(WORKDIR) -v $(KUBECONFIG):/root/.kube/config $(BUILD_IMAGE):$(OPERATOR_SDK_VERSION) /bin/bash -c 'operator-sdk test local ./test/e2e --debug --image $(E2EIMAGE) --go-test-flags "-v -mod=vendor -timeout 60m" --operator-namespace cassandra-e2e'
 
-#execute Test filter based on given Regex
+#execute Test filters based on given Regex
 ifeq (docker-e2e-test-fix-arg,$(firstword $(MAKECMDGOALS)))
   E2E_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
   $(eval $(E2E_ARGS):;@:)
 endif
+ifeq (kuttl-test-fix-arg,$(firstword $(MAKECMDGOALS)))
+  KUTTL_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  $(eval $(KUTTL_ARGS):;@:)
+endif
 
 docker-e2e-test-fix-arg:
 ifeq ($(E2E_ARGS),)
 	@echo "args are: ExecuteCleanup; RollingRestart ; ClusterScaleDown ; ClusterScaleUp ; ClusterScaleDownSimple" && exit 1
 endif
 	docker run --rm --network host --env GO111MODULE=on -v $(PWD):$(WORKDIR) -v $(KUBECONFIG):/root/.kube/config $(BUILD_IMAGE):$(OPERATOR_SDK_VERSION) /bin/bash -c 'operator-sdk test local ./test/e2e --debug --image $(E2EIMAGE) --go-test-flags "-v -timeout 60m -run ^TestCassandraCluster$$/^group$$/^$(E2E_ARGS)$$" --operator-namespace cassandra-e2e' || { kubectl get events --all-namespaces --sort-by .metadata.creationTimestamp ; exit 1; }
 
+kuttl-test-fix-arg:
+ifeq ($(KUTTL_ARGS),)
+	@echo "args are: ScaleUpAndDown" && exit 1
+endif
+	kuttl test --config ./test/e2e/kuttl/kuttl-test.yaml ./test/e2e/kuttl --test $(KUTTL_ARGS)
+
 e2e-test-fix-scale-down:
 	operator-sdk test local ./test/e2e --image $(E2EIMAGE) --go-test-flags "-v -timeout 60m -run ^TestCassandraCluster$$/^group$$/^ClusterScaleDown$$" --operator-namespace cassandra-e2e || { kubectl get events --all-namespaces --sort-by .metadata.creationTimestamp ; exit 1; }
 

test/e2e/kuttl/ScaleUpAndDown/00-assert.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cassandra-e2e-dc1-rack1
+status:
+  currentReplicas: 1
+  replicas: 1
+---
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+status:
+  cassandraRackStatus:
+    dc1-rack1:
+      cassandraLastAction:
+        name: Initializing
+        status: Done
+      phase: Running
+  lastClusterAction: Initializing
+  lastClusterActionStatus: Done
+  phase: Running

test/e2e/kuttl/ScaleUpAndDown/00-createCluster.yaml

@@ -0,0 +1,28 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: helm install casskop orange-incubator/cassandra-operator
+    namespaced: true
+---
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+spec:
+  nodesPerRacks: 1
+  cassandraImage: cassandra:latest
+  dataCapacity: "1Gi"
+  hardAntiAffinity: false
+  deletePVC: false
+  autoPilot: true
+  autoUpdateSeedList: false
+  resources:
+    requests: &requests
+      cpu: 100m
+      memory: 512Mi
+    limits: *requests
+  topology:
+    dc:
+      - name: dc1
+        rack:
+          - name: rack1

test/e2e/kuttl/ScaleUpAndDown/01-assert.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cassandra-e2e-dc1-rack1
+status:
+  currentReplicas: 2
+  replicas: 2
+---
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+spec:
+  nodesPerRacks: 2
+status:
+  cassandraRackStatus:
+    dc1-rack1:
+      cassandraLastAction:
+        name: ScaleUp
+        status: Done

test/e2e/kuttl/ScaleUpAndDown/01-scaleUp-dc.yaml

@@ -0,0 +1,11 @@
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+spec:
+  nodesPerRacks: 2
+  topology:
+    dc:
+      - name: dc1
+        rack:
+          - name: rack1

test/e2e/kuttl/ScaleUpAndDown/02-assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 500
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cassandra-e2e-dc1-rack1
+status:
+  currentReplicas: 1
+  replicas: 1
+---
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+spec:
+  nodesPerRacks: 1
+status:
+  cassandraRackStatus:
+    dc1-rack1:
+      cassandraLastAction:
+        name: ScaleDown
+        status: Done

test/e2e/kuttl/ScaleUpAndDown/02-scaleDown-dc.yaml

@@ -0,0 +1,11 @@
+apiVersion: db.orange.com/v1alpha1
+kind: CassandraCluster
+metadata:
+  name: cassandra-e2e
+spec:
+  nodesPerRacks: 1
+  topology:
+    dc:
+      - name: dc1
+        rack:
+          - name: rack1

test/e2e/kuttl/kuttl-test.yaml

@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+#crdDir: ../../../deploy/crds/
+testDirs:
+  - .
+timeout: 300
+commands:
+  - command: helm repo add orange-incubator https://orange-kubernetes-charts-incubator.storage.googleapis.com/
+    ignoreFailure: true
+  - command: kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/casskop/master/deploy/crds/db.orange.com_cassandraclusters_crd.yaml
+    ignoreFailure: true
+  - command: kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/casskop/master/deploy/crds/db.orange.com_cassandrabackups_crd.yaml
+    ignoreFailure: true
+  - command: kubectl apply -f https://raw.githubusercontent.com/Orange-OpenSource/casskop/master/deploy/crds/db.orange.com_cassandrarestores_crd.yaml
+    ignoreFailure: true

website/docs/8_contributing/1_developer_guide.md

@@ -293,6 +293,23 @@ make e2e-test-fix-arg ClusterScaleDown
 **Tip**: When tests fail, there may be resources that need to be cleaned up. Run `tools/e2e_test_cleanup.sh` to delete resources left over from tests.
 
 
+### Run kuttl tests
+
+This requires [kuttl cli](https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin) to be installed on your machine
+
+You first need to have a Kubernetes cluster set up with kubectl.
+
+Then to run all tests you can simply type :
+```
+kubectl kuttl test --config ./test/e2e/kuttl/kuttl-test.yaml ./test/e2e/kuttl/
+```
+
+This will run all testcases in the `/test/e2e/kuttl/` directory in parallel on different generated namespaces (with Casskop automatically installed on each).
+
+>If you installed only the binary of kuttl, you can omit the `kubectl` at the beginning
+
+**Tip**: You can specify a single test case to run by adding `--test TestCase` where `TestCase` is the name of one of the directories in `/test/e2e/kuttl/*here*` (like `ScaleUpAndDown` for example)
+
 ### Debug CassKop in remote in a Kubernetes cluster
 
 CassKop makes some specific calls to the Jolokia API of the CassandraNodes it deploys inside the kubernetes