Merge pull request #869 from Altinity/0.18.0

Merge 0.18.0 release

.dockerignore

@@ -1,6 +1,10 @@
-vendor/
 docs/
 .vagrant/
 .idea/
+tests/
+hack/
 *.log
 ./tests/image/cache/
+.devspace/
+devspace.yaml
+Vagrantfile

.github/workflows/build_branch.yaml

@@ -0,0 +1,29 @@
+name: build_branch
+
+on:
+  push:
+    branches:
+      - "[0-9]+.[0-9]+.[0-9]+"
+
+jobs:
+  build_master:
+    name: Build branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Build and push Docker images
+        env:
+          DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+        run: |
+          export CHO_RELEASE=$(cat release)
+
+          echo "${DOCKER_PASS}" | docker login -u $DOCKER_USER --password-stdin docker.io
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push  .
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE} --pull --push .

.github/workflows/build_master.yaml

@@ -0,0 +1,30 @@
+name: build_master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build_master:
+    name: Build master
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Build and push Docker images
+        env:
+          DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+        run: |
+          export CHO_RELEASE=latest
+
+          echo "${DOCKER_PASS}" | docker login -u $DOCKER_USER --password-stdin docker.io
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/operator/Dockerfile -t docker.io/${DOCKER_ORG}/clickhouse-operator:${CHO_RELEASE} --pull --push  .
+          docker buildx build --progress plain --platform=linux/amd64,linux/arm64 -f dockerfile/metrics-exporter/Dockerfile -t docker.io/${DOCKER_ORG}/metrics-exporter:${CHO_RELEASE} --pull --push .
+  

.github/workflows/run_tests.yaml

@@ -0,0 +1,12 @@
+name: grafana_push
+on: workflow_dispatch
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          pip3 install -U -r ./tests/image/requirements.txt
+          python3 ./tests/regression.py --only "/regression/e2e.test_operator/*"
+          

.gitignore

@@ -11,12 +11,12 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
-# IDE folders
+# IDE folders - everywhere
 .idea
-/.project
-/.pydevproject
+.project
+.pydevproject
 
-# Skip log folder
+# Skip logs and caches
 log
 *.log
 __pycache__
@@ -25,7 +25,7 @@ __pycache__
 dev/bin
 dev/log
 
-.vagrant
+/.vagrant
 /.DS_Store
 
 # Skip vendor
@@ -44,3 +44,6 @@ tests/image/cache
 
 # Skip tmp folder
 /tmp/
+
+# Ignore DevSpace cache and log folder
+/.devspace

README.md

@@ -1,6 +1,6 @@
-# ClickHouse Operator
+# Altinity Operator for ClickHouse
 
-ClickHouse Operator creates, configures and manages ClickHouse clusters running on Kubernetes.
+Operator creates, configures and manages ClickHouse clusters running on Kubernetes.
 
 [![GitHub release](https://img.shields.io/github/v/release/altinity/clickhouse-operator?include_prereleases)](https://img.shields.io/github/v/release/altinity/clickhouse-operator?include_prereleases)
 [![CircleCI](https://circleci.com/gh/Altinity/clickhouse-operator.svg?style=svg)](https://circleci.com/gh/Altinity/clickhouse-operator)
@@ -12,14 +12,12 @@ ClickHouse Operator creates, configures and manages ClickHouse clusters running
 
 ## Features
 
-The ClickHouse Operator for Kubernetes currently provides the following:
-
-- Creates ClickHouse clusters based on Custom Resource [specification][chi_max_yaml] provided
+- Creates ClickHouse clusters defined as custom resources
 - Customized storage provisioning (VolumeClaim templates)
 - Customized pod templates
 - Customized service templates for endpoints
-- ClickHouse configuration and settings (including Zookeeper integration)
-- Flexible templating
+- ClickHouse configuration management
+- ClickHouse users management
 - ClickHouse cluster scaling including automatic schema propagation
 - ClickHouse version upgrades
 - Exporting ClickHouse metrics to Prometheus
@@ -53,17 +51,17 @@ The ClickHouse Operator for Kubernetes currently provides the following:
 
 **How to contribute**
  * [How to contribute/submit a patch][contributing_manual]
+ * [How to easy development process with devspace.sh][devspace_manual]
 
 ---
-**All docs**
- * [All available docs list][all_docs_list]
+ * [Documentation index][all_docs_list]
 ---
 
 ## License
 
 Copyright (c) 2019-2219, Altinity Ltd and/or its affiliates. All rights reserved.
 
-`clickhouse-operator` is licensed under the Apache License 2.0.
+Altinity Operator for ClickHouse is licensed under the Apache License 2.0.
 
 See [LICENSE](./LICENSE) for more details.
 
@@ -83,6 +81,6 @@ See [LICENSE](./LICENSE) for more details.
 [update_operator]: ./docs/operator_upgrade.md
 [schema_migration]: ./docs/schema_migration.md
 [operator_configuration]: ./docs/operator_configuration.md
-[all_docs_list]: ./docs/README.md
 [contributing_manual]: ./CONTRIBUTING.md
-
+[devspace_manual]: ./docs/devspace.md
+[all_docs_list]: ./docs/README.md

Vagrantfile

@@ -72,6 +72,13 @@ Vagrant.configure(2) do |config|
     # alertmanager
     clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 9093, host_ip: "127.0.0.1", host: 9093
 
+    # devspace UI
+    clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 8090, host_ip: "127.0.0.1", host: 8090
+
+    # delve for devspace
+    clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 40001, host_ip: "127.0.0.1", host: 40001
+    clickhouse_operator.vm.network "forwarded_port", guest_ip: "172.16.2.99", guest: 40002, host_ip: "127.0.0.1", host: 40002
+
     clickhouse_operator.vm.host_name = "local-altinity-clickhouse-operator"
     # vagrant plugin install vagrant-disksize
     clickhouse_operator.disksize.size = '50GB'
@@ -155,9 +162,10 @@ Vagrant.configure(2) do |config|
 #    K8S_VERSION=${K8S_VERSION:-1.19.14}
 #    performance issue 1.20.x, 1.21.x
 #    https://github.com/kubernetes/kubeadm/issues/2395
-#    K8S_VERSION=${K8S_VERSION:-1.20.10}
-#    K8S_VERSION=${K8S_VERSION:-1.21.4}
-    K8S_VERSION=${K8S_VERSION:-1.22.2}
+#    K8S_VERSION=${K8S_VERSION:-1.20.14}
+#    K8S_VERSION=${K8S_VERSION:-1.21.8}
+#    K8S_VERSION=${K8S_VERSION:-1.22.5}
+    K8S_VERSION=${K8S_VERSION:-1.23.1}
     export VALIDATE_YAML=true
 
     killall kubectl || true
@@ -222,12 +230,32 @@ EOF
     export OPERATOR_IMAGE=altinity/clickhouse-operator:${OPERATOR_RELEASE}
     export METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:${OPERATOR_RELEASE}
 
+    # devspace
+    DEVSPACE_VERSION=$(curl -sL https://github.com/devspace-cloud/devspace/releases/latest -H "Accept: application/json" | jq -r .tag_name)
+    wget -c --progress=bar:force:noscroll -O /usr/local/bin/${DEVSPACE_VERSION}-devspace-linux-amd64 "https://github.com/devspace-cloud/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64"
+    wget -c --progress=bar:force:noscroll -O /usr/local/bin/${DEVSPACE_VERSION}-devspace-linux-amd64.sha256 "https://github.com/devspace-cloud/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64.sha256"
+    sed -i -E "s/\\/Users.+devspace\\-linux\\-amd64/\\/usr\\/local\\/bin\\/${DEVSPACE_VERSION}-devspace-linux-amd64/g" /usr/local/bin/${DEVSPACE_VERSION}-devspace-linux-amd64.sha256
+    sha256sum -c /usr/local/bin/${DEVSPACE_VERSION}-devspace-linux-amd64.sha256
+    cp -fv /usr/local/bin/${DEVSPACE_VERSION}-devspace-linux-amd64 /usr/local/bin/devspace
+    chmod +x /usr/local/bin/devspace
+
+    # docker build
+    export COMPANY_REPO=${COMPANY_REPO:-altinity}
+    go mod download -x
+    go mod tidy
+    go mod vendor
+    time docker buildx build -f dockerfile/operator/Dockerfile --platform=linux/amd64 --output=type=image,name=$COMPANY_REPO/clickhouse-operator:$OPERATOR_RELEASE .
+    time docker buildx build -f dockerfile/metrics-exporter/Dockerfile --platform=linux/amd64 --output=type=image,name=$COMPANY_REPO/metrics-exporter:$OPERATOR_RELEASE .
+
+
+    # install clickhouse-operator
     if ! kubectl get deployment clickhouse-operator -n "${OPERATOR_NAMESPACE}" 1>/dev/null 2>/dev/null; then
         cd /vagrant/deploy/operator/
         bash -x ./clickhouse-operator-install.sh
         cd /vagrant
     fi
 
+    # install prometheus-operator + prometheus instance + ServiceMonitor for clickhouse
     export PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-prometheus}
     cd /vagrant/deploy/prometheus/
     kubectl delete ns ${PROMETHEUS_NAMESPACE} || true
@@ -245,6 +273,7 @@ EOF
 
     cd /vagrant/
 
+    # install grafana-operator + grafana instance + GrafanaDashboard, GrafanaDatasource for clickhouse
     export GRAFANA_NAMESPACE=${GRAFANA_NAMESPACE:-grafana}
     cd /vagrant/deploy/grafana/grafana-with-grafana-operator/
     kubectl delete ns ${GRAFANA_NAMESPACE} || true
@@ -271,7 +300,7 @@ EOF
     # open http://localhost:8888/chi and check exists clickhouse installations
     kubectl --namespace="${OPERATOR_NAMESPACE}" port-forward --address 0.0.0.0 service/clickhouse-operator-metrics 8888 </dev/null &>/dev/null &
 
-    for image in $(cat ./tests/configs/test-017-multi-version.yaml | yq r - "spec.templates.podTemplates[*].spec.containers[*].image"); do
+    for image in $(cat ./tests/configs/test-017-multi-version.yaml | yq eval -e ".spec.templates.podTemplates[].spec.containers[].image"); do
         docker pull ${image}
     done
 
@@ -285,5 +314,4 @@ EOF
     # cp -fv /tmp/audit2rbac.yaml /vagrant/deploy/dev/clickhouse-operator-install-yaml-template-02-section-rbac-restricted.yaml
 
   SHELL
-
 end

cmd/metrics_exporter/app/metrics_exporter.go

@@ -90,17 +90,17 @@ func Run() {
 	log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt)
 
 	// Initialize k8s API clients
-	kubeClient, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
+	kubeClient, _, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
 
 	// Create operator instance
 	chop.New(kubeClient, chopClient, chopConfigFile)
 	log.Info(chop.Config().String(true))
 
 	exporter := metrics.StartMetricsREST(
 		metrics.NewCHAccessInfo(
-			chop.Config().CHUsername,
-			chop.Config().CHPassword,
-			chop.Config().CHPort,
+			chop.Config().ClickHouse.Access.Username,
+			chop.Config().ClickHouse.Access.Password,
+			chop.Config().ClickHouse.Access.Port,
 		),
 
 		metricsEP,

cmd/operator/app/clickhouse_operator.go

@@ -92,14 +92,14 @@ func Run() {
 		chopInformerFactoryResyncPeriod = defaultInformerFactoryResyncDebugPeriod
 	}
 
-	log.A().Info("Starting clickhouse-operator. Version:%s GitSHA:%s BuiltAt:%s", version.Version, version.GitSHA, version.BuiltAt)
+	log.F().Info("Starting clickhouse-operator. Version:%s GitSHA:%s BuiltAt:%s", version.Version, version.GitSHA, version.BuiltAt)
 
 	// Initialize k8s API clients
-	kubeClient, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
+	kubeClient, extClient, chopClient := chop.GetClientset(kubeConfigFile, masterURL)
 
 	// Create operator instance
 	chop.New(kubeClient, chopClient, chopConfigFile)
-	log.V(1).A().Info("Log options parsed")
+	log.V(1).F().Info("Log options parsed")
 	log.Info(chop.Config().String(true))
 
 	// Create Informers
@@ -117,6 +117,7 @@ func Run() {
 	// Create Controller
 	chiController := chi.NewController(
 		chopClient,
+		extClient,
 		kubeClient,
 		chopInformerFactory,
 		kubeInformerFactory,
@@ -142,7 +143,7 @@ func Run() {
 	//
 	// Start Controller
 	//
-	log.V(1).A().Info("Starting CHI controller")
+	log.V(1).F().Info("Starting CHI controller")
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
 	go func() {