Adopt Golang version ClickHouse data schema management

Signed-off-by: Yanjun Zhou <zhouya@vmware.com>
antrea-io · Sep 14, 2022 · 0244cfd · 0244cfd
1 parent 673512b
commit 0244cfd
Show file tree

Hide file tree

Showing 27 changed files with 968 additions and 746 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -43,6 +43,37 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-clickhouse-monitor:latest
+  
+  check-clickhouse-server-changes:
+    name: Check whether clickhouse-server image needs to be built based on diff
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: antrea-io/has-changes@v2
+      id: check_diff
+      with:
+        paths: plugins/clickhouse-schema-management/* build/images/Dockerfile.clickhouse-server.ubuntu
+    outputs:
+      has_changes: ${{ steps.check_diff.outputs.has_changes }}
+
+  build-clickhouse-server:
+    needs: check-clickhouse-server-changes
+    if: ${{ needs.check-clickhouse-server-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build clickhouse-server Docker image
+      run: make clickhouse-server
+    - name: Push clickhouse-server Docker image to registry
+      if: ${{ github.repository == 'antrea-io/theia' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-clickhouse-server:latest
 
   check-policy-recommendation-changes:
     name: Check whether policy-recommendation image needs to be built based on diff

diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -33,7 +33,22 @@ jobs:
         make clickhouse-monitor
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/theia-clickhouse-monitor:"${VERSION}"
-    
+
+  build-clickhouse-server:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build clickhouse-server Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make clickhouse-server
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/theia-clickhouse-server:"${VERSION}"
+
   build-policy-recommendation:
     runs-on: [ubuntu-latest]
     needs: get-version

diff --git a/.github/workflows/docker_update_theia.yml b/.github/workflows/docker_update_theia.yml
@@ -13,9 +13,6 @@ on:
       grafana-tag:
         description: 'Tag for grafana Docker image'
         required: true
-      ch-server-tag:
-        description: 'Tag for clickhouse-server Docker image'
-        required: true
       spark-operator-tag:
         description: 'Tag for spark-operator Docker image'
         required: true
@@ -50,12 +47,6 @@ jobs:
           src: docker.io/grafana/grafana:${{ github.event.inputs.grafana-tag }}
           dst: |
             docker.io/antrea/theia-grafana:${{ github.event.inputs.grafana-tag }}
-      - name: Push antrea/theia-clickhouse-server
-        uses: akhilerm/tag-push-action@v2.0.0
-        with:
-          src: docker.io/clickhouse/clickhouse-server:${{ github.event.inputs.ch-server-tag }}
-          dst: |
-            docker.io/antrea/theia-clickhouse-server:${{ github.event.inputs.ch-server-tag }}
       - name: Push antrea/theia-spark-operator
         uses: akhilerm/tag-push-action@v2.0.0
         with:

diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -63,11 +63,29 @@ jobs:
         path: clickhouse-monitor.tar
         retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
 
+  build-clickhouse-server-image:
+    name: Build ClickHouse server image to be used for Kind e2e tests
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' }}
+    runs-on: [ ubuntu-latest ]
+    steps:
+    - uses: actions/checkout@v3
+    - run: make clickhouse-server
+    - name: Save ClickHouse server image to tarball
+      run: docker save -o clickhouse-server.tar antrea/theia-clickhouse-server
+    - name: Upload ClickHouse server image for subsequent jobs
+      uses: actions/upload-artifact@v3
+      with:
+        name: clickhouse-server
+        path: clickhouse-server.tar
+        retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails
+
   test-e2e-encap:
     name: E2e tests on a Kind cluster on Linux
     needs: 
       - build-policy-recommendation-image
       - build-clickhouse-monitor-image
+      - build-clickhouse-server-image
     runs-on: [ubuntu-latest]
     steps:
       - name: Free disk space
@@ -87,12 +105,18 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: clickhouse-monitor
+      - name: Download ClickHouse server images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: clickhouse-server
       - name: Load Theia image
         run:  |
           docker load -i policy-recommendation.tar
           docker tag antrea/theia-policy-recommendation:latest projects.registry.vmware.com/antrea/theia-policy-recommendation:latest
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
+          docker load -i clickhouse-server.tar
+          docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -115,7 +139,9 @@ jobs:
 
   test-upgrade-from-N-1:
     name: Upgrade from Theia version N-1
-    needs: build-clickhouse-monitor-image
+    needs: 
+      - build-clickhouse-monitor-image
+      - build-clickhouse-server-image 
     runs-on: [ubuntu-latest]
     steps:
       - name: Free disk space
@@ -135,6 +161,14 @@ jobs:
         run:  |
           docker load -i clickhouse-monitor.tar
           docker tag antrea/theia-clickhouse-monitor:latest projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest
+      - name: Download ClickHouse server images from previous jobs
+        uses: actions/download-artifact@v3
+        with:
+          name: clickhouse-server
+      - name: Load Theia image
+        run:  |
+          docker load -i clickhouse-server.tar
+          docker tag antrea/theia-clickhouse-server:latest projects.registry.vmware.com/antrea/theia-clickhouse-server:latest
       - name: Install Kind
         run: |
           curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
@@ -164,6 +198,7 @@ jobs:
     needs:
     - build-policy-recommendation-image
     - build-clickhouse-monitor-image
+    - build-clickhouse-server-image
     - test-e2e-encap
     - test-upgrade-from-N-1
     runs-on: [ubuntu-latest]
@@ -178,3 +213,8 @@ jobs:
       uses: geekyeggo/delete-artifact@v1
       with:
         name: clickhouse-monitor
+    - name: Delete clickhouse-server
+      if: ${{ needs.build-clickhouse-server-image.result == 'success' }}
+      uses: geekyeggo/delete-artifact@v1
+      with:
+        name: clickhouse-server
diff --git a/Makefile b/Makefile
@@ -175,6 +175,18 @@ theia-manager:
 theia-manager-bin:
 	@mkdir -p $(BINDIR)
 	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/cmd/theia-manager
+.PHONY: clickhouse-server
+clickhouse-server:
+	@echo "===> Building antrea/theia-clickhouse-server Docker image <==="
+	docker build --pull -t antrea/theia-clickhouse-server:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.clickhouse-server.ubuntu $(DOCKER_BUILD_ARGS) .
+	docker tag antrea/theia-clickhouse-server:$(DOCKER_IMG_VERSION) antrea/theia-clickhouse-server
+	docker tag antrea/theia-clickhouse-server:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-clickhouse-server
+	docker tag antrea/theia-clickhouse-server:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/theia-clickhouse-server:$(DOCKER_IMG_VERSION)
+
+.PHONY: clickhouse-schema-management-plugin
+clickhouse-schema-management-plugin:
+	@mkdir -p $(BINDIR)
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-schema-management
 
 .PHONY: policy-recommendation
 policy-recommendation:

diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md
@@ -26,7 +26,7 @@ Kubernetes: `>= 1.16.0-0`
 | clickhouse.cluster.shards | int | `1` | Number of ClickHouse shards in the cluster. |
 | clickhouse.cluster.zookeeperHosts | list | `[]` | To use a pre-installed ZooKeeper for ClickHouse data replication, please provide a list of your ZooKeeper hosts. To install a customized ZooKeeper, refer to <https://github.com/Altinity/clickhouse-operator/blob/master/docs/zookeeper_setup.md> |
 | clickhouse.connectionSecret | object | `{"password":"clickhouse_operator_password","username":"clickhouse_operator"}` | Credentials to connect to ClickHouse. They will be stored in a secret. |
-| clickhouse.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-server","tag":"22.6"}` | Container image used by ClickHouse. |
+| clickhouse.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-server","tag":""}` | Container image used by ClickHouse. |
 | clickhouse.monitor.deletePercentage | float | `0.5` | The percentage of records in ClickHouse that will be deleted when the storage grows above threshold. Vary from 0 to 1. |
 | clickhouse.monitor.enable | bool | `true` | Determine whether to run a monitor to periodically check the ClickHouse memory usage and clean data. |
 | clickhouse.monitor.execInterval | string | `"1m"` | The time interval between two round of monitoring. Can be a plain integer using one of these unit suffixes ns, us (or µs), ms, s, m, h. |

diff --git a/build/charts/theia/provisioning/datasources/init.sh b/build/charts/theia/provisioning/datasources/init.sh
@@ -18,9 +18,21 @@ set -e
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-source $THIS_DIR/migrate.sh
 source $THIS_DIR/create_table.sh
 
-migrate
+# This function is kept to be compatible with version below v0.3.0
+function setDataVersion {
+    tables=$(clickhouse client -h 127.0.0.1 -q "SHOW TABLES")
+    if [[ $tables == *"migrate_version"* ]]; then
+        clickhouse client -h 127.0.0.1 -q "ALTER TABLE migrate_version DELETE WHERE version!=''"
+    else
+        clickhouse client -h 127.0.0.1 -q "CREATE TABLE migrate_version (version String) engine=MergeTree ORDER BY version"
+    fi
+    clickhouse client -h 127.0.0.1 -q "INSERT INTO migrate_version (*) VALUES ('{{ .Chart.Version }}')"
+    echo "=== Set data schema version to {{ .Chart.Version }} ==="
+}
+
+../clickhouse-schema-management
 createTable
 setDataVersion
+
diff --git a/build/charts/theia/provisioning/datasources/migrate.sh b/build/charts/theia/provisioning/datasources/migrate.sh
diff --git a/build/charts/theia/provisioning/datasources/migrators/000001_0-1-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000001_0-1-0.down.sql