Merge branch 'main' into fix/add-rocky8-build-binaries

PingCAP-QE · Sep 5, 2024 · 909fa74 · 909fa74
2 parents fd9b9b9 + 022d4cc
commit 909fa74
Show file tree

Hide file tree

Showing 25 changed files with 83 additions and 73 deletions.
diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh
@@ -14,7 +14,8 @@ function test_get_builder() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo -en "[🚢] $cm $os $ac $version release:\t"
-                    $script $cm $os $ac $version release
+                    echo "$cm $os $ac $version:"
+                    $script "$cm" "$os" "$ac" "$version" $profile
                 done
             done
         done
@@ -27,7 +28,7 @@ function test_get_builder() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo -en "[🚢] $cm $os $ac $version enterprise:\t"
-                    $script $cm $os $ac $version enterprise
+                    $script "$cm" "$os" "$ac" "$version" $profile
                 done
             done
         done
@@ -61,7 +62,7 @@ function test_get_builder_freedom_releasing() {
     for ac in $architectures; do
         for version in v1.5.0 v1.6.0; do
             echo -en "[🚢] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 
@@ -72,7 +73,7 @@ function test_get_builder_freedom_releasing() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo -en "[🚢] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 
@@ -82,7 +83,7 @@ function test_get_builder_freedom_releasing() {
     for ac in $architectures; do
         for version in v0.5.0 v0.6.0; do
             echo -en "[🚢] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 
@@ -92,7 +93,7 @@ function test_get_builder_freedom_releasing() {
         for ac in $architectures; do
             for version in v0.1.2 v0.1.3; do
                 echo -en "[🚢] $cm $os $ac $version $profile:\t"
-                $script $cm $os $ac $version $profile
+                $script "$cm" "$os" "$ac" "$version" $profile
             done
         done
     done
@@ -112,7 +113,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-                    $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -127,7 +128,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-                    $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" enterprise branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -142,7 +143,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-                    $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" failpoint branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -163,7 +164,7 @@ function test_gen_package_artifacts_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v1.5.0 v1.6.0; do
             echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -174,7 +175,7 @@ function test_gen_package_artifacts_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -185,7 +186,7 @@ function test_gen_package_artifacts_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v0.5.0 v0.6.0; do
             echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -196,7 +197,7 @@ function test_gen_package_artifacts_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v0.1.2 v0.1.3; do
             echo -en "[📃📦] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -215,13 +216,12 @@ function test_gen_package_images_script() {
         for version in $versions; do
             for ac in $architectures; do
                 echo -en "[📃💿] $cm $os $ac $version $profile:\t"
-                $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+                $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
                 shellcheck -S error packages/scripts/build-package-images.sh
             done
         done
     done
 
-    # tidb enterprise profile
     # enterprise profile
     local profile="enterprise"
     local components="tidb tikv pd tiflash"
@@ -262,7 +262,7 @@ function test_gen_package_images_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v1.6.0 v1.5.0; do
             echo -en "[📃💿] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -272,7 +272,7 @@ function test_gen_package_images_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo -en "[📃💿] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -282,7 +282,7 @@ function test_gen_package_images_script_freedom_releasing() {
     for version in v0.5.0 v0.6.0; do
         for ac in $architectures; do
             echo -en "[📃💿] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -292,7 +292,7 @@ function test_gen_package_images_script_freedom_releasing() {
     for ac in $architectures; do
         for version in v0.1.2 v0.1.3; do
             echo -en "[📃💿] $cm $os $ac $version $profile:\t"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -310,7 +310,7 @@ function test_gen_offline_package_artifacts_script() {
             for ac in $architectures; do
                 for edition in $editions; do
                     echo "$os $ac $version $edition:"
-                    $script $os $ac $version $edition
+                    $script "$os" "$ac" "$version" "$edition"
                     shellcheck -S error packages/scripts/compose-offline-packages-artifacts.sh
                 done
             done

diff --git a/.github/workflows/pull-prod-runtime-images.yaml b/.github/workflows/pull-prod-runtime-images.yaml
@@ -58,7 +58,7 @@ jobs:
         run: |
           curl -L https://github.com/estesp/manifest-tool/releases/download/v2.1.7/binaries-manifest-tool-2.1.7.tar.gz | tar -zxvf - manifest-tool-linux-amd64
           sudo install manifest-tool-linux-amd64 /usr/local/bin/manifest-tool
-      
+
       - name: Cache layers
         uses: actions/cache@v4
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,16 @@
+ci:
+  autofix_commit_msg: |
+    fix: auto fixes from pre-commit.com hooks
+
+    for more information, see https://pre-commit.ci
+  autoupdate_commit_msg: "chore: pre-commit autoupdate"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.16.1
+    hooks:
+      - id: gitleaks
diff --git a/dockerfiles-multi-stages/tiflash/Dockerfile b/dockerfiles-multi-stages/tiflash/Dockerfile
@@ -41,4 +41,4 @@ COPY --from=busybox:1.36.1 /bin/busybox /bin/busybox
 ENV LD_LIBRARY_PATH /tiflash
 COPY --from=builder /tiflash/output/tiflash /tiflash
 
-ENTRYPOINT ["/tiflash/tiflash", "server"]
+ENTRYPOINT ["/tiflash/tiflash", "server"]
diff --git a/dockerfiles/bases/build-old.sh b/dockerfiles/bases/build-old.sh
@@ -8,25 +8,25 @@ build() {
     case "${PLATFORMS}" in
     linux/amd64)
         echo "building for linux/amd64 platform..."
-        docker build --tag=$IMAGE --platform=$PLATFORMS --target amd64 . -f "$dockerfile"
+        docker build --tag="$IMAGE" --platform="$PLATFORMS" --target amd64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push $IMAGE
+            docker push "$IMAGE"
         fi
         ;;
     linux/arm64)
         echo "building for linux/arm64 platform..."
-        docker build --tag=$IMAGE --platform=$PLATFORMS --target arm64 . -f "$dockerfile"
+        docker build --tag="$IMAGE" --platform="$PLATFORMS" --target arm64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push $IMAGE
+            docker push "$IMAGE"
         fi
         ;;
     linux/arm64,linux/amd64 | linux/amd64,linux/arm64)
         echo "building for linux/arm64 and linux/amd64 platforms..."
-        docker build --tag=${IMAGE}_linux_amd64 --platform=linux/amd64 --target amd64 . -f "$dockerfile"
-        docker build --tag=${IMAGE}_linux_arm64 --platform=linux/arm64 --target arm64 . -f "$dockerfile"
+        docker build --tag="${IMAGE}_linux_amd64" --platform=linux/amd64 --target amd64 . -f "$dockerfile"
+        docker build --tag="${IMAGE}_linux_arm64" --platform=linux/arm64 --target arm64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push ${IMAGE}_linux_amd64
-            docker push ${IMAGE}_linux_arm64
+            docker push "${IMAGE}_linux_amd64"
+            docker push "${IMAGE}_linux_arm64"
 
             # compose manifest for multi-arch image.
             pushed_repo="${IMAGE%:*}"
@@ -35,14 +35,14 @@ build() {
             yq -i ".tags = [\"$tag\"]" manifest.yaml
 
             # linux/amd64
-            manifest-tool inspect --raw ${IMAGE}_linux_amd64 >manifest_linux_amd64.json
+            manifest-tool inspect --raw "${IMAGE}_linux_amd64" >manifest_linux_amd64.json
             yq -i '.manifests += [{}]' manifest.yaml
             digest=$(jq -r '.digest' manifest_linux_amd64.json)
             yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml
             yq -i '.manifests[-1].platform.os = "linux"' manifest.yaml
             yq -i '.manifests[-1].platform.architecture = "amd64"' manifest.yaml
             # linux/arm64
-            manifest-tool inspect --raw ${IMAGE}_linux_arm64 >manifest_linux_arm64.json
+            manifest-tool inspect --raw "${IMAGE}_linux_arm64" >manifest_linux_arm64.json
             yq -i '.manifests += [{}]' manifest.yaml
             digest=$(jq -r '.digest' manifest_linux_arm64.json)
             yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml

diff --git a/dockerfiles/bases/tikv-base/release-6.5.Dockerfile b/dockerfiles/bases/tikv-base/release-6.5.Dockerfile
@@ -13,4 +13,3 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \
 # set timezone
 ENV TZ=/etc/localtime
 ENV TZDIR=/usr/share/zoneinfo
-
diff --git a/dockerfiles/cd/builders/ng-monitoring/Dockerfile b/dockerfiles/cd/builders/ng-monitoring/Dockerfile
@@ -2,7 +2,7 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring 
+#   - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring
 #   - rm ng-monitoring/.dockerignore # make step depended on git metadata.
 #   - docker build -t ng-monitoring -f Dockerfile ./ng-monitoring
 

diff --git a/dockerfiles/cd/builders/tidb-dashboard/Dockerfile b/dockerfiles/cd/builders/tidb-dashboard/Dockerfile
@@ -2,7 +2,7 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord 
+#   - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord
 #   - rm tidb-dashbord/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb-dashbord -f Dockerfile ./tidb-dashbord
 

diff --git a/dockerfiles/cd/builders/tidb-operator/Dockerfile b/dockerfiles/cd/builders/tidb-operator/Dockerfile
@@ -2,7 +2,7 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb-operator.git tidb 
+#   - git clone https://github.com/pingcap/tidb-operator.git tidb
 #   - rm tidb-operator/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb-operator -f Dockerfile ./tidb-operator
 

diff --git a/dockerfiles/cd/builders/tidb/Dockerfile b/dockerfiles/cd/builders/tidb/Dockerfile
@@ -2,7 +2,7 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb.git tidb 
+#   - git clone https://github.com/pingcap/tidb.git tidb
 #   - rm tidb/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb -f Dockerfile ./tidb
 

diff --git a/dockerfiles/cd/builders/tiflash/Dockerfile b/dockerfiles/cd/builders/tiflash/Dockerfile
@@ -38,12 +38,12 @@ ENV LD ld.lld
 COPY --from=extract-conda-openssl /usr/local/opt/openssl /usr/local/opt/openssl
 
 # building and install libc++
-RUN --mount=type=cache,target=/tmp/llvm \
-    git clone https://github.com/llvm/llvm-project.git  --branch llvmorg-17.0.6 --depth 1 /tmp/llvm/llvm-project && \
-    cd /tmp/llvm/llvm-project && rm -rf build && mkdir build && \
+RUN git clone https://github.com/llvm/llvm-project.git  --branch llvmorg-17.0.6 --depth 1 /tmp/llvm-project && \
+    cd /tmp/llvm-project && rm -rf build && mkdir build && \
     cmake -G Ninja -S runtimes -B build -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" -DCMAKE_INSTALL_PREFIX=/usr -DLLVM_LIBDIR_SUFFIX=64 && \
     ninja -C build cxx cxxabi unwind && \
-    ninja -C build install-cxx install-cxxabi install-unwind
+    ninja -C build install-cxx install-cxxabi install-unwind && \
+    rm -rf /tmp/llvm-project
 
 # install rust toolchain
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none

diff --git a/dockerfiles/ci/base/Dockerfile b/dockerfiles/ci/base/Dockerfile
@@ -67,7 +67,6 @@ RUN curl -sL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \
     apt-get clean && \
     npm install -g yarn pnpm
 
-# Upgrade Git tool 
+# Upgrade Git tool
 # renovate: datasource=github-tags depName=git/git
-COPY --from=bitnami/git:2.46.0 /opt/bitnami/git/bin/git /usr/bin/git 
-
+COPY --from=bitnami/git:2.46.0 /opt/bitnami/git/bin/git /usr/bin/git
diff --git a/dockerfiles/ci/skaffold.yaml b/dockerfiles/ci/skaffold.yaml
@@ -62,7 +62,7 @@ profiles:
         path: /build/artifacts/0/kaniko/buildArgs
         value:
           # renovate: datasource=docker depName=golang
-          GOLANG_VERSION: 1.20.10      
+          GOLANG_VERSION: 1.20.10
   - name: go-1.19
     patches:
       - op: replace
@@ -72,7 +72,7 @@ profiles:
         path: /build/artifacts/0/kaniko/buildArgs
         value:
           # renovate: datasource=docker depName=golang
-          GOLANG_VERSION: 1.19.12     
+          GOLANG_VERSION: 1.19.12
 ---
 apiVersion: skaffold/v4beta6
 kind: Config

diff --git a/dockerfiles/ee-apps/nginx-s3/example.nginx.conf b/dockerfiles/ee-apps/nginx-s3/example.nginx.conf
@@ -27,7 +27,7 @@ http {
     #tcp_nopush     on;
     gzip  on;
     keepalive_timeout  65;
-    
+
     # Define the server block for the default server
     server {
         listen 80  ;
@@ -57,6 +57,6 @@ http {
             proxy_intercept_errors on;
             rewrite                .* /$url_full break;
             proxy_pass             http://$bucket;
-        }   
+        }
     }
 }
diff --git a/dockerfiles/products/le6.5-linux-amd64-debug/tiem b/dockerfiles/products/le6.5-linux-amd64-debug/tiem
@@ -12,4 +12,4 @@ COPY bin/cluster-server ./bin
 COPY bin/metadb-server ./bin
 COPY build_helper/docker_start_cmd.sh ./scripts
 EXPOSE 4116
-ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"]
+ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"]
diff --git a/dockerfiles/products/le6.5-linux-amd64/tiem b/dockerfiles/products/le6.5-linux-amd64/tiem
@@ -12,4 +12,4 @@ COPY bin/cluster-server ./bin
 COPY bin/metadb-server ./bin
 COPY build_helper/docker_start_cmd.sh ./scripts
 EXPOSE 4116
-ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"]
+ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"]
diff --git a/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer b/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer
@@ -8,4 +8,4 @@ COPY rules/*.rules.yml /tmp/
 COPY datasources/*.yaml /tmp/
 
 ENTRYPOINT ["/usr/bin/init.sh"]
-CMD ["TIDB-Cluster", "/grafana-dashboard-definitions/tidb/", "false", "/etc/prometheus"]
+CMD ["TIDB-Cluster", "/grafana-dashboard-definitions/tidb/", "false", "/etc/prometheus"]