From 022d4cc322782644524456071d764a4a9ac2d19c Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Thu, 5 Sep 2024 12:06:19 +0800 Subject: [PATCH] fix(dockerfiles/cd/builders/tiflash): fix dockerfile to avoid cache conflict with docker buildx (#398) Signed-off-by: wuhuizuo --------- Signed-off-by: wuhuizuo Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/scripts/ci.sh | 40 +++++++++---------- .../workflows/pull-prod-runtime-images.yaml | 2 +- .pre-commit-config.yaml | 16 ++++++++ dockerfiles-multi-stages/tiflash/Dockerfile | 2 +- dockerfiles/bases/build-old.sh | 20 +++++----- .../bases/tikv-base/release-6.5.Dockerfile | 1 - .../cd/builders/ng-monitoring/Dockerfile | 4 +- dockerfiles/cd/builders/pd/Dockerfile | 2 +- .../cd/builders/tidb-dashboard/Dockerfile | 4 +- .../cd/builders/tidb-operator/Dockerfile | 4 +- dockerfiles/cd/builders/tidb/Dockerfile | 4 +- dockerfiles/cd/builders/tiflash/Dockerfile | 10 ++--- dockerfiles/cd/builders/tiflow/Dockerfile | 2 +- dockerfiles/cd/builders/tikv/Dockerfile | 2 +- dockerfiles/cd/builders/tikv/fips.Dockerfile | 2 +- dockerfiles/cd/utils/ks3util/Dockerfile | 2 +- dockerfiles/cd/utils/release/Dockerfile | 2 +- dockerfiles/cd/utils/remote/Dockerfile | 2 +- dockerfiles/ci/base/Dockerfile | 7 ++-- dockerfiles/ci/release-build-base/Dockerfile | 2 +- dockerfiles/ci/skaffold.yaml | 4 +- dockerfiles/ee-apps/ats/Dockerfile | 2 +- .../ee-apps/nginx-s3/example.nginx.conf | 4 +- .../products/le6.5-linux-amd64-debug/tiem | 2 +- dockerfiles/products/le6.5-linux-amd64/tiem | 2 +- .../le6.5-linux-arm64/dm-monitor-initializer | 2 +- packages/README.md | 6 +-- packages/offline-packages.yaml.tmpl | 1 - ...compose-offline-packages-artifacts.sh.tmpl | 6 +-- .../gen-delivery-images-with-config.sh | 15 +++---- .../gen-package-artifacts-with-config.sh | 2 +- .../scripts/gen-package-images-with-config.sh | 2 +- ...gen-package-offline-package-with-config.sh | 6 +-- .../get-package-builder-with-config.sh | 2 +- 34 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh index 82d2c919..16bff948 100755 --- a/.github/scripts/ci.sh +++ b/.github/scripts/ci.sh @@ -14,7 +14,7 @@ function test_get_builder() { for os in $operating_systems; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done done @@ -26,7 +26,7 @@ function test_get_builder() { for os in $operating_systems; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done done @@ -38,7 +38,7 @@ function test_get_builder() { for ac in $architectures; do for version in v1.5.0 v1.6.0; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done # tiflow-operator @@ -47,7 +47,7 @@ function test_get_builder() { for ac in $architectures; do for version in v6.4.0-20221102-1667359250 v20221018; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done @@ -57,7 +57,7 @@ function test_get_builder() { for ac in $architectures; do for version in v0.5.0 v0.6.0; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done @@ -67,7 +67,7 @@ function test_get_builder() { for ac in $architectures; do for version in v0.1.2 v0.1.3; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile + $script "$cm" "$os" "$ac" "$version" $profile done done } @@ -85,7 +85,7 @@ function test_gen_package_artifacts_script() { for os in $operating_systems; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -98,7 +98,7 @@ function test_gen_package_artifacts_script() { for os in $operating_systems; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version enterprise branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" enterprise branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -111,7 +111,7 @@ function test_gen_package_artifacts_script() { for os in $operating_systems; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version failpoint branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" failpoint branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -125,7 +125,7 @@ function test_gen_package_artifacts_script() { for ac in $architectures; do for version in v1.5.0 v1.6.0; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -136,7 +136,7 @@ function test_gen_package_artifacts_script() { for ac in $architectures; do for version in v6.4.0-20221102-1667359250 v20221018; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -147,7 +147,7 @@ function test_gen_package_artifacts_script() { for ac in $architectures; do for version in v0.5.0 v0.6.0; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -158,7 +158,7 @@ function test_gen_package_artifacts_script() { for ac in $architectures; do for version in v0.1.2 v0.1.3; do echo "$cm $os $ac $version:" - $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-artifacts.sh done done @@ -175,7 +175,7 @@ function test_gen_package_images_script() { for cm in $components; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -186,7 +186,7 @@ function test_gen_package_images_script() { for version in $versions; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -198,7 +198,7 @@ function test_gen_package_images_script() { for ac in $architectures; do for version in v1.6.0 v1.5.0; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -208,7 +208,7 @@ function test_gen_package_images_script() { for ac in $architectures; do for version in v6.4.0-20221102-1667359250 v20221018; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -219,7 +219,7 @@ function test_gen_package_images_script() { for version in v0.5.0 v0.6.0; do for ac in $architectures; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -230,7 +230,7 @@ function test_gen_package_images_script() { for ac in $architectures; do for version in v0.1.2 v0.1.3; do echo "$cm $os $ac $version:" - $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef shellcheck -S error packages/scripts/build-package-images.sh done done @@ -248,7 +248,7 @@ function test_gen_offline_package_artifacts_script() { for ac in $architectures; do for edition in $editions; do echo "$os $ac $version $edition:" - $script $os $ac $version $edition + $script "$os" "$ac" "$version" "$edition" shellcheck -S error packages/scripts/compose-offline-packages-artifacts.sh done done diff --git a/.github/workflows/pull-prod-runtime-images.yaml b/.github/workflows/pull-prod-runtime-images.yaml index 0ec19228..52e0d13b 100644 --- a/.github/workflows/pull-prod-runtime-images.yaml +++ b/.github/workflows/pull-prod-runtime-images.yaml @@ -58,7 +58,7 @@ jobs: run: | curl -L https://github.com/estesp/manifest-tool/releases/download/v2.1.7/binaries-manifest-tool-2.1.7.tar.gz | tar -zxvf - manifest-tool-linux-amd64 sudo install manifest-tool-linux-amd64 /usr/local/bin/manifest-tool - + - name: Cache layers uses: actions/cache@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8e06d579 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +ci: + autofix_commit_msg: | + fix: auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autoupdate_commit_msg: "chore: pre-commit autoupdate" +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/gitleaks/gitleaks + rev: v8.16.1 + hooks: + - id: gitleaks diff --git a/dockerfiles-multi-stages/tiflash/Dockerfile b/dockerfiles-multi-stages/tiflash/Dockerfile index 79a3ffde..e8c7c43b 100644 --- a/dockerfiles-multi-stages/tiflash/Dockerfile +++ b/dockerfiles-multi-stages/tiflash/Dockerfile @@ -41,4 +41,4 @@ COPY --from=busybox:1.36.1 /bin/busybox /bin/busybox ENV LD_LIBRARY_PATH /tiflash COPY --from=builder /tiflash/output/tiflash /tiflash -ENTRYPOINT ["/tiflash/tiflash", "server"] \ No newline at end of file +ENTRYPOINT ["/tiflash/tiflash", "server"] diff --git a/dockerfiles/bases/build-old.sh b/dockerfiles/bases/build-old.sh index 8567247c..6aa1cb6c 100755 --- a/dockerfiles/bases/build-old.sh +++ b/dockerfiles/bases/build-old.sh @@ -8,25 +8,25 @@ build() { case "${PLATFORMS}" in linux/amd64) echo "building for linux/amd64 platform..." - docker build --tag=$IMAGE --platform=$PLATFORMS --target amd64 . -f "$dockerfile" + docker build --tag="$IMAGE" --platform="$PLATFORMS" --target amd64 . -f "$dockerfile" if $PUSH_IMAGE; then - docker push $IMAGE + docker push "$IMAGE" fi ;; linux/arm64) echo "building for linux/arm64 platform..." - docker build --tag=$IMAGE --platform=$PLATFORMS --target arm64 . -f "$dockerfile" + docker build --tag="$IMAGE" --platform="$PLATFORMS" --target arm64 . -f "$dockerfile" if $PUSH_IMAGE; then - docker push $IMAGE + docker push "$IMAGE" fi ;; linux/arm64,linux/amd64 | linux/amd64,linux/arm64) echo "building for linux/arm64 and linux/amd64 platforms..." - docker build --tag=${IMAGE}_linux_amd64 --platform=linux/amd64 --target amd64 . -f "$dockerfile" - docker build --tag=${IMAGE}_linux_arm64 --platform=linux/arm64 --target arm64 . -f "$dockerfile" + docker build --tag="${IMAGE}_linux_amd64" --platform=linux/amd64 --target amd64 . -f "$dockerfile" + docker build --tag="${IMAGE}_linux_arm64" --platform=linux/arm64 --target arm64 . -f "$dockerfile" if $PUSH_IMAGE; then - docker push ${IMAGE}_linux_amd64 - docker push ${IMAGE}_linux_arm64 + docker push "${IMAGE}_linux_amd64" + docker push "${IMAGE}_linux_arm64" # compose manifest for multi-arch image. pushed_repo="${IMAGE%:*}" @@ -35,14 +35,14 @@ build() { yq -i ".tags = [\"$tag\"]" manifest.yaml # linux/amd64 - manifest-tool inspect --raw ${IMAGE}_linux_amd64 >manifest_linux_amd64.json + manifest-tool inspect --raw "${IMAGE}_linux_amd64" >manifest_linux_amd64.json yq -i '.manifests += [{}]' manifest.yaml digest=$(jq -r '.digest' manifest_linux_amd64.json) yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml yq -i '.manifests[-1].platform.os = "linux"' manifest.yaml yq -i '.manifests[-1].platform.architecture = "amd64"' manifest.yaml # linux/arm64 - manifest-tool inspect --raw ${IMAGE}_linux_arm64 >manifest_linux_arm64.json + manifest-tool inspect --raw "${IMAGE}_linux_arm64" >manifest_linux_arm64.json yq -i '.manifests += [{}]' manifest.yaml digest=$(jq -r '.digest' manifest_linux_arm64.json) yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml diff --git a/dockerfiles/bases/tikv-base/release-6.5.Dockerfile b/dockerfiles/bases/tikv-base/release-6.5.Dockerfile index 67195b93..e5a5f853 100644 --- a/dockerfiles/bases/tikv-base/release-6.5.Dockerfile +++ b/dockerfiles/bases/tikv-base/release-6.5.Dockerfile @@ -13,4 +13,3 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \ # set timezone ENV TZ=/etc/localtime ENV TZDIR=/usr/share/zoneinfo - diff --git a/dockerfiles/cd/builders/ng-monitoring/Dockerfile b/dockerfiles/cd/builders/ng-monitoring/Dockerfile index ec8a9ec4..de3f5ae8 100644 --- a/dockerfiles/cd/builders/ng-monitoring/Dockerfile +++ b/dockerfiles/cd/builders/ng-monitoring/Dockerfile @@ -2,7 +2,7 @@ # - docker >= v20.10 # # build steps: -# - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring +# - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring # - rm ng-monitoring/.dockerignore # make step depended on git metadata. # - docker build -t ng-monitoring -f Dockerfile ./ng-monitoring @@ -10,7 +10,7 @@ FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for NG-Monitoring" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ diff --git a/dockerfiles/cd/builders/pd/Dockerfile b/dockerfiles/cd/builders/pd/Dockerfile index 1bf5e104..07122f05 100644 --- a/dockerfiles/cd/builders/pd/Dockerfile +++ b/dockerfiles/cd/builders/pd/Dockerfile @@ -9,7 +9,7 @@ FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for PD" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ diff --git a/dockerfiles/cd/builders/tidb-dashboard/Dockerfile b/dockerfiles/cd/builders/tidb-dashboard/Dockerfile index ee0c346f..5832eb51 100644 --- a/dockerfiles/cd/builders/tidb-dashboard/Dockerfile +++ b/dockerfiles/cd/builders/tidb-dashboard/Dockerfile @@ -2,7 +2,7 @@ # - docker >= v20.10 # # build steps: -# - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord +# - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord # - rm tidb-dashbord/.dockerignore # make step depended on git metadata. # - docker build -t tidb-dashbord -f Dockerfile ./tidb-dashbord @@ -10,7 +10,7 @@ FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiDB Dashboard" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ diff --git a/dockerfiles/cd/builders/tidb-operator/Dockerfile b/dockerfiles/cd/builders/tidb-operator/Dockerfile index 584b3063..f93045a7 100644 --- a/dockerfiles/cd/builders/tidb-operator/Dockerfile +++ b/dockerfiles/cd/builders/tidb-operator/Dockerfile @@ -2,7 +2,7 @@ # - docker >= v20.10 # # build steps: -# - git clone https://github.com/pingcap/tidb-operator.git tidb +# - git clone https://github.com/pingcap/tidb-operator.git tidb # - rm tidb-operator/.dockerignore # make step depended on git metadata. # - docker build -t tidb-operator -f Dockerfile ./tidb-operator @@ -11,7 +11,7 @@ ARG GOLANG_VERSION=1.21.13 FROM golang:${GOLANG_VERSION} as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiDB Operator" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" LABEL go-version="${GOLANG_VERSION}" # install packages. diff --git a/dockerfiles/cd/builders/tidb/Dockerfile b/dockerfiles/cd/builders/tidb/Dockerfile index 2274e8a0..752ceeb5 100644 --- a/dockerfiles/cd/builders/tidb/Dockerfile +++ b/dockerfiles/cd/builders/tidb/Dockerfile @@ -2,7 +2,7 @@ # - docker >= v20.10 # # build steps: -# - git clone https://github.com/pingcap/tidb.git tidb +# - git clone https://github.com/pingcap/tidb.git tidb # - rm tidb/.dockerignore # make step depended on git metadata. # - docker build -t tidb -f Dockerfile ./tidb @@ -10,7 +10,7 @@ FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiDB" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ diff --git a/dockerfiles/cd/builders/tiflash/Dockerfile b/dockerfiles/cd/builders/tiflash/Dockerfile index ecfd1b27..4ec52013 100644 --- a/dockerfiles/cd/builders/tiflash/Dockerfile +++ b/dockerfiles/cd/builders/tiflash/Dockerfile @@ -24,7 +24,7 @@ RUN zip_url=$([ "$(arch)" = "x86_64" ] && \ FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiFlash" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ @@ -38,12 +38,12 @@ ENV LD ld.lld COPY --from=extract-conda-openssl /usr/local/opt/openssl /usr/local/opt/openssl # building and install libc++ -RUN --mount=type=cache,target=/tmp/llvm \ - git clone https://github.com/llvm/llvm-project.git --branch llvmorg-17.0.6 --depth 1 /tmp/llvm/llvm-project && \ - cd /tmp/llvm/llvm-project && rm -rf build && mkdir build && \ +RUN git clone https://github.com/llvm/llvm-project.git --branch llvmorg-17.0.6 --depth 1 /tmp/llvm-project && \ + cd /tmp/llvm-project && rm -rf build && mkdir build && \ cmake -G Ninja -S runtimes -B build -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" -DCMAKE_INSTALL_PREFIX=/usr -DLLVM_LIBDIR_SUFFIX=64 && \ ninja -C build cxx cxxabi unwind && \ - ninja -C build install-cxx install-cxxabi install-unwind + ninja -C build install-cxx install-cxxabi install-unwind && \ + rm -rf /tmp/llvm-project # install rust toolchain RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none diff --git a/dockerfiles/cd/builders/tiflow/Dockerfile b/dockerfiles/cd/builders/tiflow/Dockerfile index cf2b2ff6..7b20ece2 100644 --- a/dockerfiles/cd/builders/tiflow/Dockerfile +++ b/dockerfiles/cd/builders/tiflow/Dockerfile @@ -43,7 +43,7 @@ RUN node --version && npm --version && npm install -g yarn FROM builder-dm as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiFlow" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" ########### stage: building-cdc FROM builder-cdc as building-cdc diff --git a/dockerfiles/cd/builders/tikv/Dockerfile b/dockerfiles/cd/builders/tikv/Dockerfile index 91f03a6c..0bc5bf45 100644 --- a/dockerfiles/cd/builders/tikv/Dockerfile +++ b/dockerfiles/cd/builders/tikv/Dockerfile @@ -9,7 +9,7 @@ FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiKV" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. # install packages. diff --git a/dockerfiles/cd/builders/tikv/fips.Dockerfile b/dockerfiles/cd/builders/tikv/fips.Dockerfile index b4652e74..51786e68 100644 --- a/dockerfiles/cd/builders/tikv/fips.Dockerfile +++ b/dockerfiles/cd/builders/tikv/fips.Dockerfile @@ -9,7 +9,7 @@ FROM rockylinux:9.3.20231119 as builder LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiKV with FIPS support" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN dnf install -y \ diff --git a/dockerfiles/cd/utils/ks3util/Dockerfile b/dockerfiles/cd/utils/ks3util/Dockerfile index 39c39421..aad47230 100644 --- a/dockerfiles/cd/utils/ks3util/Dockerfile +++ b/dockerfiles/cd/utils/ks3util/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.20.2 LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "utils image for CD release" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install tools: k3sutil ARG KS3UTIL_VER=2.4.2 diff --git a/dockerfiles/cd/utils/release/Dockerfile b/dockerfiles/cd/utils/release/Dockerfile index fc4bb5d2..0b5f0619 100644 --- a/dockerfiles/cd/utils/release/Dockerfile +++ b/dockerfiles/cd/utils/release/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.20.2 LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "utils image for CD release" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install tools: bash, git, jq, yq # RUN apk add --no-cache bash git jq yq uuidgen gomplate diff --git a/dockerfiles/cd/utils/remote/Dockerfile b/dockerfiles/cd/utils/remote/Dockerfile index 411cbc1d..f6c74f55 100644 --- a/dockerfiles/cd/utils/remote/Dockerfile +++ b/dockerfiles/cd/utils/remote/Dockerfile @@ -2,6 +2,6 @@ FROM alpine:3.20.2 LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "utils image for remote building" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" RUN apk add --no-cache bash openssh-client rsync diff --git a/dockerfiles/ci/base/Dockerfile b/dockerfiles/ci/base/Dockerfile index 9d2595dd..b400daa0 100644 --- a/dockerfiles/ci/base/Dockerfile +++ b/dockerfiles/ci/base/Dockerfile @@ -6,7 +6,7 @@ FROM golang:${GOLANG_VERSION} LABEL go-version "${GOLANG_VERSION}" LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "CI Base image for building/testing" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN apt-get update && \ @@ -67,7 +67,6 @@ RUN curl -sL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \ apt-get clean && \ npm install -g yarn pnpm -# Upgrade Git tool +# Upgrade Git tool # renovate: datasource=github-tags depName=git/git -COPY --from=bitnami/git:2.46.0 /opt/bitnami/git/bin/git /usr/bin/git - +COPY --from=bitnami/git:2.46.0 /opt/bitnami/git/bin/git /usr/bin/git diff --git a/dockerfiles/ci/release-build-base/Dockerfile b/dockerfiles/ci/release-build-base/Dockerfile index 3d55ed6b..f0f7f2c2 100644 --- a/dockerfiles/ci/release-build-base/Dockerfile +++ b/dockerfiles/ci/release-build-base/Dockerfile @@ -8,7 +8,7 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \ LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "CD Base image for building" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # install packages. RUN yum update --nogpgcheck -y && \ diff --git a/dockerfiles/ci/skaffold.yaml b/dockerfiles/ci/skaffold.yaml index 2bf211b0..feccd7ec 100644 --- a/dockerfiles/ci/skaffold.yaml +++ b/dockerfiles/ci/skaffold.yaml @@ -62,7 +62,7 @@ profiles: path: /build/artifacts/0/kaniko/buildArgs value: # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.20.10 + GOLANG_VERSION: 1.20.10 - name: go-1.19 patches: - op: replace @@ -72,7 +72,7 @@ profiles: path: /build/artifacts/0/kaniko/buildArgs value: # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.19.12 + GOLANG_VERSION: 1.19.12 --- apiVersion: skaffold/v4beta6 kind: Config diff --git a/dockerfiles/ee-apps/ats/Dockerfile b/dockerfiles/ee-apps/ats/Dockerfile index ff3c2ab8..8c6ee0df 100644 --- a/dockerfiles/ee-apps/ats/Dockerfile +++ b/dockerfiles/ee-apps/ats/Dockerfile @@ -2,7 +2,7 @@ FROM debian:12.6 LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" -LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" # renovate: datasource=github-tags depName=apache/trafficserver ARG TRAFFIC_SERVER_VERSION=9.2.4 diff --git a/dockerfiles/ee-apps/nginx-s3/example.nginx.conf b/dockerfiles/ee-apps/nginx-s3/example.nginx.conf index b6447535..5445cd06 100644 --- a/dockerfiles/ee-apps/nginx-s3/example.nginx.conf +++ b/dockerfiles/ee-apps/nginx-s3/example.nginx.conf @@ -27,7 +27,7 @@ http { #tcp_nopush on; gzip on; keepalive_timeout 65; - + # Define the server block for the default server server { listen 80 ; @@ -57,6 +57,6 @@ http { proxy_intercept_errors on; rewrite .* /$url_full break; proxy_pass http://$bucket; - } + } } } diff --git a/dockerfiles/products/le6.5-linux-amd64-debug/tiem b/dockerfiles/products/le6.5-linux-amd64-debug/tiem index 3d962745..851ce487 100644 --- a/dockerfiles/products/le6.5-linux-amd64-debug/tiem +++ b/dockerfiles/products/le6.5-linux-amd64-debug/tiem @@ -12,4 +12,4 @@ COPY bin/cluster-server ./bin COPY bin/metadb-server ./bin COPY build_helper/docker_start_cmd.sh ./scripts EXPOSE 4116 -ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"] diff --git a/dockerfiles/products/le6.5-linux-amd64/tiem b/dockerfiles/products/le6.5-linux-amd64/tiem index b0932587..6b41824b 100644 --- a/dockerfiles/products/le6.5-linux-amd64/tiem +++ b/dockerfiles/products/le6.5-linux-amd64/tiem @@ -12,4 +12,4 @@ COPY bin/cluster-server ./bin COPY bin/metadb-server ./bin COPY build_helper/docker_start_cmd.sh ./scripts EXPOSE 4116 -ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/usr/local/tiem/scripts/docker_start_cmd.sh"] diff --git a/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer b/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer index 1e19314c..04d5a667 100644 --- a/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer +++ b/dockerfiles/products/le6.5-linux-arm64/dm-monitor-initializer @@ -8,4 +8,4 @@ COPY rules/*.rules.yml /tmp/ COPY datasources/*.yaml /tmp/ ENTRYPOINT ["/usr/bin/init.sh"] -CMD ["TIDB-Cluster", "/grafana-dashboard-definitions/tidb/", "false", "/etc/prometheus"] \ No newline at end of file +CMD ["TIDB-Cluster", "/grafana-dashboard-definitions/tidb/", "false", "/etc/prometheus"] diff --git a/packages/README.md b/packages/README.md index 5c691cfb..0354368a 100644 --- a/packages/README.md +++ b/packages/README.md @@ -1,4 +1,4 @@ -Central declarative congfigurations for artifacts delivering. +Central declarative congfigurations for artifacts delivering. === > We use go template format to control them. @@ -6,8 +6,8 @@ Central declarative congfigurations for artifacts delivering. ## Prerequire tools -- [gomplate](https://github.com/hairyhenderson/gomplate) - > Please install the master version. +- [gomplate](https://github.com/hairyhenderson/gomplate) + > Please install the master version. - [yq](https://github.com/mikefarah/yq) - [jq](https://jqlang.github.io/jq/download/) diff --git a/packages/offline-packages.yaml.tmpl b/packages/offline-packages.yaml.tmpl index d620fa8e..0d7399cd 100644 --- a/packages/offline-packages.yaml.tmpl +++ b/packages/offline-packages.yaml.tmpl @@ -1204,4 +1204,3 @@ editions: url: "http://fileserver.pingcap.net/download/builds/pingcap/mydumper/0042e179b74fd9cd731be4550bee4c284c212be6/centos7/mydumper-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" extract: true extract_inner_path: "mydumper-{{ .Release.os }}-{{ .Release.arch }}/bin/mydumper" - diff --git a/packages/scripts/compose-offline-packages-artifacts.sh.tmpl b/packages/scripts/compose-offline-packages-artifacts.sh.tmpl index 6006c4ff..c27c4b21 100644 --- a/packages/scripts/compose-offline-packages-artifacts.sh.tmpl +++ b/packages/scripts/compose-offline-packages-artifacts.sh.tmpl @@ -141,12 +141,12 @@ function pre_check_artifact{{ $i }}() { {{- range . }} echo -n "🔍 check tiup package: {{ .name }}@{{ .src.version }} in platform $os/$arch." {{- if eq .src.version "latest" }} - tiup list {{ .name }} | grep -E '^v\d+\.\d+\.\d+\s+' | tail -1 | grep -E "($os/$arch|any/any)" > /dev/null && echo " ✅" || { + tiup list {{ .name }} | grep -E '^v\d+\.\d+\.\d+\s+' | tail -1 | grep -E "($os/$arch|any/any)" > /dev/null && echo " ✅" || { echo " ❌" exit 1 } {{- else }} - tiup list {{ .name }} | grep -E '^{{ .src.version }}\s+' | grep "$os/$arch" > /dev/null && echo " ✅" || { + tiup list {{ .name }} | grep -E '^{{ .src.version }}\s+' | grep "$os/$arch" > /dev/null && echo " ✅" || { echo " ❌" exit 1 } @@ -158,7 +158,7 @@ function pre_check_artifact{{ $i }}() { {{- range ($components | jq `map(select(.src.type == "oci"))`) }} # {{ .name }} echo -n "🔍 check file '{{ .name }}' from OCI artifact: {{ .src.url }} -> {{ .src.path }}" - check_file_in_oci_artifact {{ .src.url }} "{{ .src.path }}" && echo " ✅" || { + check_file_in_oci_artifact {{ .src.url }} "{{ .src.path }}" && echo " ✅" || { echo " ❌" exit 1 } diff --git a/packages/scripts/gen-delivery-images-with-config.sh b/packages/scripts/gen-delivery-images-with-config.sh index d074b811..70c6fb9c 100755 --- a/packages/scripts/gen-delivery-images-with-config.sh +++ b/packages/scripts/gen-delivery-images-with-config.sh @@ -9,27 +9,24 @@ main() { local image_url_with_tag="$1" local yaml_file="${2:-${PROJECT_ROOT_DIR}/packages/delivery.yaml}" local out_file="${3:-${RELEASE_SCRIPTS_DIR}/delivery-images.sh}" - rm -rf $out_file + rm -rf "$out_file" # Extract image URL and tag - local image_url=$(echo "$image_url_with_tag" | cut -d ':' -f1) - local tag=$(echo "$image_url_with_tag" | cut -d ':' -f2) + image_url="$(echo "$image_url_with_tag" | cut -d ':' -f1)" + tag=$(echo "$image_url_with_tag" | cut -d ':' -f2) # Retrieve rules for the source repository from the YAML config - local rules=$(yq ".image_copy_rules[\"$image_url\"]" "$yaml_file") - local rule_count=$(yq ".image_copy_rules[\"$image_url\"] | length" "$yaml_file") - if [ $rule_count -eq 0 ]; then + rule_count=$(yq ".image_copy_rules[\"$image_url\"] | length" "$yaml_file") + if [ "$rule_count" -eq 0 ]; then echo "🤷 none rules found for image: $image_url" exit 0 fi # Loop through each rule for the source repository for ri in $(seq 0 $((rule_count - 1))); do - echo $ri - local rule=$(yq ".image_copy_rules[\"$image_url\"][$ri]" "$yaml_file") + rule=$(yq ".image_copy_rules[\"$image_url\"][$ri]" "$yaml_file") description=$(yq '.description' <<< "$rule") dest_repos=$(yq '.dest_repositories[]' <<< "$rule") - dest_tags= # Check if the tag matches the tags regex if yq -e ".tags_regex[] | select(. | test \"$tag\")" 2>&1> /dev/null <<< "$rule" ; then diff --git a/packages/scripts/gen-package-artifacts-with-config.sh b/packages/scripts/gen-package-artifacts-with-config.sh index 621394e2..d50c7e3d 100755 --- a/packages/scripts/gen-package-artifacts-with-config.sh +++ b/packages/scripts/gen-package-artifacts-with-config.sh @@ -66,7 +66,7 @@ function main() { exit 0 fi - gomplate --context .=release-router.yaml -f $RELEASE_SCRIPTS_DIR/build-package-artifacts.sh.tmpl --chmod "755" --out $out_file + gomplate --context .=release-router.yaml -f "$RELEASE_SCRIPTS_DIR/build-package-artifacts.sh.tmpl" --chmod "755" --out "$out_file" echo "✅ Generated shell script: $out_file" } diff --git a/packages/scripts/gen-package-images-with-config.sh b/packages/scripts/gen-package-images-with-config.sh index 2ab671b8..cb172bbd 100755 --- a/packages/scripts/gen-package-images-with-config.sh +++ b/packages/scripts/gen-package-images-with-config.sh @@ -72,7 +72,7 @@ function main() { exit 0 fi - gomplate --context .=release-router.yaml -f $RELEASE_SCRIPTS_DIR/build-package-images.sh.tmpl --chmod "755" --out $out_file + gomplate --context .=release-router.yaml -f "$RELEASE_SCRIPTS_DIR/build-package-images.sh.tmpl" --chmod "755" --out "$out_file" echo "✅ Generated shell script: $out_file" } diff --git a/packages/scripts/gen-package-offline-package-with-config.sh b/packages/scripts/gen-package-offline-package-with-config.sh index 82dfb1ba..7a1e9d99 100755 --- a/packages/scripts/gen-package-offline-package-with-config.sh +++ b/packages/scripts/gen-package-offline-package-with-config.sh @@ -65,9 +65,9 @@ function main() { } function install_tiup_tool() { - local run_os="$(uname -s)" - local run_arch="$(uname -m)" - local tiup_ver="v1.15.2" + run_os="$(uname -s)" + run_arch="$(uname -m)" + tiup_ver="v1.15.2" local tiup_tarball case "${run_os}/${run_arch}" in diff --git a/packages/scripts/get-package-builder-with-config.sh b/packages/scripts/get-package-builder-with-config.sh index 75dad2e6..a3fd94eb 100755 --- a/packages/scripts/get-package-builder-with-config.sh +++ b/packages/scripts/get-package-builder-with-config.sh @@ -40,7 +40,7 @@ function main() { fi # get the builder image. - yq -e '.builders[0].image' release-package.yaml | tee $out_file + yq -e '.builders[0].image' release-package.yaml | tee "$out_file" } main "$@"