ci: make gen-code work properly (#509)

* fix gen-code

Signed-off-by: zirain <zirain2009@gmail.com>

* lint license

Signed-off-by: zirain <zirain2009@gmail.com>

* add comment

Signed-off-by: zirain <zirain2009@gmail.com>

* fix REPO_ROOT

Signed-off-by: zirain <zirain2009@gmail.com>

* add kustomize build

Signed-off-by: zirain <zirain2009@gmail.com>

* remove usage of util::install_tools

Signed-off-by: zirain <zirain2009@gmail.com>

* nit

Signed-off-by: zirain <zirain2009@gmail.com>

* rename to TOOLS_DIR

Signed-off-by: zirain <zirain2009@gmail.com>

* rename to HACK_DIR

Signed-off-by: zirain <zirain2009@gmail.com>

* fix gen-thanos

Signed-off-by: zirain <zirain2009@gmail.com>

---------

Signed-off-by: zirain <zirain2009@gmail.com>

.gitignore

@@ -31,3 +31,6 @@ node_modules/
 # ignore all local dev files
 .vscode/
 .local/
+
+.tools
+.gopath

Makefile

@@ -20,7 +20,7 @@ LDFLAGS := "-X kurator.dev/kurator/pkg/version.gitVersion=$(VERSION) \
 GO_BUILD=CGO_ENABLED=0 GOOS=$(GOOS) go build -ldflags $(LDFLAGS)
 DOCKER_BUILD=docker build --build-arg BASE_VERSION=nonroot --build-arg BASE_IMAGE=gcr.io/distroless/static
 
-FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./out -o -path ./.github  -o -path ./hack -o -path ./licenses -o -path ./vendor \) -prune -o -type f
+FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./out -o -path ./.github  -o -path ./hack -o -path ./licenses -o -path ./vendor -o -path ./.gopath \) -prune -o -type f
 XARGS = xargs -0 -r
 
 IMAGE_HUB ?= ghcr.io/kurator-dev
@@ -36,6 +36,8 @@ else
 endif
 export PATH := $(GOBIN):$(PATH)
 
+include Makefile.tools.mk
+
 .PHONY: build
 build: tidy kurator cluster-operator fleet-manager
 
@@ -107,22 +109,23 @@ fix-copyright:
 	@${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\
 		${XARGS} hack/fix_copyright_banner.sh
 
-golangci-lint:
+.PHONY: golangci-lint
+golangci-lint: $(golangci-lint) ## Run golangci-lint
 	hack/golangci-lint.sh
 
-init-gen:
-	hack/init-gen-tools.sh
+.PHONY: init-gen-tools
+init-gen-tools: $(jb) $(gojsontoyaml) $(jsonnet)
 
 .PHONY: gen-prom
-gen-prom: init-gen
+gen-prom: init-gen-tools
 	hack/gen-prom.sh manifests/jsonnet/prometheus/prometheus.jsonnet manifests/profiles/prom/
 
 .PHONY: gen-prom-thanos
-gen-prom-thanos: init-gen
+gen-prom-thanos: init-gen-tools
 	hack/gen-prom.sh manifests/jsonnet/prometheus/thanos.jsonnet manifests/profiles/prom-thanos/
 
 .PHONY: gen-thanos
-gen-thanos: init-gen
+gen-thanos: init-gen-tools
 	hack/gen-thanos.sh
 
 .PHONY: sync-crds
@@ -141,7 +144,9 @@ test: clean tidy
 clean:
 	go clean -testcache
 	go clean -cache
-	rm -rf $(OUT_BASE_PATH)
+	@rm -rf $(OUT_BASE_PATH)
+	@rm -rf .tools
+	@rm -rf .gopath
 
 .PHONY: gen
 gen: clean \
@@ -168,8 +173,13 @@ doc.build:
 	KURATOR_VERSION=$(VERSION) hack/local-docsite-build.sh
 
 PHONY: init-codegen
-init-codegen:
-	hack/init-codegen.sh
+init-codegen: $(kustomize) \
+				$(deepcopy-gen) \
+				$(client-gen) \
+				$(lister-gen) \
+				$(informer-gen) \
+				$(register-gen) \
+ 			    $(controller-gen) ## Install code generation tools
 
 .PHONY: gen-api
 gen-api: gen-code gen-crd gen-api-doc
@@ -178,12 +188,34 @@ gen-api: gen-code gen-crd gen-api-doc
 gen-crd: init-codegen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	hack/update-crdgen.sh
 
+
+PACKAGE					    := kurator.dev
+GOPATH_SHIM                 := ${PWD}/.gopath
+PACKAGE_SHIM                := $(GOPATH_SHIM)/src/$(PACKAGE)
+
+$(GOPATH_SHIM):
+	@echo Create gopath shim... >&2
+	@mkdir -p $(GOPATH_SHIM)
+
+# learn from kyverno/kyverno project, this will allow you run client-gen everywhere without put project into GOPATH
+# DO NOT REMOVE THIS `.INTERMEDIATE`
+.INTERMEDIATE: $(PACKAGE_SHIM)
+$(PACKAGE_SHIM): $(GOPATH_SHIM)
+	@echo Create package shim... >&2
+	@mkdir -p $(GOPATH_SHIM)/src/kurator.dev && ln -s -f ${PWD} $(PACKAGE_SHIM)
+
 .PHONY: gen-code
-gen-code: init-codegen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+gen-code: $(PACKAGE_SHIM) init-codegen gen-code-clean ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	hack/update-codegen.sh
 
+.PHONY: gen-code-clean
+gen-code-clean: ## Clean up generated files
+	@echo "Cleaning up generated files..."
+	@find pkg/apis -type f -name zz_generated* | xargs rm
+	@find pkg/client-go -type f -name *.go | xargs rm
+
 .PHONY: gen-api-doc
-gen-api-doc: ## Generate API documentation
+gen-api-doc: $(gen-crd-api-reference-docs) ## Generate API documentation
 	hack/gen-api-doc.sh
 
 .PHONY: release-artifacts

Makefile.tools.mk

@@ -0,0 +1,36 @@
+# Go tools directory holds the binaries of Go-based tools.
+GO           := $(shell which go)
+TOOLS_DIR    ?= $(PWD)/.tools
+
+# Catch all rules for Go-based tools.
+$(GO_TOOLS_DIR)/%:
+	@GOBIN=$(TOOLS_DIR) $(GO) install $($(notdir $@)@v)
+
+# Go-based tools.
+golangci-lint 	:= $(TOOLS_DIR)/golangci-lint
+client-gen    	:= $(TOOLS_DIR)/client-gen
+informer-gen  	:= $(TOOLS_DIR)/informer-gen
+lister-gen    	:= $(TOOLS_DIR)/lister-gen
+register-gen  	:= $(TOOLS_DIR)/register-gen
+deepcopy-gen  	:= $(TOOLS_DIR)/deepcopy-gen
+controller-gen	:= $(TOOLS_DIR)/controller-gen
+kustomize       := $(TOOLS_DIR)/kustomize
+jb				:= $(TOOLS_DIR)/jb
+gojsontoyaml    := $(TOOLS_DIR)/gojsontoyaml
+jsonnet         := $(TOOLS_DIR)/jsonnet
+
+gen-crd-api-reference-docs := $(TOOLS_DIR)/gen-crd-api-reference-docs
+
+golangci-lint@v 	:= github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
+client-gen@v    	:= k8s.io/code-generator/cmd/client-gen@v0.25.2
+informer-gen@v  	:= k8s.io/code-generator/cmd/informer-gen@v0.25.2
+lister-gen@v    	:= k8s.io/code-generator/cmd/lister-gen@v0.25.2
+register-gen@v  	:= k8s.io/code-generator/cmd/register-gen@v0.25.2
+deepcopy-gen@v  	:= k8s.io/code-generator/cmd/deepcopy-gen@v0.25.2
+controller-gen@v 	:= sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0
+kustomize@v       	:= sigs.k8s.io/kustomize/kustomize/v4@v4.5.5
+jb@v                := github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1
+gojsontoyaml@v      := github.com/brancz/gojsontoyaml@v0.1.0
+jsonnet@v           := github.com/google/go-jsonnet/cmd/jsonnet@v0.18.0
+
+gen-crd-api-reference-docs@v := github.com/ahmetb/gen-crd-api-reference-docs@45bac9a # 2023-03-28

hack/gen-api-doc.sh

@@ -7,25 +7,19 @@ set -o pipefail
 
 # For all commands, the working directory is the parent directory(repo root).
 REPO_ROOT=$(git rev-parse --show-toplevel)
-cd "${REPO_ROOT}"
-source hack/util.sh
+GEN_CRD_API_REFERENCE_DOCS=${REPO_ROOT}/.tools/gen-crd-api-reference-docs
 
 CONFIG_FILE="${REPO_ROOT}/hack/api-docs/config.json"
 TEMPLATE_DIR="${REPO_ROOT}/hack/api-docs/template"
 OUT_DIR="${REPO_ROOT}/docs/content/en/references"
 API_DIR="./pkg/apis"
-
-if ! [ -x "$(command -v gen-crd-api-reference-docs)" ]; then
-    util::install_tools github.com/ahmetb/gen-crd-api-reference-docs 45bac9a # 2023-03-28
-fi
-
 API_GROUPS=("cluster" "infra" "fleet" "apps" "backups" "pipeline")
 
 for APIGROUP in "${API_GROUPS[@]}"
 do
     OUT_FILE="${OUT_DIR}/${APIGROUP}_v1alpha1_types.html"
     echo "Generating docs for ${APIGROUP}/v1alpha1 to ${OUT_FILE}"
-    gen-crd-api-reference-docs \
+    ${GEN_CRD_API_REFERENCE_DOCS} \
       --api-dir="${API_DIR}/${APIGROUP}/v1alpha1" \
       --config="${CONFIG_FILE}" \
       --template-dir="${TEMPLATE_DIR}" \

hack/gen-chart.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+REPO_ROOT=$(git rev-parse --show-toplevel)
 
 OUT_BASE_PATH=${REPO_ROOT}/out
 CHART_OUT_PATH=${OUT_BASE_PATH}/charts

hack/gen-prom.sh

@@ -6,7 +6,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+JB="${REPO_ROOT}/.tools/jb"
 PROM_OUT_PATH=${REPO_ROOT}/out/prom
 PROM_JSONNET_FILE=${REPO_ROOT}/$1
 PROM_MANIFESTS_PATH=${REPO_ROOT}/${2}
@@ -24,12 +25,12 @@ mkdir -p "${PROM_OUT_PATH}"
 cp "${PROM_JSONNET_FILE}" "${PROM_OUT_PATH}/kube-prometheus.jsonnet"
 
 pushd "${PROM_OUT_PATH}"
-    jb init
-    jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@"${KUBE_PROM_VER}"
+    ${JB} init
+    ${JB} install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@"${KUBE_PROM_VER}"
     wget https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/"${KUBE_PROM_VER}"/build.sh -O build.sh
-    jb update
+    ${JB} update
 
-    bash build.sh kube-prometheus.jsonnet
+    PATH="${REPO_ROOT}/.tools:$PATH" bash build.sh kube-prometheus.jsonnet
 popd
 
 cp -r "${PROM_OUT_PATH}"/manifests/* "${PROM_MANIFESTS_PATH}"

hack/gen-thanos.sh

@@ -6,7 +6,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+JB="${REPO_ROOT}/.tools/jb"
 THANOS_OUT_PATH="${REPO_ROOT}/out/thanos"
 THANOS_MANIFESTS_PATH="${REPO_ROOT}/manifests/profiles/thanos"
 KUBE_THANOS_VER=${KUBE_THANOS_VER:-v0.26.0}
@@ -22,13 +23,13 @@ echo "path: ${THANOS_OUT_PATH}";
 echo "version: ${KUBE_THANOS_VER}"
 
 pushd "${THANOS_OUT_PATH}"
-    jb init
-    jb install github.com/thanos-io/kube-thanos/jsonnet/kube-thanos@"${KUBE_THANOS_VER}"
-    jb update
+    ${JB} init
+    ${JB} install github.com/thanos-io/kube-thanos/jsonnet/kube-thanos@"${KUBE_THANOS_VER}"
+    ${JB} update
 
     cp "${REPO_ROOT}/hack/build-thanos.sh" build.sh
 
-    bash build.sh thanos.jsonnet
+    PATH="${REPO_ROOT}/.tools:$PATH" bash build.sh thanos.jsonnet
 popd
 
 cp -r "${THANOS_OUT_PATH}"/manifests/* "${THANOS_MANIFESTS_PATH}"

hack/golangci-lint.sh

@@ -4,18 +4,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-GOLANGCI_LINT_PKG="github.com/golangci/golangci-lint/cmd/golangci-lint"
-GOLANGCI_LINT_VER="v1.51.2"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+GOLANGCI_LINT=${REPO_ROOT}/.tools/golangci-lint
 
-cd "${REPO_ROOT}"
-source "hack/util.sh"
-
-if ! [ -x "$(command -v golangci-lint)" ]; then
-  util::install_tools ${GOLANGCI_LINT_PKG} ${GOLANGCI_LINT_VER}
-fi
-
-if golangci-lint run -c "$REPO_ROOT/common/config/.golangci.yml"; then
+if ${GOLANGCI_LINT} run -c "$REPO_ROOT/common/config/.golangci.yml"; then
   echo 'Congratulations!  All Go source files have passed staticcheck.'
 else
   echo # print one empty line, separate from warning messages.

hack/local-dev-setup.sh

@@ -10,9 +10,9 @@ set -o pipefail
 # This script depends on utils in: ${REPO_ROOT}/hack/util.sh
 # 1. used by developer to setup develop environment quickly.
 # 2. used by e2e testing to setup test environment automatically.
-REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
-KIND_CONFIGS_ROOT=${REPO_ROOT}/kind-configs
-source "${REPO_ROOT}"/util.sh
+ROOT_DIR=$(git rev-parse --show-toplevel)/hack
+KIND_CONFIGS_ROOT=${ROOT_DIR}/kind-configs
+source "${ROOT_DIR}"/util.sh
 
 KIND_VERSION=${KIND_VERSION:-"kindest/node:v1.25.3"}
 
@@ -31,7 +31,7 @@ TEMP_PATH=$(mktemp -d)
 echo -e "Preparing kind config in path: ${TEMP_PATH}"
 #When the Enable worker option is turned on, select to copy the configuration that contains the worker.
 if [ ${ENABLE_KIND_WITH_WORKER} = "true" ]; then
-    cp -rf ${REPO_ROOT}/kind-configs-with-worker/*.yaml "${TEMP_PATH}"/
+    cp -rf ${ROOT_DIR}/kind-configs-with-worker/*.yaml "${TEMP_PATH}"/
 else
     cp -rf "${KIND_CONFIGS_ROOT}"/*.yaml "${TEMP_PATH}"/
 fi

hack/local-setup-cluster.sh

@@ -5,9 +5,9 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
-KIND_CONFIGS_ROOT=${REPO_ROOT}/kind-configs
-source "${REPO_ROOT}"/util.sh
+HACK_DIR=$(git rev-parse --show-toplevel)/hack
+KIND_CONFIGS_ROOT=${HACK_DIR}/kind-configs
+source "${HACK_DIR}"/util.sh
 
 KIND_VERSION=${KIND_VERSION:-"kindest/node:v1.25.3"}
 
@@ -20,7 +20,7 @@ TEMP_PATH=$(mktemp -d)
 echo -e "Preparing kind config in path: ${TEMP_PATH}"
 #When the Enable worker option is turned on, select to copy the configuration that contains the worker.
 if [ ${ENABLE_KIND_WITH_WORKER} = "true" ]; then
-    cp -rf ${REPO_ROOT}/kind-configs-with-worker/*.yaml "${TEMP_PATH}"/
+    cp -rf ${HACK_DIR}/kind-configs-with-worker/*.yaml "${TEMP_PATH}"/
 else
     cp -rf "${KIND_CONFIGS_ROOT}"/*.yaml "${TEMP_PATH}"/
 fi

hack/release-artifacts.sh

@@ -8,7 +8,7 @@ GOOS=${GOOS:-"linux"}
 GOARCH=${GOARCH:-"amd64"}
 
 VERSION=${VERSION:-"latest"}
-REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
+REPO_ROOT=$(git rev-parse --show-toplevel)
 OUT_BASE_PATH=${OUT_BASE_PATH:-"${REPO_ROOT}/out"}
 RELEASE_PATH="${OUT_BASE_PATH}/release-artifacts"
 CHART_PATH="${OUT_BASE_PATH}/charts"