-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(apps/prod/tekton/configs/pipelines): add a new darwin build pipe…
…line (#1370) - combine multi tasks into one task: acquire resource then build with the resource machine then release the resource. - just use deno for run for build. based on implement: PingCAP-QE/artifacts#503 Signed-off-by: wuhuizuo <wuhuizuo@126.com> Signed-off-by: wuhuizuo <wuhuizuo@126.com>
- Loading branch information
Showing
7 changed files
with
332 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
165 changes: 165 additions & 0 deletions
165
apps/prod/tekton/configs/pipelines/pingcap-build-package-darwin-v2.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Pipeline | ||
| metadata: | ||
| name: pingcap-build-package-darwin-v2 | ||
| spec: | ||
| params: | ||
| - name: git-url | ||
| type: string | ||
| - name: git-ref | ||
| type: string | ||
| default: master | ||
| - name: git-refspec | ||
| default: +refs/heads/*:refs/heads/* | ||
| type: string | ||
| - name: git-revision | ||
| default: master | ||
| type: string | ||
| - name: component | ||
| type: string | ||
| description: | | ||
| compoent name, supports: | ||
| - tidb | ||
| - tikv | ||
| - pd | ||
| - tiflash | ||
| - tiflow | ||
| - advanced-statefulset | ||
| May be it is an idea to judge it by git-url, but it maybe | ||
| not a generic way for forked repositories with custom names. | ||
| - name: os | ||
| default: darwin | ||
| - name: arch | ||
| default: amd64 | ||
| - name: profile | ||
| default: release | ||
| - name: push | ||
| default: "true" | ||
| - name: registry | ||
| description: the base OCI registry server for store artifacts, it can be set with prefix repo path. | ||
| default: "hub.pingcap.net" | ||
| - name: force-builder-image | ||
| description: > | ||
| The builder image to use for building binaries by force, if empty. | ||
| Default the task will complete one from config in artifacts.git repo. | ||
| type: string | ||
| default: "" | ||
| - name: boskos-server-url | ||
| description: The URL of the running boskos server | ||
| default: http://boskos.apps.svc | ||
| results: | ||
| - name: pushed-binaries | ||
| description: pushed binaries. | ||
| value: "$(tasks.build-binaries.results.pushed)" | ||
| workspaces: | ||
| - name: source | ||
| description: The workspace where the git repo will be cloned. | ||
| - name: dockerconfig | ||
| description: Includes a docker `config.json` | ||
| - name: git-basic-auth | ||
| description: secret containing a .gitconfig and .git-credentials file. | ||
| optional: true | ||
| - name: mac-ssh-credentials | ||
| description: secret contains ssh private key in `id_rsa` key for login mac | ||
| tasks: | ||
| - name: checkout | ||
| retries: 2 | ||
| taskRef: | ||
| name: git-clone | ||
| params: | ||
| - name: url | ||
| value: $(params.git-url) | ||
| - name: revision | ||
| value: $(params.git-revision) | ||
| - name: depth | ||
| value: "0" | ||
| - name: refspec | ||
| value: $(params.git-refspec) | ||
| - name: subdirectory | ||
| value: $(params.component) | ||
| workspaces: | ||
| - name: output | ||
| workspace: source | ||
| - name: basic-auth | ||
| workspace: git-basic-auth | ||
| - name: checkout-ext | ||
| runAfter: | ||
| - checkout | ||
| taskRef: | ||
| name: pingcap-git-clone-ext | ||
| params: | ||
| - name: depth | ||
| value: "0" | ||
| - name: refspec | ||
| value: +refs/heads/*:refs/heads/* | ||
| - name: master-directory | ||
| value: $(params.component) | ||
| workspaces: | ||
| - name: output | ||
| workspace: source | ||
| - name: basic-auth | ||
| workspace: git-basic-auth | ||
| - name: get-release-ver | ||
| runAfter: | ||
| - checkout | ||
| taskRef: | ||
| name: pingcap-get-set-release-version | ||
| workspaces: | ||
| - name: source | ||
| workspace: source | ||
| subPath: $(params.component) | ||
| - name: get-binaries-builder | ||
| taskRef: | ||
| name: pingcap-get-builder-image | ||
| params: | ||
| - name: component | ||
| value: "$(params.component)" | ||
| - name: os | ||
| value: "$(params.os)" | ||
| - name: arch | ||
| value: "$(params.arch)" | ||
| - name: profile | ||
| value: "$(params.profile)" | ||
| - name: version | ||
| value: "$(tasks.get-release-ver.results.version)" | ||
| - name: force-builder-image | ||
| value: "$(params.force-builder-image)" | ||
| - name: build-binaries | ||
| runAfter: | ||
| - checkout-ext | ||
| taskRef: | ||
| name: pingcap-build-binaries-darwin-v2 | ||
| params: | ||
| - name: os | ||
| value: "$(params.os)" | ||
| - name: arch | ||
| value: "$(params.arch)" | ||
| - name: profile | ||
| value: "$(params.profile)" | ||
| - name: component | ||
| value: "$(params.component)" | ||
| - name: version | ||
| value: "$(tasks.get-release-ver.results.version)" | ||
| - name: git-ref | ||
| value: $(params.git-ref) | ||
| - name: git-sha | ||
| value: $(tasks.checkout.results.commit) | ||
| - name: builder-image | ||
| value: "$(tasks.get-binaries-builder.results.image-url)" | ||
| - name: release-dir | ||
| value: build | ||
| - name: push | ||
| value: "$(params.push)" | ||
| - name: registry | ||
| value: "$(params.registry)" | ||
| - name: boskos-server-url | ||
| value: $(params.boskos-server-url) | ||
| workspaces: | ||
| - name: source | ||
| workspace: source | ||
| - name: dockerconfig | ||
| workspace: dockerconfig | ||
| - name: ssh-directory | ||
| workspace: mac-ssh-credentials | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
150 changes: 150 additions & 0 deletions
150
apps/prod/tekton/configs/tasks/pingcap-build-binaries-darwin-v2.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| apiVersion: tekton.dev/v1beta1 | ||
| kind: Task | ||
| metadata: | ||
| name: pingcap-build-binaries-darwin-v2 | ||
| labels: | ||
| app.kubernetes.io/version: "2.0" | ||
| annotations: | ||
| tekton.dev/platforms: "linux/amd64,linux/arm64" | ||
| spec: | ||
| description: >- | ||
| Build binaries for the component. | ||
| workspaces: | ||
| - name: source | ||
| description: source code. | ||
| - name: dockerconfig | ||
| description: Includes a docker `config.json` | ||
| mountPath: /root/.docker | ||
| optional: true | ||
| - name: ssh-directory | ||
| description: ssh credential for remote building on mac. | ||
| results: | ||
| - description: Just built and pushed artifacts, it will be a yaml content. | ||
| name: pushed | ||
| params: | ||
| - name: component | ||
| - name: os | ||
| default: darwin | ||
| - name: arch | ||
| default: amd64 | ||
| - name: version | ||
| type: string | ||
| description: the version to releasing. | ||
| - name: profile | ||
| type: string | ||
| default: release | ||
| description: > | ||
| supports: 'release' or 'failpoint'. | ||
| - name: git-ref | ||
| - name: git-sha | ||
| - name: builder-image | ||
| default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928 | ||
| - name: release-dir | ||
| default: build | ||
| - name: push | ||
| type: string | ||
| default: 'false' | ||
| - name: registry | ||
| default: hub.pingcap.net | ||
| - name: boskos-server-url | ||
| description: The URL of the running boskos server | ||
| default: "http://boskos.test-pods.svc.cluster.local" | ||
| steps: | ||
| - name: generate-build-script | ||
| image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928 | ||
| script: | | ||
| git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts | ||
| git_ref="$(params.git-ref)" | ||
| git_sha="$(params.git-sha)" | ||
| if [ "$git_sha" == "$git_ref" ]; then | ||
| git_sha="" | ||
| fi | ||
| out_script="/workspace/build-package-artifacts.sh" | ||
| /workspace/artifacts/packages/scripts/gen-package-artifacts-with-config.sh \ | ||
| $(params.component) \ | ||
| $(params.os) \ | ||
| $(params.arch) \ | ||
| $(params.version) \ | ||
| $(params.profile) \ | ||
| $git_ref \ | ||
| "$git_sha" \ | ||
| /workspace/artifacts/packages/packages.yaml.tmpl \ | ||
| "$out_script" \ | ||
| $(params.registry) | ||
| if [ -f "$out_script" ]; then | ||
| sed -i 's/fetch_file_from_oci_artifact hub.pingcap.net/fetch_file_from_oci_artifact hub-mig.pingcap.net/g' "$out_script" | ||
| cat "$out_script" | ||
| else | ||
| echo "🤷 no output script generated!" | ||
| fi | ||
| - name: prepare-remote-env-file | ||
| image: "$(params.builder-image)" | ||
| script: | | ||
| :> /workspace/remote.env | ||
| # 1. set locale lang | ||
| echo "export LC_ALL=C.UTF-8" >> /workspace/remote.env | ||
| # 2. optional get go builder in PATH env var, got the go version(x.y) from the current container with `go version`. | ||
| if go version; then | ||
| go_bin_path="/usr/local/$(go version | cut -d ' ' -f 3 | cut -d '.' -f -2)/bin" | ||
| echo "export PATH=${go_bin_path}:\$PATH" >> /workspace/remote.env | ||
| fi | ||
| # 3. set npm mirror and node options | ||
| echo 'export NPM_CONFIG_REGISTRY="https://registry.npmmirror.com"' >> /workspace/remote.env | ||
| echo 'export NODE_OPTIONS="--max_old_space_size=8192"' >> /workspace/remote.env | ||
| # 4. optional set for rust | ||
| echo 'export CARGO_NET_GIT_FETCH_WITH_CLI=true' >> /workspace/remote.env | ||
| echo 'export RUSTUP_DIST_SERVER="https://rsproxy.cn"' >> /workspace/remote.env | ||
| echo 'export RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup"' >> /workspace/remote.env | ||
| - name: build | ||
| image: docker.io/denoland/deno:alpine-2.1.3 | ||
| env: | ||
| - name: WORKSPACE_SSH_DIRECTORY_BOUND | ||
| value: $(workspaces.ssh-directory.bound) | ||
| - name: WORKSPACE_SSH_DIRECTORY_PATH | ||
| value: $(workspaces.ssh-directory.path) | ||
| workingDir: $(workspaces.source.path)/$(params.component) | ||
| # TODO: get the user and host by better way. | ||
| script: | | ||
| script="/workspace/build-package-artifacts.sh" | ||
| if [ ! -f "$script" ]; then | ||
| echo "No build script, skip it." | ||
| exit 0 | ||
| fi | ||
| if [ "${WORKSPACE_SSH_DIRECTORY_BOUND}" != "true" ] ; then | ||
| echo "No ssh directory bound, we need it." | ||
| exit 1 | ||
| fi | ||
| env_file="/workspace/remote.env" | ||
| deno run --allow-all https://github.com/PingCAP-QE/artifacts/raw/refs/heads/main/packages/scripts/build-in-darwin-boskos.ts \ | ||
| --sshInfoDir ${WORKSPACE_SSH_DIRECTORY_PATH} \ | ||
| --sourcePath $(workspaces.source.path) \ | ||
| --envFile ${env_file} \ | ||
| --scriptFile ${script} \ | ||
| --component $(params.component) \ | ||
| --boskos.serverUrl $(params.boskos-server-url) \ | ||
| --boskos.type mac-machine-$(params.arch) \ | ||
| --boskos.owner $(context.taskRun.name) \ | ||
| --releaseDir $(params.release-dir) | ||
| - name: publish | ||
| image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928 | ||
| workingDir: $(workspaces.source.path)/$(params.component) | ||
| script: | | ||
| script="/workspace/build-package-artifacts.sh" | ||
| if [ ! -f "$script" ]; then | ||
| echo "No build script, skip it." | ||
| printf '"{}"' > $(results.pushed.path) | ||
| exit 0 | ||
| fi | ||
| oras version | ||
| # try 2 times. | ||
| "$script" -p -w "$(params.release-dir)" -o $(results.pushed.path) || \ | ||
| "$script" -p -w "$(params.release-dir)" -o $(results.pushed.path) |
Oops, something went wrong.