From 8a6e746916b06f3ed78d6e0d42d4c48b46d2ede5 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 May 2024 09:56:34 +0100 Subject: [PATCH 1/7] Update lib-injection docker image tags --- .gitlab-ci.yml | 46 +++++++-------- .gitlab/build-lib-init.sh | 116 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 26 deletions(-) create mode 100644 .gitlab/build-lib-init.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb80cd236fc..5b8adfb844d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -213,47 +213,41 @@ deploy_artifacts_to_github: - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/*.jar - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/*.jar - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/*.jar - -deploy_to_docker_registries: + +generate-lib-init-tag-values: + tags: ["arch:amd64"] + image: registry.ddbuild.io/ci/auto_inject/gitlab:current stage: deploy rules: - - if: '$CI_COMMIT_TAG =~ /.*-[Rr][Cc].*/' - when: manual - allow_failure: true - if: '$POPULATE_CACHE' when: never - - if: '$CI_COMMIT_TAG =~ /^v.*/' + # We don't tag prerelease versions + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' when: on_success - when: manual allow_failure: true - trigger: - project: DataDog/public-images - branch: main - strategy: depend variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-java/dd-lib-java-init:$CI_COMMIT_SHA - IMG_DESTINATIONS: dd-lib-java-init:$CI_COMMIT_TAG - IMG_SIGNING: "false" + IMG_DESTINATION_BASE: dd-lib-java-init + script: + - ./.gitlab/build-lib-init.sh + artifacts: + reports: + dotenv: build.env -deploy_latest_tag_to_docker_registries: +deploy-lib-init-trigger: stage: deploy - rules: - - if: '$CI_COMMIT_TAG =~ /.*-[Rr][Cc].*/' - when: manual - allow_failure: true - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v1\..*/' - when: on_success - - when: manual - allow_failure: true + # needs the version from the generate-tag-values job + needs: + - job: generate-lib-init-tag-values + artifacts: true trigger: +# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing project: DataDog/public-images branch: main strategy: depend variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-java/dd-lib-java-init:$CI_COMMIT_SHA - IMG_DESTINATIONS: dd-lib-java-init:latest + IMG_SOURCES: ghcr.io/datadog/dd-trace-java/dd-lib-java-init:$CI_COMMIT_TAG + IMG_DESTINATIONS: $IMG_DESTINATIONS IMG_SIGNING: "false" create_key: diff --git a/.gitlab/build-lib-init.sh b/.gitlab/build-lib-init.sh new file mode 100644 index 00000000000..0161af08b10 --- /dev/null +++ b/.gitlab/build-lib-init.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +set -e + +# Safety checks to make sure we have required values +if [ -z "$CI_COMMIT_TAG" ]; then + echo "Error: CI_COMMIT_TAG was not provided" + exit 1 +fi + +if [ -z "$CI_COMMIT_SHA" ]; then + echo "Error: CI_COMMIT_SHA was not provided" + exit 1 +fi + +if [ -z "$IMG_DESTINATION_BASE" ]; then + echo "Error: IMG_DESTINATION_BASE. This should be set to the destination docker image, excluding the tag name, e.g. dd-lib-dotnet-init" + exit 1 +fi + +# If this is a pre-release release, we don't publish +if echo "$CI_COMMIT_TAG" | grep -q "-" > /dev/null; then + echo "Error: This is a pre-release version, should not publish images: $CI_COMMIT_TAG" + exit 1 +fi + +# Calculate the tags we use for floating major and minor versions +MAJOR_MINOR_VERSION="$(sed -nE 's/^(v[0-9]+\.[0-9]+)\.[0-9]+$/\1/p' <<< ${CI_COMMIT_TAG})" +MAJOR_VERSION="$(sed -nE 's/^(v[0-9]+)\.[0-9]+\.[0-9]+$/\1/p' <<< ${CI_COMMIT_TAG})" + +# Make sure we have all the tags +git fetch --tags + +# We need to determine whether this is is the latest tag and whether it's the latest major or not +# So we fetch all tags and sort them to find both the latest, and the latest in this major. +# 'sort' technically gets prerelease versions in the wrong order here, but we explicitly +# exclude them anyway, as they're ignored for the purposes of determining the 'latest' tags. +LATEST_TAG="$(git tag | grep -v '-' | sort -V -r | head -n 1)" +LATEST_MAJOR_TAG="$(git tag -l "$MAJOR_VERSION.*" | grep -v '-' | sort -V -r | head -n 1)" +echo "This tag: $CI_COMMIT_TAG" +echo "Latest repository tag: $LATEST_TAG" +echo "Latest repository tag for this major: $LATEST_MAJOR_TAG" +echo "---------" + +# GNU sort -C (silent) reports via exit code whether the data is already in sorted order +# We use this to check whether the current tag is greater than (or equal to) the latest tag +if printf '%s\n' "$LATEST_TAG" "$CI_COMMIT_TAG" | sort -C -V; then + # The current tag is the latest in the repository + IS_LATEST_TAG=1 +else + IS_LATEST_TAG=0 +fi + +if printf '%s\n' "$LATEST_MAJOR_TAG" "$CI_COMMIT_TAG" | sort -C -V; then + # The current tag is the latest for this major version in the repository + IS_LATEST_MAJOR_TAG=1 +else + IS_LATEST_MAJOR_TAG=0 +fi + +# print everything for debugging purposes +echo "Calculated values:" +echo "MAJOR_MINOR_VERSION=${MAJOR_MINOR_VERSION}" +echo "MAJOR_VERSION=${MAJOR_VERSION}" +echo "IS_LATEST_TAG=${IS_LATEST_TAG}" +echo "IS_LATEST_MAJOR_TAG=${IS_LATEST_MAJOR_TAG}" +echo "---------" + +# Final check that everything is ok +# We should have a major_minor version +if [ -z "$MAJOR_MINOR_VERSION" ]; then + echo "Error: Could not determine major_minor version for stable release, this should not happen" + exit 1 +fi + +# if this is a latest major tag, we should have a major version +if [ "$IS_LATEST_MAJOR_TAG" -eq 1 ] && [ -z "$MAJOR_VERSION" ]; then + echo "Error: Could not determine major version for latest major release, this should not happen" + exit 1 +fi + +# Generate the final variables, and save them into build.env so they can be read by the trigger job +set_image_tags() { + SUFFIX="$1" + VARIABLE_SUFFIX="${SUFFIX:+_$SUFFIX}" # add a '_' prefix + TAG_SUFFIX="${SUFFIX:+-$SUFFIX}" # add a '-' prefix + + # We always add this tag, regardless of the version + DESTINATIONS="${IMG_DESTINATION_BASE}:${CI_COMMIT_TAG}${TAG_SUFFIX}" + + # We always add the major_minor tag (we never release 2.5.2 _after_ 2.5.3, for example) + DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:${MAJOR_MINOR_VERSION}${TAG_SUFFIX}" + + # Only latest-major releases get the major tag + if [ "$IS_LATEST_MAJOR_TAG" -eq 1 ]; then + DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:${MAJOR_VERSION}${TAG_SUFFIX}" + fi + + # Only latest releases get the latest tag + if [ "$IS_LATEST_TAG" -eq 1 ]; then + DESTINATIONS="${DESTINATIONS},${IMG_DESTINATION_BASE}:latest${TAG_SUFFIX}" + fi + + # Save the value to the build.env file + echo "IMG_DESTINATIONS${VARIABLE_SUFFIX}=${DESTINATIONS}" + echo "IMG_DESTINATIONS${VARIABLE_SUFFIX}=${DESTINATIONS}" >> build.env +} + +# Calculate the non-suffixed tags +set_image_tags + +# For each suffix, calculate the tags +for ADDITIONAL_TAG_SUFFIX in ${ADDITIONAL_TAG_SUFFIXES//,/ } +do + set_image_tags "$ADDITIONAL_TAG_SUFFIX" +done \ No newline at end of file From 61b18ebfdde62d01bb17d9259e237ed4ca2856a8 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 May 2024 09:56:45 +0100 Subject: [PATCH 2/7] git update-index --chmod=+x .\.gitlab\build-lib-init.sh --- .gitlab/build-lib-init.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .gitlab/build-lib-init.sh diff --git a/.gitlab/build-lib-init.sh b/.gitlab/build-lib-init.sh old mode 100644 new mode 100755 From 92efccb8ea8d1e0e65f4fb2feb1e0d7547c8050f Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 May 2024 09:57:17 +0100 Subject: [PATCH 3/7] TESTING --- .gitlab-ci.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b8adfb844d..c2bb0acdd50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,16 +218,10 @@ generate-lib-init-tag-values: tags: ["arch:amd64"] image: registry.ddbuild.io/ci/auto_inject/gitlab:current stage: deploy - rules: - - if: '$POPULATE_CACHE' - when: never - # We don't tag prerelease versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - - when: manual - allow_failure: true + needs: [] variables: IMG_DESTINATION_BASE: dd-lib-java-init + CI_COMMIT_TAG: v1.23.2 script: - ./.gitlab/build-lib-init.sh artifacts: @@ -241,8 +235,8 @@ deploy-lib-init-trigger: - job: generate-lib-init-tag-values artifacts: true trigger: -# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing - project: DataDog/public-images + project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing +# project: DataDog/public-images branch: main strategy: depend variables: From a9bf19786064b19b89604c46ecc7e387b4f1556d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 May 2024 09:57:24 +0100 Subject: [PATCH 4/7] Revert "TESTING" This reverts commit 92efccb8ea8d1e0e65f4fb2feb1e0d7547c8050f. --- .gitlab-ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2bb0acdd50..5b8adfb844d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,10 +218,16 @@ generate-lib-init-tag-values: tags: ["arch:amd64"] image: registry.ddbuild.io/ci/auto_inject/gitlab:current stage: deploy - needs: [] + rules: + - if: '$POPULATE_CACHE' + when: never + # We don't tag prerelease versions + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' + when: on_success + - when: manual + allow_failure: true variables: IMG_DESTINATION_BASE: dd-lib-java-init - CI_COMMIT_TAG: v1.23.2 script: - ./.gitlab/build-lib-init.sh artifacts: @@ -235,8 +241,8 @@ deploy-lib-init-trigger: - job: generate-lib-init-tag-values artifacts: true trigger: - project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing -# project: DataDog/public-images +# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing + project: DataDog/public-images branch: main strategy: depend variables: From fce428beebf24110549f2b6c224129e20992b676 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 22 May 2024 15:02:30 +0100 Subject: [PATCH 5/7] Improve tag search check --- .gitlab/build-lib-init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/build-lib-init.sh b/.gitlab/build-lib-init.sh index 0161af08b10..6f47cfaaee1 100755 --- a/.gitlab/build-lib-init.sh +++ b/.gitlab/build-lib-init.sh @@ -35,8 +35,8 @@ git fetch --tags # So we fetch all tags and sort them to find both the latest, and the latest in this major. # 'sort' technically gets prerelease versions in the wrong order here, but we explicitly # exclude them anyway, as they're ignored for the purposes of determining the 'latest' tags. -LATEST_TAG="$(git tag | grep -v '-' | sort -V -r | head -n 1)" -LATEST_MAJOR_TAG="$(git tag -l "$MAJOR_VERSION.*" | grep -v '-' | sort -V -r | head -n 1)" +LATEST_TAG="$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V -r | head -n 1)" +LATEST_MAJOR_TAG="$(git tag -l "$MAJOR_VERSION.*" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V -r | head -n 1)" echo "This tag: $CI_COMMIT_TAG" echo "Latest repository tag: $LATEST_TAG" echo "Latest repository tag for this major: $LATEST_MAJOR_TAG" From 5d1c4bb3e20be2423a7a5942ddaf29b76e068120 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 29 May 2024 15:47:10 +0200 Subject: [PATCH 6/7] Revert accidental change to IMG_SOURCES --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b8adfb844d..35c7bb4ac50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -246,7 +246,7 @@ deploy-lib-init-trigger: branch: main strategy: depend variables: - IMG_SOURCES: ghcr.io/datadog/dd-trace-java/dd-lib-java-init:$CI_COMMIT_TAG + IMG_SOURCES: ghcr.io/datadog/dd-trace-java/dd-lib-java-init:$CI_COMMIT_SHA IMG_DESTINATIONS: $IMG_DESTINATIONS IMG_SIGNING: "false" From c722dcab0f77002aca810af7e1df8a0d9fd70b0f Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Fri, 31 May 2024 07:41:09 +0200 Subject: [PATCH 7/7] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35c7bb4ac50..b01f84ee14b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,7 +229,7 @@ generate-lib-init-tag-values: variables: IMG_DESTINATION_BASE: dd-lib-java-init script: - - ./.gitlab/build-lib-init.sh + - .gitlab/build-lib-init.sh artifacts: reports: dotenv: build.env