Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick to 7.12: [PACKAGING] Push docker images with the architecture in the version (#24121) #24159

Merged
merged 1 commit into from
Feb 23, 2021
Merged

Cherry-pick to 7.12: [PACKAGING] Push docker images with the architecture in the version (#24121) #24159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 46 additions & 23 deletions .ci/packaging.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,9 @@ pipeline {
withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
deleteDir()
release()
pushCIDockerImages()
dir("${BASE_DIR}"){
pushCIDockerImages(arch: 'amd64')
}
}
prepareE2ETestForPackage("${BEATS_FOLDER}")
}
Expand Down Expand Up @@ -234,7 +236,9 @@ pipeline {
withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
deleteWorkspace()
release()
pushCIDockerImages()
dir("${BASE_DIR}"){
pushCIDockerImages(arch: 'arm64')
}
}
}
post {
Expand Down Expand Up @@ -270,27 +274,37 @@ pipeline {
}
}

def pushCIDockerImages(){
/**
* @param arch what architecture
*/
def pushCIDockerImages(Map args = [:]) {
def arch = args.get('arch', 'amd64')
catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
if (env?.BEATS_FOLDER?.endsWith('auditbeat')) {
tagAndPush('auditbeat')
tagAndPush(beatName: 'auditbeat', arch: arch)
} else if (env?.BEATS_FOLDER?.endsWith('filebeat')) {
tagAndPush('filebeat')
tagAndPush(beatName: 'filebeat', arch: arch)
} else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
tagAndPush('heartbeat')
tagAndPush(beatName: 'heartbeat', arch: arch)
} else if ("${env.BEATS_FOLDER}" == "journalbeat"){
tagAndPush('journalbeat')
tagAndPush(beatName: 'journalbeat', arch: arch)
} else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
tagAndPush('metricbeat')
tagAndPush(beatName: 'metricbeat', arch: arch)
} else if ("${env.BEATS_FOLDER}" == "packetbeat"){
tagAndPush('packetbeat')
tagAndPush(beatName: 'packetbeat', arch: arch)
} else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
tagAndPush('elastic-agent')
tagAndPush(beatName: 'elastic-agent', arch: arch)
}
}
}

def tagAndPush(beatName){
/**
* @param beatName name of the Beat
* @param arch what architecture
*/
def tagAndPush(Map args = [:]) {
def beatName = args.beatName
def arch = args.get('arch', 'amd64')
def libbetaVer = env.BEAT_VERSION
def aliasVersion = ""
if("${env.SNAPSHOT}" == "true"){
Expand All @@ -307,14 +321,22 @@ def tagAndPush(beatName){

dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")

// supported tags
def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
if (!isPR() && aliasVersion != "") {
tags << aliasVersion
}
// supported image flavours
def variants = ["", "-oss", "-ubi8"]
variants.each { variant ->
doTagAndPush(beatName, variant, libbetaVer, tagName)
doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")

if (!isPR() && aliasVersion != "") {
doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
tags.each { tag ->
// TODO:
// For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
// they support the versioning with the architecture
if ("${arch}" == "amd64") {
doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
}
doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
}
}
}
Expand All @@ -325,18 +347,19 @@ def tagAndPush(beatName){
* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
*/
def doTagAndPush(beatName, variant, sourceTag, targetTag) {
def doTagAndPush(Map args = [:]) {
def beatName = args.beatName
def variant = args.variant
def sourceTag = args.sourceTag
def targetTag = args.targetTag
def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"

def iterations = 0
retryWithSleep(retries: 3, seconds: 5, backoff: true) {
iterations++
def status = sh(label: "Change tag and push ${targetName}", script: """
docker tag ${sourceName} ${targetName}
docker push ${targetName}
""", returnStatus: true)

def status = sh(label: "Change tag and push ${targetName}",
script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}",
returnStatus: true)
if ( status > 0 && iterations < 3) {
error("tag and push failed for ${beatName}, retry")
} else if ( status > 0 ) {
Expand Down
12 changes: 12 additions & 0 deletions .ci/scripts/docker-tag-push.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -exuo pipefail
MSG="parameter missing."
SOURCE_IMAGE=${1:?$MSG}
TARGET_IMAGE=${2:?$MSG}

if docker image inspect "${SOURCE_IMAGE}" &> /dev/null ; then
docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"
docker push "${TARGET_IMAGE}"
else
echo "docker image ${SOURCE_IMAGE} does not exist"
fi
67 changes: 40 additions & 27 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,23 +321,26 @@ def uploadPackages(bucketUri, beatsFolder){
/**
* Push the docker images for the given beat.
* @param beatsFolder beats folder
* @param arch what architecture
*/
def pushCIDockerImages(beatsFolder){
def pushCIDockerImages(Map args = [:]) {
def arch = args.get('arch', 'amd64')
def beatsFolder = args.beatsFolder
catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
if (beatsFolder.endsWith('auditbeat')) {
tagAndPush('auditbeat')
tagAndPush(beatName: 'auditbeat', arch: arch)
} else if (beatsFolder.endsWith('filebeat')) {
tagAndPush('filebeat')
tagAndPush(beatName: 'filebeat', arch: arch)
} else if (beatsFolder.endsWith('heartbeat')) {
tagAndPush('heartbeat')
tagAndPush(beatName: 'heartbeat', arch: arch)
} else if ("${beatsFolder}" == "journalbeat"){
tagAndPush('journalbeat')
tagAndPush(beatName: 'journalbeat', arch: arch)
} else if (beatsFolder.endsWith('metricbeat')) {
tagAndPush('metricbeat')
tagAndPush(beatName: 'metricbeat', arch: arch)
} else if ("${beatsFolder}" == "packetbeat"){
tagAndPush('packetbeat')
tagAndPush(beatName: 'packetbeat', arch: arch)
} else if ("${beatsFolder}" == "x-pack/elastic-agent") {
tagAndPush('elastic-agent')
tagAndPush(beatName: 'elastic-agent', arch: arch)
}
}
}
Expand All @@ -346,7 +349,9 @@ def pushCIDockerImages(beatsFolder){
* Tag and push all the docker images for the given beat.
* @param beatName name of the Beat
*/
def tagAndPush(beatName){
def tagAndPush(Map args = [:]) {
def beatName = args.beatName
def arch = args.get('arch', 'amd64')
def libbetaVer = env.VERSION
if("${env?.SNAPSHOT.trim()}" == "true"){
aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
Expand All @@ -360,41 +365,40 @@ def tagAndPush(beatName){
tagName = "pr-${env.CHANGE_ID}"
}

// supported tags
def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
if (!isPR() && aliasVersion != "") {
tags << aliasVersion
}
// supported image flavours
def variants = ["", "-oss", "-ubi8"]
variants.each { variant ->
doTagAndPush(beatName, variant, libbetaVer, tagName)
doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")

if (!isPR() && aliasVersion != "") {
doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
tags.each { tag ->
doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
}
}
}

/**
* Tag and push the given sourceTag docker image with the tag name targetTag.
* @param beatName name of the Beat
* @param variant name of the variant used to build the docker image name
* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
*/
def doTagAndPush(beatName, variant, sourceTag, targetTag) {
def doTagAndPush(Map args = [:]) {
def beatName = args.beatName
def variant = args.variant
def sourceTag = args.sourceTag
def targetTag = args.targetTag
def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"

def iterations = 0
retryWithSleep(retries: 3, seconds: 5, backoff: true) {
iterations++
def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash
docker images
if docker image inspect "${sourceName}" &> /dev/null ; then
docker tag ${sourceName} ${targetName}
docker push ${targetName}
else
echo 'docker image ${sourceName} does not exist'
fi
""", returnStatus: true)
def status = sh(label: "Change tag and push ${targetName}",
script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}",
returnStatus: true)
if ( status > 0 && iterations < 3) {
error("tag and push failed for ${beatName}, retry")
} else if ( status > 0 ) {
Expand Down Expand Up @@ -462,6 +466,7 @@ def target(Map args = [:]) {
def isMage = args.get('isMage', false)
def isE2E = args.e2e?.get('enabled', false)
def isPackaging = args.get('package', false)
def dockerArch = args.get('dockerArch', 'amd64')
withNode(args.label) {
withGithubNotify(context: "${context}") {
withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
Expand All @@ -482,7 +487,7 @@ def target(Map args = [:]) {
// TODO:
// push docker images should happen only after the e2e?
if (isPackaging) {
pushCIDockerImages("${directory}")
pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
}
}
}
Expand Down Expand Up @@ -918,7 +923,15 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
}
if(args?.content?.containsKey('packaging-linux')) {
steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true)
steps.packagingLinux(context: args.context,
command: args.content.get('packaging-linux'),
directory: args.project,
label: args.label,
isMage: true,
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'amd64')
}
if(args?.content?.containsKey('k8sTest')) {
steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
Expand Down