From 01480cad829d8f5bd53c68485c3b68e7546a9709 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 5 Apr 2024 21:54:17 -0400 Subject: [PATCH] One JDK to rule them all (#6865) * reduce number of jvms provided * consistent sha length --------- Signed-off-by: Justin Florentine Signed-off-by: amsmota --- .github/workflows/docker.yml | 12 +- build.gradle | 176 +++++++------------------ docker/{openjdk-latest => }/Dockerfile | 0 docker/graalvm/Dockerfile | 49 ------- docker/openj9-jdk-17/Dockerfile | 72 ---------- docker/openjdk-17-debug/Dockerfile | 63 --------- docker/openjdk-17/Dockerfile | 53 -------- docker/test.sh | 6 +- 8 files changed, 52 insertions(+), 379 deletions(-) rename docker/{openjdk-latest => }/Dockerfile (100%) delete mode 100644 docker/graalvm/Dockerfile delete mode 100644 docker/openj9-jdk-17/Dockerfile delete mode 100644 docker/openjdk-17-debug/Dockerfile delete mode 100644 docker/openjdk-17/Dockerfile diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 592adc141c6..5cb78f215d4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,16 +21,8 @@ jobs: uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: cache-disabled: true - - name: hadoLint_openj9-jdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile - - name: hadoLint_openjdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - name: hadoLint_openjdk_17_debug - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - name: hadoLint_openjdk_latest - run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - name: hadoLint_graalvm - run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + - name: hadoLint + run: docker run --rm -i hadolint/hadolint < docker/Dockerfile buildDocker: needs: hadolint permissions: diff --git a/build.gradle b/build.gradle index f4493ddb6e2..fb37b350d56 100644 --- a/build.gradle +++ b/build.gradle @@ -704,13 +704,6 @@ def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project def dockerOrgName = project.hasProperty('dockerOrgName') ? project.getProperty("dockerOrgName") : "hyperledger" def dockerArtifactName = project.hasProperty("dockerArtifactName") ? project.getProperty("dockerArtifactName") : "besu" def dockerImageName = "${dockerOrgName}/${dockerArtifactName}" -def dockerVariants = project.hasProperty("dockerVariants") ? project.getProperty("dockerVariants").split(",") : [ - "openjdk-17", - "openjdk-17-debug", - "openj9-jdk-17", - "graalvm", - "openjdk-latest" -] // rename the top level dir from besu- to besu and this makes it really // simple for use in docker @@ -738,34 +731,21 @@ task distDocker { def dockerBuildDir = "build/docker-besu/" doLast { - for (def jvmVariant in dockerVariants) { - copy { - from file("${projectDir}/docker/${jvmVariant}/Dockerfile") - into(dockerBuildDir) - } - exec { - def image = "${dockerImageName}:${dockerBuildVersion}-${jvmVariant}" - def dockerPlatform = "" - if (project.hasProperty('docker-platform')){ - dockerPlatform = "--platform ${project.getProperty('docker-platform')}" - println "Building for platform ${project.getProperty('docker-platform')}" - } - def gitDetails = getGitCommitDetails(10) - executable shell - workingDir dockerBuildDir - args "-c", "docker build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." - } - } - - // tag the "default" (which is the variant in the zero position) - exec { - executable shell - args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:${dockerBuildVersion}'" + copy { + from file("${projectDir}/docker/Dockerfile") + into(dockerBuildDir) } - // create a static tag for the benchmark target exec { + def image = "${dockerImageName}:${dockerBuildVersion}" + def dockerPlatform = "" + if (project.hasProperty('docker-platform')){ + dockerPlatform = "--platform ${project.getProperty('docker-platform')}" + println "Building for platform ${project.getProperty('docker-platform')}" + } + def gitDetails = getGitCommitDetails(7) executable shell - args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:benchmark'" + workingDir dockerBuildDir + args "-c", "docker build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." } } } @@ -779,14 +759,13 @@ task testDocker { } doLast { - for (def variant in dockerVariants) { - exec { - def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}" - workingDir "${projectDir}/docker/${variant}" - executable shell - args "-c", "../test.sh ${image}-${variant}" - } + exec { + def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}" + workingDir "${projectDir}/docker" + executable shell + args "-c", "./test.sh ${image}" } + } } @@ -805,22 +784,9 @@ task dockerUpload { } doLast { - for (def variant in dockerVariants) { - def variantImage = "${image}-${variant}" - exec { - def archVariantImage = "${variantImage}-${architecture}" - def cmd = "docker tag '${variantImage}' '${archVariantImage}' && docker push '${archVariantImage}'" - additionalTags.each { tag -> cmd += " && docker tag '${variantImage}' '${dockerImageName}:${tag.trim()}-${variant}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${variant}-${architecture}'" } - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - } - exec { - def archImage = "${image}-${architecture}" - def cmd = "docker tag ${image} ${archImage} && docker push '${archImage}'" - additionalTags.each { tag -> cmd += " && docker tag '${image}' '${dockerImageName}:${tag.trim()}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${architecture}'" } + def archVariantImage = "${image}-${architecture}" + def cmd = "docker tag '${image}' '${archVariantImage}' && docker push '${archVariantImage}'" println "Executing '${cmd}'" executable shell args "-c", cmd @@ -834,25 +800,9 @@ task dockerUploadRelease { doLast { for (def architecture in archs) { - for (def variant in dockerVariants) { - def variantImage = "${image}-${variant}" - exec { - def cmd = "docker pull '${variantImage}-${architecture}' && docker tag '${variantImage}-${architecture}' '${dockerImageName}:latest-${variant}-${architecture}'" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - exec { - def cmd = "docker push '${dockerImageName}:latest-${variant}-${architecture}'" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - } exec { - def archImage = "${image}-${architecture}" - def cmd = "docker pull '${archImage}' && docker tag ${archImage} '${dockerImageName}:latest-${architecture}'" + def cmd = "docker pull '${image}-${architecture}' && docker tag '${image}-${architecture}' '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" executable shell args "-c", cmd @@ -863,54 +813,17 @@ task dockerUploadRelease { executable shell args "-c", cmd } - } - } -} - -task manifestDocker { - def image = "${dockerImageName}:${dockerBuildVersion}" - def archs = ["arm64", "amd64"] - def tags = ["${image}"] - if (project.hasProperty('branch') && project.property('branch') == 'main') { - tags.add("${dockerImageName}:develop") - } - - if (!isInterimBuild(dockerBuildVersion)) { - tags.add("${dockerImageName}:" + dockerBuildVersion.split(/\./)[0..1].join('.')) - } - - doLast { - for (baseTag in tags) { - for (def variant in dockerVariants) { - def variantImage = "${baseTag}-${variant}" - def targets = "" - archs.forEach { arch -> targets += "'${variantImage}-${arch}' " } - - exec { - def cmd = "docker manifest create '${variantImage}' ${targets}" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - exec { - def cmd = "docker manifest push '${variantImage}'" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - } exec { - def targets = "" - archs.forEach { arch -> targets += "'${baseTag}-${arch}' " } - def cmd = "docker manifest create '${baseTag}' ${targets}" + def archImage = "${image}-${architecture}" + def cmd = "docker pull '${archImage}' && docker tag ${archImage} '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" executable shell args "-c", cmd } exec { - def cmd = "docker manifest push '${baseTag}'" + def cmd = "docker push '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" executable shell args "-c", cmd @@ -919,29 +832,34 @@ task manifestDocker { } } +task manifestDocker { + def image = "${dockerImageName}:${dockerBuildVersion}" + def archs = [ + "arm64", + "amd64"] //TODO: this assumes dockerUpload task has already been run on 2 different archs! + doLast { + exec { + def targets = "" + archs.forEach { arch -> targets += "'${image}-${arch}' " } + def cmd = "docker manifest create '${image}' ${targets}" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + exec { + def cmd = "docker manifest push '${image}'" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + } +} + task manifestDockerRelease { def archs = ["arm64", "amd64"] def baseTag = "${dockerImageName}:latest"; doLast { - for (def variant in dockerVariants) { - def variantImage = "${baseTag}-${variant}" - def targets = "" - archs.forEach { arch -> targets += "'${variantImage}-${arch}' " } - - exec { - def cmd = "docker manifest create '${variantImage}' ${targets} --amend" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - exec { - def cmd = "docker manifest push '${variantImage}'" - println "Executing '${cmd}'" - executable shell - args "-c", cmd - } - } exec { def targets = "" diff --git a/docker/openjdk-latest/Dockerfile b/docker/Dockerfile similarity index 100% rename from docker/openjdk-latest/Dockerfile rename to docker/Dockerfile diff --git a/docker/graalvm/Dockerfile b/docker/graalvm/Dockerfile deleted file mode 100644 index d4009b9c929..00000000000 --- a/docker/graalvm/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ - -FROM ghcr.io/graalvm/graalvm-ce:ol9-java17 -ARG VERSION="dev" - -RUN adduser --home /opt/besu besu && \ - chown besu:besu /opt/besu && \ - chmod 0755 /opt/besu - -USER besu -WORKDIR /opt/besu - -COPY --chown=besu:besu besu /opt/besu/ -RUN chmod -R 755 /opt/besu - -# Expose services ports -# 8545 HTTP JSON-RPC -# 8546 WS JSON-RPC -# 8547 HTTP GraphQL -# 8550 HTTP ENGINE JSON-RPC -# 8551 WS ENGINE JSON-RPC -# 30303 P2P -EXPOSE 8545 8546 8547 8550 8551 30303 - -# defaults for host interfaces -ENV BESU_RPC_HTTP_HOST 0.0.0.0 -ENV BESU_RPC_WS_HOST 0.0.0.0 -ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0 -ENV BESU_PID_PATH "/tmp/pid" - -ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION" - -ENV OLDPATH="${PATH}" -ENV PATH="/opt/besu/bin:${OLDPATH}" - -ENTRYPOINT ["besu"] -HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]" - -# Build-time metadata as defined at http://label-schema.org -ARG BUILD_DATE -ARG VCS_REF -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="Besu" \ - org.label-schema.description="Enterprise Ethereum client" \ - org.label-schema.url="https://besu.hyperledger.org/" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/hyperledger/besu.git" \ - org.label-schema.vendor="Hyperledger" \ - org.label-schema.version=$VERSION \ - org.label-schema.schema-version="1.0" diff --git a/docker/openj9-jdk-17/Dockerfile b/docker/openj9-jdk-17/Dockerfile deleted file mode 100644 index bbbe52cb1eb..00000000000 --- a/docker/openj9-jdk-17/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -FROM ubuntu:22.04 - -ARG VERSION="dev" -ARG TARGETPLATFORM - -RUN apt-get update && \ - apt-get install --no-install-recommends -q --assume-yes curl=7* libjemalloc-dev=5.* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN ARCH=$(uname -m) && \ - if [ "$ARCH" = "aarch64" ]; then \ - curl -kL -o jdk-17.tar.gz https://github.com/ibmruntimes/semeru17-binaries/releases/download/jdk-17.0.9%2B9_openj9-0.41.0/ibm-semeru-open-jre_aarch64_linux_17.0.9_9_openj9-0.41.0.tar.gz ; \ - elif [ "$(uname -s)" = "Darwin" ] && [ "$(uname -m)" = "arm64" ]; then \ - curl -kL -o jdk-17.tar.gz https://github.com/ibmruntimes/semeru17-binaries/releases/download/jdk-17.0.9%2B9_openj9-0.41.0/ibm-semeru-open-jre_aarch64_mac_17.0.9_9_openj9-0.41.0.tar.gz ; \ - elif [ "$ARCH" = "x86_64" ]; then \ - curl -kL -o jdk-17.tar.gz https://github.com/ibmruntimes/semeru17-binaries/releases/download/jdk-17.0.9%2B9_openj9-0.41.0/ibm-semeru-open-jre_x64_linux_17.0.9_9_openj9-0.41.0.tar.gz ; \ - else \ - echo "Unsupported platform: $ARCH"; exit 1; \ - fi - -RUN tar -xzf jdk-17.tar.gz && \ - rm jdk-17.tar.gz && \ - mv jdk-17.0.9+9-jre /usr/bin/ && \ - update-alternatives --install "/usr/bin/java" "java" "/usr/bin/jdk-17.0.9+9-jre/bin/java" 1 && \ - adduser --disabled-password --gecos "" --home /opt/besu besu && \ - chown besu:besu /opt/besu && \ - chmod 0755 /opt/besu - -ENV JAVA_HOME /usr/bin/jdk-17.0.9+9-jre/ -RUN export JAVA_HOME - -USER besu -WORKDIR /opt/besu - -COPY --chown=besu:besu besu /opt/besu/ - -# Expose services ports -# 8545 HTTP JSON-RPC -# 8546 WS JSON-RPC -# 8547 HTTP GraphQL -# 8550 HTTP ENGINE JSON-RPC -# 8551 WS ENGINE JSON-RPC -# 30303 P2P -EXPOSE 8545 8546 8547 8550 8551 30303 - -# defaults for host interfaces -ENV BESU_RPC_HTTP_HOST 0.0.0.0 -ENV BESU_RPC_WS_HOST 0.0.0.0 -ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0 -ENV BESU_PID_PATH "/tmp/pid" - -ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION" - -ENV OLDPATH="${PATH}" -ENV PATH="/opt/besu/bin:${OLDPATH}" - -ENTRYPOINT ["besu"] -HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]" - -# Build-time metadata as defined at http://label-schema.org -ARG BUILD_DATE -ARG VCS_REF -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="Besu" \ - org.label-schema.description="Enterprise Ethereum client" \ - org.label-schema.url="https://besu.hyperledger.org/" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/hyperledger/besu.git" \ - org.label-schema.vendor="Hyperledger" \ - org.label-schema.version=$VERSION \ - org.label-schema.schema-version="1.0" diff --git a/docker/openjdk-17-debug/Dockerfile b/docker/openjdk-17-debug/Dockerfile deleted file mode 100644 index c08f1ea4b9f..00000000000 --- a/docker/openjdk-17-debug/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ - -FROM ubuntu:22.04 - -ARG VERSION="dev" -RUN apt-get update && \ - apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909* && \ - apt-get install --no-install-recommends -q --assume-yes curl=7* wget=1.21* jq=1.6* net-tools=1.60* openjdk-17-jdk-headless=17* libjemalloc-dev=5.* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - adduser --disabled-password --gecos "" --home /opt/besu besu && \ - chown besu:besu /opt/besu && \ - chmod 0755 /opt/besu - -USER besu -WORKDIR /opt/besu - -COPY --chown=besu:besu besu /opt/besu/ - -# Expose services ports -# 5005 JDWP for attaching remote debuggers -# 8545 HTTP JSON-RPC -# 8546 WS JSON-RPC -# 8547 HTTP GraphQL -# 8550 HTTP ENGINE JSON-RPC -# 8551 WS ENGINE JSON-RPC -# 9545 Metrics -# 1098 JMX / JMX-RMI -# 30303 P2P -EXPOSE 5005 8545 8546 8547 8550 8551 9545 1098 30303 - -# defaults for host interfaces -ENV BESU_RPC_HTTP_HOST 0.0.0.0 -ENV BESU_RPC_WS_HOST 0.0.0.0 -ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0 -ENV BESU_METRICS_HOST 0.0.0.0 -ENV BESU_JMX_HOST 0.0.0.0 -ENV BESU_PID_PATH "/tmp/pid" - -#debug options for maximum observability -ENV BESU_LOGGING "INFO" -ENV JDWP_OPTS "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" -ENV JAVA_OPTS "${JDWP_OPTS} " - -ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION" - -ENV OLDPATH="${PATH}" -ENV PATH="/opt/besu/bin:${OLDPATH}" - -ENTRYPOINT ["besu"] -HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]" - -# Build-time metadata as defined at http://label-schema.org -ARG BUILD_DATE -ARG VCS_REF -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="Besu" \ - org.label-schema.description="Enterprise Ethereum client" \ - org.label-schema.url="https://besu.hyperledger.org/" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/hyperledger/besu.git" \ - org.label-schema.vendor="Hyperledger" \ - org.label-schema.version=$VERSION \ - org.label-schema.schema-version="1.0" diff --git a/docker/openjdk-17/Dockerfile b/docker/openjdk-17/Dockerfile deleted file mode 100644 index bb9f09c410a..00000000000 --- a/docker/openjdk-17/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ - -FROM ubuntu:22.04 -ARG VERSION="dev" - -RUN apt-get update && \ - apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909* && \ - apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - adduser --disabled-password --gecos "" --home /opt/besu besu && \ - chown besu:besu /opt/besu && \ - chmod 0755 /opt/besu - -USER besu -WORKDIR /opt/besu - -COPY --chown=besu:besu besu /opt/besu/ - -# Expose services ports -# 8545 HTTP JSON-RPC -# 8546 WS JSON-RPC -# 8547 HTTP GraphQL -# 8550 HTTP ENGINE JSON-RPC -# 8551 WS ENGINE JSON-RPC -# 30303 P2P -EXPOSE 8545 8546 8547 8550 8551 30303 - -# defaults for host interfaces -ENV BESU_RPC_HTTP_HOST 0.0.0.0 -ENV BESU_RPC_WS_HOST 0.0.0.0 -ENV BESU_GRAPHQL_HTTP_HOST 0.0.0.0 -ENV BESU_PID_PATH "/tmp/pid" - -ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION" - -ENV OLDPATH="${PATH}" -ENV PATH="/opt/besu/bin:${OLDPATH}" - -ENTRYPOINT ["besu"] -HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]" - -# Build-time metadata as defined at http://label-schema.org -ARG BUILD_DATE -ARG VCS_REF -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="Besu" \ - org.label-schema.description="Enterprise Ethereum client" \ - org.label-schema.url="https://besu.hyperledger.org/" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/hyperledger/besu.git" \ - org.label-schema.vendor="Hyperledger" \ - org.label-schema.version=$VERSION \ - org.label-schema.schema-version="1.0" diff --git a/docker/test.sh b/docker/test.sh index 34ce7d3a64e..1d30224f45a 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -export TEST_PATH=../tests +export TEST_PATH=./tests export GOSS_PATH=$TEST_PATH/goss-linux-${architecture} export GOSS_OPTS="$GOSS_OPTS --format junit" export GOSS_FILES_STRATEGY=cp @@ -12,7 +12,7 @@ i=0 ## Checks on the Dockerfile GOSS_FILES_PATH=$TEST_PATH/00 \ bash $TEST_PATH/dgoss dockerfile $DOCKER_IMAGE $DOCKER_FILE \ -> ../reports/00.xml || i=`expr $i + 1` +> ./reports/00.xml || i=`expr $i + 1` # fail fast if we dont pass static checks if [[ $i != 0 ]]; then exit $i; fi @@ -25,6 +25,6 @@ bash $TEST_PATH/dgoss run $DOCKER_IMAGE \ --rpc-http-enabled \ --rpc-ws-enabled \ --graphql-http-enabled \ -> ../reports/01.xml || i=`expr $i + 1` +> ./reports/01.xml || i=`expr $i + 1` exit $i