From d275c2567a9b0ae5d59955d59c5f2fac6cfbf5dd Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 28 Oct 2024 13:38:09 +0100 Subject: [PATCH 1/4] WIP: Build hdfs-utils from feat/extended-request branch --- hadoop/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 28e65634c..7bad8457e 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -103,8 +103,8 @@ WORKDIR /stackable # labels to build a rackID from. # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils. -RUN curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC . && \ - cd hdfs-utils-${HDFS_UTILS} && \ +RUN curl "https://github.com/stackabletech/hdfs-utils/archive/refs/heads/feat/extended-request.tar.gz" | tar -xzC . && \ + cd hdfs-utils-feat-extended-request && \ mvn --no-transfer-progress clean package -P hadoop-${PRODUCT} -DskipTests -Dmaven.javadoc.skip=true && \ mkdir -p /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib && \ cp target/hdfs-utils-$HDFS_UTILS.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar && \ From 609d1d06e53518042ba0dbf75449dbcaa0291140 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 5 Nov 2024 13:28:54 +0100 Subject: [PATCH 2/4] chore: Bump to hdfs-utils 0.4.0 --- CHANGELOG.md | 1 + hadoop/Dockerfile | 4 ++-- hadoop/versions.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bec3f7b19..69cf2d5fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ All notable changes to this project will be documented in this file. - Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890], [#897]). - ci: Bump `stackabletech/actions` to 0.1.0 ([#901], [#903], [#907], [#910]). - ubi-rust-builder: Bump Rust toolchain to 1.81.0 ([#902]). +- hadoop: Bump to `hdfs-utils` 0.4.0 ([#XXX]). ### Removed diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 7bad8457e..28e65634c 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -103,8 +103,8 @@ WORKDIR /stackable # labels to build a rackID from. # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils. -RUN curl "https://github.com/stackabletech/hdfs-utils/archive/refs/heads/feat/extended-request.tar.gz" | tar -xzC . && \ - cd hdfs-utils-feat-extended-request && \ +RUN curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC . && \ + cd hdfs-utils-${HDFS_UTILS} && \ mvn --no-transfer-progress clean package -P hadoop-${PRODUCT} -DskipTests -Dmaven.javadoc.skip=true && \ mkdir -p /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib && \ cp target/hdfs-utils-$HDFS_UTILS.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar && \ diff --git a/hadoop/versions.py b/hadoop/versions.py index 93e4a3dab..8126e6434 100644 --- a/hadoop/versions.py +++ b/hadoop/versions.py @@ -6,7 +6,7 @@ "async_profiler": "2.9", "jmx_exporter": "1.0.1", "protobuf": "3.7.1", - "hdfs_utils": "0.3.0", + "hdfs_utils": "0.4.0", }, { "product": "3.3.6", @@ -15,7 +15,7 @@ "async_profiler": "2.9", "jmx_exporter": "1.0.1", "protobuf": "3.7.1", - "hdfs_utils": "0.3.0", + "hdfs_utils": "0.4.0", }, { "product": "3.4.0", @@ -24,6 +24,6 @@ "async_profiler": "2.9", "jmx_exporter": "1.0.1", "protobuf": "3.7.1", - "hdfs_utils": "0.3.0", + "hdfs_utils": "0.4.0", }, ] From 6c1aa60bff312671cae1f3b2bc47130623feda6e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 5 Nov 2024 13:37:15 +0100 Subject: [PATCH 3/4] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73eaa50ab..93a278f7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ All notable changes to this project will be documented in this file. - ci: Bump `stackabletech/actions` to 0.2.0 ([#901], [#903], [#907], [#910], [#913]). - ubi-rust-builder: Bump Rust toolchain to 1.81.0 ([#902]). - ci: Handle release builds in the same build workflows ([#913]). -- hadoop: Bump to `hdfs-utils` 0.4.0 ([#XXX]). +- hadoop: Bump to `hdfs-utils` 0.4.0 ([#914]). ### Removed @@ -100,6 +100,7 @@ All notable changes to this project will be documented in this file. [#910]: https://github.com/stackabletech/docker-images/pull/910 [#912]: https://github.com/stackabletech/docker-images/pull/912 [#913]: https://github.com/stackabletech/docker-images/pull/913 +[#914]: https://github.com/stackabletech/docker-images/pull/914 ## [24.7.0] - 2024-07-24 From 81994b82be9da8f7e67d42f2df20539e5853f240 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 5 Nov 2024 14:07:49 +0100 Subject: [PATCH 4/4] fix: Fix compilation by compiling using Java 17 --- hadoop/Dockerfile | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 28e65634c..646147fe3 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5 # check=error=true -FROM stackable/image/java-devel AS builder +FROM stackable/image/java-devel AS hadoop-builder ARG PRODUCT ARG ASYNC_PROFILER @@ -95,6 +95,17 @@ ARG HDFS_UTILS ARG PRODUCT ARG STACKABLE_USER_UID +# Starting with hdfs-utils 0.4.0 we need to use Java 17 for compilation. +# We can not simply use java-devel with Java 17, as it is also used to compile Hadoop in this +# Dockerfile, which needs Java 11. So we need to also use the java-devel image in version 11 and +# install Java 17 ourselves. +# The adptiom yum repo is already added by the java-devel Dockerfile. +RUN microdnf update && \ + microdnf install -y temurin-17-jdk && \ + microdnf clean all && \ + rm -rf /var/cache/yum +ENV JAVA_HOME="/usr/lib/jvm/temurin-17-jdk" + USER ${STACKABLE_USER_UID} WORKDIR /stackable @@ -125,9 +136,9 @@ LABEL name="Apache Hadoop" \ summary="The Stackable image for Apache Hadoop." \ description="This image is deployed by the Stackable Operator for Apache Hadoop / HDFS." -COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/async-profiler /stackable/async-profiler/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/jmx /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/async-profiler /stackable/async-profiler/ COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/