Adjust tarball creation to be reproducible

Dockerfile-builder.template

@@ -13,6 +13,8 @@ RUN set -eux; \
 		musl-dev \
 		patch \
 		tzdata \
+# busybox's tar ironically does not maintain mtime of directories correctly (which we need for SOURCE_DATE_EPOCH / reproducibility)
+		tar \
 	;
 {{ ) else ( -}}
 FROM debian:bookworm-slim
@@ -234,22 +236,45 @@ RUN set -eux; \
 	curl -fL -o busybox.tar.bz2 "https://busybox.net/downloads/$tarball"; \
 	echo "$BUSYBOX_SHA256 *busybox.tar.bz2" | sha256sum -c -; \
 	gpg --batch --verify busybox.tar.bz2.sig busybox.tar.bz2; \
-	mkdir -p /usr/src/busybox; \
-	tar -xf busybox.tar.bz2 -C /usr/src/busybox --strip-components 1; \
-	rm busybox.tar.bz2*
+# Alpine... 😅
+	mkdir -p /usr/src; \
+	tar -xf busybox.tar.bz2 -C /usr/src "busybox-$BUSYBOX_VERSION"; \
+	mv "/usr/src/busybox-$BUSYBOX_VERSION" /usr/src/busybox; \
+	rm busybox.tar.bz2*; \
+	\
+# save the tarball's filesystem timestamp persistently (in case building busybox modifies it) so we can use it for reproducible rootfs later
+	SOURCE_DATE_EPOCH="$(stat -c '%Y' /usr/src/busybox | tee /usr/src/busybox.SOURCE_DATE_EPOCH)"; \
+	date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \
+	touch -t "$date" /usr/src/busybox.SOURCE_DATE_EPOCH; \
+# for logging validation/edification
+	date --date "@$SOURCE_DATE_EPOCH" --rfc-2822
 
 WORKDIR /usr/src/busybox
 
+# add tiny script to clamp/fixup timestamps for reproducibility (so it can be run in each layer that touches "rootfs" -- overlay does not seem able to represent a timestamp-only change to an otherwise empty directory like "/dev" otherwise)
+RUN set -eux; \
+	SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \
+	date="$(date -d "@$SOURCE_DATE_EPOCH" '+%Y%m%d%H%M.%S')"; \
+	echo "exec find rootfs -depth -newer /usr/src/busybox.SOURCE_DATE_EPOCH -exec ls -ld '{}' + -exec touch -ht '$date' '{}' + -exec ls -ld '{}' +" > clamp.sh; \
+	chmod +x clamp.sh; \
+	mkdir rootfs; \
+	./clamp.sh
+
 RUN set -eux; \
 	\
+# build date/time gets embedded in the BusyBox binary -- SOURCE_DATE_EPOCH should override that
+	SOURCE_DATE_EPOCH="$(cat /usr/src/busybox.SOURCE_DATE_EPOCH)"; \
+	export SOURCE_DATE_EPOCH; \
+# (has to be set in the config stage for making sure "AUTOCONF_TIMESTAMP" is embedded correctly)
+	\
 	setConfs=' \
 		CONFIG_AR=y \
 		CONFIG_FEATURE_AR_CREATE=y \
 		CONFIG_FEATURE_AR_LONG_FILENAMES=y \
 # CONFIG_LAST_SUPPORTED_WCHAR: see https://github.com/docker-library/busybox/issues/13 (UTF-8 input)
 		CONFIG_LAST_SUPPORTED_WCHAR=0 \
 {{ if env.variant == "glibc" then ( -}}
-# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... :cry:
+# As long as we rely on libnss (see below), we have to have libc.so anyhow, so we've removed CONFIG_STATIC here... 😭
 {{ ) else ( -}}
 		CONFIG_STATIC=y \
 {{ ) end -}}
@@ -361,7 +386,10 @@ RUN set -eux; \
 {{ ) else "" end -}}
 	chroot rootfs /bin/getconf _NPROCESSORS_ONLN; \
 	\
-	chroot rootfs /bin/busybox --install /bin
+# TODO make this create symlinks instead so the output tarball is cleaner (but "-s" outputs absolute symlinks which is kind of annoying to deal with -- we should also consider letting busybox determine the "install paths"; see "busybox --list-full")
+	chroot rootfs /bin/busybox --install /bin; \
+	\
+	./clamp.sh
 
 # install a few extra files from buildroot (/etc/passwd, etc)
 RUN set -eux; \
@@ -404,37 +432,47 @@ RUN set -eux; \
 			} \
 			printf "chmod %s %s\n", $3, $1; \
 		} \
-	' ../buildroot/system/device_table.txt | sh -eux
+	' ../buildroot/system/device_table.txt | sh -eux; \
+	\
+	./clamp.sh
 
 # create missing home directories and ensure /usr/bin/env exists
 RUN set -eux; \
-	cd rootfs; \
-	for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \
-		user="${userHome%%=*}"; \
-		home="${userHome#*=}"; \
-		home="./${home#/}"; \
-		if [ ! -d "$home" ]; then \
-			mkdir -p "$home"; \
-			chown "$user" "$home"; \
-			chmod 755 "$home"; \
+	( \
+		cd rootfs; \
+		for userHome in $(awk -F ':' '{ print $3 ":" $4 "=" $6 }' etc/passwd); do \
+			user="${userHome%%=*}"; \
+			home="${userHome#*=}"; \
+			home="./${home#/}"; \
+			if [ ! -d "$home" ]; then \
+				mkdir -p "$home"; \
+				chown "$user" "$home"; \
+				chmod 755 "$home"; \
+			fi; \
+		done; \
+		if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \
+			mkdir -p usr/bin; \
+			ln -s ../../bin/env usr/bin/; \
 		fi; \
-	done; \
-	if [ ! -s usr/bin/env ] && [ -s bin/env ]; then \
-		mkdir -p usr/bin; \
-		ln -s ../../bin/env usr/bin/; \
-	fi
+	); \
+	\
+	./clamp.sh
 
 # test and make sure it works
 RUN chroot rootfs /usr/bin/env sh -xec 'true'
 
 # ensure correct timezone (UTC)
 RUN set -eux; \
 	ln -vL /usr/share/zoneinfo/UTC rootfs/etc/localtime; \
-	[ "$(chroot rootfs date +%Z)" = 'UTC' ]
+	[ "$(chroot rootfs date +%Z)" = 'UTC' ]; \
+	\
+	./clamp.sh
 
 # test and make sure DNS works too
 RUN cp -L /etc/resolv.conf rootfs/etc/; \
 	chroot rootfs /bin/sh -xec 'nslookup google.com'; \
-	rm rootfs/etc/resolv.conf
+	rm rootfs/etc/resolv.conf; \
+	\
+	./clamp.sh
 
 # vim:set ft=dockerfile:

build.sh

@@ -13,7 +13,15 @@ for dir; do
 	(
 		set -x
 		docker build -t "$base-builder" -f "$dir/Dockerfile.builder" "$dir"
-		docker run --rm "$base-builder" tar cC rootfs . | xz -T0 -z9 > "$dir/busybox.tar.xz"
+		docker run --rm "$base-builder" \
+			tar \
+				--create \
+				--directory rootfs \
+				--numeric-owner \
+				--transform 's,^./,,' \
+				--sort name \
+				. \
+			| xz -T0 -z9 > "$dir/busybox.tar.xz"
 		docker build -t "$base-test" "$dir"
 		docker run --rm "$base-test" sh -xec 'true'