Initial Builder: oci-import support

.github/workflows/ci.yml

@@ -47,8 +47,11 @@ jobs:
     strategy: ${{ fromJson(needs.generate-jobs.outputs.strategy) }}
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
+    env:
+      BASHBREW_ARCH: amd64 # TODO consider using "$BASHBREW_SCRIPTS/bashbrew-host-arch.sh" ? (would make it harder to force i386 in our matrix too, so explicit is probably better)
     steps:
       - uses: actions/checkout@v3
+      - uses: docker-library/bashbrew@HEAD # build.sh needs bashbrew
       - name: Prepare Environment
         run: ${{ matrix.runs.prepare }}
       - name: Pull Dependencies

Dockerfile.template

@@ -1,3 +1,5 @@
+# this isn't used for the official published images anymore, but is included for backwards compatibility
+# see https://github.com/docker-library/bashbrew/issues/51
 FROM scratch
-ADD busybox.tar.xz /
+ADD busybox.tar.gz /
 CMD ["sh"]

apply-templates.sh

@@ -35,16 +35,21 @@ for version; do
 	variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)"
 	eval "variants=( $variants )"
 
+	# TODO somehow make sure this deletes any content we're not generating (without accidentally deleting potentialy generated tarballs for the things we *do* care about)
+
 	for variant in "${variants[@]}"; do
 		export variant
 
 		echo "processing $version/$variant ..."
 
+		mkdir -p "$version/$variant"
+
 		{
 			generated_warning
 			gawk -f "$jqt" Dockerfile-builder.template
 		} > "$version/$variant/Dockerfile.builder"
 
 		cp Dockerfile.template "$version/$variant/Dockerfile"
+		ln -svfT amd64/rootfs.tar.gz "$version/$variant/busybox.tar.gz"
 	done
 done

build.sh

@@ -8,11 +8,39 @@ if [ "$#" -eq 0 ]; then
 	eval "set -- $dirs"
 fi
 
+[ -n "$BASHBREW_ARCH" ]
+platformString="$(bashbrew cat --format '{{ ociPlatform arch }}' <(echo 'Maintainers: empty hack (@example)'))"
+platform="$(bashbrew cat --format '{{ ociPlatform arch | json }}' <(echo 'Maintainers: empty hack (@example)'))"
+
 for dir; do
-	base="busybox:${dir////-}"
+	variant="$(basename "$dir")"
+	base="busybox:${dir////-}-$BASHBREW_ARCH"
+
+	froms="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile.builder")"
+	for from in "$froms"; do
+		if ! bashbrew remote arches --json "$from" | jq -e '.arches | has(env.BASHBREW_ARCH)' > /dev/null; then
+			echo >&2 "warning: '$base' is 'FROM $from' which does not support '$BASHBREW_ARCH'; skipping"
+			continue 2
+		fi
+	done
+
 	(
 		set -x
-		docker build -t "$base-builder" -f "$dir/Dockerfile.builder" "$dir"
+
+		# TODO save the output of "bashbrew remote arches" above so we can "--build-context" here?
+		docker buildx build \
+			--progress=plain \
+			--platform "$platformString" \
+			--pull \
+			--load \
+			--tag "$base-builder" \
+			--file "$dir/Dockerfile.builder" \
+			"$dir"
+
+		oci="$dir/$BASHBREW_ARCH"
+		rm -rf "$oci"
+		mkdir "$oci" "$oci/blobs" "$oci/blobs/sha256"
+
 		docker run --rm "$base-builder" \
 			tar \
 				--create \
@@ -22,8 +50,89 @@ for dir; do
 				--sort name \
 				--mtime /usr/src/busybox.SOURCE_DATE_EPOCH --clamp-mtime \
 				. \
-			| xz -T0 -z9 > "$dir/busybox.tar.xz"
-		sha256sum "$dir/busybox.tar.xz"
+				> "$oci/rootfs.tar"
+
+		# if we gzip separately, we can calculate the diffid without decompressing
+		diffId="$(sha256sum "$oci/rootfs.tar" | cut -d' ' -f1)"
+		diffId="sha256:$diffId"
+
+		# we need to use the container's gzip so it's more likely reproducible over time (and using busybox's own gzip is a cute touch 😀)
+		docker run -i --rm "$base-builder" chroot rootfs gzip -c < "$oci/rootfs.tar" > "$oci/rootfs.tar.gz"
+		rm "$oci/rootfs.tar"
+		rootfs="$(sha256sum "$oci/rootfs.tar.gz" | cut -d' ' -f1)"
+		ln -svfT --relative "$oci/rootfs.tar.gz" "$oci/blobs/sha256/$rootfs"
+		rootfsSize="$(stat --format '%s' --dereference "$oci/blobs/sha256/$rootfs")"
+		rootfs="sha256:$rootfs"
+
+		SOURCE_DATE_EPOCH="$(docker run --rm "$base-builder" cat /usr/src/busybox.SOURCE_DATE_EPOCH)"
+		createdBy="$(docker run --rm --env variant="$variant" "$base-builder" sh -euc '. /etc/os-release && echo "BusyBox $BUSYBOX_VERSION ($variant)${BUILDROOT_VERSION:+, Buildroot $BUILDROOT_VERSION}, ${NAME%% *} $VERSION_ID"')"
+		jq -n --tab --arg SOURCE_DATE_EPOCH "$SOURCE_DATE_EPOCH" --arg diffId "$diffId" --arg createdBy "$createdBy" --argjson platform "$platform" '
+			($SOURCE_DATE_EPOCH | tonumber | strftime("%Y-%m-%dT%H:%M:%SZ")) as $created
+			| {
+				config: {
+					Cmd: [ "sh" ],
+				},
+				created: $created,
+				history: [ {
+					created: $created,
+					created_by: $createdBy,
+				} ],
+				rootfs: {
+					type: "layers",
+					diff_ids: [ $diffId ],
+				},
+			} + $platform
+		' > "$oci/image-config.json"
+		config="$(sha256sum "$oci/image-config.json" | cut -d' ' -f1)"
+		ln -svfT --relative "$oci/image-config.json" "$oci/blobs/sha256/$config"
+		configSize="$(stat --format '%s' --dereference "$oci/blobs/sha256/$config")"
+		config="sha256:$config"
+
+		version="$(cut <<<"$createdBy" -d' ' -f2)" # a better way to scrape the BusyBox version?  maybe this is fine (want to avoid yet another container run)
+		jq -n --tab --arg version "$version" --arg variant "$variant" --arg config "$config" --arg configSize "$configSize" --arg rootfs "$rootfs" --arg rootfsSize "$rootfsSize" '
+			{
+				schemaVersion: 2,
+				mediaType: "application/vnd.oci.image.manifest.v1+json",
+				config: {
+					mediaType: "application/vnd.oci.image.config.v1+json",
+					digest: $config,
+					size: ($configSize | tonumber),
+				},
+				layers: [ {
+					mediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
+					digest: $rootfs,
+					size: ($rootfsSize | tonumber),
+				} ],
+				annotations: {
+					"org.opencontainers.image.url": "https://github.com/docker-library/busybox",
+					"org.opencontainers.image.version": ($version + "-" + $variant),
+				},
+			}
+		' > "$oci/image-manifest.json"
+		manifest="$(sha256sum "$oci/image-manifest.json" | cut -d' ' -f1)"
+		ln -svfT --relative "$oci/image-manifest.json" "$oci/blobs/sha256/$manifest"
+		manifestSize="$(stat --format '%s' --dereference "$oci/blobs/sha256/$manifest")"
+		manifest="sha256:$manifest"
+
+		jq -nc '{ imageLayoutVersion:"1.0.0" }' > "$oci/oci-layout"
+		jq -n --tab --arg version "$version" --arg variant "$variant" --arg manifest "$manifest" --arg manifestSize "$manifestSize" --argjson platform "$platform" '
+			{
+				schemaVersion: 2,
+				mediaType: "application/vnd.oci.image.index.v1+json",
+				manifests: [ {
+					mediaType: "application/vnd.oci.image.manifest.v1+json",
+					digest: $manifest,
+					size: ($manifestSize | tonumber),
+					platform: $platform,
+					annotations: {
+						"org.opencontainers.image.ref.name": ("busybox:" + $version + "-" + $variant),
+						"io.containerd.image.name": ("busybox:" + $version + "-" + $variant),
+					},
+				} ],
+			}
+		' > "$oci/index.json"
+
+		ln -svfT --relative "$oci/rootfs.tar.gz" "$dir/busybox.tar.gz"
 		docker build -t "$base-test" "$dir"
 		docker run --rm "$base-test" sh -xec 'true'
 

generate-stackbrew-library.sh

@@ -34,6 +34,8 @@ Maintainers: Tianon Gravi <admwiggin@gmail.com> (@tianon),
              Joseph Ferguson <yosifkit@gmail.com> (@yosifkit)
 GitRepo: $gitHubUrl.git
 GitCommit: $selfCommit
+Builder: oci-import
+File: index.json
 EOH
 for arch in "${arches[@]}"; do
 	commit="${archCommits[$arch]}"
@@ -97,6 +99,7 @@ for version; do
 	fi
 	versionAliases+=( latest )
 
+	actualArches=()
 	declare -A archLatestDir=()
 	for variant in "${variants[@]}"; do
 		dir="$version/$variant"
@@ -107,25 +110,31 @@ for version; do
 		variantArches=()
 		for arch in "${arches[@]}"; do
 			archCommit="${archCommits[$arch]}"
-			if wget --quiet --spider -O /dev/null -o /dev/null "$rawGitUrl/$archCommit/$dir/busybox.tar.xz"; then
+			if wget --quiet --spider -O /dev/null -o /dev/null "$rawGitUrl/$archCommit/$dir/$arch/rootfs.tar.gz"; then
 				variantArches+=( "$arch" )
-				: "${archLatestDir[$arch]:=$dir}" # record the first supported directory per architecture for "latest" and friends
+				if [ -z "${archLatestDir[$arch]:-}" ]; then
+					# record the first supported directory per architecture for "latest" and friends
+					archLatestDir["$arch"]="$dir/$arch"
+					actualArches+=( "$arch" )
+				fi
 			fi
 		done
 
-		if _tags "${variantAliases[@]}"; then
+		if [ "${#variantArches[@]}" -gt 0 ] && _tags "${variantAliases[@]}"; then
 			cat <<-EOE
 				Architectures: $(join ', ' "${variantArches[@]}")
-				Directory: $dir
 			EOE
+			for arch in "${variantArches[@]}"; do
+				echo "$arch-Directory: $dir/$arch"
+			done
 		fi
 	done
 
-	if _tags "${versionAliases[@]}"; then
+	if [ "${#actualArches[@]}" -gt 0 ] && _tags "${versionAliases[@]}"; then
 		cat <<-EOE
-			Architectures: $(join ', ' "${arches[@]}")
+			Architectures: $(join ', ' "${actualArches[@]}")
 		EOE
-		for arch in "${arches[@]}"; do
+		for arch in "${actualArches[@]}"; do
 			archDir="${archLatestDir[$arch]}"
 			cat <<-EOA
 				${arch}-Directory: $archDir

latest-1/glibc/amd64/blobs/sha256/1c10e93757391bd459976f6382e0c0e03a0af3f239cd745424b06d3b4737f60e

@@ -0,0 +1 @@
+../../image-config.json

latest-1/glibc/amd64/blobs/sha256/be454047891e51dc9c733f98c5fdeae2e1e82b1f12a2bd052b4c22e1161f1bc0

@@ -0,0 +1 @@
+../../rootfs.tar.gz

latest-1/glibc/amd64/blobs/sha256/da78e77ef445a26ffe297167a7e1e046766a57d2ebcf9b19723894e2a636aee1

@@ -0,0 +1 @@
+../../image-manifest.json

latest-1/glibc/amd64/image-config.json

@@ -0,0 +1,22 @@
+{
+	"config": {
+		"Cmd": [
+			"sh"
+		]
+	},
+	"created": "2021-12-26T16:56:57Z",
+	"history": [
+		{
+			"created": "2021-12-26T16:56:57Z",
+			"created_by": "BusyBox 1.35.0 (glibc), Debian 12"
+		}
+	],
+	"rootfs": {
+		"type": "layers",
+		"diff_ids": [
+			"sha256:be94c013eea8cbd66c1fc201cd544144cf5befd123bececbe4ca2453e29314b6"
+		]
+	},
+	"architecture": "amd64",
+	"os": "linux"
+}

latest-1/glibc/amd64/image-manifest.json

@@ -0,0 +1,20 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.manifest.v1+json",
+	"config": {
+		"mediaType": "application/vnd.oci.image.config.v1+json",
+		"digest": "sha256:1c10e93757391bd459976f6382e0c0e03a0af3f239cd745424b06d3b4737f60e",
+		"size": 372
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+			"digest": "sha256:be454047891e51dc9c733f98c5fdeae2e1e82b1f12a2bd052b4c22e1161f1bc0",
+			"size": 2148915
+		}
+	],
+	"annotations": {
+		"org.opencontainers.image.url": "https://github.com/docker-library/busybox",
+		"org.opencontainers.image.version": "1.35.0-glibc"
+	}
+}

latest-1/glibc/amd64/index.json

@@ -0,0 +1,19 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.index.v1+json",
+	"manifests": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:da78e77ef445a26ffe297167a7e1e046766a57d2ebcf9b19723894e2a636aee1",
+			"size": 610,
+			"platform": {
+				"architecture": "amd64",
+				"os": "linux"
+			},
+			"annotations": {
+				"org.opencontainers.image.ref.name": "busybox:1.35.0-glibc",
+				"io.containerd.image.name": "busybox:1.35.0-glibc"
+			}
+		}
+	]
+}

latest-1/glibc/amd64/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}

latest-1/glibc/busybox.tar.gz

@@ -0,0 +1 @@
+amd64/rootfs.tar.gz

latest-1/musl/amd64/blobs/sha256/7e3712170429f1cb4fc838e6ec80548d6af2778301f3c88a57e4178d7aa16818

@@ -0,0 +1 @@
+../../rootfs.tar.gz

latest-1/musl/amd64/blobs/sha256/ab169714e570fc081439d21c6fefccdc40841c722a5ced84af57da7ccacce8e3

@@ -0,0 +1 @@
+../../image-manifest.json

latest-1/musl/amd64/blobs/sha256/f36b741eb9675622ca20443a43b42d5dae106e346115faf4b4604a51e63bd421

@@ -0,0 +1 @@
+../../image-config.json

latest-1/musl/amd64/image-config.json

@@ -0,0 +1,22 @@
+{
+	"config": {
+		"Cmd": [
+			"sh"
+		]
+	},
+	"created": "2021-12-26T16:56:57Z",
+	"history": [
+		{
+			"created": "2021-12-26T16:56:57Z",
+			"created_by": "BusyBox 1.35.0 (musl), Alpine 3.19.1"
+		}
+	],
+	"rootfs": {
+		"type": "layers",
+		"diff_ids": [
+			"sha256:e81af897a6ed42bf107335af397aee8b165cb545b01ebdda4cbd9fff72c51fdb"
+		]
+	},
+	"architecture": "amd64",
+	"os": "linux"
+}

latest-1/musl/amd64/image-manifest.json

@@ -0,0 +1,20 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.manifest.v1+json",
+	"config": {
+		"mediaType": "application/vnd.oci.image.config.v1+json",
+		"digest": "sha256:f36b741eb9675622ca20443a43b42d5dae106e346115faf4b4604a51e63bd421",
+		"size": 375
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+			"digest": "sha256:7e3712170429f1cb4fc838e6ec80548d6af2778301f3c88a57e4178d7aa16818",
+			"size": 849546
+		}
+	],
+	"annotations": {
+		"org.opencontainers.image.url": "https://github.com/docker-library/busybox",
+		"org.opencontainers.image.version": "1.35.0-musl"
+	}
+}

latest-1/musl/amd64/index.json

@@ -0,0 +1,19 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.index.v1+json",
+	"manifests": [
+		{
+			"mediaType": "application/vnd.oci.image.manifest.v1+json",
+			"digest": "sha256:ab169714e570fc081439d21c6fefccdc40841c722a5ced84af57da7ccacce8e3",
+			"size": 608,
+			"platform": {
+				"architecture": "amd64",
+				"os": "linux"
+			},
+			"annotations": {
+				"org.opencontainers.image.ref.name": "busybox:1.35.0-musl",
+				"io.containerd.image.name": "busybox:1.35.0-musl"
+			}
+		}
+	]
+}

latest-1/musl/amd64/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}