Test: rebuild disks from bootable containers when the BIB container changes

.github/workflows/tests.yml

@@ -127,7 +127,7 @@ jobs:
     steps:
 
       - name: Install build and test dependencies
-        run: dnf -y install python3-pytest
+        run: dnf -y install python3-pytest podman skopeo
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
@@ -146,7 +146,7 @@ jobs:
     steps:
 
       - name: Install build and test dependencies
-        run: dnf -y install python3-pylint git-core grep
+        run: dnf -y install python3-pylint git-core grep python3-pytest
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4

test/scripts/boot-image

@@ -129,7 +129,7 @@ def boot_container(distro, arch, image_type, image_path, manifest_id):
                    "--security-opt", "label=type:unconfined_t",
                    "-v", f"{tmpdir}:/output",
                    "-v", f"{config_file}:/config.json",
-                   "quay.io/centos-bootc/bootc-image-builder:latest",
+                   testlib.BIB_REF,
                    "--type=ami",
                    "--config=/config.json",
                    container_ref]
@@ -184,6 +184,7 @@ def main():
     image_path = find_image_file(search_path)
 
     print(f"Testing image at {image_path}")
+    bib_image_id = ""
     match image_type:
         case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami":
             boot_ami(distro, arch, image_type, image_path)
@@ -193,6 +194,10 @@ def main():
                 build_info = json.load(info_fp)
             manifest_id = build_info["manifest-checksum"]
             boot_container(distro, arch, image_type, image_path, manifest_id)
+            # get the image ID from the local store so we know it's the one we used in the test
+            # (in case the container in the registry was updated while we were testing)
+            bib_image_id = testlib.skopeo_inspect_id(f"containers-storage:{testlib.BIB_REF}",
+                                                     testlib.host_container_arch())
         case _:
             # skip
             print(f"{image_type} boot tests are not supported yet")
@@ -205,6 +210,8 @@ def main():
     with open(info_file_path, encoding="utf-8") as info_fp:
         build_info = json.load(info_fp)
     build_info["boot-success"] = True
+    if bib_image_id:
+        build_info["bib-id"] = bib_image_id
     with open(info_file_path, "w", encoding="utf-8") as info_fp:
         json.dump(build_info, info_fp, indent=2)
 

test/scripts/imgtestlib.py

@@ -17,6 +17,8 @@
 SCHUTZFILE = "Schutzfile"
 OS_RELEASE_FILE = "/etc/os-release"
 
+BIB_REF = "quay.io/centos-bootc/bootc-image-builder:latest"
+
 # ostree containers are pushed to the CI registry to be reused by dependants
 OSTREE_CONTAINERS = [
     "iot-container",
@@ -33,6 +35,10 @@
     "iot-bootable-container",
 ]
 
+BIB_TYPES = [
+    "iot-bootable-container"
+]
+
 
 # base and terraform bits copied from main .gitlab-ci.yml
 # needed for status reporting and defining the runners
@@ -275,6 +281,17 @@ def check_for_build(manifest_fname, build_info_path, errors):
     # boot testing supported: check if it's been tested, otherwise queue it for rebuild and boot
     if dl_config.get("boot-success", False):
         print("  This image was successfully boot tested")
+
+        # check if it's a BIB type and compare image IDs
+        if image_type in BIB_TYPES:
+            booted_id = dl_config.get("bib-id", None)
+            current_id = skopeo_inspect_id(f"docker://{BIB_REF}", host_container_arch())
+            if booted_id != current_id:
+                print(f"Container disk image was built with bootc-image-builder {booted_id}")
+                print(f"  Testing {current_id}")
+                print("  Adding config to build pipeline.")
+                return True
+
         return False
     print("  Boot test success not found.")
 
@@ -401,3 +418,62 @@ def rng_seed_env():
         raise RuntimeError("'rngseed' not found in Schutzfile")
 
     return {"OSBUILD_TESTING_RNG_SEED": str(seed)}
+
+
+def host_container_arch():
+    host_arch = os.uname().machine
+    match host_arch:
+        case "x86_64":
+            return "amd64"
+        case "aarch64":
+            return "arm64"
+    return host_arch
+
+
+def is_manifest_list(data):
+    """Inspect a manifest determine if it's a multi-image manifest-list."""
+    media_type = data.get("mediaType")
+    #  Check if mediaType is set according to docker or oci specifications
+    if media_type in ("application/vnd.docker.distribution.manifest.list.v2+json",
+                      "application/vnd.oci.image.index.v1+json"):
+        return True
+
+    # According to the OCI spec, setting mediaType is not mandatory. So, if it is not set at all, check for the
+    # existence of manifests
+    if media_type is None and data.get("manifests") is not None:
+        return True
+
+    return False
+
+
+def skopeo_inspect_id(image_name: str, arch: str) -> str:
+    """
+    Returns the image ID (config digest) of the container image. If the image resolves to a manifest list, the config
+    digest of the given architecture is resolved.
+
+    Runs with 'sudo' when inspecting a local container because in our tests we need to read the root container storage.
+    """
+    cmd = ["skopeo", "inspect", "--raw", image_name]
+    if image_name.startswith("containers-storage"):
+        cmd = ["sudo"] + cmd
+    out, _ = runcmd(cmd)
+    data = json.loads(out)
+    if not is_manifest_list(data):
+        return data["config"]["digest"]
+
+    for manifest in data.get("manifests", []):
+        platform = manifest.get("platform", {})
+        img_arch = platform.get("architecture", "")
+        img_ostype = platform.get("os", "")
+
+        if arch != img_arch or img_ostype != "linux":
+            continue
+
+        image_no_tag = ":".join(image_name.split(":")[:-1])
+        manifest_digest = manifest["digest"]
+        arch_image_name = f"{image_no_tag}@{manifest_digest}"
+        # inspect the arch-specific manifest to get the image ID (config digest)
+        return skopeo_inspect_id(arch_image_name, arch)
+
+    # don't error out, just return an empty string and let the caller handle it
+    return ""

test/scripts/test_imgtestlib.py

@@ -1,7 +1,21 @@
 import os
+import subprocess as sp
+import tempfile
+
+import pytest
 
 import imgtestlib as testlib
 
+TEST_ARCHES = ["amd64", "arm64"]
+
+
+def can_sudo_nopw() -> bool:
+    """
+    Check if we can run sudo without a password.
+    """
+    job = sp.run(["sudo", "-n", "true"], capture_output=True, check=False)
+    return job.returncode == 0
+
 
 def test_runcmd():
     stdout, stderr = testlib.runcmd(["/bin/echo", "hello"])
@@ -32,3 +46,49 @@ def test_path_generators():
         "inforoot/osbuild-104-1.fc39.noarch/abc123/info.json"
     assert testlib.gen_build_info_s3("fedora-39", "aarch64", "abc123") == \
         testlib.S3_BUCKET + "/images/builds/fedora-39/aarch64/osbuild-104-1.fc39.noarch/abc123/"
+
+
+test_container = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test"
+
+# manifest IDs for
+#  registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test:latest
+manifest_ids = {
+    "amd64": "sha256:601c98c8148720ec5c29b8e854a1d5d88faddbc443eca12920d76cf993d7290e",
+    "arm64": "sha256:1a19a94647b1379fed8c23eb7553327cb604ba546eb93f9f6c1e6d11911c8beb",
+}
+
+# image IDs for
+#  registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test:latest
+image_ids = {
+    "amd64": "sha256:dbb63178dc9157068107961f11397df3fb62c02fa64f697d571bf84aad71cb99",
+    "arm64": "sha256:62d2a7b3bf9e0b4f3aba22553d6971227b5a39f7f408d46347b1ee74eb97cb20",
+}
+
+
+@pytest.mark.parametrize("arch", TEST_ARCHES)
+def test_skopeo_inspect_id_manifest_list(arch):
+    transport = "docker://"
+    image_id = image_ids[arch]
+    assert testlib.skopeo_inspect_id(f"{transport}{test_container}:latest", arch) == image_id
+
+
+@pytest.mark.parametrize("arch", TEST_ARCHES)
+def test_skopeo_inspect_image_manifest(arch):
+    transport = "docker://"
+    manifest_id = manifest_ids[arch]
+    image_id = image_ids[arch]
+    # arch arg to skopeo_inspect_id doesn't matter here
+    assert testlib.skopeo_inspect_id(f"{transport}{test_container}@{manifest_id}", arch) == image_id
+
+
+@pytest.mark.skipif(not can_sudo_nopw(), reason="requires passwordless sudo")
+@pytest.mark.parametrize("arch", TEST_ARCHES)
+@pytest.mark.skip("disabled")  # disabled: fails in github action - needs work
+def test_skopeo_inspect_localstore(arch):
+    transport = "containers-storage:"
+    image = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test:latest"
+    with tempfile.TemporaryDirectory() as tmpdir:
+        testlib.runcmd(["sudo", "podman", "pull", f"--arch={arch}", "--storage-driver=vfs", f"--root={tmpdir}", image])
+
+        # arch arg to skopeo_inspect_id doesn't matter here
+        assert testlib.skopeo_inspect_id(f"{transport}[vfs@{tmpdir}]{image}", arch) == image_ids[arch]