diff --git a/README.md b/README.md index 5d8640c2..e86b74a8 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Found an issue in the build for SCALE? Please report it on our [Jira bugtracker] ## Requirements - Debian 10 or later (VM or Bare-Metal) - - 16GB of RAM + - 20GB of RAM - At least 15GB of free disk space In addition to the host, you will want to pre-install the following packages: diff --git a/scale_build/extensions.py b/scale_build/extensions.py index 33fd61db..089f9510 100644 --- a/scale_build/extensions.py +++ b/scale_build/extensions.py @@ -13,129 +13,163 @@ logger = logging.getLogger(__name__) -BINARIES = ("apt", "apt-config", "apt-key", "dpkg") -TEMPORARY_PACKAGES = ["gcc", "make", "pkg-config"] -PERMANENT_PACKAGES = ["libvulkan1", "nvidia-container-toolkit", "vulkan-validationlayers"] -HEADERS = {"User-Agent": "curl/7.88.1"} -EXTENSIONS_CHROOT = os.path.join(TMPFS, "extensions_chroot") -EXTENSIONS_CHROOT_BASE = os.path.join(TMPFS, "extensions_chroot_base") - def build_extensions(rootfs_image, dst_dir): - for path in [EXTENSIONS_CHROOT, EXTENSIONS_CHROOT_BASE]: - if os.path.exists(path): - shutil.rmtree(path) - os.makedirs(path) - - run(["unsquashfs", "-dest", EXTENSIONS_CHROOT, rootfs_image]) - run(["unsquashfs", "-dest", EXTENSIONS_CHROOT_BASE, rootfs_image]) - - kernel_version = get_kernel_version(EXTENSIONS_CHROOT) - - os.makedirs(os.path.join(EXTENSIONS_CHROOT, "proc"), exist_ok=True) - run(["mount", "proc", os.path.join(EXTENSIONS_CHROOT, "proc"), "-t", "proc"]) - os.makedirs(os.path.join(EXTENSIONS_CHROOT, "sys"), exist_ok=True) - run(["mount", "sysfs", os.path.join(EXTENSIONS_CHROOT, "sys"), "-t", "sysfs"]) - os.makedirs(os.path.join(EXTENSIONS_CHROOT, "packages"), exist_ok=True) - run(["mount", "--bind", PKG_DIR, os.path.join(EXTENSIONS_CHROOT, "packages")]) - try: - shutil.copyfile("/etc/resolv.conf", f"{EXTENSIONS_CHROOT}/etc/resolv.conf") - - for binary in BINARIES: - os.unlink(os.path.join(EXTENSIONS_CHROOT, f"usr/local/bin/{binary}")) - os.chmod(os.path.join(EXTENSIONS_CHROOT, f"usr/bin/{binary}"), 0o755) - - add_nvidia_repository() - run_in_chroot(["apt", "update"], chroot=EXTENSIONS_CHROOT) - run_in_chroot(["apt", "-y", "install"] + TEMPORARY_PACKAGES + PERMANENT_PACKAGES, chroot=EXTENSIONS_CHROOT) - - install_nvidia_driver(kernel_version) - - run_in_chroot(["apt", "-y", "remove"] + TEMPORARY_PACKAGES, chroot=EXTENSIONS_CHROOT) - run_in_chroot(["apt", "-y", "autoremove"], chroot=EXTENSIONS_CHROOT) - finally: - run(["umount", os.path.join(EXTENSIONS_CHROOT, "packages")]) - run(["umount", os.path.join(EXTENSIONS_CHROOT, "sys")]) - run(["umount", os.path.join(EXTENSIONS_CHROOT, "proc")]) + chroot = os.path.join(TMPFS, "extensions_chroot") + chroot_base = os.path.join(TMPFS, "extensions_chroot_base") + + if os.path.exists(chroot_base): + shutil.rmtree(chroot_base) + os.makedirs(chroot_base) + run(["unsquashfs", "-dest", chroot_base, rootfs_image]) + + for klass, name in [(DevToolsExtension, "dev-tools"), (NvidiaExtension, "nvidia")]: + klass(rootfs_image, chroot_base, chroot).build(name, f"{dst_dir}/{name}.raw") + + +class Extension: + def __init__(self, base_image: str, chroot_base: str, chroot: str): + """ + :param base_image: rootfs squashfs image path + :param chroot_base: a path where `base_image` is extracted + (it will be used to compare which files were modified and should be included in the extension image) + :param chroot: a path which will be used as chroot for extension install + """ + self.base_image = base_image + self.chroot_base = chroot_base + self.chroot = chroot + + def build(self, name, dst_path): + if os.path.exists(self.chroot): + shutil.rmtree(self.chroot) + os.makedirs(self.chroot) + + run(["unsquashfs", "-dest", self.chroot, self.base_image]) + + os.makedirs(os.path.join(self.chroot, "proc"), exist_ok=True) + run(["mount", "proc", os.path.join(self.chroot, "proc"), "-t", "proc"]) + os.makedirs(os.path.join(self.chroot, "sys"), exist_ok=True) + run(["mount", "sysfs", os.path.join(self.chroot, "sys"), "-t", "sysfs"]) + os.makedirs(os.path.join(self.chroot, "packages"), exist_ok=True) + run(["mount", "--bind", PKG_DIR, os.path.join(self.chroot, "packages")]) + try: + shutil.copyfile("/etc/resolv.conf", f"{self.chroot}/etc/resolv.conf") + + self.build_impl() + finally: + run(["umount", os.path.join(self.chroot, "packages")]) + run(["umount", os.path.join(self.chroot, "sys")]) + run(["umount", os.path.join(self.chroot, "proc")]) + + self.build_extension(name, dst_path) + + def build_impl(self): + raise NotImplementedError + + def build_extension(self, name, dst_path): + changed_files = [ + os.path.relpath(filename, self.chroot) + for filename in map( + lambda filename: os.path.join(os.getcwd(), filename), + run( + ["rsync", "-avn", "--out-format=%f", f"{self.chroot}/", f"{self.chroot_base}/"], + log=False, + ).stdout.split("\n") + ) + if os.path.abspath(filename).startswith(os.path.abspath(self.chroot)) + ] + + sysext_files = [f for f in changed_files if f.startswith("usr/") and not (f.startswith("usr/src/"))] + + for root, dirs, files in os.walk(self.chroot, topdown=False): + for f in files: + path = os.path.relpath(os.path.abspath(os.path.join(root, f)), self.chroot) + if path not in sysext_files: + os.unlink(os.path.join(root, f)) + + for d in dirs: + try: + os.rmdir(os.path.join(root, d)) + except NotADirectoryError: + os.unlink(os.path.join(root, d)) # It's a symlink + except OSError as e: + if e.errno == errno.ENOTEMPTY: + pass + else: + raise + + os.makedirs(f"{self.chroot}/usr/lib/extension-release.d", exist_ok=True) + with open(f"{self.chroot}/usr/lib/extension-release.d/extension-release.{name}", "w") as f: + f.write("ID=_any\n") + + run(["mksquashfs", self.chroot, dst_path, "-comp", "xz"]) + + def run(self, cmd: list[str]): + run_in_chroot(cmd, chroot=self.chroot) + + +class DevToolsExtension(Extension): + def build_impl(self): + # Make `install-dev-tools` think that this is not necessary + os.unlink(os.path.join(self.chroot, "usr/local/libexec/disable-rootfs-protection")) + + self.run(["install-dev-tools"]) + + +class NvidiaExtension(Extension): + binaries = ("apt", "apt-config", "apt-key", "dpkg") + temporary_packages = ["gcc", "make", "pkg-config"] + permanent_packages = ["libvulkan1", "nvidia-container-toolkit", "vulkan-validationlayers"] + headers = {"User-Agent": "curl/7.88.1"} + + def build_impl(self): + kernel_version = get_kernel_version(self.chroot) + + for binary in self.binaries: + os.unlink(os.path.join(self.chroot, f"usr/local/bin/{binary}")) + os.chmod(os.path.join(self.chroot, f"usr/bin/{binary}"), 0o755) + + self.add_nvidia_repository() + self.run(["apt", "update"]) + self.run(["apt", "-y", "install"] + self.temporary_packages + self.permanent_packages) + + self.install_nvidia_driver(kernel_version) + + self.run(["apt", "-y", "remove"] + self.temporary_packages) + self.run(["apt", "-y", "autoremove"]) + + def add_nvidia_repository(self): + r = requests.get("https://nvidia.github.io/libnvidia-container/gpgkey") + r.raise_for_status() - build_extension("nvidia", f"{dst_dir}/nvidia.raw") + with open(f"{self.chroot}/key.gpg", "w") as f: + f.write(r.text) + self.run(["gpg", "-o", "/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg", "--dearmor", "/key.gpg"]) -def add_nvidia_repository(): - r = requests.get("https://nvidia.github.io/libnvidia-container/gpgkey") - r.raise_for_status() + with open(f"{self.chroot}/etc/apt/sources.list.d/nvidia-container-toolkit.list", "w") as f: + f.write("deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] " + "https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /") - with open(f"{EXTENSIONS_CHROOT}/key.gpg", "w") as f: - f.write(r.text) + def download_nvidia_driver(self): + prefix = "https://download.nvidia.com/XFree86/Linux-x86_64" - run_in_chroot(["gpg", "-o", "/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg", "--dearmor", "/key.gpg"], - chroot=EXTENSIONS_CHROOT) + version = get_manifest()["extensions"]["nvidia"]["current"] + filename = f"NVIDIA-Linux-x86_64-{version}-no-compat32.run" + result = f"{self.chroot}/{filename}" - with open(f"{EXTENSIONS_CHROOT}/etc/apt/sources.list.d/nvidia-container-toolkit.list", "w") as f: - f.write("deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] " - "https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /") + with requests.get(f"{prefix}/{version}/{filename}", headers=self.headers, stream=True, timeout=10) as r: + r.raise_for_status() + with open(result, "wb") as f: + shutil.copyfileobj(r.raw, f) + os.chmod(result, 0o755) + return result -def download_nvidia_driver(): - prefix = "https://download.nvidia.com/XFree86/Linux-x86_64" + def install_nvidia_driver(self, kernel_version): + driver = self.download_nvidia_driver() - version = get_manifest()["extensions"]["nvidia"]["current"] - filename = f"NVIDIA-Linux-x86_64-{version}-no-compat32.run" - result = f"{EXTENSIONS_CHROOT}/{filename}" + self.run([f"/{os.path.basename(driver)}", "--skip-module-load", "--silent", f"--kernel-name={kernel_version}", + "--allow-installation-with-running-driver", "--no-rebuild-initramfs"]) - with requests.get(f"{prefix}/{version}/{filename}", headers=HEADERS, stream=True, timeout=10) as r: - r.raise_for_status() - with open(result, "wb") as f: - shutil.copyfileobj(r.raw, f) - - os.chmod(result, 0o755) - return result - - -def install_nvidia_driver(kernel_version): - driver = download_nvidia_driver() - - run_in_chroot([f"/{os.path.basename(driver)}", "--skip-module-load", "--silent", f"--kernel-name={kernel_version}", - "--allow-installation-with-running-driver", "--no-rebuild-initramfs"], - chroot=EXTENSIONS_CHROOT) - - os.unlink(driver) - - -def build_extension(name, dst_path): - changed_files = [ - os.path.relpath(filename, EXTENSIONS_CHROOT) - for filename in map( - lambda filename: os.path.join(os.getcwd(), filename), - run( - ["rsync", "-avn", "--out-format=%f", f"{EXTENSIONS_CHROOT}/", f"{EXTENSIONS_CHROOT_BASE}/"], - log=False, - ).stdout.split("\n") - ) - if os.path.abspath(filename).startswith(os.path.abspath(EXTENSIONS_CHROOT)) - ] - - sysext_files = [f for f in changed_files if f.startswith("usr/") and not (f.startswith("usr/src/"))] - - for root, dirs, files in os.walk(EXTENSIONS_CHROOT, topdown=False): - for f in files: - path = os.path.relpath(os.path.abspath(os.path.join(root, f)), EXTENSIONS_CHROOT) - if path not in sysext_files: - os.unlink(os.path.join(root, f)) - - for d in dirs: - try: - os.rmdir(os.path.join(root, d)) - except NotADirectoryError: - os.unlink(os.path.join(root, d)) # It's a symlink - except OSError as e: - if e.errno == errno.ENOTEMPTY: - pass - else: - raise - - os.makedirs(f"{EXTENSIONS_CHROOT}/usr/lib/extension-release.d", exist_ok=True) - with open(f"{EXTENSIONS_CHROOT}/usr/lib/extension-release.d/extension-release.{name}", "w") as f: - f.write("ID=_any\n") - - run(["mksquashfs", EXTENSIONS_CHROOT, dst_path, "-comp", "xz"]) + os.unlink(driver) diff --git a/scale_build/image/bootstrap.py b/scale_build/image/bootstrap.py index 92a9a1c3..b2edee86 100644 --- a/scale_build/image/bootstrap.py +++ b/scale_build/image/bootstrap.py @@ -15,7 +15,7 @@ def setup_chroot_basedir(bootstrapdir_obj): if os.path.exists(CHROOT_BASEDIR): shutil.rmtree(CHROOT_BASEDIR) os.makedirs(TMPFS, exist_ok=True) - run(['mount', '-t', 'tmpfs', '-o', 'size=16G', 'tmpfs', TMPFS]) + run(['mount', '-t', 'tmpfs', '-o', 'size=20G', 'tmpfs', TMPFS]) bootstrapdir_obj.restore_cache(CHROOT_BASEDIR) run(['mount', 'proc', os.path.join(CHROOT_BASEDIR, 'proc'), '-t', 'proc']) run(['mount', 'sysfs', os.path.join(CHROOT_BASEDIR, 'sys'), '-t', 'sysfs']) diff --git a/scale_build/image/update.py b/scale_build/image/update.py index 2e504833..ffba6273 100644 --- a/scale_build/image/update.py +++ b/scale_build/image/update.py @@ -236,3 +236,9 @@ def build_extensions(): tf.close() run(["mksquashfs", CHROOT_BASEDIR, tf.name, "-one-file-system"]) do_build_extensions(tf.name, sysext_extensions_dir) + + external_extesions_dir = os.path.join(RELEASE_DIR, "extensions") + os.makedirs(external_extesions_dir, exist_ok=True) + for external_extension in ["dev-tools.raw"]: + shutil.move(os.path.join(sysext_extensions_dir, external_extension), + os.path.join(external_extesions_dir, external_extension)) diff --git a/scale_build/update_image.py b/scale_build/update_image.py index cac80c43..31ea31e0 100644 --- a/scale_build/update_image.py +++ b/scale_build/update_image.py @@ -38,7 +38,7 @@ def build_update_image_impl(): with LoggingContext('rootfs-packages', 'w'): setup_chroot_basedir(package_bootstrap_obj) - # These files will be overwritten so we should make sure that new build does not have any entities that + # These files will be overwritten, so we should make sure that new build does not have any entities that # are not in our reference files. for reference_file, diff in compare_reference_files(cut_nonexistent_user_group_membership=True): if any(line.startswith('+') for line in diff):