diff --git a/CHANGELOG b/CHANGELOG index 64d13ce9..9ea7e137 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,6 +15,7 @@ Additions: - Added barebones module awareness - Added logging output in dedicated logging folders - Added docker backend module +- Added podman backend module Removed: diff --git a/docs/compatibility/software.rst b/docs/compatibility/software.rst index d21f5322..8876fc41 100644 --- a/docs/compatibility/software.rst +++ b/docs/compatibility/software.rst @@ -4,14 +4,13 @@ Software Compatibility Container backends ------------------- -As of now, `singularity `_, `shifter `_ and `docker `_ are supported in **e4s-cl**. +As of now, `singularity `_, `shifter `_, `docker `_ and `podman `_ are supported in **e4s-cl**. More container technologies can be supported. Create an issue on github or write a dedicated module in :code:`e4s_cl/cf/containers`. Refer to :code:`e4s_cl/cf/containers/__init__.py` for details. .. warning:: Using **docker** with MPI - Several MPI implementations expect their processes to inherit opened file descriptors; because of docker's client-daemon architecture, this is not possible. - + Several MPI implementations expect their processes to inherit opened file descriptors; because of docker's client-daemon architecture, this is not possible.To use docker images with MPI, it is encouraged to used podman `_. Process launchers ------------------ diff --git a/packages/e4s_cl/cf/containers/podman.py b/packages/e4s_cl/cf/containers/podman.py new file mode 100644 index 00000000..640efbec --- /dev/null +++ b/packages/e4s_cl/cf/containers/podman.py @@ -0,0 +1,159 @@ +""" +Podman container manager support +""" + +import os +from pathlib import Path +from e4s_cl.error import InternalError +from e4s_cl.util import which, run_subprocess +from e4s_cl.logger import get_logger +from e4s_cl.cf.pipe import NamedPipe, ENV_VAR_NAMED +from e4s_cl.cf.containers import Container, FileOptions, BackendNotAvailableError + +LOGGER = get_logger(__name__) + +NAME = 'podman' +EXECUTABLES = ['podman'] +MIMES = [] + + +def opened_fds(): + """ + -> set[int] + + Returns a list of all the opened file descriptors opened by the current + process + """ + fds = [] + + for file in Path('/proc/self/fd').glob('*'): + if not file.exists(): + continue + + try: + fd_no = int(file.name) + except ValueError: + continue + + fds.append(fd_no) + + return fds + + +class FDFiller: + """ + Context manager that will "fill" the opened file descriptors to have a + contiguous list, and make every fd inheritable + """ + + def __init__(self): + """ + Initialize by creating a buffer of opened files + """ + self.__opened_files = [] + + def __enter__(self): + """ + Create as many open files as necessary + """ + fds = opened_fds() + + # Make every existing file descriptor inheritable + for fd in fds: + try: + os.set_inheritable(fd, True) + except OSError as err: + if err.errno == 9: + continue + + # Compute all the missing numbers in the list + missing = set(range(max(fds))) - set(fds) + + while missing: + # Open files towards /dev/null + null = open('/dev/null', 'w', encoding='utf-8') + + if null.fileno() not in missing: + raise InternalError(f"Unexpected fileno: {null.fileno()}") + + try: + # Set the file as inheritable + os.set_inheritable(null.fileno(), True) + except OSError as err: + if err.errno == 9: + continue + + # It is not missing anymore + missing.discard(null.fileno()) + self.__opened_files.append(null) + + LOGGER.debug("Created %d file descriptors: %s", + len(self.__opened_files), + [f.fileno() for f in self.__opened_files]) + + def __exit__(self, type_, value, traceback): + for file in self.__opened_files: + file.close() + + +class PodmanContainer(Container): + """ + Podman container object + """ + + def _fd_number(self): + """ + Podman requires the --preserve-fds=K option to pass file descriptors; + K being the amount (in addition of 0,1,2) of fds to pass. It also is + strict on the existence and inheritance flag of those descriptors, and + will not function if any one of them is invalid/uninheritable. + """ + + LOGGER.debug("Max fd: %d (%s)", max(opened_fds()), opened_fds()) + return max(opened_fds()) - 3 + + def _working_dir(self): + return ['--workdir', os.getcwd()] + + def _format_bound(self): + + def _format(): + fifo = os.environ.get(ENV_VAR_NAMED, '') + if fifo: + yield f"--mount=type=bind,src={fifo},dst={fifo},ro=false" + + for src, dst, opt in self.bound: + yield f"--mount=type=bind,src={src.as_posix()},dst={dst.as_posix()}{',ro=true' if (opt == FileOptions.READ_ONLY) else ''}" + + return list(_format()) + + def _prepare(self, command): + + return [ + self.executable, # absolute path to podman + 'run', # Run a container + '--rm', # Remove when done + '--ipc=host', # Use host IPC /!\ + '--env-host', # Pass host environment /!\ + f"--preserve-fds={self._fd_number()}", # Inherit file descriptors /!\ + *self._working_dir(), # Work in the same CWD + *self._format_bound(), # Bound files options + self.image, + *command + ] + + def run(self, command): + """ + def run(self, command: list[str]): + """ + + if not which(self.executable): + raise BackendNotAvailableError(self.executable) + + container_cmd = self._prepare(command) + + with FDFiller(): + return run_subprocess(container_cmd, env=self.env) + + +CLASS = PodmanContainer diff --git a/packages/e4s_cl/cf/containers/singularity.py b/packages/e4s_cl/cf/containers/singularity.py index e380b1a2..5c5c6da3 100644 --- a/packages/e4s_cl/cf/containers/singularity.py +++ b/packages/e4s_cl/cf/containers/singularity.py @@ -37,13 +37,24 @@ def __setup__(self): #self.bind_file('/dev', option=FileOptions.READ_WRITE) #self.bind_file('/tmp', option=FileOptions.READ_WRITE) + def _format_bound(self): + """ + Format a list of files to a compatible bind option of singularity + """ + + def _format(): + for source, dest, options_val in self.bound: + yield f"{source}:{dest}:{OPTION_STRINGS[options_val]}" + + self.env.update({"SINGULARITY_BIND": ','.join(_format())}) + def _prepare(self, command) -> list[str]: self.add_ld_library_path("/.singularity.d/libs") self.env.update( {'SINGULARITYENV_LD_PRELOAD': ":".join(self.ld_preload)}) self.env.update( {'SINGULARITYENV_LD_LIBRARY_PATH': ":".join(self.ld_lib_path)}) - self.format_bound() + self._format_bound() nvidia_flag = ['--nv'] if self._has_nvidia() else [] return [ @@ -51,25 +62,6 @@ def _prepare(self, command) -> list[str]: self.image, *command ] - def run(self, command): - if not which(self.executable): - raise BackendNotAvailableError(self.executable) - - container_cmd = self._prepare(command) - - return run_subprocess(container_cmd, env=self.env) - - def format_bound(self): - """ - Format a list of files to a compatible bind option of singularity - """ - - def _format(): - for source, dest, options_val in self.bound: - yield f"{source}:{dest}:{OPTION_STRINGS[options_val]}" - - self.env.update({"SINGULARITY_BIND": ','.join(_format())}) - def bind_env_var(self, key, value): self.env.update({f"SINGULARITYENV_{key}": value}) @@ -79,5 +71,14 @@ def _has_nvidia(self): return False return True + def run(self, command): + if not which(self.executable): + raise BackendNotAvailableError(self.executable) + + container_cmd = self._prepare(command) + + return run_subprocess(container_cmd, env=self.env) + + CLASS = SingularityContainer diff --git a/packages/e4s_cl/cli/commands/__execute.py b/packages/e4s_cl/cli/commands/__execute.py index 3d141498..9fd93057 100644 --- a/packages/e4s_cl/cli/commands/__execute.py +++ b/packages/e4s_cl/cli/commands/__execute.py @@ -258,6 +258,9 @@ def _path(library: HostLibrary): code = container.run(command) + if code: + LOGGER.critical("Container command failed with error code %d", code) + params.teardown() return code diff --git a/packages/e4s_cl/util.py b/packages/e4s_cl/util.py index 2242b2e3..a2f1bd5d 100644 --- a/packages/e4s_cl/util.py +++ b/packages/e4s_cl/util.py @@ -154,6 +154,8 @@ def run_subprocess(cmd: list[str], if log_file := getattr(process_logger.handlers[0], 'baseFilename', None): LOGGER.error("See %s for details.", log_file) + else: + LOGGER.debug("Process %d returned %d", pid, returncode) del process_logger