diff --git a/docs/config.md b/docs/config.md index 0486ab7a8e..f271f38a2d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1347,6 +1347,12 @@ The following settings are available: `singularity.noHttps` : Pull the Singularity image with http protocol (default: `false`). +`singularity.oci` +: :::{versionadded} 23.11.0-edge + ::: +: Enable OCI-mode the allows the use of native OCI-compatible containers with Singularity. See [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`). + + `singularity.pullTimeout` : The amount of time the Singularity pull can last, exceeding which the process is terminated (default: `20 min`). diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy index 8b93ac5cfe..78e8caef71 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy @@ -55,6 +55,10 @@ class ContainerConfig extends LinkedHashMap { get('engine') } + boolean singularityOciMode() { + getEngine()=='singularity' && get('oci')?.toString() == 'true' + } + List getEnvWhitelist() { def result = get('envWhitelist') if( !result ) diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy index aafa544913..524ce6d9e3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy @@ -67,6 +67,8 @@ class ContainerHandler { final normalizedImageName = normalizeSingularityImageName(imageName) if( !config.isEnabled() || !normalizedImageName ) return normalizedImageName + if( normalizedImageName.startsWith('docker://') && config.singularityOciMode() ) + return normalizedImageName final requiresCaching = normalizedImageName =~ IMAGE_URL_PREFIX if( ContainerInspectMode.active() && requiresCaching ) return imageName @@ -192,7 +194,7 @@ class ContainerHandler { } - public static final Pattern IMAGE_URL_PREFIX = ~/^[^\/:\. ]+:\/\/(.*)/ + public static final Pattern IMAGE_URL_PREFIX = ~/^[^\/:. ]+:\/\/(.*)/ /** * Normalize Singularity image name resolving the absolute path or diff --git a/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy index 5c2d74abfc..6f69c8e116 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy @@ -39,6 +39,8 @@ class SingularityBuilder extends ContainerBuilder { private String runCmd0 + private Boolean oci + SingularityBuilder(String name) { this.image = name this.homeMount = defaultHomeMount() @@ -92,6 +94,9 @@ class SingularityBuilder extends ContainerBuilder { if( params.containsKey('readOnlyInputs') ) this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true' + if( params.oci!=null ) + oci = params.oci.toString() == 'true' + return this } @@ -117,9 +122,12 @@ class SingularityBuilder extends ContainerBuilder { if( !homeMount ) result << '--no-home ' - if( newPidNamespace ) + if( newPidNamespace && !oci ) result << '--pid ' + if( oci != null ) + result << (oci ? '--oci ' : '--no-oci ') + if( autoMounts ) { makeVolumes(mounts, result) } @@ -145,6 +153,11 @@ class SingularityBuilder extends ContainerBuilder { protected CharSequence appendEnv(StringBuilder result) { makeEnv('TMP',result) .append(' ') makeEnv('TMPDIR',result) .append(' ') + // add magic variables required by singularity to run in OCI-mode + if( oci ) { + result .append('${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ') + result .append('${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} ') + } super.appendEnv(result) } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index bc21186d1a..d53ad6aea1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -480,8 +480,12 @@ class BashWrapperBuilder { */ if( containerBuilder ) { String cmd = env ? 'eval $(nxf_container_env); ' + launcher : launcher - if( env && !containerConfig.entrypointOverride() ) { - if( containerBuilder instanceof SingularityBuilder ) + // wrap the command with an extra bash invocation either : + // - to propagate the container environment or + // - to change in the task work directory as required by singularity + final needChangeTaskWorkDir = containerBuilder instanceof SingularityBuilder + if( (env || needChangeTaskWorkDir) && !containerConfig.entrypointOverride() ) { + if( needChangeTaskWorkDir ) cmd = 'cd $PWD; ' + cmd cmd = "/bin/bash -c \"$cmd\"" } diff --git a/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy index 6c87437eb7..99ed6439fe 100644 --- a/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy @@ -61,6 +61,27 @@ class ContainerConfigTest extends Specification { } + + def 'should validate oci mode' () { + + when: + def cfg = new ContainerConfig(OPTS) + then: + cfg.singularityOciMode() == EXPECTED + + where: + OPTS | EXPECTED + [:] | false + [oci:false] | false + [oci:true] | false + [engine:'apptainer', oci:true] | false + [engine:'docker', oci:true] | false + [engine:'singularity'] | false + [engine:'singularity', oci:false] | false + [engine:'singularity', oci:true] | true + + } + def 'should get fusion options' () { when: def cfg = new ContainerConfig(OPTS) diff --git a/modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy index 43129de399..7459c9de21 100644 --- a/modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy @@ -200,11 +200,12 @@ class ContainerHandlerTest extends Specification { result == 'shifter://image' } + @Unroll def 'test normalize method for singularity' () { given: def BASE = Paths.get('/abs/path/') - def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, baseDir: BASE)) + def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, oci:OCI, baseDir: BASE)) when: def result = handler.normalizeImageName(IMAGE) @@ -215,17 +216,21 @@ class ContainerHandlerTest extends Specification { result == EXPECTED where: - IMAGE | NORMALIZED | X | EXPECTED - null | null | 0 | null - '' | null | 0 | null - '/abs/path/bar.img' | '/abs/path/bar.img' | 0 | '/abs/path/bar.img' - '/abs/path bar.img' | '/abs/path bar.img' | 0 | '/abs/path\\ bar.img' - 'file:///abs/path/bar.img' | '/abs/path/bar.img' | 0 | '/abs/path/bar.img' - 'foo.img' | Paths.get('foo.img').toAbsolutePath().toString() | 0 | Paths.get('foo.img').toAbsolutePath().toString() - 'shub://busybox' | 'shub://busybox' | 1 | '/path/to/busybox' - 'docker://library/busybox' | 'docker://library/busybox' | 1 | '/path/to/busybox' - 'foo' | 'docker://foo' | 1 | '/path/to/foo' - 'library://pditommaso/foo/bar.sif:latest' | 'library://pditommaso/foo/bar.sif:latest' | 1 | '/path/to/foo-bar-latest.img' + IMAGE | NORMALIZED | OCI | X | EXPECTED + null | null | false | 0 | null + '' | null | false | 0 | null + '/abs/path/bar.img' | '/abs/path/bar.img' | false | 0 | '/abs/path/bar.img' + '/abs/path bar.img' | '/abs/path bar.img' | false | 0 | '/abs/path\\ bar.img' + 'file:///abs/path/bar.img' | '/abs/path/bar.img' | false | 0 | '/abs/path/bar.img' + 'foo.img' | Paths.get('foo.img').toAbsolutePath().toString() | false | 0 | Paths.get('foo.img').toAbsolutePath().toString() + 'shub://busybox' | 'shub://busybox' | false | 1 | '/path/to/busybox' + 'docker://library/busybox' | 'docker://library/busybox' | false | 1 | '/path/to/busybox' + 'foo' | 'docker://foo' | false | 1 | '/path/to/foo' + 'library://pditommaso/foo/bar.sif:latest' | 'library://pditommaso/foo/bar.sif:latest' | false | 1 | '/path/to/foo-bar-latest.img' + and: + 'docker://library/busybox' | 'docker://library/busybox' | true | 0 | 'docker://library/busybox' + 'shub://busybox' | 'shub://busybox' | true | 1 | '/path/to/busybox' + } def 'should not invoke caching when engine is disabled' () { diff --git a/modules/nextflow/src/test/groovy/nextflow/container/SingularityBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/container/SingularityBuilderTest.groovy index 661f459a29..9aad74ffb9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/container/SingularityBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/container/SingularityBuilderTest.groovy @@ -154,6 +154,11 @@ class SingularityBuilderTest extends Specification { .build() .runCommand == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} singularity exec --no-home ubuntu' + new SingularityBuilder('ubuntu') + .params(oci: true) + .build() + .runCommand == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} singularity exec --no-home --oci -B "$PWD" ubuntu' + } def 'should mount home directory if specified' () { diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index 4a8c012502..03fb0616ea 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -996,6 +996,20 @@ class BashWrapperBuilderTest extends Specification { binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid' } + def 'should create wrapper with singularity and no env'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'docker://ubuntu:latest', + environment: [:], + containerConfig: [enabled: true, engine: 'singularity'] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${NXF_TASK_WORKDIR:+SINGULARITYENV_NXF_TASK_WORKDIR="$NXF_TASK_WORKDIR"} singularity exec --no-home --pid -B /work/dir docker://ubuntu:latest /bin/bash -c "cd $PWD; /bin/bash -ue /work/dir/.command.sh"' + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid' + } + def 'should create wrapper with singularity legacy entry'() { when: def binding = newBashWrapperBuilder( @@ -1010,6 +1024,20 @@ class BashWrapperBuilderTest extends Specification { binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid' } + def 'should create wrapper with singularity oci mode'() { + when: + def binding = newBashWrapperBuilder( + containerEnabled: true, + containerImage: 'docker://ubuntu:latest', + environment: [PATH: '/path/to/bin:$PATH', FOO: 'xxx'], + containerConfig: [enabled: true, engine: 'singularity', oci: true] as ContainerConfig ).makeBinding() + + then: + binding.launch_cmd == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} ${NXF_TASK_WORKDIR:+SINGULARITYENV_NXF_TASK_WORKDIR="$NXF_TASK_WORKDIR"} singularity exec --no-home --oci -B /work/dir docker://ubuntu:latest /bin/bash -c "cd $PWD; eval $(nxf_container_env); /bin/bash -ue /work/dir/.command.sh"' + binding.cleanup_cmd == "" + binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid' + } + def 'should create task and container env' () { given: def ENV = [FOO: 'hello', BAR: 'hello world', PATH: '/some/path:$PATH'] diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle index f4deb73c9c..93d749fba3 100644 --- a/plugins/nf-wave/build.gradle +++ b/plugins/nf-wave/build.gradle @@ -36,7 +36,7 @@ dependencies { api 'org.apache.commons:commons-lang3:3.12.0' api 'com.google.code.gson:gson:2.10.1' api 'org.yaml:snakeyaml:2.0' - api 'io.seqera:wave-utils:0.7.9' + api 'io.seqera:wave-utils:0.8.0' testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.codehaus.groovy:groovy:3.0.19" diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy index a79d169b2f..28c64973f3 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy @@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j import io.seqera.wave.plugin.WaveClient import nextflow.Global import nextflow.Session +import nextflow.container.ContainerConfig import nextflow.container.resolver.ContainerInfo import nextflow.container.resolver.ContainerResolver import nextflow.container.resolver.DefaultContainerResolver @@ -52,8 +53,7 @@ class WaveContainerResolver implements ContainerResolver { return client0 = new WaveClient( Global.session as Session ) } - private String getContainerEngine0(TaskRun task) { - final config = task.getContainerConfig() + private String getContainerEngine0(ContainerConfig config) { final result = config.getEngine() if( result ) return result @@ -68,13 +68,15 @@ class WaveContainerResolver implements ContainerResolver { return defaultResolver.resolveImage(task, imageName) final freeze = client().config().freezeMode() - final engine = getContainerEngine0(task) - final nativeSingularityBuild = freeze && engine in SINGULARITY_LIKE + final config = task.getContainerConfig() + final engine = getContainerEngine0(config) + final singularityOciMode = config.singularityOciMode() + final singularitySpec = freeze && engine in SINGULARITY_LIKE && !singularityOciMode if( !imageName ) { // when no image name is provided the module bundle should include a // Dockerfile or a Conda recipe or a Spack recipe to build // an image on-fly with an automatically assigned name - return waveContainer(task, null, nativeSingularityBuild) + return waveContainer(task, null, singularitySpec) } if( engine in DOCKER_LIKE ) { @@ -90,7 +92,7 @@ class WaveContainerResolver implements ContainerResolver { return defaultResolver.resolveImage(task, imageName) } // fetch the wave container name - final image = waveContainer(task, imageName, nativeSingularityBuild) + final image = waveContainer(task, imageName, singularitySpec) // oras prefixed container are served directly if( image && image.target.startsWith("oras://") ) return image diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy index c0259af468..6842819bb2 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy @@ -534,6 +534,7 @@ class WaveClientTest extends Specification { && micromamba install -y -n base conda-forge::procps-ng \\ && micromamba clean -a -y USER root + ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" '''.stripIndent() and: !assets.moduleResources @@ -572,6 +573,7 @@ class WaveClientTest extends Specification { && micromamba install -y -n base conda-forge::procps-ng \\ && micromamba clean -a -y USER root + ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" '''.stripIndent() and: !assets.moduleResources @@ -647,6 +649,7 @@ class WaveClientTest extends Specification { && micromamba install -y -n base conda-forge::procps-ng \\ && micromamba clean -a -y USER root + ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" '''.stripIndent() and: assets.condaFile == condaFile