Add support for Singularity OCI mode (#4440)

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
Signed-off-by: Dr Marco Claudio De La Pierre <marco.delapierre@gmail.com>
Co-authored-by: Dr Marco Claudio De La Pierre <marco.delapierre@gmail.com>

docs/config.md

@@ -1347,6 +1347,12 @@ The following settings are available:
 `singularity.noHttps`
 : Pull the Singularity image with http protocol (default: `false`).
 
+`singularity.oci`
+: :::{versionadded} 23.11.0-edge
+  :::
+: Enable OCI-mode the allows the use of native OCI-compatible containers with Singularity. See [Singularity documentation](https://docs.sylabs.io/guides/4.0/user-guide/oci_runtime.html#oci-mode) for more details and requirements (default: `false`).
+
+
 `singularity.pullTimeout`
 : The amount of time the Singularity pull can last, exceeding which the process is terminated (default: `20 min`).
 

modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy

@@ -55,6 +55,10 @@ class ContainerConfig extends LinkedHashMap {
         get('engine')
     }
 
+    boolean singularityOciMode() {
+        getEngine()=='singularity' && get('oci')?.toString() == 'true'
+    }
+
     List<String> getEnvWhitelist() {
         def result = get('envWhitelist')
         if( !result )

modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy

@@ -67,6 +67,8 @@ class ContainerHandler {
             final normalizedImageName = normalizeSingularityImageName(imageName)
             if( !config.isEnabled() || !normalizedImageName )
                 return normalizedImageName
+            if( normalizedImageName.startsWith('docker://') && config.singularityOciMode() )
+                return normalizedImageName
             final requiresCaching = normalizedImageName =~ IMAGE_URL_PREFIX
             if( ContainerInspectMode.active() && requiresCaching )
                 return imageName
@@ -192,7 +194,7 @@ class ContainerHandler {
     }
 
 
-    public static final Pattern IMAGE_URL_PREFIX = ~/^[^\/:\. ]+:\/\/(.*)/
+    public static final Pattern IMAGE_URL_PREFIX = ~/^[^\/:. ]+:\/\/(.*)/
 
     /**
      * Normalize Singularity image name resolving the absolute path or

modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy

@@ -39,6 +39,8 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
 
     private String runCmd0
 
+    private Boolean oci
+
     SingularityBuilder(String name) {
         this.image = name
         this.homeMount = defaultHomeMount()
@@ -92,6 +94,9 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
         if( params.containsKey('readOnlyInputs') )
             this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true'
 
+        if( params.oci!=null )
+            oci = params.oci.toString() == 'true'
+
         return this
     }
 
@@ -117,9 +122,12 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
         if( !homeMount )
             result << '--no-home '
 
-        if( newPidNamespace )
+        if( newPidNamespace && !oci )
             result << '--pid '
 
+        if( oci != null )
+            result << (oci ? '--oci ' : '--no-oci ')
+
         if( autoMounts ) {
             makeVolumes(mounts, result)
         }
@@ -145,6 +153,11 @@ class SingularityBuilder extends ContainerBuilder<SingularityBuilder> {
     protected CharSequence appendEnv(StringBuilder result) {
         makeEnv('TMP',result) .append(' ')
         makeEnv('TMPDIR',result) .append(' ')
+        // add magic variables required by singularity to run in OCI-mode
+        if( oci ) {
+            result .append('${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ')
+            result .append('${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} ')
+        }
         super.appendEnv(result)
     }
 

modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy

@@ -480,8 +480,12 @@ class BashWrapperBuilder {
          */
         if( containerBuilder ) {
             String cmd = env ? 'eval $(nxf_container_env); ' + launcher : launcher
-            if( env && !containerConfig.entrypointOverride() ) {
-                if( containerBuilder instanceof SingularityBuilder )
+            // wrap the command with an extra bash invocation either :
+            // - to propagate the container environment or
+            // - to change in the task work directory as required by singularity
+            final needChangeTaskWorkDir = containerBuilder instanceof SingularityBuilder
+            if( (env || needChangeTaskWorkDir) && !containerConfig.entrypointOverride() ) {
+                if( needChangeTaskWorkDir )
                     cmd = 'cd $PWD; ' + cmd
                 cmd = "/bin/bash -c \"$cmd\""
             }

modules/nextflow/src/test/groovy/nextflow/container/ContainerConfigTest.groovy

@@ -61,6 +61,27 @@ class ContainerConfigTest extends Specification {
 
     }
 
+
+    def 'should validate oci mode' () {
+
+        when:
+        def cfg = new ContainerConfig(OPTS)
+        then:
+        cfg.singularityOciMode() == EXPECTED
+
+        where:
+        OPTS                                | EXPECTED
+        [:]                                 | false
+        [oci:false]                         | false
+        [oci:true]                          | false
+        [engine:'apptainer', oci:true]      | false
+        [engine:'docker', oci:true]         | false
+        [engine:'singularity']              | false
+        [engine:'singularity', oci:false]   | false
+        [engine:'singularity', oci:true]    | true
+
+    }
+
     def 'should get fusion options' () {
         when:
         def cfg = new ContainerConfig(OPTS)

modules/nextflow/src/test/groovy/nextflow/container/ContainerHandlerTest.groovy

@@ -200,11 +200,12 @@ class ContainerHandlerTest extends Specification {
         result == 'shifter://image'
 
     }
+
     @Unroll
     def 'test normalize method for singularity' () {
         given:
         def BASE = Paths.get('/abs/path/')
-        def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, baseDir: BASE))
+        def handler = Spy(new ContainerHandler(engine: 'singularity', enabled: true, oci:OCI, baseDir: BASE))
 
         when:
         def result = handler.normalizeImageName(IMAGE)
@@ -215,17 +216,21 @@ class ContainerHandlerTest extends Specification {
         result == EXPECTED
 
         where:
-        IMAGE                                       | NORMALIZED                                        | X           | EXPECTED
-        null                                        | null                                              |           0 | null
-        ''                                          | null                                              |           0 | null
-        '/abs/path/bar.img'                         | '/abs/path/bar.img'                               |           0 | '/abs/path/bar.img'
-        '/abs/path bar.img'                         | '/abs/path bar.img'                               |           0 | '/abs/path\\ bar.img'
-        'file:///abs/path/bar.img'                  | '/abs/path/bar.img'                               |           0 | '/abs/path/bar.img'
-        'foo.img'                                   | Paths.get('foo.img').toAbsolutePath().toString() |       0 | Paths.get('foo.img').toAbsolutePath().toString()
-        'shub://busybox'                            | 'shub://busybox'                                  |           1 | '/path/to/busybox'
-        'docker://library/busybox'                  | 'docker://library/busybox'                        |           1 | '/path/to/busybox'
-        'foo'                                       | 'docker://foo'                                    |           1 | '/path/to/foo'
-        'library://pditommaso/foo/bar.sif:latest'   | 'library://pditommaso/foo/bar.sif:latest'         |           1 | '/path/to/foo-bar-latest.img'
+        IMAGE                                       | NORMALIZED                                        | OCI   | X           | EXPECTED
+        null                                        | null                                              | false |           0 | null
+        ''                                          | null                                              | false |           0 | null
+        '/abs/path/bar.img'                         | '/abs/path/bar.img'                               | false |           0 | '/abs/path/bar.img'
+        '/abs/path bar.img'                         | '/abs/path bar.img'                               | false |           0 | '/abs/path\\ bar.img'
+        'file:///abs/path/bar.img'                  | '/abs/path/bar.img'                               | false |           0 | '/abs/path/bar.img'
+        'foo.img'                                   | Paths.get('foo.img').toAbsolutePath().toString()  | false |           0 | Paths.get('foo.img').toAbsolutePath().toString()
+        'shub://busybox'                            | 'shub://busybox'                                  | false |           1 | '/path/to/busybox'
+        'docker://library/busybox'                  | 'docker://library/busybox'                        | false |           1 | '/path/to/busybox'
+        'foo'                                       | 'docker://foo'                                    | false |           1 | '/path/to/foo'
+        'library://pditommaso/foo/bar.sif:latest'   | 'library://pditommaso/foo/bar.sif:latest'         | false |           1 | '/path/to/foo-bar-latest.img'
+        and:
+        'docker://library/busybox'                  | 'docker://library/busybox'                        | true  |           0 | 'docker://library/busybox'
+        'shub://busybox'                            | 'shub://busybox'                                  | true  |           1 | '/path/to/busybox'
+
     }
 
     def 'should not invoke caching when engine is disabled' () {

modules/nextflow/src/test/groovy/nextflow/container/SingularityBuilderTest.groovy

@@ -154,6 +154,11 @@ class SingularityBuilderTest extends Specification {
                 .build()
                 .runCommand == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} singularity exec --no-home ubuntu'
 
+        new SingularityBuilder('ubuntu')
+            .params(oci: true)
+            .build()
+            .runCommand == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} singularity exec --no-home --oci -B "$PWD" ubuntu'
+
     }
 
     def 'should mount home directory if specified' () {

modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy

@@ -996,6 +996,20 @@ class BashWrapperBuilderTest extends Specification {
         binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid'
     }
 
+    def 'should create wrapper with singularity and no env'() {
+        when:
+        def binding = newBashWrapperBuilder(
+            containerEnabled: true,
+            containerImage: 'docker://ubuntu:latest',
+            environment: [:],
+            containerConfig: [enabled: true, engine: 'singularity'] as ContainerConfig ).makeBinding()
+
+        then:
+        binding.launch_cmd == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${NXF_TASK_WORKDIR:+SINGULARITYENV_NXF_TASK_WORKDIR="$NXF_TASK_WORKDIR"} singularity exec --no-home --pid -B /work/dir docker://ubuntu:latest /bin/bash -c "cd $PWD; /bin/bash -ue /work/dir/.command.sh"'
+        binding.cleanup_cmd == ""
+        binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid'
+    }
+
     def 'should create wrapper with singularity legacy entry'() {
         when:
         def binding = newBashWrapperBuilder(
@@ -1010,6 +1024,20 @@ class BashWrapperBuilderTest extends Specification {
         binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid'
     }
 
+    def 'should create wrapper with singularity oci mode'() {
+        when:
+        def binding = newBashWrapperBuilder(
+            containerEnabled: true,
+            containerImage: 'docker://ubuntu:latest',
+            environment: [PATH: '/path/to/bin:$PATH', FOO: 'xxx'],
+            containerConfig: [enabled: true, engine: 'singularity', oci: true] as ContainerConfig ).makeBinding()
+
+        then:
+        binding.launch_cmd == 'set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} ${XDG_RUNTIME_DIR:+XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"} ${DBUS_SESSION_BUS_ADDRESS:+DBUS_SESSION_BUS_ADDRESS="$DBUS_SESSION_BUS_ADDRESS"} ${NXF_TASK_WORKDIR:+SINGULARITYENV_NXF_TASK_WORKDIR="$NXF_TASK_WORKDIR"} singularity exec --no-home --oci -B /work/dir docker://ubuntu:latest /bin/bash -c "cd $PWD; eval $(nxf_container_env); /bin/bash -ue /work/dir/.command.sh"'
+        binding.cleanup_cmd == ""
+        binding.kill_cmd == '[[ "$pid" ]] && nxf_kill $pid'
+    }
+
     def 'should create task and container env' () {
         given:
         def ENV = [FOO: 'hello', BAR: 'hello world', PATH: '/some/path:$PATH']

plugins/nf-wave/build.gradle

@@ -36,7 +36,7 @@ dependencies {
     api 'org.apache.commons:commons-lang3:3.12.0'
     api 'com.google.code.gson:gson:2.10.1'
     api 'org.yaml:snakeyaml:2.0'
-    api 'io.seqera:wave-utils:0.7.9'
+    api 'io.seqera:wave-utils:0.8.0'
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation "org.codehaus.groovy:groovy:3.0.19"

plugins/nf-wave/src/main/io/seqera/wave/plugin/resolver/WaveContainerResolver.groovy

@@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j
 import io.seqera.wave.plugin.WaveClient
 import nextflow.Global
 import nextflow.Session
+import nextflow.container.ContainerConfig
 import nextflow.container.resolver.ContainerInfo
 import nextflow.container.resolver.ContainerResolver
 import nextflow.container.resolver.DefaultContainerResolver
@@ -52,8 +53,7 @@ class WaveContainerResolver implements ContainerResolver {
         return client0 = new WaveClient( Global.session as Session )
     }
 
-    private String getContainerEngine0(TaskRun task) {
-        final config = task.getContainerConfig()
+    private String getContainerEngine0(ContainerConfig config) {
         final result = config.getEngine()
         if( result )
             return result
@@ -68,13 +68,15 @@ class WaveContainerResolver implements ContainerResolver {
             return defaultResolver.resolveImage(task, imageName)
 
         final freeze = client().config().freezeMode()
-        final engine = getContainerEngine0(task)
-        final nativeSingularityBuild = freeze && engine in SINGULARITY_LIKE
+        final config = task.getContainerConfig()
+        final engine = getContainerEngine0(config)
+        final singularityOciMode = config.singularityOciMode()
+        final singularitySpec = freeze && engine in SINGULARITY_LIKE && !singularityOciMode
         if( !imageName ) {
             // when no image name is provided the module bundle should include a
             // Dockerfile or a Conda recipe or a Spack recipe to build
             // an image on-fly with an automatically assigned name
-            return waveContainer(task, null, nativeSingularityBuild)
+            return waveContainer(task, null, singularitySpec)
         }
 
         if( engine in DOCKER_LIKE ) {
@@ -90,7 +92,7 @@ class WaveContainerResolver implements ContainerResolver {
                 return defaultResolver.resolveImage(task, imageName)
             }
             // fetch the wave container name
-            final image = waveContainer(task, imageName, nativeSingularityBuild)
+            final image = waveContainer(task, imageName, singularitySpec)
             // oras prefixed container are served directly
             if( image && image.target.startsWith("oras://") )
                 return image

plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy

@@ -534,6 +534,7 @@ class WaveClientTest extends Specification {
                     && micromamba install -y -n base conda-forge::procps-ng \\
                     && micromamba clean -a -y
                 USER root
+                ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
                     '''.stripIndent()
         and:
         !assets.moduleResources
@@ -572,6 +573,7 @@ class WaveClientTest extends Specification {
                     && micromamba install -y -n base conda-forge::procps-ng \\
                     && micromamba clean -a -y
                 USER root
+                ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
                     '''.stripIndent()
         and:
         !assets.moduleResources
@@ -647,6 +649,7 @@ class WaveClientTest extends Specification {
                     && micromamba install -y -n base conda-forge::procps-ng \\
                     && micromamba clean -a -y
                 USER root
+                ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
                     '''.stripIndent()
         and:
         assets.condaFile == condaFile