Fix closure rendering in yaml and json config (#5408) [ci fast]

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Signed-off-by: Ben Sherman <bentshermann@gmail.com> Co-authored-by: Ben Sherman <bentshermann@gmail.com>
nextflow-io · Oct 18, 2024 · d3a85ce · d3a85ce
1 parent 711864f
commit d3a85ce
Show file tree

Hide file tree

Showing 4 changed files with 130 additions and 6 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy
@@ -18,8 +18,6 @@ package nextflow.config
 
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
-import groovy.transform.PackageScope
-
 /**
  * Placeholder class that replacing closure definitions in the nextflow configuration
  * file in order to print the closure content itself
@@ -28,7 +26,6 @@ import groovy.transform.PackageScope
  */
 @EqualsAndHashCode
 @CompileStatic
-@PackageScope
 class ConfigClosurePlaceholder {
 
     private String str

diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy
@@ -22,10 +22,10 @@ import groovy.json.JsonOutput
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
+import nextflow.config.ConfigClosurePlaceholder
 import org.codehaus.groovy.runtime.InvokerHelper
 import org.yaml.snakeyaml.DumperOptions
 import org.yaml.snakeyaml.Yaml
-
 /**
  * Helper method to handle configuration object
  *
@@ -356,9 +356,32 @@ class ConfigHelper {
         for( Map.Entry it : config ) {
             if( it.value instanceof Map && !it.value )
                 continue
-            result.put(it.key, it.value)
+            result.put(it.key, normaliseObject0(it.value))
         }
         return result
     }
+
+    private static Object normaliseObject0(Object value) {
+        if( value instanceof Map ) {
+            final map = value as Map
+            final ret = new LinkedHashMap(map.size())
+            for( Map.Entry entry : map.entrySet() ) {
+                ret.put(entry.key, normaliseObject0(entry.value))
+            }
+            return ret
+        }
+        if( value instanceof Collection ) {
+            final lis = value as List
+            final ret = new ArrayList(lis.size())
+            for( Object it : lis )
+                ret.add(normaliseObject0(it))
+            return ret
+        }
+        else if( value instanceof ConfigClosurePlaceholder ) {
+            return value.toString()
+        }
+        else
+            return value
+    }
 }
 
diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy
@@ -490,4 +490,101 @@ class CmdConfigTest extends Specification {
         folder?.deleteDir()
         SysEnv.pop()
     }
+
+    def 'should render json config output' () {
+        given:
+        def folder = Files.createTempDirectory('test')
+        def CONFIG = folder.resolve('nextflow.config')
+
+        CONFIG.text = '''
+        manifest {
+            author = 'me'
+            mainScript = 'foo.nf'
+        }
+        
+        process {
+          cpus = 4 
+          queue = 'cn-el7'
+          memory = { 10.GB }   
+          ext.other = { 10.GB * task.attempt }
+        }
+        '''
+        def buffer = new ByteArrayOutputStream()
+        // command definition
+        def cmd = new CmdConfig(outputFormat: 'json')
+        cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()]))
+        cmd.stdout = buffer
+        cmd.args = [ '.' ]
+
+        when:
+        cmd.run()
+
+        then:
+        buffer.toString() == '''
+        {
+            "manifest": {
+                "author": "me",
+                "mainScript": "foo.nf"
+            },
+            "process": {
+                "cpus": 4,
+                "queue": "cn-el7",
+                "memory": "{ 10.GB }",
+                "ext": {
+                    "other": "{ 10.GB * task.attempt }"
+                }
+            }
+        }
+        '''
+            .stripIndent().leftTrim()
+
+        cleanup:
+        folder.deleteDir()
+    }
+
+    def 'should render yaml config output' () {
+        given:
+        def folder = Files.createTempDirectory('test')
+        def CONFIG = folder.resolve('nextflow.config')
+
+        CONFIG.text = '''
+        manifest {
+            author = 'me'
+            mainScript = 'foo.nf'
+        }
+        
+        process {
+          cpus = 4 
+          queue = 'cn-el7'
+          memory = { 10.GB }   
+          ext.other = { 10.GB * task.attempt }
+        }
+        '''
+        def buffer = new ByteArrayOutputStream()
+        // command definition
+        def cmd = new CmdConfig(outputFormat: 'yaml')
+        cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()]))
+        cmd.stdout = buffer
+        cmd.args = [ '.' ]
+
+        when:
+        cmd.run()
+
+        then:
+        buffer.toString() == '''
+            manifest:
+              author: me
+              mainScript: foo.nf
+            process:
+              cpus: 4
+              queue: cn-el7
+              memory: '{ 10.GB }'
+              ext:
+                other: '{ 10.GB * task.attempt }'
+            '''
+                .stripIndent().leftTrim()
+
+        cleanup:
+        folder.deleteDir()
+    }
 }
diff --git a/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy
@@ -19,6 +19,7 @@ package nextflow.util
 import java.nio.file.Files
 import java.nio.file.Paths
 
+import nextflow.config.ConfigClosurePlaceholder
 import spock.lang.Specification
 import spock.lang.Unroll
 
@@ -250,6 +251,7 @@ class ConfigHelperTest extends Specification {
         def config = new ConfigObject()
         config.process.queue = 'long'
         config.process.executor = 'slurm'
+        config.process.memory = new ConfigClosurePlaceholder('{ 1.GB }')
         config.docker.enabled = true
         config.zeta.'quoted-attribute'.foo = 1
 
@@ -263,6 +265,7 @@ class ConfigHelperTest extends Specification {
                 },
                 "process": {
                     "executor": "slurm",
+                    "memory": "{ 1.GB }",
                     "queue": "long"
                 },
                 "zeta": {
@@ -281,7 +284,8 @@ class ConfigHelperTest extends Specification {
             {
                 "process": {
                     "queue": "long",
-                    "executor": "slurm"
+                    "executor": "slurm",
+                    "memory": "{ 1.GB }"
                 },
                 "docker": {
                     "enabled": true
@@ -300,6 +304,7 @@ class ConfigHelperTest extends Specification {
         def config = new ConfigObject()
         config.process.queue = 'long'
         config.process.executor = 'slurm'
+        config.process.memory = new ConfigClosurePlaceholder('{ 1.GB }')
         config.docker.enabled = true
         config.zeta.'quoted-attribute'.foo = 1
 
@@ -311,6 +316,7 @@ class ConfigHelperTest extends Specification {
               enabled: true
             process:
               executor: slurm
+              memory: '{ 1.GB }'
               queue: long
             zeta:
               quoted-attribute:
@@ -325,6 +331,7 @@ class ConfigHelperTest extends Specification {
             process:
               queue: long
               executor: slurm
+              memory: '{ 1.GB }'
             docker:
               enabled: true
             zeta: