From d3a85cebbb0e1e2b032aaf4bbdcf515af6e27459 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 18 Oct 2024 15:03:04 +0200 Subject: [PATCH] Fix closure rendering in yaml and json config (#5408) [ci fast] Signed-off-by: Paolo Di Tommaso Signed-off-by: Ben Sherman Co-authored-by: Ben Sherman --- .../config/ConfigClosurePlaceholder.groovy | 3 - .../groovy/nextflow/util/ConfigHelper.groovy | 27 +++++- .../groovy/nextflow/cli/CmdConfigTest.groovy | 97 +++++++++++++++++++ .../nextflow/util/ConfigHelperTest.groovy | 9 +- 4 files changed, 130 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy index 2780ef34e3..1ceb074923 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy @@ -18,8 +18,6 @@ package nextflow.config import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode -import groovy.transform.PackageScope - /** * Placeholder class that replacing closure definitions in the nextflow configuration * file in order to print the closure content itself @@ -28,7 +26,6 @@ import groovy.transform.PackageScope */ @EqualsAndHashCode @CompileStatic -@PackageScope class ConfigClosurePlaceholder { private String str diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy index ad63c2f81f..d077760614 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy @@ -22,10 +22,10 @@ import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j +import nextflow.config.ConfigClosurePlaceholder import org.codehaus.groovy.runtime.InvokerHelper import org.yaml.snakeyaml.DumperOptions import org.yaml.snakeyaml.Yaml - /** * Helper method to handle configuration object * @@ -356,9 +356,32 @@ class ConfigHelper { for( Map.Entry it : config ) { if( it.value instanceof Map && !it.value ) continue - result.put(it.key, it.value) + result.put(it.key, normaliseObject0(it.value)) } return result } + + private static Object normaliseObject0(Object value) { + if( value instanceof Map ) { + final map = value as Map + final ret = new LinkedHashMap(map.size()) + for( Map.Entry entry : map.entrySet() ) { + ret.put(entry.key, normaliseObject0(entry.value)) + } + return ret + } + if( value instanceof Collection ) { + final lis = value as List + final ret = new ArrayList(lis.size()) + for( Object it : lis ) + ret.add(normaliseObject0(it)) + return ret + } + else if( value instanceof ConfigClosurePlaceholder ) { + return value.toString() + } + else + return value + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy index cce8f9b0fd..a2f97b7496 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy @@ -490,4 +490,101 @@ class CmdConfigTest extends Specification { folder?.deleteDir() SysEnv.pop() } + + def 'should render json config output' () { + given: + def folder = Files.createTempDirectory('test') + def CONFIG = folder.resolve('nextflow.config') + + CONFIG.text = ''' + manifest { + author = 'me' + mainScript = 'foo.nf' + } + + process { + cpus = 4 + queue = 'cn-el7' + memory = { 10.GB } + ext.other = { 10.GB * task.attempt } + } + ''' + def buffer = new ByteArrayOutputStream() + // command definition + def cmd = new CmdConfig(outputFormat: 'json') + cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()])) + cmd.stdout = buffer + cmd.args = [ '.' ] + + when: + cmd.run() + + then: + buffer.toString() == ''' + { + "manifest": { + "author": "me", + "mainScript": "foo.nf" + }, + "process": { + "cpus": 4, + "queue": "cn-el7", + "memory": "{ 10.GB }", + "ext": { + "other": "{ 10.GB * task.attempt }" + } + } + } + ''' + .stripIndent().leftTrim() + + cleanup: + folder.deleteDir() + } + + def 'should render yaml config output' () { + given: + def folder = Files.createTempDirectory('test') + def CONFIG = folder.resolve('nextflow.config') + + CONFIG.text = ''' + manifest { + author = 'me' + mainScript = 'foo.nf' + } + + process { + cpus = 4 + queue = 'cn-el7' + memory = { 10.GB } + ext.other = { 10.GB * task.attempt } + } + ''' + def buffer = new ByteArrayOutputStream() + // command definition + def cmd = new CmdConfig(outputFormat: 'yaml') + cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()])) + cmd.stdout = buffer + cmd.args = [ '.' ] + + when: + cmd.run() + + then: + buffer.toString() == ''' + manifest: + author: me + mainScript: foo.nf + process: + cpus: 4 + queue: cn-el7 + memory: '{ 10.GB }' + ext: + other: '{ 10.GB * task.attempt }' + ''' + .stripIndent().leftTrim() + + cleanup: + folder.deleteDir() + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy index 7674b97054..985038ca1f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/util/ConfigHelperTest.groovy @@ -19,6 +19,7 @@ package nextflow.util import java.nio.file.Files import java.nio.file.Paths +import nextflow.config.ConfigClosurePlaceholder import spock.lang.Specification import spock.lang.Unroll @@ -250,6 +251,7 @@ class ConfigHelperTest extends Specification { def config = new ConfigObject() config.process.queue = 'long' config.process.executor = 'slurm' + config.process.memory = new ConfigClosurePlaceholder('{ 1.GB }') config.docker.enabled = true config.zeta.'quoted-attribute'.foo = 1 @@ -263,6 +265,7 @@ class ConfigHelperTest extends Specification { }, "process": { "executor": "slurm", + "memory": "{ 1.GB }", "queue": "long" }, "zeta": { @@ -281,7 +284,8 @@ class ConfigHelperTest extends Specification { { "process": { "queue": "long", - "executor": "slurm" + "executor": "slurm", + "memory": "{ 1.GB }" }, "docker": { "enabled": true @@ -300,6 +304,7 @@ class ConfigHelperTest extends Specification { def config = new ConfigObject() config.process.queue = 'long' config.process.executor = 'slurm' + config.process.memory = new ConfigClosurePlaceholder('{ 1.GB }') config.docker.enabled = true config.zeta.'quoted-attribute'.foo = 1 @@ -311,6 +316,7 @@ class ConfigHelperTest extends Specification { enabled: true process: executor: slurm + memory: '{ 1.GB }' queue: long zeta: quoted-attribute: @@ -325,6 +331,7 @@ class ConfigHelperTest extends Specification { process: queue: long executor: slurm + memory: '{ 1.GB }' docker: enabled: true zeta: