From fc5e2c2a8e4608c19d7072f622f3368024420c47 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Fri, 29 Nov 2024 18:47:38 +0000 Subject: [PATCH 1/3] Incorrect CPU value in Azure example (#5549) [ci skip] Signed-off-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> --- docs/azure.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/azure.md b/docs/azure.md index 6087c18ec2..61269e43ac 100644 --- a/docs/azure.md +++ b/docs/azure.md @@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob process.machineType = "Standard_D*d_v5,Standard_E*d_v5" ``` -For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`: +For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`: ```nextflow process EXAMPLE_PROCESS { machineType "Standard_E*d_v5" - cpus 16 + cpus 8 memory 8.GB script: From ab13ce58b2b176afd6cabbeb7149d8f0611a77f8 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 27 Nov 2024 22:30:30 +0100 Subject: [PATCH 2/3] Update changelog [ci skip] Signed-off-by: Paolo Di Tommaso --- changelog.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/changelog.txt b/changelog.txt index 3420a4041e..0be1d17adf 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,20 @@ NEXTFLOW CHANGE-LOG =================== +24.10.2 - 27 Nov 2024 +- Prevent NPE with null AWS Batch response [3d491934] +- Fix overlapping conda lock file (#5540) [df66deaa] +- Fix missing wave response (#5547) [eb85cda8] +- Bump nf-wave@1.7.4 [93d09404] +- Bump nf-amazon@2.9.2 [469a35dd] + +24.10.1 - 18 Nov 2024 +- Fix overlapping file lock exception (#5489) [a2566d54] +- Fix isContainerReady when wave is disabled (#5509) [c69e3711] +- Bump nf-wave@1.7.3 [e7709a0f] +- Bump nf-azure@1.10.2 [54496ac4] +- Bump nf-amazon@2.9.1 [fa227933] +- Bump netty-common to version 4.1.115.Final [90623c1e] + 24.10.0 - 27 Oct 2024 - Add `manifest.contributors` config option (#5322) [cf0f9690] - Add wave mirror and scan config [92e69776] From 3c8e602d487a24654da8046ada3f7dc12e0fe39f Mon Sep 17 00:00:00 2001 From: Jorge Ejarque Date: Mon, 2 Dec 2024 18:42:00 +0100 Subject: [PATCH 3/3] Detecting errors in data unstaging (#5345) Signed-off-by: jorgee Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../executor/SimpleFileCopyStrategy.groovy | 2 +- .../nextflow/executor/command-run.txt | 27 ++++++++++++++++--- .../executor/BashWrapperBuilderTest.groovy | 4 +-- .../SimpleFileCopyStrategyTest.groovy | 6 ++--- .../executor/test-bash-wrapper-with-trace.txt | 20 +++++++++++--- .../nextflow/executor/test-bash-wrapper.txt | 20 +++++++++++--- .../BashWrapperBuilderWithS3Test.groovy | 2 +- .../BashWrapperBuilderWithAzTest.groovy | 2 +- .../GoogleLifeSciencesHelper.groovy | 2 +- .../GoogleLifeSciencesHelperTest.groovy | 2 +- .../google/lifesciences/bash-wrapper-gcp.txt | 20 +++++++++++--- validation/awsbatch-unstage-fail.config | 12 +++++++++ validation/awsbatch.sh | 9 ++++++- .../Dockerfile | 11 ++++++++ .../fake_aws/bin/aws | 9 +++++++ validation/test-aws-unstage-fail.nf | 16 +++++++++++ 16 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 validation/awsbatch-unstage-fail.config create mode 100644 validation/test-aws-unstage-fail-container/Dockerfile create mode 100755 validation/test-aws-unstage-fail-container/fake_aws/bin/aws create mode 100644 validation/test-aws-unstage-fail.nf diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index e1a92b172c..6d9dcbd538 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -183,7 +183,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy { return """\ IFS=\$'\\n' for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do - ${stageOutCommand('$name', targetDir, mode)} || true + ${stageOutCommand('$name', targetDir, mode)} done unset IFS""".stripIndent(true) } diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt index 2bf34b617a..26cbf6a829 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt @@ -99,7 +99,13 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + ## Capture possible errors. + ## Can be caused either by the task script, unstage script or after script if defined + local last_err=$? + ## capture the task error first or fallback to unstage error + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status {{exit_file}} set +u {{cleanup_cmd}} @@ -121,13 +127,26 @@ nxf_stage() { {{stage_inputs}} } -nxf_unstage() { +nxf_unstage_outputs() { true - {{unstage_controls}} - [[ ${nxf_main_ret:=0} != 0 ]] && return {{unstage_outputs}} } +nxf_unstage_controls() { + true + {{unstage_controls}} +} + +nxf_unstage() { + ## Deactivate fast failure to allow uploading stdout and stderr files later + if [[ ${nxf_main_ret:=0} == 0 ]]; then + ## Data unstaging redirecting stdout and stderr with append mode + (set -e -o pipefail; (nxf_unstage_outputs | tee -a {{stdout_file}}) 3>&1 1>&2 2>&3 | tee -a {{stderr_file}}) + nxf_unstage_ret=$? + fi + nxf_unstage_controls +} + nxf_main() { trap on_exit EXIT trap on_term TERM INT USR2 diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index 5df66b6369..e54bd97d72 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -559,7 +559,7 @@ class BashWrapperBuilderTest extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_fs_copy "$name" /work/dir || true + nxf_fs_copy "$name" /work/dir done unset IFS '''.stripIndent().rightTrim() @@ -576,7 +576,7 @@ class BashWrapperBuilderTest extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_fs_move "$name" /another/dir || true + nxf_fs_move "$name" /another/dir done unset IFS '''.stripIndent().rightTrim() diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy index 29cbb35697..6361e4f394 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy @@ -270,7 +270,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_copy "$name" /target/work\\ dir || true + nxf_fs_copy "$name" /target/work\\ dir done unset IFS ''' @@ -293,7 +293,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_move "$name" /target/store || true + nxf_fs_move "$name" /target/store done unset IFS ''' @@ -315,7 +315,7 @@ class SimpleFileCopyStrategyTest extends Specification { script == ''' IFS=$'\\n' for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do - nxf_fs_rsync "$name" /target/work\\'s || true + nxf_fs_rsync "$name" /target/work\\'s done unset IFS ''' diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt index ef1380e7cf..1de9614e11 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt @@ -270,7 +270,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -289,9 +292,20 @@ nxf_stage() { true } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt index f465b5b9b4..3bb4f34fe5 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt @@ -81,7 +81,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -100,9 +103,20 @@ nxf_stage() { true } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy index 3e213444cb..4f90e22aa2 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy @@ -58,7 +58,7 @@ class BashWrapperBuilderWithS3Test extends Specification { binding.unstage_outputs == '''\ IFS=$'\\n' for name in $(eval "ls -1d test.bam test.bai bla\\ nk.txt" | sort | uniq); do - nxf_s3_upload $name s3://some/buck\\ et || true + nxf_s3_upload $name s3://some/buck\\ et done unset IFS '''.stripIndent().rightTrim() diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy index eb72d25163..1969345af5 100644 --- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy @@ -47,7 +47,7 @@ class BashWrapperBuilderWithAzTest extends Specification { binding.unstage_outputs == """\ IFS=\$'\\n' for name in \$(eval "ls -1d test.bam test.bai" | sort | uniq); do - nxf_az_upload \$name '${AzHelper.toHttpUrl(target)}' || true + nxf_az_upload \$name '${AzHelper.toHttpUrl(target)}' done unset IFS """.stripIndent().rightTrim() diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy index be08d069e3..7352770b32 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy @@ -365,7 +365,7 @@ class GoogleLifeSciencesHelper { final remoteTaskDir = getRemoteTaskDir(workDir) def result = 'set -x; ' result += "trap 'err=\$?; exec 1>&2; gsutil -m -q cp -R $localTaskDir/${TaskRun.CMD_LOG} ${remoteTaskDir}/${TaskRun.CMD_LOG} || true; [[ \$err -gt 0 || \$GOOGLE_LAST_EXIT_STATUS -gt 0 || \$NXF_DEBUG -gt 0 ]] && { ls -lah $localTaskDir || true; gsutil -m -q cp -R /google/ ${remoteTaskDir}; } || rm -rf $localTaskDir; exit \$err' EXIT; " - result += "{ cd $localTaskDir; bash ${TaskRun.CMD_RUN} nxf_unstage; } >> $localTaskDir/${TaskRun.CMD_LOG} 2>&1" + result += "{ cd $localTaskDir; bash ${TaskRun.CMD_RUN} nxf_unstage;} >> $localTaskDir/${TaskRun.CMD_LOG} 2>&1" return result } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy index 35cda62f0b..9db824a902 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy @@ -548,7 +548,7 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification { def unstage = helper.getUnstagingScript(dir) then: unstage == - 'set -x; trap \'err=$?; exec 1>&2; gsutil -m -q cp -R /work/dir/.command.log gs://my-bucket/work/dir/.command.log || true; [[ $err -gt 0 || $GOOGLE_LAST_EXIT_STATUS -gt 0 || $NXF_DEBUG -gt 0 ]] && { ls -lah /work/dir || true; gsutil -m -q cp -R /google/ gs://my-bucket/work/dir; } || rm -rf /work/dir; exit $err\' EXIT; { cd /work/dir; bash .command.run nxf_unstage; } >> /work/dir/.command.log 2>&1' + 'set -x; trap \'err=$?; exec 1>&2; gsutil -m -q cp -R /work/dir/.command.log gs://my-bucket/work/dir/.command.log || true; [[ $err -gt 0 || $GOOGLE_LAST_EXIT_STATUS -gt 0 || $NXF_DEBUG -gt 0 ]] && { ls -lah /work/dir || true; gsutil -m -q cp -R /google/ gs://my-bucket/work/dir; } || rm -rf /work/dir; exit $err\' EXIT; { cd /work/dir; bash .command.run nxf_unstage;} >> /work/dir/.command.log 2>&1' } @Unroll diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt index 70a68452aa..c7382062a1 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt @@ -168,7 +168,10 @@ nxf_fs_fcp() { } on_exit() { - exit_status=${nxf_main_ret:=$?} + local last_err=$? + local exit_status=${nxf_main_ret:=0} + [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} + [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status > {{folder}}/.exitcode set +u exit $exit_status @@ -192,12 +195,23 @@ nxf_stage() { nxf_parallel "${downloads[@]}" } -nxf_unstage() { +nxf_unstage_outputs() { + true +} + +nxf_unstage_controls() { true gsutil -m -q cp -R .command.out gs://bucket/work/dir/.command.out || true gsutil -m -q cp -R .command.err gs://bucket/work/dir/.command.err || true gsutil -m -q cp -R .exitcode gs://bucket/work/dir/.exitcode || true - [[ ${nxf_main_ret:=0} != 0 ]] && return +} + +nxf_unstage() { + if [[ ${nxf_main_ret:=0} == 0 ]]; then + (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) + nxf_unstage_ret=$? + fi + nxf_unstage_controls } nxf_main() { diff --git a/validation/awsbatch-unstage-fail.config b/validation/awsbatch-unstage-fail.config new file mode 100644 index 0000000000..81b96579d7 --- /dev/null +++ b/validation/awsbatch-unstage-fail.config @@ -0,0 +1,12 @@ +/* + * do not include plugin requirements otherwise latest + * published version will be downloaded instead of using local build + */ + +workDir = 's3://nextflow-ci/work' +process.executor = 'awsbatch' +process.queue = 'nextflow-ci' +process.container = 'quay.io/nextflow/test-aws-unstage-fail:1.0' +aws.region = 'eu-west-1' +aws.batch.maxTransferAttempts = 3 +aws.batch.delayBetweenAttempts = '5 sec' diff --git a/validation/awsbatch.sh b/validation/awsbatch.sh index d58727e7e8..b73571cbd6 100644 --- a/validation/awsbatch.sh +++ b/validation/awsbatch.sh @@ -7,6 +7,13 @@ get_abs_filename() { export NXF_CMD=${NXF_CMD:-$(get_abs_filename ../launch.sh)} +# Execution should fail ignoring +$NXF_CMD run test-aws-unstage-fail.nf -c awsbatch-unstage-fail.config || true +[[ `grep -c "Error executing process > 'test (1)'" .nextflow.log` == 1 ]] || false +[[ `grep -c " Essential container in task exited" .nextflow.log` == 1 ]] || false +[[ `grep -cozP "Command exit status:\n 1" .nextflow.log` == 1 ]] || false +[[ `grep -c "Producing a failure in aws" .nextflow.log` == 2 ]] || false + $NXF_CMD run test-complexpaths.nf -c awsbatch.config [[ -d foo ]] || false [[ -e 'foo/.alpha' ]] || false @@ -73,4 +80,4 @@ $NXF_CMD run nextflow-io/hello \ -process.array 10 \ -with-wave \ -with-fusion \ - -c awsbatch.config \ No newline at end of file + -c awsbatch.config diff --git a/validation/test-aws-unstage-fail-container/Dockerfile b/validation/test-aws-unstage-fail-container/Dockerfile new file mode 100644 index 0000000000..0dd281ba58 --- /dev/null +++ b/validation/test-aws-unstage-fail-container/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu + +RUN apt-get update && apt-get -y install curl unzip && apt-get clean + + +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && ./aws/install && rm -rf aws* + +ADD fake_aws /fake_aws + +ENV PATH=/fake_aws/bin/:$PATH diff --git a/validation/test-aws-unstage-fail-container/fake_aws/bin/aws b/validation/test-aws-unstage-fail-container/fake_aws/bin/aws new file mode 100755 index 0000000000..80985d9e08 --- /dev/null +++ b/validation/test-aws-unstage-fail-container/fake_aws/bin/aws @@ -0,0 +1,9 @@ +#!/bin/bash + +if [[ "$*" == *".command."* ]] || [[ "$*" == *".exitcode"* ]]; then + /usr/local/bin/aws $@ +else + >&2 echo "Producing a failure in aws $@" + exit 2 +fi + diff --git a/validation/test-aws-unstage-fail.nf b/validation/test-aws-unstage-fail.nf new file mode 100644 index 0000000000..96bcb9af1e --- /dev/null +++ b/validation/test-aws-unstage-fail.nf @@ -0,0 +1,16 @@ +process test { + input: + val i + output: + file("test${i}") + file("test_2_${i}") + script: + """ + dd if=/dev/urandom of=test${i} bs=1K count=90 + dd if=/dev/urandom of=test_2_${i} bs=1K count=90 + """ +} + +workflow { + Channel.of(1) | test +}