diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3cf322c98d..9a8675a013 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,6 +90,7 @@ jobs: - name: Test if: steps.changed-files.outputs.any_changed == 'true' run: | + env | sort # configure test env if [[ "$GOOGLE_SECRET" ]]; then echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json @@ -158,6 +159,11 @@ jobs: - name: Run tests run: | env | sort + # configure test env + if [[ "$GOOGLE_SECRET" ]]; then + echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json + export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json + fi cat $HOME/.nextflow/scm make clean assemble install bash test-ci.sh diff --git a/.github/workflows/seqeradocs-changelog.yml b/.github/workflows/seqeradocs-changelog.yml new file mode 100644 index 0000000000..f41058fe71 --- /dev/null +++ b/.github/workflows/seqeradocs-changelog.yml @@ -0,0 +1,62 @@ +name: Push release change log to Seqera Docs + +on: + release: + types: [published] + workflow_dispatch: + inputs: + release_name: + description: "Release version (e.g. 1.0.0)" + required: true + release_body: + description: "Release changelog content" + required: true + +jobs: + update-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Clone seqeralabs/docs + run: | + git clone https://github.com/seqeralabs/docs.git seqeralabs-docs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create changelog file + run: | + mkdir -p seqeralabs-docs/changelog/nextflow + cat << EOF > seqeralabs-docs/changelog/nextflow/${{ github.event.release.name || inputs.release_name }}.mdx + --- + title: Nextflow ${{ github.event.release.name || inputs.release_name }} + date: $(date +%Y-%m-%d) + tags: [nextflow] + --- + + ${{ github.event.release.body || inputs.release_body }} + EOF + + - uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.DOCS_BOT_APP_ID }} + private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }} + owner: seqeralabs + repositories: docs + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ steps.generate-token.outputs.token }} + branch-token: ${{ steps.generate-token.outputs.token }} + path: seqeralabs-docs + commit-message: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}" + title: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}" + body: | + This PR adds the changelog for Nextflow ${{ github.event.release.name || inputs.release_name }} to the Seqera documentation. + + This is an automated PR created from the Nextflow repository. + branch: changelog-nextflow-${{ github.event.release.name || inputs.release_name }} + base: master + delete-branch: true diff --git a/VERSION b/VERSION index 21651351e2..2b70d664e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.10.0 +24.11.0-edge diff --git a/changelog.txt b/changelog.txt index 3420a4041e..54fba39ad4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,61 @@ NEXTFLOW CHANGE-LOG =================== +24.11.0-edge - 3 Dec 2024 +- Add GHA to submit dependencies to dependabot (#5440) [80395a6d] +- Add NotFoundException to retry condition for Google Batch [aa4d19cc] +- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae] +- Add `env()` function (#5506) [fa0e8e0f] +- Add more scientists to run name generator (#5447) [38d9eda0] +- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0] +- Add RepositoryProvider.revision now public property (#5500) [f0a4c526] +- Deprecate process `shell` block (#5508) [6f527551] +- Detecting errors in data unstaging (#5345) [3c8e602d] +- Disable virtual threads on CI tests [ci slip] [69d07dbc] +- Fix Fargate warning on memory check (#5475) [bdf0ad00] +- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8] +- Fix missing wave response (#5547) [ee252173] +- Fix overlapping conda lock file (#5540) [9248c04d] +- Fix overlapping conda lock exception (#5489) [eaaeb3de] +- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621] +- Fix Wave GCP integration test (#5490) [ad56c89b] +- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db] +- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a] +- Improve docs for using the GPU accelerator directive (#5488) [4b908524] +- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f] +- Prevent NPE with null AWS Batch response [12fc1d60] +- Target Java 17 as minimal Java version (#5045) [0140f954] +- Update 'nexus-staging' plugin to latest version (#5462) [07934513] +- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0] +- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659] +- Update process snippets to comply with strict syntax (#5526) [be1694bf] +- Update Wave dependencies [09ccd295] +- Bump aws sdk 1.12.777 (#5458) [8bad0b4b] +- Bump bouncycastle to jdk18on:1.78.1 (#5467) [cd8c385f] +- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df] +- Bump commons-io:2.15.1 [767e4a0a] +- Bump groovy 4.0.24 [dd71ad31] +- Bump netty-common to version 4.1.115.Final [d1bbd3d0] +- Bump nf-amazon@2.10.0 [2b653b07] +- Bump nf-azure@1.11.0 [6af7198d] +- Bump nf-google@1.16.0 [9494f970] +- Bump nf-google@1.8.0 [7e2c8d82] +- Bump protobuf-java:3.25.5 to nf-google [488b7906] + +24.10.2 - 27 Nov 2024 +- Prevent NPE with null AWS Batch response [3d491934] +- Fix overlapping conda lock file (#5540) [df66deaa] +- Fix missing wave response (#5547) [eb85cda8] +- Bump nf-wave@1.7.4 [93d09404] +- Bump nf-amazon@2.9.2 [469a35dd] + +24.10.1 - 18 Nov 2024 +- Fix overlapping file lock exception (#5489) [a2566d54] +- Fix isContainerReady when wave is disabled (#5509) [c69e3711] +- Bump nf-wave@1.7.3 [e7709a0f] +- Bump nf-azure@1.10.2 [54496ac4] +- Bump nf-amazon@2.9.1 [fa227933] +- Bump netty-common to version 4.1.115.Final [90623c1e] + 24.10.0 - 27 Oct 2024 - Add `manifest.contributors` config option (#5322) [cf0f9690] - Add wave mirror and scan config [92e69776] diff --git a/docs/_static/dag.mmd b/docs/_static/dag.mmd index 0963e2c106..989a536ae9 100644 --- a/docs/_static/dag.mmd +++ b/docs/_static/dag.mmd @@ -1,3 +1,17 @@ +%%{ + init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#B6ECE2', + 'primaryTextColor': '#160F26', + 'primaryBorderColor': '#065647', + 'lineColor': '#545555', + 'clusterBkg': '#BABCBD22', + 'clusterBorder': '#DDDEDE', + 'fontFamily': 'arial' + } + } +}%% flowchart TB subgraph " " v0["Channel.fromFilePairs"] diff --git a/docs/aws.md b/docs/aws.md index 5c703ada9c..84ab5c448f 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -46,53 +46,54 @@ Minimal permissions policies to be attached to the AWS account used by Nextflow - To use AWS Batch: ```json - "batch:DescribeJobQueues" "batch:CancelJob" - "batch:SubmitJob" - "batch:ListJobs" "batch:DescribeComputeEnvironments" - "batch:TerminateJob" + "batch:DescribeJobDefinitions" + "batch:DescribeJobQueues" "batch:DescribeJobs" + "batch:ListJobs" "batch:RegisterJobDefinition" - "batch:DescribeJobDefinitions" + "batch:SubmitJob" + "batch:TagResource" + "batch:TerminateJob" ``` - To view [EC2](https://aws.amazon.com/ec2/) instances: ```json - "ecs:DescribeTasks" + "ec2:DescribeInstanceAttribute" "ec2:DescribeInstances" + "ec2:DescribeInstanceStatus" "ec2:DescribeInstanceTypes" - "ec2:DescribeInstanceAttribute" "ecs:DescribeContainerInstances" - "ec2:DescribeInstanceStatus" + "ecs:DescribeTasks" ``` - To pull container images from [ECR](https://aws.amazon.com/ecr/) repositories: ```json - "ecr:GetAuthorizationToken" "ecr:BatchCheckLayerAvailability" - "ecr:GetDownloadUrlForLayer" - "ecr:GetRepositoryPolicy" - "ecr:DescribeRepositories" - "ecr:ListImages" - "ecr:DescribeImages" "ecr:BatchGetImage" + "ecr:DescribeImages" + "ecr:DescribeImageScanFindings" + "ecr:DescribeRepositories" + "ecr:GetAuthorizationToken" + "ecr:GetDownloadUrlForLayer" "ecr:GetLifecyclePolicy" "ecr:GetLifecyclePolicyPreview" + "ecr:GetRepositoryPolicy" + "ecr:ListImages" "ecr:ListTagsForResource" - "ecr:DescribeImageScanFindings" ``` :::{note} If you are running Fargate or Fargate Spot, you may need the following policies in addition to the listed above: ```json + "ec2:DescribeSubnets" "ecs:CreateCluster" "ecs:DeleteCluster" "ecs:DescribeClusters" "ecs:ListClusters" - "ec2:DescribeSubnets" ``` ::: @@ -538,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue and the Batch Execution Role. :::{note} -This feature requires the use {ref}`Wave ` container provisioning service. +Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs. +To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}Wave documentation for more details. ::: ## Advanced configuration diff --git a/docs/azure.md b/docs/azure.md index 6087c18ec2..61269e43ac 100644 --- a/docs/azure.md +++ b/docs/azure.md @@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob process.machineType = "Standard_D*d_v5,Standard_E*d_v5" ``` -For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`: +For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`: ```nextflow process EXAMPLE_PROCESS { machineType "Standard_E*d_v5" - cpus 16 + cpus 8 memory 8.GB script: diff --git a/docs/cache-and-resume.md b/docs/cache-and-resume.md index e0bf78cca1..7184909aa2 100644 --- a/docs/cache-and-resume.md +++ b/docs/cache-and-resume.md @@ -148,6 +148,8 @@ process gather { input: tuple val(id), file(foo) tuple val(id), file(bar) + + script: """ merge_command $foo $bar """ @@ -168,6 +170,8 @@ workflow { process gather { input: tuple val(id), file(foo), file(bar) + + script: """ merge_command $foo $bar """ diff --git a/docs/channel.md b/docs/channel.md index 7c5c6404ac..8eb4c1b576 100644 --- a/docs/channel.md +++ b/docs/channel.md @@ -45,6 +45,7 @@ process foo { output: path 'x.txt' + script: """ echo $x > x.txt """ diff --git a/docs/cli.md b/docs/cli.md index 016c4f2075..a6d0b24544 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -243,6 +243,36 @@ $ nextflow run --files "*.fasta" ``` ::: +Parameters specified on the command line can be also specified in a params file using the `-params-file` option. + +```bash +nextflow run main.nf -params-file pipeline_params.yml +``` + +The `-params-file` option loads parameters for your Nextflow pipeline from a JSON or YAML file. Parameters defined in the file are equivalent to specifying them directly on the command line. For example, instead of specifying parameters on the command line: + +```bash +nextflow run main.nf --alpha 1 --beta foo +``` + +Parameters can be represented in YAML format: + +```yaml +alpha: 1 +beta: 'foo' +``` + +Or in JSON format: + +```json +{ + "alpha": 1, + "beta": "foo" +} +``` + +The parameters specified in a params file are merged with the resolved configuration. The values provided via a params file overwrite those of the same name in the Nextflow configuration file, but not those specified on the command line. + ## Managing projects Nextflow seamlessly integrates with popular Git providers, including [BitBucket](http://bitbucket.org/), [GitHub](http://github.com), and [GitLab](http://gitlab.com) for managing Nextflow pipelines as version-controlled Git repositories. diff --git a/docs/conda.md b/docs/conda.md index bb7189aa05..8c7405c1a4 100644 --- a/docs/conda.md +++ b/docs/conda.md @@ -6,7 +6,7 @@ Nextflow has built-in support for Conda that allows the configuration of workflow dependencies using Conda recipes and environment files. -This allows Nextflow applications to use popular tool collections such as [Bioconda](https://bioconda.github.io) whilst taking advantage of the configuration flexibility provided by Nextflow. +This allows Nextflow applications to use popular tool collections such as [Bioconda](https://bioconda.github.io) and the [Python Package index](https://pypi.org/), whilst taking advantage of the configuration flexibility provided by Nextflow. ## Prerequisites @@ -22,7 +22,7 @@ Dependencies are specified by using the {ref}`process-conda` directive, providin Conda environments are stored on the file system. By default, Nextflow instructs Conda to save the required environments in the pipeline work directory. The same environment may be created/saved multiple times across multiple executions when using different work directories. ::: -You can specify the directory where the Conda environments are stored using the `conda.cacheDir` configuration property. When using a computing cluster, make sure to use a shared file system path accessible from all compute nodes. See the {ref}`configuration page ` for details about Conda configuration. +You can specify the directory where the Conda environments are stored using the `conda.cacheDir` configuration property. When using a computing cluster, make sure to use a shared file system path accessible from all compute nodes. See the {ref}`configuration page ` for details about Conda configuration. :::{warning} The Conda environment feature is not supported by executors that use remote object storage as a work directory. For example, AWS Batch. @@ -49,9 +49,10 @@ Conda package names can specified using the `conda` directive. Multiple package process foo { conda 'bwa samtools multiqc' - ''' + script: + """ your_command --here - ''' + """ } ``` @@ -61,6 +62,7 @@ The usual Conda package syntax and naming conventions can be used. The version o The name of the channel where a package is located can be specified prefixing the package with the channel name as shown here `bioconda::bwa=0.7.15`. +(conda-env-files)= ### Use Conda environment files Conda environments can also be defined using one or more Conda environment files. This is a file that lists the required packages and channels structured using the YAML format. For example: @@ -76,20 +78,6 @@ dependencies: - bwa=0.7.15 ``` -This other example shows how to leverage a Conda environment file to install Python packages from the [PyPI repository](https://pypi.org/)), through the `pip` package manager (which must also be explicitly listed as a required package): - -```yaml -name: my-env-2 -channels: - - defaults -dependencies: - - pip - - pip: - - numpy - - pandas - - matplotlib -``` - Read the Conda documentation for more details about how to create [environment files](https://conda.io/docs/user-guide/tasks/manage-environments.html#creating-an-environment-file-manually). The path of an environment file can be specified using the `conda` directive: @@ -98,9 +86,10 @@ The path of an environment file can be specified using the `conda` directive: process foo { conda '/some/path/my-env.yaml' - ''' + script: + """ your_command --here - ''' + """ } ``` @@ -108,7 +97,26 @@ process foo { The environment file name **must** have a `.yml` or `.yaml` extension or else it won't be properly recognised. ::: -Alternatively, it is possible to provide the dependencies using a plain text file, just listing each package name as a separate line. For example: +(conda-pypi)= +### Python Packages from PyPI + +Conda environment files can also be used to install Python packages from the [PyPI repository](https://pypi.org/), through the `pip` package manager (which must also be explicitly listed as a required package): + +```yaml +name: my-env-2 +channels: + - defaults +dependencies: + - pip + - pip: + - numpy + - pandas + - matplotlib +``` + +### Conda text files + +It is possible to provide dependencies by listing each package name as a separate line in a plain text file. For example: ``` bioconda::star=2.5.4a @@ -120,6 +128,43 @@ bioconda::multiqc=1.4 Like before, the extension matters. Make sure the dependencies file has a `.txt` extension. ::: +### Conda lock files + +The final way to provide packages to Conda is with [Conda lock files](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#identical-conda-envs). + +These are generated from existing Conda environments using the following command: + +```bash +conda list --explicit > spec-file.txt +``` + +or if using Mamba / Micromamba: + +```bash +micromamba env export --explicit > spec-file.txt +``` + +Conda lock files can also be downloaded from [Wave](https://seqera.io/wave/) build pages. + +These files include every package and their dependencies. As such, no Conda environment resolution step is needed. This is faster and more reproducible. + +The files contain package URLs and an optional md5hash for each download to confirm identity: + +``` +# micromamba env export --explicit +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h77fa898_7.conda#abf3fec87c2563697defa759dec3d639 +https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h77fa898_7.conda#72ec1b1b04c4d15d4204ece1ecea5978 +# .. and so on +``` + +To use with Nextflow, simply set the `conda` directive to the lock file path. + ### Use existing Conda environments If you already have a local Conda environment, you can use it in your workflow specifying the installation directory of such environment by using the `conda` directive: @@ -128,9 +173,10 @@ If you already have a local Conda environment, you can use it in your workflow s process foo { conda '/path/to/an/existing/env/directory' - ''' + script: + """ your_command --here - ''' + """ } ``` diff --git a/docs/config.md b/docs/config.md index ef1ac5a97c..05fbc1130b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -113,6 +113,15 @@ The following constants are globally available in a Nextflow configuration file: `projectDir` : The directory where the main script is located. +## Functions + +The following functions are globally available in a Nextflow configuration file: + +`env( name )` +: :::{versionadded} 24.11.0-edge + ::: +: Get the value of the environment variable with the specified name in the Nextflow launch environment. + (config-params)= ## Parameters @@ -129,6 +138,8 @@ params { } ``` +See {ref}`cli-params` for information about how to modify these on the command line. + (config-process)= ## Process configuration diff --git a/docs/container.md b/docs/container.md index 1d0ad99e51..ba8a563828 100644 --- a/docs/container.md +++ b/docs/container.md @@ -94,7 +94,7 @@ Read the {ref}`Process scope ` section to learn more about proce Nextflow is able to transparently pull remote container images stored in any Docker compatible registry. -By default when a container name is specified, Nextflow checks if an image file with that name exists in the local file system. If that image file exists, it's used to execute the container. If a matching file does not exist, Nextflow automatically tries to pull an image with the specified name from the container registry. +By default, when a container name is specified, Nextflow checks if an image file with that name exists in the local file system. If that image file exists, it's used to execute the container. If a matching file does not exist, Nextflow automatically tries to pull an image with the specified name from the container registry. If you want Nextflow to check only for local file images, prefix the container name with the `file://` pseudo-protocol. For example: @@ -107,7 +107,7 @@ apptainer.enabled = true Use three `/` slashes to specify an **absolute** file path, otherwise the path will be interpreted as relative to the workflow launch directory. ::: -To pull images from Apptainer Hub or a third party Docker registry, simply prefix the image name with the `shub://`, `docker://` or `docker-daemon://` pseudo-protocol as required by Apptainer. For example: +To pull images from Apptainer Hub or a third party Docker registry, prefix the image name with the `shub://`, `docker://` or `docker-daemon://` pseudo-protocol as required by Apptainer. For example: ```groovy process.container = 'docker://quay.io/biocontainers/multiqc:1.3--py35_2' @@ -120,11 +120,11 @@ You do not need to specify `docker://` to pull from a Docker repository. Nextflo This feature requires the `apptainer` tool to be installed where the workflow execution is launched (as opposed to the compute nodes). ::: -Nextflow caches those images in the `apptainer` directory in the pipeline work directory by default. However it is suggested to provide a centralised cache directory by using either the `NXF_APPTAINER_CACHEDIR` environment variable or the `apptainer.cacheDir` setting in the Nextflow config file. +Nextflow caches Apptainer images in the `apptainer` directory, in the pipeline work directory, by default. However, it is recommended to provide a centralized cache directory using the `NXF_APPTAINER_CACHEDIR` environment variable or the `apptainer.cacheDir` setting in the Nextflow config file. -:::{versionadded} 21.09.0-edge -When looking for a Apptainer image file, Nextflow first checks the *library* directory, and if the image file is not found, the *cache* directory is used s usual. The library directory can be defined either using the `NXF_APPTAINER_LIBRARYDIR` environment variable or the `apptainer.libraryDir` configuration setting (the latter overrides the former). -::: +Nextflow uses the library directory to determine the location of Apptainer containers. The library directory can be defined using the `apptainer.libraryDir` configuration setting or the `NXF_APPTAINER_LIBRARYDIR` environment variable. The configuration file option overrides the environment variable if both are set. + +Nextflow first checks the library directory when searching for the image. If the image is not found it then checks the cache directory. The main difference between the library directory and the cache directory is that the first is assumed to be a read-only container repository, while the latter is expected to be writable path where container images can added for caching purposes. :::{warning} When using a compute cluster, the Apptainer cache directory must reside in a shared filesystem accessible to all compute nodes. @@ -293,17 +293,19 @@ It is possible to specify a different Docker image for each process definition i process foo { container 'image_name_1' - ''' + script: + """ do this - ''' + """ } process bar { container 'image_name_2' - ''' + script: + """ do that - ''' + """ } ``` @@ -380,17 +382,19 @@ It is possible to specify a different container image for each process definitio process foo { container 'image_name_1' - ''' + script: + """ do this - ''' + """ } process bar { container 'image_name_2' - ''' + script: + """ do that - ''' + """ } ``` @@ -649,11 +653,11 @@ process.container = 'library://library/default/alpine:3.8' The `library://` pseudo-protocol allows you to import Singularity images from a local Docker installation instead of downloading them from a Docker registry. This feature requires the `singularity` tool to be installed where the workflow execution is launched (as opposed to the compute nodes). -Nextflow caches the images in `${NXF_WORK}/singularity` by default. However, it is recommended to define a centralised cache directory using either the `NXF_SINGULARITY_CACHEDIR` environment variable or the `singularity.cacheDir` setting in the Nextflow config file. +Nextflow caches Singularity images in the `singularity` directory, in the pipeline work directory, by default. However, it is recommended to provide a centralized cache directory using the `NXF_SINGULARITY_CACHEDIR` environment variable or the `singularity.cacheDir` setting in the Nextflow config file. -:::{versionadded} 21.09.0-edge -When looking for a Singularity image file, Nextflow first checks the *library* directory, and if the image file is not found, the *cache* directory is used as usual. The library directory can be defined either using the `NXF_SINGULARITY_LIBRARYDIR` environment variable or the `singularity.libraryDir` configuration setting (the latter overrides the former). -::: +Nextflow uses the library directory to determine the location of Singularity images. The library directory can be defined using the `singularity.libraryDir` configuration setting or the `NXF_SINGULARITY_LIBRARYDIR` environment variable. The configuration file option overrides the environment variable if both are set. + +Nextflow first checks the library directory when searching for the image. If the image is not found it then checks the cache directory. The main difference between the library directory and the cache directory is that the first is assumed to be a read-only container repository, while the latter is expected to be writable path where container images can added for caching purposes. :::{warning} When using a compute cluster, the Singularity cache directory must reside in a shared filesystem accessible to all compute nodes. diff --git a/docs/developer/nextflow.ast.md b/docs/developer/nextflow.ast.md index 0fe680e344..2b3bad9150 100644 --- a/docs/developer/nextflow.ast.md +++ b/docs/developer/nextflow.ast.md @@ -32,6 +32,7 @@ process splitLetters { output: path 'chunk_*' + script: """ printf '${params.str}' | split -b 6 - chunk_ """ @@ -43,6 +44,7 @@ process convertToUpper { output: stdout + script: """ cat $x | tr '[a-z]' '[A-Z]' """ @@ -62,6 +64,7 @@ process( splitLetters( { output: path('chunk_*') + script: """ printf '${params.str}' | split -b 6 - chunk_ """ @@ -73,6 +76,7 @@ process( convertToUpper( { output: stdout + script: """ cat $x | tr '[a-z]' '[A-Z]' """ diff --git a/docs/developer/plugins.md b/docs/developer/plugins.md index e058eec355..5a95ec8f0f 100644 --- a/docs/developer/plugins.md +++ b/docs/developer/plugins.md @@ -141,6 +141,8 @@ You can then use this executor in your pipeline: ```nextflow process foo { executor 'my-executor' + + // ... } ``` diff --git a/docs/dsl1.md b/docs/dsl1.md index e7503fda91..04b82fec76 100644 --- a/docs/dsl1.md +++ b/docs/dsl1.md @@ -29,6 +29,7 @@ process splitLetters { output: file 'chunk_*' into letters + script: """ printf '${params.str}' | split -b 6 - chunk_ """ @@ -41,6 +42,7 @@ process convertToUpper { output: stdout result + script: """ cat $x | tr '[a-z]' '[A-Z]' """ diff --git a/docs/fusion.md b/docs/fusion.md index 925393bc23..bf15304522 100644 --- a/docs/fusion.md +++ b/docs/fusion.md @@ -214,7 +214,7 @@ Then you can run your pipeline using the following command: nextflow run -work-dir s3:///scratch ``` -Replace `` and `` with a pipeline script and bucket or your choice, for example: +Replace `` and `` with a pipeline script and bucket of your choice, for example: ```bash nextflow run https://github.com/nextflow-io/rnaseq-nf -work-dir s3://nextflow-ci/scratch @@ -222,7 +222,7 @@ nextflow run https://github.com/nextflow-io/rnaseq-nf -work-dir s3://nextflow-ci :::{warning} The option `fusion.exportStorageCredentials` leaks the AWS credentials on the task launcher script created by Nextflow. -This option should only be used for development purposes. +This option should only be used for testing and development purposes. ::: ### Local execution with Minio @@ -267,11 +267,53 @@ Then you can run your pipeline using the following command: nextflow run -work-dir s3://foobar/scratch ``` -Replace `` with a pipeline script and bucket or your choice: +Replace `` with a pipeline script and bucket of your choice: :::{warning} The option `fusion.exportStorageCredentials` leaks the AWS credentials on the task launcher script created by Nextflow. -This option should only be used for development purposes. +This option should only be used for testing and development purposes. +::: + +### Local execution with Oracle Object Storage + +Fusion file system and Nextflow are compatible with [Oracle Object Storage](https://www.oracle.com/cloud/storage/object-storage/). + +:::{note} +This capability relies on the S3-like API compatibility provided by Oracle storage and not by a native support in +Nextflow and Fusion. As such it may not fully work and support all Nextflow and Fusion features. +::: + +This configuration requires the execution of your pipeline tasks using Docker or a similar container engine. + +The following should be included in your Nextflow configuration file: + +```groovy +aws.region = '' +aws.accessKey = '' +aws.secretKey = '' +aws.client.endpoint = 'https://.compat.objectstorage..oraclecloud.com' +aws.client.s3PathStyleAccess = true +aws.client.protocol = 'https' +aws.client.signerOverride = 'AWSS3V4SignerType' +docker.enabled = true +docker.containerOptions = '-e FUSION_AWS_REGION=' +fusion.enabled = true +fusion.exportStorageCredentials = true +wave.enabled = true +tower.accessToken = '' // optional +``` + +Then you can run your pipeline using the following command: + +```bash +nextflow run -work-dir s3:///scratch +``` + +In the above snippet replace the placeholders `` and `` with your [Oracle Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2), +and the placeholders `` and `` with the namespace and region of your Oracle bucket. + +:::{warning} +The `fusion.exportStorageCredentials` option leaks the Oracle credentials to the Nextflow task launcher script and should only be used for testing and development purposes. ::: ## Advanced settings diff --git a/docs/google.md b/docs/google.md index b187b35685..703948c103 100644 --- a/docs/google.md +++ b/docs/google.md @@ -104,6 +104,7 @@ process myTask { cpus 8 memory '40 GB' + script: """ your_command --here """ @@ -112,6 +113,7 @@ process myTask { process anotherTask { machineType 'n1-highmem-8' + script: """ your_command --here """ @@ -130,6 +132,7 @@ process myTask { memory '20 GB' machineType 'n2-*,c2-*,m3-*' + script: """ your_command --here """ @@ -148,6 +151,7 @@ process myTask { memory '20 GB' machineType 'template://my-template' + script: """ your_command --here """ @@ -341,16 +345,18 @@ process custom_resources_task { memory '40 GB' disk '200 GB' + script: """ - + your_command --here """ } process predefined_resources_task { machineType 'n1-highmem-8' + script: """ - + your_command --here """ } ``` diff --git a/docs/install.md b/docs/install.md index efee6f883d..246ce57fcb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -119,15 +119,25 @@ NXF_VER=24.06.0-edge nextflow info ## Standalone distribution -Nextflow has a set of {ref}`core plugins ` which are downloaded at runtime by default. There is also a standalone distribution (i.e. the `all` distribution) which comes pre-packaged with all core plugins. This distribution is mainly useful for offline environments. +The Nextflow standalone distribution (i.e. the `dist` distribution) consists of self-contained `nextflow` executable file +that includes all the application dependencies for core functionalities, and it can run without downloading third parties +libraries. This distribution is mainly useful for offline environments. -The installer for the `all` distribution can be found on the [GitHub releases page](https://github.com/nextflow-io/nextflow/releases), under the "Assets" section for a specific release. The installation procedure is the same as for the standard distribution, only using this URL instead of `https://get.nextflow.io`: +Note however the support for cloud services e.g. AWS, Seqera Platform, Wave, etc. still require the download +of the corresponding Nextflow plugins. -```bash -export NXF_VER=23.10.0 -curl -s https://github.com/nextflow-io/nextflow/releases/download/v$NXF_VER/nextflow-$NXF_VER-all -``` +To use the standalone distribution: -:::{warning} -The `all` distribution does not support third-party plugins. Only the {ref}`core plugins ` are supported. -::: +1. Download it from the [GitHub releases page](https://github.com/nextflow-io/nextflow/releases), under the "Assets" section for a specific + +2. Grant execution permissions to the downloaded file e.g. + + ``` + chmod -x nextflow-24.10.1-dist + ``` + +3. Then you can use it as a drop-in replacement for `nextflow` command. For example: + + ``` + ./nextflow-24.10.1-dist run hello + ``` diff --git a/docs/metrics.md b/docs/metrics.md index 8ccb43bbaa..eac8ae0e3d 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -21,6 +21,7 @@ In the first example, let's consider the simple use case in which a process perf process CpuUsageEx1 { cpus 2 + script: """ stress -c 1 -t 10 # compute square-root of random numbers during 10s using 1 CPU """ @@ -35,6 +36,7 @@ In the second example, some time will be spent performing pure computation and s process CpuUsageEx2 { cpus 1 + script: """ stress -c 1 -t 10 # compute square-root of random numbers during 10s using 1 CPU stress -c 1 -t 5 # compute square-root of random numbers during 5s using 1 CPU @@ -57,6 +59,7 @@ The third example is similar to the second one except that the pure computation process CpuUsageEx3 { cpus 2 + script: """ stress -c 2 -t 10 # compute square-root of random numbers during 10s using 2 CPUs sleep 10 # use no CPU during 10s @@ -232,6 +235,7 @@ The first and second programs are executed in `foo` and `bar` processes respecti process foo { memory '1.5 GB' + script: """ memory_vmem_1GiB_ram_0Gib """ @@ -240,6 +244,7 @@ process foo { process bar { memory '1.5 GB' + script: """ memory_vmem_1GiB_ram_1Gib """ diff --git a/docs/overview.md b/docs/overview.md index a393c823de..44b1a3cb8a 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -33,6 +33,7 @@ process blastSearch { output: path "top_hits.txt" + script: """ blastp -db $db -query $query -outfmt 6 > blast_result cat blast_result | head -n 10 | cut -f 2 > top_hits.txt @@ -47,6 +48,7 @@ process extractTopHits { output: path "sequences.txt" + script: """ blastdbcmd -db $db -entry_batch $top_hits > sequences.txt """ diff --git a/docs/process.md b/docs/process.md index d84112d99a..871ecea093 100644 --- a/docs/process.md +++ b/docs/process.md @@ -161,9 +161,9 @@ In the above example, the process will execute one of several scripts depending ### Template -Process scripts can be externalised to **template** files, which can be reused across different processes and tested independently from the overall pipeline execution. +Process scripts can be externalized to **template** files, which allows them to be reused across different processes and tested independently from the pipeline execution. -A template is simply a shell script file that Nextflow is able to execute by using the `template` function as shown below: +A template can be used in place of an embedded script using the `template` function in the script section: ```nextflow process templateExample { @@ -179,9 +179,9 @@ workflow { } ``` -By default, Nextflow looks for the `my_script.sh` template file in the `templates` directory located alongside the Nextflow script and/or the module script in which the process is defined. Any other location can be specified by using an absolute template path. +By default, Nextflow looks for the template script in the `templates` directory located alongside the Nextflow script in which the process is defined. An absolute path can be used to specify a different location. However, this practice is discouraged because it hinders pipeline portability. -The template script may contain any code that can be executed by the underlying environment. For example: +An example template script is provided below: ```bash #!/bin/bash @@ -190,18 +190,32 @@ echo $STR echo "process completed" ``` -:::{tip} -The dollar character (`$`) is interpreted as a Nextflow variable when the script is run as a Nextflow template, whereas it is evaluated as a Bash variable when run as a Bash script. This can be very useful for testing your script independently from Nextflow execution. You only need to provide a Bash environment variable for each of the Nextflow variables that are referenced in your script. For example, it would be possible to execute the above script with the following command in the terminal: `STR='foo' bash templates/my_script.sh` -::: +Variables prefixed with the dollar character (`$`) are interpreted as Nextflow variables when the template script is executed by Nextflow and Bash variables when executed directly. For example, the above script can be executed from the command line by providing each input as an environment variable: + +```bash +STR='foo' bash templates/my_script.sh +``` + +The following caveats should be considered: + +- Template scripts are recommended only for Bash scripts. Languages that do not prefix variables with `$` (e.g. Python and R) can't be executed directly as a template script. + +- Variables escaped with `\$` will be interpreted as Bash variables when executed by Nextflow, but will not be interpreted as variables when executed from the command line. This practice should be avoided to ensure that the template script behaves consistently. + +- Template variables are evaluated even if they are commented out in the template script. If a template variable is missing, it will cause the pipeline to fail regardless of where it occurs in the template. :::{tip} -As a best practice, the template script should not contain any `\$` escaped variables, because these variables will not be evaluated properly when the script is executed directly. +Template scripts are generally discouraged due to the caveats described above. The best practice for using a custom script is to embed it in the process definition at first and move it to a separate file with its own command line interface once the code matures. ::: (process-shell)= ### Shell +:::{deprecated} 24.11.0-edge +Use the `script` block instead. Consider using the {ref}`VS Code extension `, which provides syntax highlighting and error checking to distinguish Nextflow variables from Bash variables in the process script. +::: + The `shell` block is a string expression that defines the script that is executed by the process. It is an alternative to the {ref}`process-script` definition with one important difference: it uses the exclamation mark `!` character, instead of the usual dollar `$` character, to denote Nextflow variables. This way, it is possible to use both Nextflow and Bash variables in the same script without having to escape the latter, which makes process scripts easier to read and maintain. For example: @@ -227,7 +241,7 @@ In the above example, `$USER` is treated as a Bash variable, while `!{str}` is t :::{note} - Shell script definitions require the use of single-quote `'` delimited strings. When using double-quote `"` delimited strings, dollar variables are interpreted as Nextflow variables as usual. See {ref}`string-interpolation`. - Variables prefixed with `!` must always be enclosed in curly brackets, i.e. `!{str}` is a valid variable whereas `!str` is ignored. -- Shell scripts support the use of the {ref}`process-template` mechanism. The same rules are applied to the variables defined in the script template. +- Shell scripts support the use of the {ref}`process-template` mechanism. The same rules are applied to the variables defined in the template script. ::: (process-native)= @@ -335,7 +349,10 @@ process basicExample { input: val x - "echo process job $x" + script: + """ + echo process job $x + """ } workflow { @@ -364,7 +381,10 @@ process basicExample { input: val x - "echo process job $x" + script: + """ + echo process job $x + """ } workflow { @@ -384,7 +404,10 @@ process blastThemAll { input: path query_file - "blastp -query ${query_file} -db nr" + script: + """ + blastp -query ${query_file} -db nr + """ } workflow { @@ -418,7 +441,10 @@ process blastThemAll { input: path 'query.fa' - "blastp -query query.fa -db nr" + script: + """ + blastp -query query.fa -db nr + """ } workflow { @@ -440,6 +466,7 @@ process foo { input: path x + script: """ your_command --in $x """ @@ -487,7 +514,10 @@ process blastThemAll { input: path 'seq' - "echo seq*" + script: + """ + echo seq* + """ } workflow { @@ -526,7 +556,10 @@ process blastThemAll { input: path 'seq?.fa' - "cat seq1.fa seq2.fa seq3.fa" + script: + """ + cat seq1.fa seq2.fa seq3.fa + """ } workflow { @@ -549,6 +582,7 @@ process simpleCount { val x path "${x}.fa" + script: """ cat ${x}.fa | grep '>' """ @@ -572,6 +606,7 @@ process printEnv { input: env 'HELLO' + script: ''' echo $HELLO world! ''' @@ -598,6 +633,7 @@ process printAll { input: stdin + script: """ cat - """ @@ -630,6 +666,7 @@ process tupleExample { input: tuple val(x), path('input.txt') + script: """ echo "Processing $x" cat input.txt > copy @@ -655,6 +692,7 @@ process alignSequences { path seq each mode + script: """ t_coffee -in $seq -mode $mode > result """ @@ -679,6 +717,7 @@ process alignSequences { each mode each path(lib) + script: """ t_coffee -in $seq -mode $mode -lib $lib > result """ @@ -818,6 +857,7 @@ process foo { output: val x + script: """ echo $x > file """ @@ -869,6 +909,7 @@ process randomNum { output: path 'result.txt' + script: ''' echo $RANDOM > result.txt ''' @@ -909,9 +950,10 @@ process splitLetters { output: path 'chunk_*' - ''' + script: + """ printf 'Hola' | split -b 1 - chunk_ - ''' + """ } workflow { @@ -956,6 +998,7 @@ process align { output: path "${species}.aln" + script: """ t_coffee -in $seq > ${species}.aln """ @@ -1056,9 +1099,10 @@ process foo { output: path 'result.txt', hidden: true - ''' + script: + """ echo 'another new line' >> result.txt - ''' + """ } ``` @@ -1069,10 +1113,11 @@ process foo { output: tuple path('last_result.txt'), path('result.txt', hidden: true) - ''' + script: + """ echo 'another new line' >> result.txt echo 'another new line' > last_result.txt - ''' + """ } ``` ::: @@ -1089,6 +1134,7 @@ process FOO { path 'hello.txt', emit: hello path 'bye.txt', emit: bye + script: """ echo "hello" > hello.txt echo "bye" > bye.txt @@ -1205,7 +1251,7 @@ process foo { script: """ - < your job here > + your_command --here """ } ``` @@ -1245,7 +1291,9 @@ process foo { maxRetries 3 script: - + """ + your_command --here + """ } ``` @@ -1268,7 +1316,9 @@ process foo { maxRetries 3 script: - + """ + your_command --here + """ } ``` In the above example, the {ref}`process-memory` is set according to previous trace record metrics. In the first attempt, when no trace metrics are available, it is set to one GB. In the subsequent attempts, it doubles the previously allocated memory. See {ref}`trace-report` for more information about trace records. @@ -1284,9 +1334,9 @@ process foo { maxRetries 5 script: - ''' + """ your_command --here - ''' + """ } ``` diff --git a/docs/reference/channel.md b/docs/reference/channel.md index fa1cb16825..13e2bfd414 100644 --- a/docs/reference/channel.md +++ b/docs/reference/channel.md @@ -428,11 +428,15 @@ A process output can be assigned to a topic using the `topic` option on an outpu process foo { output: val('foo'), topic: my_topic + + // ... } process bar { output: val('bar'), topic: my_topic + + // ... } ``` diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 43ab14c4e2..591d1fa151 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1172,29 +1172,7 @@ The `run` command is used to execute a local pipeline script or remote pipeline $ nextflow run main.nf -params-file pipeline_params.yml ``` - For example, the following params file in YAML format: - - ```yaml - alpha: 1 - beta: 'foo' - ``` - - Or in JSON format: - - ```json - { - "alpha": 1, - "beta": "foo" - } - ``` - - Is equivalent to the following command line: - - ```console - $ nextflow run main.nf --alpha 1 --beta foo - ``` - - The parameters specified with this mechanism are merged with the resolved configuration (base configuration and profiles). The values provided via a params file overwrite those of the same name in the Nextflow configuration file. + See {ref}`cli-params` for more information about writing custom parameters files. ### `self-update` diff --git a/docs/reference/config.md b/docs/reference/config.md index 464ca480ba..0efb1ce6ae 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -52,6 +52,9 @@ The following settings are available: `apptainer.envWhitelist` : Comma separated list of environment variable names to be included in the container environment. +`apptainer.libraryDir` +: Directory where remote Apptainer images are retrieved. When using a computing cluster it must be a shared folder accessible to all compute nodes. + `apptainer.noHttps` : Pull the Apptainer image with http protocol (default: `false`). @@ -1375,6 +1378,9 @@ The following settings are available: `singularity.envWhitelist` : Comma separated list of environment variable names to be included in the container environment. +`singularity.libraryDir` +: Directory where remote Singularity images are retrieved. When using a computing cluster it must be a shared folder accessible to all compute nodes. + `singularity.noHttps` : Pull the Singularity image with http protocol (default: `false`). diff --git a/docs/reference/process.md b/docs/reference/process.md index 646bebbdf4..95a9456ef8 100644 --- a/docs/reference/process.md +++ b/docs/reference/process.md @@ -19,8 +19,11 @@ The following task properties are defined in the process body: : *Available only in `exec:` blocks* : The task unique hash ID. +`task.id` +: The pipeline-level task index. Corresponds to `task_id` in the {ref}`execution trace `. + `task.index` -: The task index (corresponds to `task_id` in the {ref}`execution trace `). +: The process-level task index. `task.name` : *Available only in `exec:` blocks* @@ -232,6 +235,10 @@ The above examples will request 4 GPUs of type `nvidia-tesla-k80`. This directive is only used by certain executors. Refer to the {ref}`executor-page` page to see which executors support this directive. ::: +:::{note} +Additional options may be required to fully enable the use of accelerators. When using containers with GPUs, you must pass the GPU drivers through to the container. For Docker, this requires the option `--gpus all` in the docker run command. For Apptainer/Singularity, this requires the option `--nv`. The specific implementation details depend on the accelerator and container type being used. +::: + :::{note} The accelerator `type` option depends on the target execution platform. Refer to the platform-specific documentation for details on the available accelerators: @@ -262,6 +269,7 @@ process cpu_task { spack 'blast-plus@2.13.0' arch 'linux/x86_64', target: 'cascadelake' + script: """ blastp -query input_sequence -num_threads ${task.cpus} """ @@ -304,9 +312,10 @@ process cpu_task { executor 'slurm' array 100 - ''' + script: + """ your_command --here - ''' + """ } ``` @@ -354,6 +363,7 @@ For example: process foo { beforeScript 'source /cluster/bin/setup' + script: """ echo bar """ @@ -373,9 +383,7 @@ The cache is enabled by default, but you can disable it for a specific process b ```nextflow process noCacheThis { cache false - - script: - + // ... } ``` @@ -444,9 +452,10 @@ Nextflow automatically sets up an environment for the given package names listed process foo { conda 'bwa=0.7.15' - ''' + script: + """ your_command --here - ''' + """ } ``` @@ -468,8 +477,9 @@ For example: process runThisInDocker { container 'dockerbox:tag' + script: """ - + your_command --here """ } ``` @@ -498,9 +508,10 @@ process runThisWithDocker { output: path 'output.txt' - ''' + script: + """ your_command --data /db > output.txt - ''' + """ } ``` @@ -519,6 +530,7 @@ process big_job { cpus 8 executor 'sge' + script: """ blastp -query input_sequence -num_threads ${task.cpus} """ @@ -542,7 +554,9 @@ process sayHello { debug true script: - "echo Hello" + """ + echo Hello + """ } ``` @@ -563,8 +577,9 @@ process big_job { disk '2 GB' executor 'cirrus' + script: """ - your task script here + your_command --here """ } ``` @@ -624,8 +639,7 @@ For example: process ignoreAnyError { errorStrategy 'ignore' - script: - + // ... } ``` @@ -637,8 +651,7 @@ The `retry` error strategy allows you to re-submit for execution a process retur process retryIfFail { errorStrategy 'retry' - script: - + // ... } ``` @@ -684,8 +697,7 @@ The following example shows how to set the process's executor: process doSomething { executor 'sge' - script: - + // ... } ``` @@ -707,6 +719,7 @@ process mapping { path genome tuple val(sampleId), path(reads) + script: """ STAR --genomeDir $genome --readFilesIn $reads ${task.ext.args ?: ''} """ @@ -720,6 +733,8 @@ The `ext` directive can be set in the process definition: ```nextflow process mapping { ext version: '2.5.3', args: '--foo --bar' + + // ... } ``` @@ -778,9 +793,10 @@ The `label` directive allows the annotation of processes with mnemonic identifie process bigTask { label 'big_mem' - ''' - - ''' + script: + """ + your_command --here + """ } ``` @@ -801,7 +817,7 @@ See also: [resourceLabels](#resourcelabels) :::{versionadded} 19.07.0 ::: -The `machineType` can be used to specify a predefined Google Compute Platform [machine type](https://cloud.google.com/compute/docs/machine-types) when running using the {ref}`Google Life Sciences ` executor. +The `machineType` can be used to specify a predefined Google Compute Platform [machine type](https://cloud.google.com/compute/docs/machine-types) when running using the {ref}`Google Batch ` or {ref}`Google Life Sciences ` executor, or when using the autopools feature of the {ref}`Azure Batch executor`. This directive is optional and if specified overrides the cpus and memory directives: @@ -809,8 +825,9 @@ This directive is optional and if specified overrides the cpus and memory direct process foo { machineType 'n1-highmem-8' + script: """ - + your_command --here """ } ``` @@ -833,10 +850,11 @@ process foo { maxSubmitAwait '10 mins' maxRetries 3 queue "${task.submitAttempt==1 : 'spot-compute' : 'on-demand-compute'}" + script: - ''' - your_job --here - ''' + """ + your_command --here + """ } ``` @@ -855,6 +873,7 @@ process retryIfFail { errorStrategy 'retry' maxErrors 5 + script: """ echo 'do this as that .. ' """ @@ -879,9 +898,10 @@ If you want to execute a process in a sequential manner, set this directive to o process doNotParallelizeIt { maxForks 1 - ''' - - ''' + script: + """ + your_command --here + """ } ``` @@ -896,6 +916,7 @@ process retryIfFail { errorStrategy 'retry' maxRetries 3 + script: """ echo 'do this as that .. ' """ @@ -919,8 +940,9 @@ process big_job { memory '2 GB' executor 'sge' + script: """ - your task script here + your_command --here """ } ``` @@ -953,6 +975,7 @@ In a process definition you can use the `module` directive to load a specific mo process basicExample { module 'ncbi-blast/2.2.27' + script: """ blastp -query """ @@ -965,6 +988,7 @@ You can repeat the `module` directive for each module you need to load. Alternat process manyModules { module 'ncbi-blast/2.2.27:t_coffee/10.0:clustalw/2.1' + script: """ blastp -query """ @@ -983,6 +1007,7 @@ process big_job { penv 'smp' executor 'sge' + script: """ blastp -query input_sequence -num_threads ${task.cpus} """ @@ -1005,9 +1030,10 @@ For example: process your_task { pod env: 'FOO', value: 'bar' - ''' + script: + """ echo $FOO - ''' + """ } ``` @@ -1202,9 +1228,10 @@ process foo { output: path 'chunk_*' - ''' + script: + """ printf 'Hola' | split -b 1 - chunk_ - ''' + """ } ``` @@ -1227,9 +1254,10 @@ process foo { output: path 'chunk_*' - ''' + script: + """ printf 'Hola' | split -b 1 - chunk_ - ''' + """ } ``` @@ -1299,8 +1327,9 @@ process grid_job { queue 'long' executor 'sge' + script: """ - your task script here + your_command --here """ } ``` @@ -1332,9 +1361,10 @@ The `resourceLabels` directive allows you to specify custom name-value pairs tha process my_task { resourceLabels region: 'some-region', user: 'some-username' - ''' - - ''' + script: + """ + your_command --here + """ } ``` @@ -1370,9 +1400,9 @@ process my_task { resourceLimits cpus: 24, memory: 768.GB, time: 72.h script: - ''' + """ your_command --here - ''' + """ } ``` @@ -1410,9 +1440,10 @@ process simpleTask { output: path 'data_out' - ''' - - ''' + script: + """ + your_command --here + """ } ``` @@ -1449,9 +1480,10 @@ The `shell` directive allows you to define a custom shell command for process sc process doMoreThings { shell '/bin/bash', '-euo', 'pipefail' - ''' - your_command_here - ''' + script: + """ + your_command --here + """ } ``` @@ -1473,9 +1505,10 @@ Nextflow automatically sets up an environment for the given package names listed process foo { spack 'bwa@0.7.15' - ''' + script: + """ your_command --here - ''' + """ } ``` @@ -1580,6 +1613,7 @@ process foo { input: val code + script: """ echo $code """ @@ -1610,8 +1644,9 @@ The `time` directive allows you to define how long a process is allowed to run. process big_job { time '1h' + script: """ - your task script here + your_command --here """ } ``` diff --git a/docs/reference/stdlib.md b/docs/reference/stdlib.md index 8c09f1ac96..633e51b0eb 100644 --- a/docs/reference/stdlib.md +++ b/docs/reference/stdlib.md @@ -197,6 +197,11 @@ The following functions are available in Nextflow scripts: `branchCriteria( closure )` : Create a branch criteria to use with the {ref}`operator-branch` operator. +`env( name )` +: :::{versionadded} 24.11.0-edge + ::: +: Get the value of the environment variable with the specified name in the Nextflow launch environment. + `error( message = null )` : Throw a script runtime error with an optional error message. diff --git a/docs/reference/syntax.md b/docs/reference/syntax.md index bb7cd098dd..06d7824527 100644 --- a/docs/reference/syntax.md +++ b/docs/reference/syntax.md @@ -187,7 +187,7 @@ process sayHello { } ``` -A process may define additional sections for *directives*, *inputs*, *outputs*, *script*, *shell*, *exec*, and *stub*: +A process may define additional sections for *directives*, *inputs*, *outputs*, *script*, *exec*, and *stub*: ```nextflow process greet { @@ -202,7 +202,7 @@ process greet { output: stdout - script: // or shell: or exec: + script: // or exec: """ echo '${greeting}, ${name}!' """ @@ -214,7 +214,7 @@ process greet { } ``` -- A process must define a script, shell, or exec section (see below). All other sections are optional. Directives do not have an explicit section label, but must be defined first. +- A process must define a script or exec section (see below). All other sections are optional. Directives do not have an explicit section label, but must be defined first. - The `script:` section label can be omitted only when there are no other sections in the body. @@ -222,19 +222,9 @@ process greet { Each section may contain one or more statements. For directives, inputs, and outputs, these statements must be [function calls](#function-call). See {ref}`process-reference` for the set of available input qualifiers, output qualifiers, and directives. -The script section can be substituted with a shell or exec section: +The script section can be substituted with an exec section: ```nextflow -process greetShell { - input: - val greeting - - shell: - ''' - echo '!{greeting}, ${USER}!' - ''' -} - process greetExec { input: val greeting @@ -248,7 +238,7 @@ process greetExec { } ``` -The script, shell, and stub sections must return a string in the same manner as a [function](#function). +The script and stub sections must return a string in the same manner as a [function](#function). See {ref}`process-page` for more information on the semantics of each process section. @@ -356,7 +346,7 @@ Variables declared in a function, as well as the parameters of that function, ex Workflow inputs exist for the entire workflow body. Variables declared in the main section exist for the main, emit, and publish sections. Named outputs are not considered variable declarations and therefore do not have any scope. -Process input variables exist for the entire process body. Variables declared in the process script, shell, exec, and stub sections exist only in their respective section, with one exception -- variables declared without the `def` keyword also exist in the output section. +Process input variables exist for the entire process body. Variables declared in the process script, exec, and stub sections exist only in their respective section, with one exception -- variables declared without the `def` keyword also exist in the output section. Variables declared in an if or else branch exist only within that branch: @@ -632,16 +622,6 @@ A *slashy string* is enclosed by slashes instead of quotes: /no escape!/ ``` -Slashy strings can also span multiple lines: - -```nextflow -/ -Patterns in the code, -Symbols dance to match and find, -Logic unconfined. -/ -``` - :::{note} A slashy string cannot be empty because it would become a line comment. ::: @@ -767,10 +747,12 @@ myList[0] ### Property expression -A property expression consists of an *object expression* and a *property*, separated by a dot: +A property expression consists of an *object expression* and a *property*, separated by a *dot*, *spread dot*, or *safe dot*: ```nextflow -file.text +myFile.text // dot +myFiles*.text // spread dot: myFiles.collect { myFile -> myFile.text } +myFile?.text // safe dot: myFile != null ? myFile.text : null ``` The property must be an identifier or string literal. @@ -783,10 +765,12 @@ A function call consists of a name and argument list: printf('Hello %s!\n', 'World') ``` -A *method call* consists of an *object expression* and a function call separated by a dot: +A *method call* consists of an *object expression* and a function call separated by a *dot*, *spread dot*, or *safe dot*: ```nextflow -myList.size() +myFile.getText() // dot +myFiles*.getText() // spread dot: myFiles.collect { myFile -> myFile.getText() } +myFile?.getText() // safe dot: myFile != null ? myFile.getText() : null ``` The argument list may contain any number of *positional arguments* and *named arguments*: @@ -954,4 +938,5 @@ The following legacy features were excluded from this page because they are depr - The `addParams` and `params` clauses of include declarations. See {ref}`module-params` for more information. - The `when:` section of a process definition. See {ref}`process-when` for more information. +- The `shell:` section of a process definition. See {ref}`process-shell` for more information. - The implicit `it` closure parameter. See {ref}`script-closure` for more information. diff --git a/docs/secrets.md b/docs/secrets.md index 393992b18d..b20ce13134 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -56,6 +56,7 @@ process someJob { secret 'MY_ACCESS_KEY' secret 'MY_SECRET_KEY' + script: """ your_command --access \$MY_ACCESS_KEY --secret \$MY_SECRET_KEY """ diff --git a/docs/snippets/grouptuple-groupkey.nf b/docs/snippets/grouptuple-groupkey.nf index fa9f08c6fd..5c8fce8e7a 100644 --- a/docs/snippets/grouptuple-groupkey.nf +++ b/docs/snippets/grouptuple-groupkey.nf @@ -1,12 +1,13 @@ -chr_frequency = ["chr1": 2, "chr2": 3] - Channel.of( - ['region1', 'chr1', '/path/to/region1_chr1.vcf'], - ['region2', 'chr1', '/path/to/region2_chr1.vcf'], - ['region1', 'chr2', '/path/to/region1_chr2.vcf'], - ['region2', 'chr2', '/path/to/region2_chr2.vcf'], - ['region3', 'chr2', '/path/to/region3_chr2.vcf'] + ['chr1', ['/path/to/region1_chr1.vcf', '/path/to/region2_chr1.vcf']], + ['chr2', ['/path/to/region1_chr2.vcf', '/path/to/region2_chr2.vcf', '/path/to/region3_chr2.vcf']], ) - .map { region, chr, vcf -> tuple( groupKey(chr, chr_frequency[chr]), vcf ) } + .flatMap { chr, vcfs -> + vcfs.collect { vcf -> + tuple(groupKey(chr, vcfs.size()), vcf) // preserve group size with key + } + } + .view { v -> "scattered: ${v}" } .groupTuple() - .view() \ No newline at end of file + .map { key, vcfs -> tuple(key.getGroupTarget(), vcfs) } // unwrap group key + .view { v -> "gathered: ${v}" } \ No newline at end of file diff --git a/docs/snippets/grouptuple-groupkey.out b/docs/snippets/grouptuple-groupkey.out index e97159c872..be3f00185b 100644 --- a/docs/snippets/grouptuple-groupkey.out +++ b/docs/snippets/grouptuple-groupkey.out @@ -1,2 +1,7 @@ -[chr1, [/path/to/region1_chr1.vcf, /path/to/region2_chr1.vcf]] -[chr2, [/path/to/region1_chr2.vcf, /path/to/region2_chr2.vcf, /path/to/region3_chr2.vcf]] \ No newline at end of file +scattered: [chr1, /path/to/region1_chr1.vcf] +scattered: [chr1, /path/to/region2_chr1.vcf] +scattered: [chr2, /path/to/region1_chr2.vcf] +scattered: [chr2, /path/to/region2_chr2.vcf] +scattered: [chr2, /path/to/region3_chr2.vcf] +gathered: [chr1, [/path/to/region1_chr1.vcf, /path/to/region2_chr1.vcf]] +gathered: [chr2, [/path/to/region1_chr2.vcf, /path/to/region2_chr2.vcf, /path/to/region3_chr2.vcf]] \ No newline at end of file diff --git a/docs/spack.md b/docs/spack.md index f721fb6971..922947beb0 100644 --- a/docs/spack.md +++ b/docs/spack.md @@ -49,6 +49,7 @@ Spack package names can specified using the `spack` directive. Multiple package process foo { spack 'bwa samtools py-multiqc' + script: ''' your_command --here ''' @@ -93,6 +94,7 @@ The path of an environment file can be specified using the `spack` directive: process foo { spack '/some/path/my-env.yaml' + script: ''' your_command --here ''' @@ -111,6 +113,7 @@ If you already have a local Spack environment, you can use it in your workflow s process foo { spack '/path/to/an/existing/env/directory' + script: ''' your_command --here ''' diff --git a/docs/vscode.md b/docs/vscode.md index 9e5e58115f..3c81c37617 100644 --- a/docs/vscode.md +++ b/docs/vscode.md @@ -230,38 +230,6 @@ if (aligner == 'bowtie2') { } ``` -**Slashy dollar strings** - -Groovy supports a wide variety of strings, including multi-line strings, dynamic strings, slashy strings, multi-line dynamic slashy strings, and more. - -The Nextflow language specification supports single- and double-quoted strings, multi-line strings, and slashy strings. Dynamic slashy strings are not supported: - -```groovy -def logo = /--cl-config 'custom_logo: "${multiqc_logo}"'/ -``` - -Use a double-quoted string instead: - -```nextflow -def logo = "--cl-config 'custom_logo: \"${multiqc_logo}\"'" -``` - -Slashy dollar strings are not supported: - -```groovy -$/ -echo "Hello world!" -/$ -``` - -Use a multi-line string instead: - -```nextflow -""" -echo "Hello world!" -""" -``` - **Implicit environment variables** In Nextflow DSL1 and DSL2, you can reference environment variables directly in strings: @@ -276,6 +244,14 @@ The Nextflow language specification does not support implicit environment variab println "PWD = ${System.getenv('PWD')}" ``` +:::{versionadded} 24.11.0-edge +The `env()` function can be used instead of `System.getenv()`: + +```nextflow +println "PWD = ${env('PWD')}" +``` +::: + ### Restricted syntax The following patterns are still supported but have been restricted, i.e. some syntax variants have been removed. @@ -326,6 +302,62 @@ To ease the migration of existing scripts, the language server only reports warn Type annotations and static type checking will be addressed in a future version of the Nextflow language specification. ::: +**Strings** + +Groovy supports a wide variety of strings, including multi-line strings, dynamic strings, slashy strings, multi-line dynamic slashy strings, and more. + +The Nextflow language specification supports single- and double-quoted strings, multi-line strings, and slashy strings. + +Slashy strings cannot be interpolated: + +```nextflow +def id = 'SRA001' +assert 'SRA001.fastq' ~= /${id}\.f(?:ast)?q/ +``` + +Use a double-quoted string instead: + +```nextflow +def id = 'SRA001' +assert 'SRA001.fastq' ~= "${id}\\.f(?:ast)?q" +``` + +Slashy strings cannot span multiple lines: + +```groovy +/ +Patterns in the code, +Symbols dance to match and find, +Logic unconfined. +/ +``` + +Use a multi-line string instead: + +```nextflow +""" +Patterns in the code, +Symbols dance to match and find, +Logic unconfined. +""" +``` + +Dollar slashy strings are not supported: + +```groovy +$/ +echo "Hello world!" +/$ +``` + +Use a multi-line string instead: + +```nextflow +""" +echo "Hello world!" +""" +``` + **Process env inputs/outputs** In Nextflow DSL1 and DSL2, the name of a process `env` input/output can be specified with or without quotes: @@ -335,6 +367,8 @@ process PROC { input: env FOO env 'BAR' + + // ... } ``` @@ -345,6 +379,8 @@ process PROC { input: env 'FOO' env 'BAR' + + // ... } ``` @@ -432,6 +468,10 @@ The `each` process input is deprecated. Use the `combine` or `cross` operator to The process `when` section is deprecated. Use conditional logic, such as an `if` statement or the `filter` operator, to control the process invocation in the calling workflow. +**Process shell section** + +The process `shell` section is deprecated. Use the `script` block instead. The VS Code extension provides syntax highlighting and error checking to help distinguish between Nextflow variables and Bash variables. + ### Configuration syntax See {ref}`config-syntax` for a comprehensive description of the configuration language. @@ -465,7 +505,7 @@ includeConfig ({ return 'large.config' else return '/dev/null' -})() +}()) ``` The include source is a closure that is immediately invoked. It includes a different config file based on the return value of the closure. Including `/dev/null` is equivalent to including nothing. diff --git a/docs/wave.md b/docs/wave.md index 668de765f0..c27c034f6b 100644 --- a/docs/wave.md +++ b/docs/wave.md @@ -90,6 +90,8 @@ conda.channels = 'conda-forge,bioconda' ``` ::: +Packages from the [Python Package Index](https://pypi.org/) can also be added to a Conda `environment.yml` file. See {ref}`Conda and PyPI ` for more information. + (wave-singularity)= ### Build Singularity native images diff --git a/docs/your-first-script.md b/docs/your-first-script.md index e0ded98a7c..b5101d4b9c 100644 --- a/docs/your-first-script.md +++ b/docs/your-first-script.md @@ -55,6 +55,7 @@ process convertToUpper { output: stdout + script: """ rev $x """ diff --git a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy index 3f1257a01c..e7dc15884c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy @@ -57,6 +57,17 @@ class Nextflow { private static final Random random = new Random() + /** + * Get the value of an environment variable from the launch environment. + * + * @param name + * The environment variable name to be referenced + * @return + * The value associate with the specified variable name or {@code null} if the variable does not exist. + */ + static String env(String name) { + return SysEnv.get(name) + } static private fileNamePattern( FilePatternSplitter splitter, Map opts ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 9b7ff555e7..f394245259 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -71,6 +71,7 @@ import nextflow.util.Barrier import nextflow.util.ConfigHelper import nextflow.util.Duration import nextflow.util.HistoryFile +import nextflow.util.LoggerHelper import nextflow.util.NameGenerator import nextflow.util.SysHelper import nextflow.util.ThreadPoolManager @@ -787,10 +788,11 @@ class Session implements ISession { */ void abort(Throwable cause = null) { if( aborted ) return - if( !(cause instanceof ScriptCompilationException) ) + if( cause !instanceof ScriptCompilationException ) log.debug "Session aborted -- Cause: ${cause?.message ?: cause ?: '-'}" aborted = true error = cause + LoggerHelper.aborted = true try { // log the dataflow network status def status = dumpNetworkStatus() diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy index 7d1ff901f5..15234c6039 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy @@ -22,6 +22,7 @@ import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap +import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j @@ -32,7 +33,6 @@ import nextflow.file.FileMutex import nextflow.util.CacheHelper import nextflow.util.Duration import nextflow.util.Escape -import org.yaml.snakeyaml.Yaml /** * Handle Conda environment creation and caching * @@ -166,6 +166,18 @@ class CondaCache { str.endsWith('.txt') && !str.contains('\n') } + static protected String sipHash(CharSequence data) { + Hashing + .sipHash24() + .newHasher() + .putUnencodedChars(data) + .hash() + .toString() + } + + static protected String sipHash(Path path) { + sipHash(path.toAbsolutePath().normalize().toString()) + } /** * Get the path on the file system where store a Conda environment @@ -188,11 +200,8 @@ class CondaCache { try { final path = condaEnv as Path content = path.text - final yaml = (Map)new Yaml().load(content) - if( yaml.name ) - name = yaml.name - else - name = path.baseName + name = 'env-' + sipHash(path) + } catch( NoSuchFileException e ) { throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") @@ -205,7 +214,7 @@ class CondaCache { try { final path = condaEnv as Path content = path.text - name = path.baseName + name = 'env-' + sipHash(path) } catch( NoSuchFileException e ) { throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") @@ -242,8 +251,8 @@ class CondaCache { * @return the conda environment prefix {@link Path} */ @PackageScope - Path createLocalCondaEnv(String condaEnv) { - final prefixPath = condaPrefixPath(condaEnv) + Path createLocalCondaEnv(String condaEnv, Path prefixPath) { + if( prefixPath.isDirectory() ) { log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath" return prefixPath @@ -351,17 +360,18 @@ class CondaCache { */ @PackageScope DataflowVariable getLazyImagePath(String condaEnv) { - - if( condaEnv in condaPrefixPaths ) { + final prefixPath = condaPrefixPath(condaEnv) + final condaEnvPath = prefixPath.toString() + if( condaEnvPath in condaPrefixPaths ) { log.trace "${binaryName} found local environment `$condaEnv`" - return condaPrefixPaths[condaEnv] + return condaPrefixPaths[condaEnvPath] } synchronized (condaPrefixPaths) { - def result = condaPrefixPaths[condaEnv] + def result = condaPrefixPaths[condaEnvPath] if( result == null ) { - result = new LazyDataflowVariable({ createLocalCondaEnv(condaEnv) }) - condaPrefixPaths[condaEnv] = result + result = new LazyDataflowVariable({ createLocalCondaEnv(condaEnv, prefixPath) }) + condaPrefixPaths[condaEnvPath] = result } else { log.trace "${binaryName} found local cache for environment `$condaEnv` (2)" diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy index 46a6f182da..a97e23f19a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy @@ -16,12 +16,12 @@ package nextflow.config -import ch.artecat.grengine.Grengine -import groovy.transform.Memoized - import java.nio.file.NoSuchFileException import java.nio.file.Path +import ch.artecat.grengine.Grengine +import groovy.transform.Memoized +import nextflow.SysEnv import nextflow.exception.IllegalConfigException import nextflow.file.FileHelper import org.codehaus.groovy.control.CompilerConfiguration @@ -74,6 +74,18 @@ abstract class ConfigBase extends Script { this.configStack = stack } + /** + * Get the value of an environment variable from the launch environment. + * + * @param name + * The environment variable name to be referenced + * @return + * The value associate with the specified variable name or {@code null} if the variable does not exist. + */ + String env(String name) { + return SysEnv.get(name) + } + /** * Implements the config file include */ diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index e1a92b172c..6d9dcbd538 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -183,7 +183,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy { return """\ IFS=\$'\\n' for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do - ${stageOutCommand('$name', targetDir, mode)} || true + ${stageOutCommand('$name', targetDir, mode)} done unset IFS""".stripIndent(true) } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy index c8ecc872a4..cb78900b33 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy @@ -345,8 +345,7 @@ class TaskConfig extends LazyMap implements Cloneable { int getMaxRetries() { def result = get('maxRetries') - def defResult = getErrorStrategy() == ErrorStrategy.RETRY ? 1 : 0 - result ? result as int : defResult + result ? result as int : 1 } int getMaxErrors() { diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index a1d48e5316..4b5fbf2791 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -309,6 +309,8 @@ class TaskProcessor { this.ownerScript = script this.config = config this.taskBody = taskBody + if( taskBody.isShell ) + log.warn "Process ${name} > the `shell` block is deprecated, use `script` instead" this.name = name this.maxForks = config.maxForks && config.maxForks>0 ? config.maxForks as int : 0 this.forksCount = maxForks ? new LongAdder() : null diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index f3926c0b60..38b4ba4782 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -790,7 +790,7 @@ class TaskRun implements Cloneable { * @param body A {@code BodyDef} object instance */ void resolve(BodyDef body) { - processor.session.stubRun + processor.session.stubRun && config.getStubBlock() ? resolveStub(config.getStubBlock()) : resolveBody(body) } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy index 75a6d2798e..dc87b2f827 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy @@ -73,11 +73,23 @@ abstract class RepositoryProvider { return this } + String getRevision() { + return this.revision + } + RepositoryProvider setRevision(String revision) { this.revision = revision return this } + String getProject() { + return this.project + } + + ProviderConfig getConfig() { + return this.config + } + boolean hasCredentials() { getUser() && getPassword() } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index df7ce27ec6..089eb781da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -126,7 +126,7 @@ class ProcessConfig implements Map, Cloneable { debug: false, cacheable: true, shell: BashWrapperBuilder.BASH, - maxRetries: 0, + maxRetries: 1, maxErrors: -1, errorStrategy: ErrorStrategy.TERMINATE ] diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy index 75795e5667..9a23cd77fd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy @@ -16,10 +16,6 @@ package nextflow.util -import ch.qos.logback.core.encoder.Encoder -import ch.qos.logback.core.spi.FilterAttachable -import ch.qos.logback.core.spi.LifeCycle - import static nextflow.Const.* import java.lang.reflect.Field @@ -28,6 +24,7 @@ import java.nio.file.FileAlreadyExistsException import java.nio.file.NoSuchFileException import java.nio.file.Path import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger import java.util.regex.Pattern import ch.qos.logback.classic.Level @@ -42,13 +39,16 @@ import ch.qos.logback.core.ConsoleAppender import ch.qos.logback.core.CoreConstants import ch.qos.logback.core.FileAppender import ch.qos.logback.core.LayoutBase +import ch.qos.logback.core.encoder.Encoder import ch.qos.logback.core.encoder.LayoutWrappingEncoder import ch.qos.logback.core.filter.Filter import ch.qos.logback.core.joran.spi.NoAutoStart import ch.qos.logback.core.rolling.FixedWindowRollingPolicy import ch.qos.logback.core.rolling.RollingFileAppender import ch.qos.logback.core.rolling.TriggeringPolicyBase +import ch.qos.logback.core.spi.FilterAttachable import ch.qos.logback.core.spi.FilterReply +import ch.qos.logback.core.spi.LifeCycle import ch.qos.logback.core.util.FileSize import groovy.transform.CompileStatic import groovy.transform.PackageScope @@ -85,6 +85,10 @@ import org.slf4j.MarkerFactory @CompileStatic class LoggerHelper { + static volatile boolean aborted + + static final AtomicInteger errCount = new AtomicInteger() + static private Logger log = LoggerFactory.getLogger(LoggerHelper) static public Marker STICKY = MarkerFactory.getMarker('sticky') @@ -419,6 +423,10 @@ class LoggerHelper { return FilterReply.NEUTRAL; } + // print to console only the very first error log and ignore the others + if( aborted && event.level==Level.ERROR && errCount.getAndIncrement()>0 ) + return FilterReply.DENY; + def logger = event.getLoggerName() def level = event.getLevel() for( int i=0; i
+%%{
+  init: {
+    'theme': 'base',
+    'themeVariables': {
+      'primaryColor': '#B6ECE2',
+      'primaryTextColor': '#160F26',
+      'primaryBorderColor': '#065647',
+      'lineColor': '#545555',
+      'clusterBkg': '#BABCBD22',
+      'clusterBorder': '#DDDEDE',
+      'fontFamily': 'arial'
+    }
+  }
+}%%
 REPLACE_WITH_NETWORK_DATA