Merge branch 'master' into docs-user-guide

Signed-off-by: Christopher Hakkaart <chris.hakkaart@seqera.io>
nextflow-io · Dec 4, 2024 · 62f2404 · 62f2404
2 parents c839def + fc63128
commit 62f2404
Show file tree

Hide file tree

Showing 95 changed files with 1,061 additions and 377 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -90,6 +90,7 @@ jobs:
       - name: Test
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
+            env | sort
             # configure test env
             if [[ "$GOOGLE_SECRET" ]]; then
             echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
@@ -158,6 +159,11 @@ jobs:
       - name: Run tests
         run: |
           env | sort
+          # configure test env
+          if [[ "$GOOGLE_SECRET" ]]; then
+          echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
+          export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json
+          fi
           cat $HOME/.nextflow/scm
           make clean assemble install
           bash test-ci.sh

diff --git a/.github/workflows/seqeradocs-changelog.yml b/.github/workflows/seqeradocs-changelog.yml
@@ -0,0 +1,62 @@
+name: Push release change log to Seqera Docs
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      release_name:
+        description: "Release version (e.g. 1.0.0)"
+        required: true
+      release_body:
+        description: "Release changelog content"
+        required: true
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Clone seqeralabs/docs
+        run: |
+          git clone https://github.com/seqeralabs/docs.git seqeralabs-docs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create changelog file
+        run: |
+          mkdir -p seqeralabs-docs/changelog/nextflow
+          cat << EOF > seqeralabs-docs/changelog/nextflow/${{ github.event.release.name || inputs.release_name }}.mdx
+          ---
+          title: Nextflow ${{ github.event.release.name || inputs.release_name }}
+          date: $(date +%Y-%m-%d)
+          tags: [nextflow]
+          ---
+
+          ${{ github.event.release.body || inputs.release_body }}
+          EOF
+
+      - uses: actions/create-github-app-token@v1
+        id: generate-token
+        with:
+          app-id: ${{ secrets.DOCS_BOT_APP_ID }}
+          private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
+          owner: seqeralabs
+          repositories: docs
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          branch-token: ${{ steps.generate-token.outputs.token }}
+          path: seqeralabs-docs
+          commit-message: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          title: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          body: |
+            This PR adds the changelog for Nextflow ${{ github.event.release.name || inputs.release_name }} to the Seqera documentation.
+
+            This is an automated PR created from the Nextflow repository.
+          branch: changelog-nextflow-${{ github.event.release.name || inputs.release_name }}
+          base: master
+          delete-branch: true
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-24.10.0
+24.11.0-edge
diff --git a/changelog.txt b/changelog.txt
@@ -1,5 +1,61 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.11.0-edge - 3 Dec 2024
+- Add GHA to submit dependencies to dependabot (#5440) [80395a6d]
+- Add NotFoundException to retry condition for Google Batch [aa4d19cc]
+- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae]
+- Add `env()` function (#5506) [fa0e8e0f]
+- Add more scientists to run name generator (#5447) [38d9eda0]
+- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0]
+- Add RepositoryProvider.revision now public property (#5500) [f0a4c526]
+- Deprecate process `shell` block (#5508) [6f527551]
+- Detecting errors in data unstaging (#5345) [3c8e602d]
+- Disable virtual threads on CI tests [ci slip] [69d07dbc]
+- Fix Fargate warning on memory check (#5475) [bdf0ad00]
+- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8]
+- Fix missing wave response (#5547) [ee252173]
+- Fix overlapping conda lock file (#5540) [9248c04d]
+- Fix overlapping conda lock exception (#5489) [eaaeb3de]
+- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621]
+- Fix Wave GCP integration test (#5490) [ad56c89b]
+- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db]
+- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a]
+- Improve docs for using the GPU accelerator directive (#5488) [4b908524]
+- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f]
+- Prevent NPE with null AWS Batch response [12fc1d60]
+- Target Java 17 as minimal Java version (#5045) [0140f954]
+- Update 'nexus-staging' plugin to latest version (#5462) [07934513]
+- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0]
+- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659]
+- Update process snippets to comply with strict syntax (#5526) [be1694bf]
+- Update Wave dependencies [09ccd295]
+- Bump aws sdk 1.12.777 (#5458) [8bad0b4b]
+- Bump bouncycastle to jdk18on:1.78.1  (#5467) [cd8c385f]
+- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df]
+- Bump commons-io:2.15.1 [767e4a0a]
+- Bump groovy 4.0.24 [dd71ad31]
+- Bump netty-common to version 4.1.115.Final [d1bbd3d0]
+- Bump nf-amazon@2.10.0 [2b653b07]
+- Bump nf-azure@1.11.0 [6af7198d]
+- Bump nf-google@1.16.0 [9494f970]
+- Bump nf-google@1.8.0 [7e2c8d82]
+- Bump protobuf-java:3.25.5 to nf-google [488b7906]
+
+24.10.2 - 27 Nov 2024
+- Prevent NPE with null AWS Batch response [3d491934]
+- Fix overlapping conda lock file (#5540) [df66deaa]
+- Fix missing wave response (#5547) [eb85cda8]
+- Bump nf-wave@1.7.4 [93d09404]
+- Bump nf-amazon@2.9.2 [469a35dd]
+
+24.10.1 - 18 Nov 2024
+- Fix overlapping file lock exception (#5489) [a2566d54]
+- Fix isContainerReady when wave is disabled (#5509) [c69e3711]
+- Bump nf-wave@1.7.3 [e7709a0f]
+- Bump nf-azure@1.10.2 [54496ac4]
+- Bump nf-amazon@2.9.1 [fa227933]
+- Bump netty-common to version 4.1.115.Final [90623c1e]
+
 24.10.0 - 27 Oct 2024
 - Add `manifest.contributors` config option (#5322) [cf0f9690]
 - Add wave mirror and scan config [92e69776]

diff --git a/docs/_static/dag.mmd b/docs/_static/dag.mmd
@@ -1,3 +1,17 @@
+%%{
+    init: {
+        'theme': 'base',
+        'themeVariables': {
+            'primaryColor': '#B6ECE2',
+            'primaryTextColor': '#160F26',
+            'primaryBorderColor': '#065647',
+            'lineColor': '#545555',
+            'clusterBkg': '#BABCBD22',
+            'clusterBorder': '#DDDEDE',
+            'fontFamily': 'arial'
+        }
+    }
+}%%
 flowchart TB
     subgraph " "
     v0["Channel.fromFilePairs"]

diff --git a/docs/aws.md b/docs/aws.md
@@ -46,53 +46,54 @@ Minimal permissions policies to be attached to the AWS account used by Nextflow
 - To use AWS Batch:
 
   ```json
-  "batch:DescribeJobQueues"
   "batch:CancelJob"
-  "batch:SubmitJob"
-  "batch:ListJobs"
   "batch:DescribeComputeEnvironments"
-  "batch:TerminateJob"
+  "batch:DescribeJobDefinitions"
+  "batch:DescribeJobQueues"
   "batch:DescribeJobs"
+  "batch:ListJobs"
   "batch:RegisterJobDefinition"
-  "batch:DescribeJobDefinitions"
+  "batch:SubmitJob"
+  "batch:TagResource"
+  "batch:TerminateJob"
   ```
 
 - To view [EC2](https://aws.amazon.com/ec2/) instances:
 
   ```json
-  "ecs:DescribeTasks"
+  "ec2:DescribeInstanceAttribute"
   "ec2:DescribeInstances"
+  "ec2:DescribeInstanceStatus"
   "ec2:DescribeInstanceTypes"
-  "ec2:DescribeInstanceAttribute"
   "ecs:DescribeContainerInstances"
-  "ec2:DescribeInstanceStatus"
+  "ecs:DescribeTasks"
   ```
 
 - To pull container images from [ECR](https://aws.amazon.com/ecr/) repositories:
 
   ```json
-  "ecr:GetAuthorizationToken"
   "ecr:BatchCheckLayerAvailability"
-  "ecr:GetDownloadUrlForLayer"
-  "ecr:GetRepositoryPolicy"
-  "ecr:DescribeRepositories"
-  "ecr:ListImages"
-  "ecr:DescribeImages"
   "ecr:BatchGetImage"
+  "ecr:DescribeImages"
+  "ecr:DescribeImageScanFindings"
+  "ecr:DescribeRepositories"
+  "ecr:GetAuthorizationToken"
+  "ecr:GetDownloadUrlForLayer"
   "ecr:GetLifecyclePolicy"
   "ecr:GetLifecyclePolicyPreview"
+  "ecr:GetRepositoryPolicy"
+  "ecr:ListImages"
   "ecr:ListTagsForResource"
-  "ecr:DescribeImageScanFindings"
   ```
 
 :::{note}
 If you are running Fargate or Fargate Spot, you may need the following policies in addition to the listed above:
   ```json
+  "ec2:DescribeSubnets"
   "ecs:CreateCluster"
   "ecs:DeleteCluster"
   "ecs:DescribeClusters"
   "ecs:ListClusters"
-  "ec2:DescribeSubnets"
   ```
 :::
 
@@ -538,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue
 and the Batch Execution Role.
 
 :::{note}
-This feature requires the use {ref}`Wave <wave-page>` container provisioning service.
+Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs.
+To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}Wave <wave-page> documentation for more details.
 :::
 
 ## Advanced configuration

diff --git a/docs/azure.md b/docs/azure.md
@@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob
 process.machineType = "Standard_D*d_v5,Standard_E*d_v5"
 ```
 
-For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`:
+For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`:
 
 ```nextflow
 process EXAMPLE_PROCESS {
     machineType "Standard_E*d_v5"
-    cpus 16
+    cpus 8
     memory 8.GB
 
     script:

diff --git a/docs/cache-and-resume.md b/docs/cache-and-resume.md
@@ -148,6 +148,8 @@ process gather {
     input:
     tuple val(id), file(foo)
     tuple val(id), file(bar)
+
+    script:
     """
     merge_command $foo $bar
     """
@@ -168,6 +170,8 @@ workflow {
 process gather {
     input:
     tuple val(id), file(foo), file(bar)
+
+    script:
     """
     merge_command $foo $bar
     """

diff --git a/docs/channel.md b/docs/channel.md
@@ -45,6 +45,7 @@ process foo {
   output:
   path 'x.txt'
 
+  script:
   """
   echo $x > x.txt
   """

diff --git a/docs/cli.md b/docs/cli.md
@@ -243,6 +243,36 @@ $ nextflow run <pipeline> --files "*.fasta"
 ```
 :::
 
+Parameters specified on the command line can be also specified in a params file using the `-params-file` option.
+
+```bash
+nextflow run main.nf -params-file pipeline_params.yml
+```
+
+The `-params-file` option loads parameters for your Nextflow pipeline from a JSON or YAML file. Parameters defined in the file are equivalent to specifying them directly on the command line. For example, instead of specifying parameters on the command line:
+
+```bash
+nextflow run main.nf --alpha 1 --beta foo
+```
+
+Parameters can be represented in YAML format:
+
+```yaml
+alpha: 1
+beta: 'foo'
+```
+
+Or in JSON format:
+
+```json
+{
+  "alpha": 1,
+  "beta": "foo"
+}
+```
+
+The parameters specified in a params file are merged with the resolved configuration. The values provided via a params file overwrite those of the same name in the Nextflow configuration file, but not those specified on the command line.
+
 ## Managing projects
 
 Nextflow seamlessly integrates with popular Git providers, including [BitBucket](http://bitbucket.org/), [GitHub](http://github.com), and [GitLab](http://gitlab.com) for managing Nextflow pipelines as version-controlled Git repositories.
-Original file line number
+Diff line change
@@ Expand Up / @@ -45,6 +45,7 @@ process foo { @@
       output:
       path 'x.txt'
+      script:
       """
       echo $x > x.txt
       """
@@ Expand Down @@