Merge branch 'master' into 5464-setting-default-maxretries-and-errors…

…trategy-and-override-for-some-processes

.github/workflows/build.yml

@@ -90,6 +90,7 @@ jobs:
       - name: Test
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
+            env | sort
             # configure test env
             if [[ "$GOOGLE_SECRET" ]]; then
             echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
@@ -158,6 +159,11 @@ jobs:
       - name: Run tests
         run: |
           env | sort
+          # configure test env
+          if [[ "$GOOGLE_SECRET" ]]; then
+          echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
+          export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json
+          fi
           cat $HOME/.nextflow/scm
           make clean assemble install
           bash test-ci.sh

docs/config.md

@@ -113,6 +113,15 @@ The following constants are globally available in a Nextflow configuration file:
 `projectDir`
 : The directory where the main script is located.
 
+## Functions
+
+The following functions are globally available in a Nextflow configuration file:
+
+`env( name )`
+: :::{versionadded} 24.11.0-edge
+  :::
+: Get the value of the environment variable with the specified name in the Nextflow launch environment.
+
 (config-params)=
 
 ## Parameters

docs/container.md

@@ -123,7 +123,7 @@ This feature requires the `apptainer` tool to be installed where the workflow ex
 Nextflow caches those images in the `apptainer` directory in the pipeline work directory by default. However it is suggested to provide a centralised cache directory by using either the `NXF_APPTAINER_CACHEDIR` environment variable or the `apptainer.cacheDir` setting in the Nextflow config file.
 
 :::{versionadded} 21.09.0-edge
-When looking for a Apptainer image file, Nextflow first checks the *library* directory, and if the image file is not found, the *cache* directory is used s usual. The library directory can be defined either using the `NXF_APPTAINER_LIBRARYDIR` environment variable or the `apptainer.libraryDir` configuration setting (the latter overrides the former).
+When looking for a Apptainer image file, Nextflow first checks the *library* directory, and if the image file is not found, the *cache* directory is used as usual. The library directory can be defined either using the `NXF_APPTAINER_LIBRARYDIR` environment variable or the `apptainer.libraryDir` configuration setting (the latter overrides the former).
 :::
 
 :::{warning}

docs/developer/plugins.md

@@ -18,7 +18,7 @@ dependencies {
     compileOnly 'org.pf4j:pf4j:3.4.1'
 
     testImplementation project(':nextflow')
-    testImplementation "org.codehaus.groovy:groovy:4.0.23"
+    testImplementation "org.codehaus.groovy:groovy:4.0.24"
     testImplementation "org.codehaus.groovy:groovy-nio:4.0.23"
 }
 ```

docs/executor.md

@@ -252,10 +252,22 @@ The `local` executor is used by default. It runs the pipeline processes on the c
 
 The `local` executor is useful for developing and testing a pipeline script on your computer, before switching to a cluster or cloud environment with production data.
 
+Resource requests and other job characteristics can be controlled via the following process directives:
+
+- {ref}`process-cpus`
+- {ref}`process-memory`
+- {ref}`process-time`
+- {ref}`process-container`
+- {ref}`process-containerOptions`
+
 :::{note}
 While the `local` executor limits the number of concurrent tasks based on requested vs available resources, it does not enforce task resource requests. In other words, it is possible for a local task to use more CPUs and memory than it requested, in which case it may starve other tasks. An exception to this behavior is when using {ref}`container-docker` or {ref}`container-podman` containers, in which case the resource requests are enforced by the container runtime.
 :::
 
+The local executor supports two types of tasks:
+- Script tasks (processes with a `script` or `shell` block) - executed via a Bash wrapper
+- Native tasks (processes with an `exec` block) - executed directly in the JVM.
+
 (lsf-executor)=
 
 ## LSF

docs/fusion.md

@@ -214,15 +214,15 @@ Then you can run your pipeline using the following command:
 nextflow run <YOUR PIPELINE> -work-dir s3://<YOUR BUCKET>/scratch
 ```
 
-Replace `<YOUR PIPELINE>` and `<YOUR BUCKET>` with a pipeline script and bucket or your choice, for example:
+Replace `<YOUR PIPELINE>` and `<YOUR BUCKET>` with a pipeline script and bucket of your choice, for example:
 
 ```bash
 nextflow run https://github.com/nextflow-io/rnaseq-nf -work-dir s3://nextflow-ci/scratch
 ```
 
 :::{warning}
 The option `fusion.exportStorageCredentials` leaks the AWS credentials on the task launcher script created by Nextflow.
-This option should only be used for development purposes.
+This option should only be used for testing and development purposes.
 :::
 
 ### Local execution with Minio
@@ -267,11 +267,53 @@ Then you can run your pipeline using the following command:
 nextflow run <YOUR PIPELINE> -work-dir s3://foobar/scratch
 ```
 
-Replace `<YOUR PIPELINE>` with a pipeline script and bucket or your choice:
+Replace `<YOUR PIPELINE>` with a pipeline script and bucket of your choice:
 
 :::{warning}
 The option `fusion.exportStorageCredentials` leaks the AWS credentials on the task launcher script created by Nextflow.
-This option should only be used for development purposes.
+This option should only be used for testing and development purposes.
+:::
+
+### Local execution with Oracle Object Storage
+
+Fusion file system and Nextflow are compatible with [Oracle Object Storage](https://www.oracle.com/cloud/storage/object-storage/).
+
+:::{note}
+This capability relies on the S3-like API compatibility provided by Oracle storage and not by a native support in
+Nextflow and Fusion. As such it may not fully work and support all Nextflow and Fusion features.
+:::
+
+This configuration requires the execution of your pipeline tasks using Docker or a similar container engine.
+
+The following should be included in your Nextflow configuration file:
+
+```groovy
+aws.region = '<YOUR_REGION>'
+aws.accessKey = '<YOUR_ACCESS_KEY>'
+aws.secretKey = '<YOUR_SECRET_KEY>'
+aws.client.endpoint = 'https://<YOUR_BUCKET_NAMESPACE>.compat.objectstorage.<YOUR_REGION>.oraclecloud.com'
+aws.client.s3PathStyleAccess = true
+aws.client.protocol = 'https'
+aws.client.signerOverride = 'AWSS3V4SignerType'
+docker.enabled = true
+docker.containerOptions = '-e FUSION_AWS_REGION=<YOUR_REGION>'
+fusion.enabled = true
+fusion.exportStorageCredentials = true
+wave.enabled = true
+tower.accessToken = '<YOUR_PLATFORM_ACCESS_TOKEN>' // optional
+```
+
+Then you can run your pipeline using the following command:
+
+```bash
+nextflow run <YOUR_PIPELINE> -work-dir s3://<YOUR_BUCKET>/scratch
+```
+
+In the above snippet replace the placeholders `<YOUR_ACCESS_KEY>` and `<YOUR_SECRET_KEY>` with your [Oracle Customer Secret Key](https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/managingcredentials.htm#Working2),
+and the placeholders `<YOUR_BUCKET_NAMESPACE>` and `<YOUR_REGION>` with the namespace and region of your Oracle bucket.
+
+:::{warning}
+The `fusion.exportStorageCredentials` option leaks the Oracle credentials to the Nextflow task launcher script and should only be used for testing and development purposes.
 :::
 
 ## Advanced settings

docs/install.md

@@ -119,15 +119,16 @@ NXF_VER=24.06.0-edge nextflow info
 
 ## Standalone distribution
 
-Nextflow has a set of {ref}`core plugins <plugins-core>` which are downloaded at runtime by default. There is also a standalone distribution (i.e. the `all` distribution) which comes pre-packaged with all core plugins. This distribution is mainly useful for offline environments.
+The Nextflow standalone distribution (i.e. the `dist` distribution) consists of self-contained `nextflow` executable file
+that includes all the application dependencies for core functionalities, and it can run without downloading third parties
+libraries. This distribution is mainly useful for offline environments.
 
-The installer for the `all` distribution can be found on the [GitHub releases page](https://github.com/nextflow-io/nextflow/releases), under the "Assets" section for a specific release. The installation procedure is the same as for the standard distribution, only using this URL instead of `https://get.nextflow.io`:
+Note however the support for cloud services e.g. AWS, Seqera Platform, Wave, etc. still require the download
+of the corresponding Nextflow plugins.
+
+The installer for the `dist` distribution can be found on the [GitHub releases page](https://github.com/nextflow-io/nextflow/releases), under the "Assets" section for a specific release. The installation procedure is the same as for the standard distribution, only using this URL instead of `https://get.nextflow.io`:
 
 ```bash
-export NXF_VER=23.10.0
-curl -s https://github.com/nextflow-io/nextflow/releases/download/v$NXF_VER/nextflow-$NXF_VER-all
+export NXF_VER=24.10.0
+curl -s https://github.com/nextflow-io/nextflow/releases/download/v$NXF_VER/nextflow-$NXF_VER-dist
 ```
-
-:::{warning}
-The `all` distribution does not support third-party plugins. Only the {ref}`core plugins <plugins-core>` are supported.
-:::

docs/process.md

@@ -161,9 +161,9 @@ In the above example, the process will execute one of several scripts depending
 
 ### Template
 
-Process scripts can be externalised to **template** files, which can be reused across different processes and tested independently from the overall pipeline execution.
+Process scripts can be externalized to **template** files, which allows them to be reused across different processes and tested independently from the pipeline execution.
 
-A template is simply a shell script file that Nextflow is able to execute by using the `template` function as shown below:
+A template can be used in place of an embedded script using the `template` function in the script section:
 
 ```nextflow
 process templateExample {
@@ -179,9 +179,9 @@ workflow {
 }
 ```
 
-By default, Nextflow looks for the `my_script.sh` template file in the `templates` directory located alongside the Nextflow script and/or the module script in which the process is defined. Any other location can be specified by using an absolute template path.
+By default, Nextflow looks for the template script in the `templates` directory located alongside the Nextflow script in which the process is defined. An absolute path can be used to specify a different location. However, this practice is discouraged because it hinders pipeline portability.
 
-The template script may contain any code that can be executed by the underlying environment. For example:
+An example template script is provided below:
 
 ```bash
 #!/bin/bash
@@ -190,12 +190,22 @@ echo $STR
 echo "process completed"
 ```
 
-:::{tip}
-The dollar character (`$`) is interpreted as a Nextflow variable when the script is run as a Nextflow template, whereas it is evaluated as a Bash variable when run as a Bash script. This can be very useful for testing your script independently from Nextflow execution. You only need to provide a Bash environment variable for each of the Nextflow variables that are referenced in your script. For example, it would be possible to execute the above script with the following command in the terminal: `STR='foo' bash templates/my_script.sh`
-:::
+Variables prefixed with the dollar character (`$`) are interpreted as Nextflow variables when the template script is executed by Nextflow and Bash variables when executed directly. For example, the above script can be executed from the command line by providing each input as an environment variable:
+
+```bash
+STR='foo' bash templates/my_script.sh
+```
+
+The following caveats should be considered:
+
+- Template scripts are recommended only for Bash scripts. Languages that do not prefix variables with `$` (e.g. Python and R) can't be executed directly as a template script.
+
+- Variables escaped with `\$` will be interpreted as Bash variables when executed by Nextflow, but will not be interpreted as variables when executed from the command line. This practice should be avoided to ensure that the template script behaves consistently.
+
+- Template variables are evaluated even if they are commented out in the template script. If a template variable is missing, it will cause the pipeline to fail regardless of where it occurs in the template.
 
 :::{tip}
-As a best practice, the template script should not contain any `\$` escaped variables, because these variables will not be evaluated properly when the script is executed directly.
+Template scripts are generally discouraged due to the caveats described above. The best practice for using a custom script is to embed it in the process definition at first and move it to a separate file with its own command line interface once the code matures.
 :::
 
 (process-shell)=
@@ -227,7 +237,7 @@ In the above example, `$USER` is treated as a Bash variable, while `!{str}` is t
 :::{note}
 - Shell script definitions require the use of single-quote `'` delimited strings. When using double-quote `"` delimited strings, dollar variables are interpreted as Nextflow variables as usual. See {ref}`string-interpolation`.
 - Variables prefixed with `!` must always be enclosed in curly brackets, i.e. `!{str}` is a valid variable whereas `!str` is ignored.
-- Shell scripts support the use of the {ref}`process-template` mechanism. The same rules are applied to the variables defined in the script template.
+- Shell scripts support the use of the {ref}`process-template` mechanism. The same rules are applied to the variables defined in the template script.
 :::
 
 (process-native)=

docs/reference/process.md

@@ -19,8 +19,11 @@ The following task properties are defined in the process body:
 : *Available only in `exec:` blocks*
 : The task unique hash ID.
 
+`task.id`
+: The pipeline-level task index. Corresponds to `task_id` in the {ref}`execution trace <trace-report>`.
+
 `task.index`
-: The task index (corresponds to `task_id` in the {ref}`execution trace <trace-report>`).
+: The process-level task index.
 
 `task.name`
 : *Available only in `exec:` blocks*
@@ -232,6 +235,10 @@ The above examples will request 4 GPUs of type `nvidia-tesla-k80`.
 This directive is only used by certain executors. Refer to the {ref}`executor-page` page to see which executors support this directive.
 :::
 
+:::{note}
+Additional options may be required to fully enable the use of accelerators. When using containers with GPUs, you must pass the GPU drivers through to the container. For Docker, this requires the option `--gpus all` in the docker run command. For Apptainer/Singularity, this requires the option `--nv`. The specific implementation details depend on the accelerator and container type being used.
+:::
+
 :::{note}
 The accelerator `type` option depends on the target execution platform. Refer to the platform-specific documentation for details on the available accelerators:
 

docs/reference/stdlib.md

@@ -197,6 +197,11 @@ The following functions are available in Nextflow scripts:
 `branchCriteria( closure )`
 : Create a branch criteria to use with the {ref}`operator-branch` operator.
 
+`env( name )`
+: :::{versionadded} 24.11.0-edge
+  :::
+: Get the value of the environment variable with the specified name in the Nextflow launch environment.
+
 `error( message = null )`
 : Throw a script runtime error with an optional error message.
 

docs/vscode.md

@@ -275,6 +275,14 @@ The Nextflow language specification does not support implicit environment variab
 println "PWD = ${System.getenv('PWD')}"
 ```
 
+:::{versionadded} 24.11.0-edge
+The `env()` function can be used instead of `System.getenv()`:
+
+```nextflow
+println "PWD = ${env('PWD')}"
+```
+:::
+
 ### Restricted syntax
 
 The following patterns are still supported but have been restricted, i.e. some syntax variants have been removed.

modules/nextflow/build.gradle

@@ -20,7 +20,7 @@ compileGroovy {
 dependencies {
     api(project(':nf-commons'))
     api(project(':nf-httpfs'))
-    api "org.apache.groovy:groovy:4.0.23"
+    api "org.apache.groovy:groovy:4.0.24"
     api "org.apache.groovy:groovy-nio:4.0.23"
     api "org.apache.groovy:groovy-xml:4.0.23"
     api "org.apache.groovy:groovy-json:4.0.23"
@@ -47,8 +47,8 @@ dependencies {
     api 'jline:jline:2.9'
     api 'org.pf4j:pf4j:3.12.0'
     api 'dev.failsafe:failsafe:3.1.0'
-    api 'org.bouncycastle:bcprov-ext-jdk15on:1.70'
-    api 'org.bouncycastle:bcpkix-jdk15on:1.70'
+    api 'org.bouncycastle:bcprov-ext-jdk18on:1.78.1'
+    api 'org.bouncycastle:bcpkix-jdk18on:1.78.1'
 
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
 

modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy

@@ -57,6 +57,17 @@ class Nextflow {
 
     private static final Random random = new Random()
 
+    /**
+     * Get the value of an environment variable from the launch environment.
+     *
+     * @param name
+     *      The environment variable name to be referenced
+     * @return
+     *      The value associate with the specified variable name or {@code null} if the variable does not exist.
+     */
+    static String env(String name) {
+        return SysEnv.get(name)
+    }
 
     static private fileNamePattern( FilePatternSplitter splitter, Map opts ) {
 

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -71,6 +71,7 @@ import nextflow.util.Barrier
 import nextflow.util.ConfigHelper
 import nextflow.util.Duration
 import nextflow.util.HistoryFile
+import nextflow.util.LoggerHelper
 import nextflow.util.NameGenerator
 import nextflow.util.SysHelper
 import nextflow.util.ThreadPoolManager
@@ -787,10 +788,11 @@ class Session implements ISession {
      */
     void abort(Throwable cause = null) {
         if( aborted ) return
-        if( !(cause instanceof ScriptCompilationException) )
+        if( cause !instanceof ScriptCompilationException )
             log.debug "Session aborted -- Cause: ${cause?.message ?: cause ?: '-'}"
         aborted = true
         error = cause
+        LoggerHelper.aborted = true
         try {
             // log the dataflow network status
             def status = dumpNetworkStatus()