Add workflow.failOnIgnore config option (#4686) [ci fast]

This commit adds support for the `workflow.failOnIgnore` configuration option that causes a workflow to report a failure exit status when one or more tasks report an error that is ignored due to the `ignore` error strategy. Signed-off-by: Ben Sherman <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Harshil Patel <[email protected]>
nextflow-io · Jun 17, 2024 · 172edc3 · 172edc3
1 parent ab5b3eb
commit 172edc3
Show file tree

Hide file tree

Showing 11 changed files with 93 additions and 11 deletions.
diff --git a/docs/config.md b/docs/config.md
@@ -1773,6 +1773,15 @@ The following settings are available:
 `wave.strategy`
 : The strategy to be used when resolving ambiguous Wave container requirements (default: `'container,dockerfile,conda,spack'`).
 
+### Scope `workflow`
+
+The `workflow` scope provides workflow execution options.
+
+`workflow.failOnIgnore`
+: :::{versionadded} 24.05.0-edge
+  :::
+: When `true`, the pipeline will exit with a non-zero exit code if any failed tasks are ignored using the `ignore` error strategy.
+
 (config-miscellaneous)=
 
 ### Miscellaneous

diff --git a/docs/metadata.md b/docs/metadata.md
@@ -58,6 +58,11 @@ The following table lists the properties that can be accessed on the `workflow`
 : *Available only in the `workflow.onComplete` and `workflow.onError` handlers*
 : Exit status of the task that caused the workflow execution to fail.
 
+`workflow.failOnIgnore`
+: :::{versionadded} 24.05.0-edge
+  :::
+: Whether the `workflow.failOnIgnore` config option was enabled.
+
 `workflow.fusion.enabled`
 : Whether Fusion is enabled.
 

diff --git a/docs/process.md b/docs/process.md
@@ -1652,16 +1652,19 @@ The `errorStrategy` directive allows you to define how an error condition is man
 The following error strategies are available:
 
 `terminate` (default)
-: Terminate the execution as soon as an error condition is reported. Pending jobs are killed.
+: When a task fails, terminate the pipeline immediately. Pending and running jobs are killed.
 
 `finish`
-: Initiate an orderly pipeline shutdown when an error condition is raised, waiting for the completion of any submitted jobs.
+: When a task fails, wait for pending and running tasks to finish and then terminate the pipeline.
 
 `ignore`
-: Ignore process execution errors.
+: Ignore all task failures and complete the pipeline execution successfully.
+: :::{versionadded} 24.05.0-edge
+  When the `workflow.failOnIgnore` config option is set to `true`, the pipeline will return a non-zero exit code if one or more failed tasks were ignored.
+  :::
 
 `retry`
-: Re-submit any process that returns an error condition.
+: When a task fails, retry it.
 
 When setting the `errorStrategy` directive to `ignore` the process doesn't stop on an error condition, it just reports a message notifying you of the error event.
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
@@ -223,6 +223,8 @@ class Session implements ISession {
 
     private Barrier monitorsBarrier = new Barrier()
 
+    private volatile boolean failOnIgnore
+
     private volatile boolean cancelled
 
     private volatile boolean aborted
@@ -824,7 +826,7 @@ class Session implements ISession {
 
     boolean isCancelled() { cancelled }
 
-    boolean isSuccess() { !aborted && !cancelled }
+    boolean isSuccess() { !aborted && !cancelled && !failOnIgnore }
 
     void processRegister(TaskProcessor process) {
         log.trace ">>> barrier register (process: ${process.name})"
@@ -864,6 +866,10 @@ class Session implements ISession {
         config.navigate('nextflow.enable.moduleBinaries', false) as boolean
     }
 
+    boolean failOnIgnore() {
+        config.navigate('workflow.failOnIgnore', false) as boolean
+    }
+
     @PackageScope VersionNumber getCurrentVersion() {
         new VersionNumber(BuildInfo.version)
     }
@@ -1047,6 +1053,12 @@ class Session implements ISession {
         final trace = handler.safeTraceRecord()
         cache.putTaskAsync(handler, trace)
 
+        // set the pipeline to return non-exit code if specified
+        if( handler.task.errorAction == ErrorStrategy.IGNORE && failOnIgnore() ) {
+            log.debug "Setting fail-on-ignore flag due to ignored task '${handler.task.lazyName()}'"
+            failOnIgnore = true
+        }
+
         // notify the event to the observers
         for( int i=0; i<observers.size(); i++ ) {
             final observer = observers.get(i)

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy
@@ -23,10 +23,10 @@ package nextflow.processor
  */
 enum ErrorStrategy {
 
-    TERMINATE(false),    // on error, terminate the pipeline execution killing all pending and running tasks
-    FINISH(false),       // on error, terminate the pipeline execution awaiting for previously submitted task to complete
-    IGNORE(true),       // on error, ignore it an go-on
-    RETRY(true);        // on error, retry
+    TERMINATE(false),       // on error, terminate the pipeline execution, killing all pending and running tasks
+    FINISH(false),          // on error, terminate the pipeline execution, waiting for pending and running tasks to complete
+    IGNORE(true),           // on error, ignore it and continue
+    RETRY(true);            // on error, retry
 
     final boolean soft
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy
@@ -1072,8 +1072,10 @@ class TaskProcessor {
                 errorStrategy = checkErrorStrategy(task, error, taskErrCount, procErrCount, submitRetries)
                 if( errorStrategy.soft ) {
                     def msg = "[$task.hashLog] NOTE: ${submitTimeout ? submitErrMsg : error.message}"
-                    if( errorStrategy == IGNORE ) msg += " -- Error is ignored"
-                    else if( errorStrategy == RETRY ) msg += " -- Execution is retried (${submitTimeout ? submitRetries : taskErrCount})"
+                    if( errorStrategy == IGNORE )
+                        msg += " -- Error is ignored"
+                    else if( errorStrategy == RETRY )
+                        msg += " -- Execution is retried (${submitTimeout ? submitRetries : taskErrCount})"
                     log.info msg
                     task.failed = true
                     task.errorAction = errorStrategy

diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
@@ -227,6 +227,11 @@ class WorkflowMetadata {
      */
     Manifest manifest
 
+    /**
+     * whenever it should terminate with a failure when one or more task execution failed in an error strategy
+     */
+    boolean failOnIgnore
+
     private Session session
 
     final private List<Closure> onCompleteActions = []
@@ -268,6 +273,7 @@ class WorkflowMetadata {
         this.manifest = session.getManifest()
         this.wave = new WaveMetadata(session)
         this.fusion = new FusionMetadata(session)
+        this.failOnIgnore = session.failOnIgnore()
 
         // check if there's a onComplete action in the config file
         registerConfigAction(session.config.workflow as Map)

diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowMetadataTest.groovy
@@ -101,6 +101,7 @@ class WorkflowMetadataTest extends Specification {
         metadata.homeDir == Paths.get(System.getProperty('user.home'))
         metadata.manifest.version == '1.0.0'
         metadata.manifest.nextflowVersion == '>=0.31.1'
+        !metadata.failOnIgnore
 
         when:
         metadata.invokeOnComplete()
@@ -116,12 +117,14 @@ class WorkflowMetadataTest extends Specification {
         session.resumeMode >> true
         session.stubRun >> true
         session.preview >> true
+        session.failOnIgnore() >> true
         metadata = new WorkflowMetadata(session, script)
         then:
         metadata.profile == 'foo_profile'
         metadata.resume
         metadata.stubRun
         metadata.preview
+        metadata.failOnIgnore
     }
 
     def foo_test_method() {

diff --git a/tests/checks/error-ignore-then-fail.nf/.checks b/tests/checks/error-ignore-then-fail.nf/.checks
@@ -0,0 +1,12 @@
+set -e
+
+set +e
+echo ''
+$NXF_CMD -q run $NXF_SCRIPT -c .config  > stdout
+status=$?
+set -e
+
+[ $status -ne 0 ] || false
+
+[[ `< .nextflow.log grep -c 'Submitted process > foo'` == 1 ]] || false
+[[ `< .nextflow.log grep -c 'Error is ignored'` == 1 ]] || false
diff --git a/tests/checks/error-ignore-then-fail.nf/.config b/tests/checks/error-ignore-then-fail.nf/.config
@@ -0,0 +1 @@
+workflow.failOnIgnore = true
diff --git a/tests/error-ignore-then-fail.nf b/tests/error-ignore-then-fail.nf
@@ -0,0 +1,29 @@
+#!/usr/bin/env nextflow
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+process foo {
+    errorStrategy 'ignore'
+
+    script:
+    '''
+    exit 1
+    '''
+}
+
+workflow {
+    foo()
+}