Prevent default output/error logs for job arrays (#5152)

Signed-off-by: Ben Sherman <[email protected]> Signed-off-by: Paolo Di Tommaso <[email protected]> Co-authored-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Aug 5, 2024 · 71f6ed0 · 71f6ed0
1 parent 82c97f8
commit 71f6ed0
Show file tree

Hide file tree

Showing 16 changed files with 43 additions and 23 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy
@@ -49,10 +49,8 @@ class CrgExecutor extends SgeExecutor {
 
         result << '-N' << getJobNameFor(task)
 
-        if( task !instanceof TaskArrayRun ) {
-            result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
-            result << '-j' << 'y'
-        }
+        result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
+        result << '-j' << 'y'
 
         result << '-terse' << ''    // note: directive need to be returned as pairs
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/LsfExecutor.groovy
@@ -70,9 +70,7 @@ class LsfExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
      */
     protected List<String> getDirectives(TaskRun task, List<String> result) {
 
-        if( task !instanceof TaskArrayRun ) {
-            result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString()
-        }
+        result << '-o' << (task.isArray() ?  '/dev/null' : task.workDir.resolve(TaskRun.CMD_LOG).toString())
 
         // add other parameters (if any)
         if( task.config.queue ) {

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy
@@ -51,10 +51,8 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
 
         result << '-N' << getJobNameFor(task)
 
-        if( task !instanceof TaskArrayRun ) {
-            result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
-            result << '-j' << 'oe'
-        }
+        result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
+        result << '-j' << 'oe'
 
         // the requested queue name
         if( task.config.queue ) {

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy
@@ -58,10 +58,8 @@ class PbsProExecutor extends PbsExecutor {
 
         result << '-N' << getJobNameFor(task)
 
-        if( task !instanceof TaskArrayRun ) {
-            result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
-            result << '-j' << 'oe'
-        }
+        result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
+        result << '-j' << 'oe'
 
         // the requested queue name
         if( task.config.queue ) {

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy
@@ -45,10 +45,8 @@ class SgeExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
 
         result << '-N' << getJobNameFor(task)
 
-        if( task !instanceof TaskArrayRun ) {
-            result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
-            result << '-j' << 'y'
-        }
+        result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
+        result << '-j' << 'y'
 
         result << '-terse' << ''    // note: directive need to be returned as pairs
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy
@@ -62,10 +62,8 @@ class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
 
         result << '-J' << getJobNameFor(task)
 
-        if( task !instanceof TaskArrayRun ) {
-            // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE
-            result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG))
-        }
+        // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE
+        result << '-o' << (task.isArray() ? '/dev/null' : quote(task.workDir.resolve(TaskRun.CMD_LOG)))
 
         result << '--no-requeue' << '' // note: directive need to be returned as pairs
 

diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy
@@ -52,4 +52,9 @@ class TaskArrayRun extends TaskRun {
         return false
     }
 
+    @Override
+    final boolean isArray() {
+        return true
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy
@@ -734,6 +734,10 @@ class TaskRun implements Cloneable {
         return processor.executor?.isContainerNative() ?: false
     }
 
+    boolean isArray() {
+        return false
+    }
+
     boolean isContainerEnabled() {
         return getContainerConfig().isEnabled() && getContainer()!=null
     }

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy
@@ -351,6 +351,8 @@ class CrgExecutorTest extends Specification {
         executor.getHeaders(task) == '''
                     #$ -t 1-5
                     #$ -N nf-mapping_tag
+                    #$ -o /dev/null
+                    #$ -j y
                     #$ -terse
                     #$ -notify
                     '''

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/LsfExecutorTest.groovy
@@ -324,6 +324,7 @@ class LsfExecutorTest extends Specification {
         }
         then:
         executor.getHeaders(taskArray) == '''
+                #BSUB -o /dev/null
                 #BSUB -J "nf-mapping_hola[1-5]"
                 '''
                 .stripIndent().leftTrim()

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy
@@ -174,6 +174,8 @@ class PbsExecutorTest extends Specification {
         executor.getHeaders(taskArray) == '''
                 #PBS -J 0-4
                 #PBS -N nf-task_name
+                #PBS -o /dev/null
+                #PBS -j oe
                 '''
                 .stripIndent().leftTrim()
     }

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/PbsProExecutorTest.groovy
@@ -201,6 +201,8 @@ class PbsProExecutorTest extends Specification {
         executor.getDirectives(task, []) == [
                 '-J', '0-4',
                 '-N', 'nf-foo',
+                '-o', '/dev/null',
+                '-j', 'oe'
         ]
     }
 

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SgeExecutorTest.groovy
@@ -207,6 +207,8 @@ class SgeExecutorTest extends Specification {
         executor.getHeaders(taskArray) == '''
                 #$ -t 1-5
                 #$ -N nf-the_task_name
+                #$ -o /dev/null
+                #$ -j y
                 #$ -terse
                 #$ -notify
                 '''

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SlurmExecutorTest.groovy
@@ -230,6 +230,7 @@ class SlurmExecutorTest extends Specification {
         executor.getHeaders(taskArray) == '''
                 #SBATCH --array 0-4
                 #SBATCH -J nf-the_task_name
+                #SBATCH -o /dev/null
                 #SBATCH --no-requeue
                 #SBATCH --signal B:USR2@30
                 '''

diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskArrayRunTest.groovy
@@ -46,4 +46,11 @@ class TaskArrayRunTest extends Specification {
         info == new ContainerInfo('ubuntu','ubuntu','ubuntu')
     }
 
+    def 'should be an array' () {
+        given:
+        def task = new TaskArrayRun()
+        expect:
+        task.isArray()
+    }
+
 }
diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskRunTest.groovy
@@ -885,4 +885,10 @@ class TaskRunTest extends Specification {
         info == new ContainerInfo('ubuntu','ubuntu','ubuntu')
     }
 
+    def 'should not be an array' () {
+        given:
+        def task = new TaskRun()
+        expect:
+        !task.isArray()
+    }
 }