Merge branch 'master' into apptainer-oci-mode

nextflow-io · Dec 13, 2023 · c6a7f30 · c6a7f30
2 parents 2193a45 + 887f06f
commit c6a7f30
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 42 deletions.
diff --git a/docs/config.md b/docs/config.md
@@ -1149,8 +1149,8 @@ Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHu
 
 The `notification` scope allows you to define the automatic sending of a notification email message when the workflow execution terminates.
 
-`notification.binding`
-: A map modelling the variables in the template file.
+`notification.attributes`
+: A map object modelling the variables that can be used in the template file.
 
 `notification.enabled`
 : Enables the sending of a notification message when the workflow execution completes.

diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy
@@ -489,10 +489,6 @@ class Session implements ISession {
         checkConfig()
         notifyFlowBegin()
 
-        if( !NextflowMeta.instance.isDsl2() ) {
-            return
-        }
-
         // bridge any dataflow queue into a broadcast channel
         CH.broadcast()
 
@@ -1080,11 +1076,15 @@ class Session implements ISession {
     }
 
     void notifyFlowBegin() {
-        observers.each { trace -> trace.onFlowBegin() }
+        for( TraceObserver trace : observers ) {
+            trace.onFlowBegin()
+        }
     }
 
     void notifyFlowCreate() {
-        observers.each { trace -> trace.onFlowCreate(this) }
+        for( TraceObserver trace : observers ) {
+            trace.onFlowCreate(this)
+        }
     }
 
     void notifyFilePublish(Path destination, Path source=null) {

diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
@@ -463,10 +463,10 @@ class WorkflowMetadata {
      */
     protected void safeMailNotification() {
         try {
-            def notifier = new WorkflowNotifier()
-            notifier.workflow = this
-            notifier.config = session.config
-            notifier.variables = NF.binding.variables
+            final notifier = new WorkflowNotifier(
+                workflow: this,
+                config: session.config,
+                variables: NF.binding.variables )
             notifier.sendNotification()
         }
         catch (Exception e) {

diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
@@ -110,9 +110,9 @@ class WorkflowNotifier {
         List<File> templates = []
         normaliseTemplate0(notification.template, templates)
         if (templates) {
-            final binding = normaliseBindings0(notification.binding)
+            final binding = normaliseBindings0(notification.attributes)
 
-            templates.each { file ->
+            for( File file : templates ) {
                 def content = loadMailTemplate(file, binding)
                 def plain = file.extension == 'txt'
                 if (plain) {
@@ -135,7 +135,7 @@ class WorkflowNotifier {
     protected Map normaliseBindings0(binding) {
 
         if (binding == null)
-            return null
+            return Map.of()
 
         if (binding instanceof Map)
             return binding
@@ -218,8 +218,10 @@ class WorkflowNotifier {
 
     private String loadMailTemplate0(InputStream source, Map binding) {
         def map = new HashMap()
-        map.putAll(variables)
-        map.putAll(binding)
+        if( variables )
+            map.putAll(variables)
+        if( binding )
+            map.putAll(binding)
 
         def template = new GStringTemplateEngine().createTemplate(new InputStreamReader(source))
         template.make(map).toString()

diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy
@@ -18,6 +18,7 @@ package nextflow.cli
 import java.nio.file.Files
 
 import nextflow.plugin.Plugins
+import spock.lang.IgnoreIf
 import spock.lang.Requires
 import spock.lang.Specification
 /**
@@ -26,6 +27,7 @@ import spock.lang.Specification
  */
 class CmdCloneTest extends Specification {
 
+    @IgnoreIf({System.getenv('NXF_SMOKE')})
     @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')})
     def testClone() {
 

diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/GiteaRepositoryProviderTest.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.scm
 
+import spock.lang.IgnoreIf
 import spock.lang.Requires
 import spock.lang.Specification
 
@@ -76,6 +77,7 @@ class GiteaRepositoryProviderTest extends Specification {
 
     }
 
+    @IgnoreIf({System.getenv('NXF_SMOKE')})
     @Requires({System.getenv('NXF_GITEA_ACCESS_TOKEN')})
     def 'should read file content'() {
 

diff --git a/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/WorkflowNotifierTest.groovy
@@ -16,6 +16,7 @@
 
 package nextflow.script
 
+import java.nio.file.Files
 import java.nio.file.Paths
 import java.time.Instant
 import java.time.OffsetDateTime
@@ -256,7 +257,7 @@ class WorkflowNotifierTest extends Specification {
         when:
         workflow.success = false
         workflow.runName = 'bar'
-        mail = notifier.createMail([to:'[email protected]', from:'[email protected]', template: ['/some/file.txt', '/other/file.html'], binding: [one:1, two:2]])
+        mail = notifier.createMail([to:'[email protected]', from:'[email protected]', template: ['/some/file.txt', '/other/file.html'], attributes: [one:1, two:2]])
         then:
         1 * notifier.loadMailTemplate(new File('/some/file.txt'), [one:1, two:2]) >> 'TEXT template'
         1 * notifier.loadMailTemplate(new File('/other/file.html'), [one:1, two:2]) >> 'HTML template'
@@ -268,6 +269,40 @@ class WorkflowNotifierTest extends Specification {
 
     }
 
+    def 'should create notification mail with custom template' () {
+        given:
+        def folder  = Files.createTempDirectory('test')
+        def template = folder.resolve('template.txt').toFile(); template.text = 'Hello world!'
+
+        and:
+        Mail mail
+        def workflow = new WorkflowMetadata()
+        def notifier = Spy(WorkflowNotifier)
+        notifier.@workflow = workflow
+
+        /*
+         * create success completion  *default* notification email
+         */
+        when:
+        workflow.success = true
+        workflow.runName = 'foo'
+        mail = notifier.createMail([to:'[email protected]', from:'[email protected]', template: template])
+        then:
+        0 * notifier.loadDefaultTextTemplate()
+        0 * notifier.loadDefaultHtmlTemplate()
+        1 * notifier.loadMailTemplate(template, [:])
+        and:
+        mail.to == '[email protected]'
+        mail.from == '[email protected]'
+        mail.subject == 'Workflow completion [foo] - SUCCEED'
+        mail.text == 'Hello world!'
+        !mail.body 
+        !mail.attachments
+
+        cleanup:
+        folder?.deleteDir()
+    }
+
 
     def 'should send notification email' () {
 

diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy
@@ -397,7 +397,7 @@ class PluginsFacade implements PluginStateListener {
         final bucketDir = config.bucketDir as String
         final executor = Bolts.navigate(config, 'process.executor')
 
-        if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') )
+        if( executor == 'awsbatch' || workDir?.startsWith('s3://') || bucketDir?.startsWith('s3://') || SysEnv.containsKey('NXF_ENABLE_AWS_SES') )
             plugins << defaultPlugins.getPlugin('nf-amazon')
 
         if( executor == 'google-lifesciences' || executor == 'google-batch' || workDir?.startsWith('gs://') || bucketDir?.startsWith('gs://')  )

diff --git a/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy b/modules/nf-commons/src/main/nextflow/util/StringUtils.groovy
@@ -56,7 +56,7 @@ class StringUtils {
         return m.matches() ? m.group(1).toLowerCase() : null
     }
 
-    static private Pattern multilinePattern = ~/"?(password|token|secret|license)"?\s?[:=]\s?"?(\w+)"?/
+    static private Pattern multilinePattern = ~/["']?(password|token|secret|license)["']?\s?[:=]\s?["']?(\w+)["']?/
 
     static String stripSecrets(String message) {
         if (message == null) {

diff --git a/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy b/modules/nf-commons/src/test/nextflow/util/StringUtilsTest.groovy
@@ -100,13 +100,15 @@ class StringUtilsTest extends Specification {
         StringUtils.stripSecrets(SECRET) == EXPECTED
 
         where:
-        SECRET                                  | EXPECTED
-        'Hi\n here is the "password" : "1234"'  | 'Hi\n here is the "password" : "********"'
-        'Hi\n here is the password : "1"'       | 'Hi\n here is the password : "********"'
-        'Hi\n here is the password : "1"'       | 'Hi\n here is the password : "********"'
-        'Hi\n "password" :"1" \n "token": "123"'| 'Hi\n "password" :"********" \n "token": "********"'
-        'Hi\n password :"1"\nsecret: "345"'     | 'Hi\n password :"********"\nsecret: "********"'
-        'secret="abc" password:"1" more text'   | 'secret="********" password:"********" more text'
+        SECRET                                          | EXPECTED
+        'Hi\n here is the "password" : "1234"'          | 'Hi\n here is the "password" : "********"'
+        'Hi\n here is the password : "1"'               | 'Hi\n here is the password : "********"'
+        'Hi\n here is the password : \'1\''             | 'Hi\n here is the password : \'********\''
+        'Hi\n "password" :"1" \n "token": "123"'        | 'Hi\n "password" :"********" \n "token": "********"'
+        'Hi\n "password" :\'1\' \n "token": "123"'      | 'Hi\n "password" :\'********\' \n "token": "********"'
+        'Hi\n \'password\' :\'1\' \n \'token\': \'123\''| 'Hi\n \'password\' :\'********\' \n \'token\': \'********\''
+        'Hi\n password :"1"\nsecret: "345"'             | 'Hi\n password :"********"\nsecret: "********"'
+        'secret="abc" password:"1" more text'           | 'secret="********" password:"********" more text'
     }
 
     @Unroll

diff --git a/...nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy b/...nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy
@@ -440,10 +440,8 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification {
 
     def 'should create pipeline actions with keepalive' () {
         given:
-        def helper = Spy(GoogleLifeSciencesHelper)
-        helper.config = Mock(GoogleLifeSciencesConfig) {
-            getKeepAliveOnFailure() >> true
-        }
+        def config = new GoogleLifeSciencesConfig(keepAliveOnFailure: true)
+        def helper = Spy(new GoogleLifeSciencesHelper(config: config))
         and:
         def req = Mock(GoogleLifeSciencesSubmitRequest)
 

diff --git a/...ogle/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy b/...ogle/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesTaskHandlerTest.groovy
@@ -198,19 +198,19 @@ class GoogleLifeSciencesTaskHandlerTest extends GoogleSpecification {
         given:
         def workDir = mockGsPath('gs://my-bucket/work/dir')
         and:
+        def config = new GoogleLifeSciencesConfig(
+            project: 'my-project',
+            zones: ['my-zone'],
+            regions: ['my-region'],
+            preemptible: true,
+            bootDiskSize: MemoryUnit.of('20 GB'),
+            usePrivateAddress: true,
+            cpuPlatform: 'Intel Skylake'
+        )
+        and:
         def executor = Mock(GoogleLifeSciencesExecutor) {
             getHelper() >> Mock(GoogleLifeSciencesHelper)
-            getConfig() >> {
-                Mock(GoogleLifeSciencesConfig) {
-                    getProject() >> 'my-project'
-                    getZones() >> ['my-zone']
-                    getRegions() >> ['my-region']
-                    getPreemptible() >> true
-                    getBootDiskSize() >> MemoryUnit.of('20 GB')
-                    getUsePrivateAddress() >> true
-                    getCpuPlatform() >> 'Intel Skylake'
-                }
-            }
+            getConfig() >> config
         }
         and:
         def task = Mock(TaskRun)