Add setting to enable the use of sync command

Signed-off-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Sep 28, 2023 · f0d5cc5 · f0d5cc5
1 parent 2b9ae6a
commit f0d5cc5
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 5 deletions.
diff --git a/docs/config.md b/docs/config.md
@@ -1588,6 +1588,11 @@ The following environment variables control the configuration of the Nextflow ru
   :::
 : Disables the requirement for Wave service when enabling the Fusion file system.
 
+`NXF_ENABLE_FS_SYNC`
+: :::{versionadded} 23.10.0
+  :::
+: When enabled the job script will execute Linux `sync` command on job completion. This may be useful to synchronize the job state over shared file systems (default: `false`)
+
 `NXF_ENABLE_SECRETS`
 : :::{versionadded} 21.09.0-edge
   :::

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy
@@ -527,7 +527,7 @@ class BashWrapperBuilder {
     }
 
     String getSyncCmd() {
-        if ( SysEnv.get( 'NXF_DISABLE_FS_SYNC' ) != "true" ) {
+        if ( SysEnv.get( 'NXF_ENABLE_FS_SYNC' ) == "true" ) {
             return 'sync || true'
         }
         return null

diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy
@@ -425,6 +425,19 @@ class BashWrapperBuilderTest extends Specification {
         folder?.deleteDir()
     }
 
+    def 'should include sync command' () {
+        given:
+        SysEnv.push([NXF_ENABLE_FS_SYNC: 'true'])
+
+        when:
+        def binding = newBashWrapperBuilder().makeBinding()
+        then:
+        binding.sync_cmd == 'sync || true'
+
+        cleanup:
+        SysEnv.pop()
+    }
+
     def 'should unstage outputs' () {
 
         given:
@@ -486,6 +499,7 @@ class BashWrapperBuilderTest extends Specification {
         binding.containsKey('container_env')
         binding.task_env == null
         binding.container_env == null
+        binding.sync_cmd == null
 
         when:
         binding = newBashWrapperBuilder(environment: [FOO:'aa', BAR:'bb']).makeBinding()

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt
@@ -271,7 +271,6 @@ on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
-    sync || true
     exit $exit_status
 }
 

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt
@@ -82,7 +82,6 @@ on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
-    sync || true
     exit $exit_status
 }
 

diff --git a/...e/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy b/...e/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy
@@ -82,7 +82,7 @@ class GoogleLifeSciencesScriptLauncherTest extends GoogleSpecification {
         def wrapper =  builder.buildNew0()
 
         then:
-        wrapper == load('bash-wrapper-gcp.txt', [folder: WORK_DIR.toString(), sync_cmd: 'sync || true' ])
+        wrapper == load('bash-wrapper-gcp.txt', [folder: WORK_DIR.toString()])
 
         cleanup:
         folder?.deleteDir()

diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt
@@ -169,7 +169,6 @@ on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf -- $exit_status > {{folder}}/.exitcode
     set +u
-    {{sync_cmd}}
     exit $exit_status
 }