Add support for rclone as stageOutMode option

This commit add support for rclone as stageOutMode used by Nextflow to control how output files are copied from the local scratch directory to the task work dir. The use of rclone is may provide more efficient copy of large data files over network file system. Signed-off-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Jan 14, 2023 · fde2ac2 · fde2ac2
1 parent f333954
commit fde2ac2
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 1 deletion.
diff --git a/docs/process.rst b/docs/process.rst
@@ -2285,6 +2285,7 @@ Value   Description
 copy    Output files are copied from the scratch directory to the work directory.
 move    Output files are moved from the scratch directory to the work directory.
 rsync   Output files are copied from the scratch directory to the work directory by using the ``rsync`` utility.
+rclone  Output files are copied from the scratch directory to the work directory by using the `rclone <https://rclone.org>`_ utility (note: it must be available in your cluster computing nodes, requires version ``23.01.0-edge`` or later).
 ======= ==================
 
 See also: `scratch`_.

diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy
@@ -259,7 +259,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy {
      * @return A shell copy or move command string
      */
 
-    static final List<String> VALID_STAGE_OUT_MODES = ['copy', 'move', 'rsync']
+    static final List<String> VALID_STAGE_OUT_MODES = ['copy', 'move', 'rsync', 'rclone']
 
     protected String stageOutCommand( String source, Path targetDir, String mode ) {
         def scheme = getPathScheme(targetDir)

diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt
@@ -91,6 +91,10 @@ nxf_fs_rsync() {
   rsync -rRl $1 $2
 }
 
+nxf_fs_rclone() {
+  rclone copyto $1 $2/$1
+}
+
 on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf $exit_status {{exit_file}}

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt
@@ -255,6 +255,10 @@ nxf_fs_rsync() {
   rsync -rRl $1 $2
 }
 
+nxf_fs_rclone() {
+  rclone copyto $1 $2/$1
+}
+
 on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf $exit_status > {{folder}}/.exitcode

diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt
@@ -70,6 +70,10 @@ nxf_fs_rsync() {
   rsync -rRl $1 $2
 }
 
+nxf_fs_rclone() {
+  rclone copyto $1 $2/$1
+}
+
 on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf $exit_status > {{folder}}/.exitcode

diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt
@@ -155,6 +155,10 @@ nxf_fs_rsync() {
   rsync -rRl $1 $2
 }
 
+nxf_fs_rclone() {
+  rclone copyto $1 $2/$1
+}
+
 on_exit() {
     exit_status=${nxf_main_ret:=$?}
     printf $exit_status > {{folder}}/.exitcode