Add requirements for container images to run with Nextflow

docs/container.rst

@@ -1,4 +1,4 @@
-.. _container-page:
+.)f _container-page:
 
 **********
 Containers
@@ -11,6 +11,11 @@ platform that supports a container runtime. Furthermore, the same pipeline can b
 executed with any of the supported container runtimes, depending on which runtimes are available
 in the target compute environment.
 
+.. note::
+  When creating your container image to use with Nextflow, be aware that Nextflow runs a background
+  process for each job in the target environment. Make sure ``bash`` and ``ps`` are installed in your
+  image, along with other tools required for collecting metrics (See the list :ref:`here <trace-required-packages>`).
+
 .. _container-charliecloud:
 
 Charliecloud

docs/tracing.rst

@@ -127,7 +127,6 @@ were used. These plots are very helpful to check that task resources are used ef
 
 Learn more about how resource usage is computed in the :ref:`Metrics documentation <metrics-page>`.
 
-
 Tasks
 -----
 
@@ -136,6 +135,8 @@ and many other metrics. You can see an example below:
 
 .. image:: images/report-tasks-min.png
 
+.. _trace-required-packages:
+
 .. note::
   Nextflow collects these metrics through a background process for each job in the target environment.
   Make sure the following tools are available in the environment where tasks are executed: ``awk``, ``date``, ``grep``, ``ps``, ``sed``, ``tail``, ``tee``.

modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy

@@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j
  *
  * @author Paolo Di Tommaso <[email protected]>
  * @author Patrick Hüther <[email protected]>
+ * @author Laurent Modolo <[email protected]>
  */
 @CompileStatic
 @Slf4j
@@ -46,6 +47,9 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
         if( params.containsKey('runOptions') )
             addRunOptions(params.runOptions.toString())
 
+        if( params.containsKey('readOnlyInputs') )
+            this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true'
+
         return this
     }
 
@@ -58,7 +62,9 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
     CharliecloudBuilder build(StringBuilder result) {
         assert image
 
-        result << 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env '
+        result << 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env '
+        if (!readOnlyInputs)
+            result << '-w '
 
         appendEnv(result)
 
@@ -78,9 +84,23 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
         return this
     }
 
+    protected String getRoot(String path) {
+        def rootPath = path.split("/")
+
+        if (rootPath.size() >= 1)
+            rootPath = "/${rootPath[1]}"
+        else
+            throw new IllegalArgumentException("Not a valid working directory value: ${path}")
+
+        return rootPath
+    }
+
     @Override
     protected String composeVolumePath(String path, boolean readOnly = false) {
-        return "-b ${escape(path)}"
+        def mountCmd = "-b ${escape(path)}"
+        if (readOnlyInputs)
+            mountCmd = "-b ${getRoot(escape(path))}"
+        return mountCmd
     }
 
     @Override

modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy

@@ -171,7 +171,8 @@ class CharliecloudCache {
             return localPath
         }
 
-        final file = new File("${localPath.parent.parent.parent}/.${localPath.name}.lock")
+        // final file = new File("${localPath.parent.parent.parent}/.${localPath.name}.lock")
+        final file = new File("${localPath.parent.parent.parent}/.ch-pulling.lock")
         final wait = "Another Nextflow instance is pulling the image $imageUrl with Charliecloud -- please wait until the download completes"
         final err =  "Unable to acquire exclusive lock after $pullTimeout on file: $file"
 

modules/nextflow/src/test/groovy/nextflow/container/CharliecloudBuilderTest.groovy

@@ -26,6 +26,7 @@ import spock.lang.Unroll
  *
  * @author Paolo Di Tommaso <[email protected]>
  * @author Patrick Hüther <[email protected]>
+ * @author Laurent Modolo <[email protected]>
  */
 class CharliecloudBuilderTest extends Specification {
 
@@ -84,6 +85,16 @@ class CharliecloudBuilderTest extends Specification {
         cmd = new CharliecloudBuilder('ubuntu').params(entry:'/bin/sh').build().getRunCommand('bwa --this --that file.fastq')
         then:
         cmd == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
+
+        when:
+        cmd = new CharliecloudBuilder('ubuntu').params(readOnlyInputs:true).build().getRunCommand('bwa --this --that file.fastq')
+        then:
+        cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
+
+        when:
+        cmd = new CharliecloudBuilder('ubuntu').params(readOnlyInputs:false).build().getRunCommand('bwa --this --that file.fastq')
+        then:
+        cmd == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
     }
 
     @Unroll