Add fusion.exportStorageCredentials option [ci fast]

Signed-off-by: Paolo Di Tommaso <[email protected]>
nextflow-io · May 13, 2023 · acb6aed · acb6aed
1 parent c3d89d9
commit acb6aed
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 11 deletions.
diff --git a/docs/fusion.md b/docs/fusion.md
@@ -66,7 +66,7 @@ docker {
 
 fusion {
     enabled = true
-    exportAwsAccessKeys = true
+    exportStorageCredentials = true
 }
 
 wave {
@@ -172,6 +172,29 @@ aws.batch.volumes = '/path/to/ec2/nvme:/tmp'
 process.scratch = false
 ```
 
-## More examples
+## Advanced settings
+
+The following configuration options are available:
+
+`fusion.enabled`
+: Enable/disable the use of Fusion file system.
+
+`fusion.exportStorageCredentials`
+: When `true` the access credentials required by the underlying object storage are exported the pipeline jobs execution environment
+(requires version `23.05.0-edge` or later).
+
+`fusion.containerConfigUrl`
+: The URL from where the container layer provisioning the Fusion client is downloaded. 
+
+`fusion.logLevel`
+: The level of logging emitted by the Fusion client.
+
+`fusion.logOutput`
+: Where the logging output is written. 
+
+`tagsEnabled`
+: Enable/disable the tagging of files created in the underlying object storage via the Fusion client (default: `true`).
+
+`tagsPattern`
+: The pattern that determines how tags are applied to files created via the Fusion client (default: `[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)`)
 
-Check out the [Wave showcase repository](https://github.com/seqeralabs/wave-showcase) for more examples on how to use Fusion file system.
diff --git a/docs/wave.md b/docs/wave.md
@@ -197,6 +197,3 @@ The following configuration options are available:
 `wave.strategy`
 : The strategy to be used when resolving ambiguous Wave container requirements (default: `'container,dockerfile,conda,spack'`).
 
-### More examples
-
-See the [Wave showcase repository](https://github.com/seqeralabs/wave-showcase) for more Wave containers configuration examples.
diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy
@@ -34,15 +34,22 @@ class FusionConfig {
 
     final private Boolean enabled
     final private String containerConfigUrl
-    final private Boolean exportAwsAccessKeys
+    @Deprecated final private Boolean exportAwsAccessKeys
+    final private Boolean exportStorageCredentials
     final private String logOutput
     final private String logLevel
     final private boolean tagsEnabled
     final private String tagsPattern
 
     boolean enabled() { enabled }
 
-    boolean exportAwsAccessKeys() { exportAwsAccessKeys }
+    @Deprecated boolean exportAwsAccessKeys() { exportAwsAccessKeys }
+
+    boolean exportStorageCredentials() {
+        return exportStorageCredentials!=null
+            ? exportStorageCredentials
+            : exportAwsAccessKeys
+    }
 
     String logLevel() { logLevel }
 
@@ -59,6 +66,7 @@ class FusionConfig {
     FusionConfig(Map opts, Map<String,String> env=System.getenv()) {
         this.enabled = opts.enabled
         this.exportAwsAccessKeys = opts.exportAwsAccessKeys
+        this.exportStorageCredentials = opts.exportStorageCredentials
         this.containerConfigUrl = opts.containerConfigUrl?.toString() ?: env.get('FUSION_CONTAINER_CONFIG_URL')
         this.logLevel = opts.logLevel
         this.logOutput = opts.logOutput

diff --git a/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy b/modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy
@@ -61,7 +61,7 @@ class FusionConfigTest extends Specification {
     @Unroll
     def 'should get export aws key' () {
         expect:
-        new FusionConfig(OPTS).exportAwsAccessKeys() == EXPECTED
+        new FusionConfig(OPTS).exportStorageCredentials() == EXPECTED
 
         where:
         OPTS                            | EXPECTED

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/fusion/AwsFusionEnv.groovy
@@ -39,7 +39,7 @@ class AwsFusionEnv implements FusionEnv {
         final result = new HashMap<String,String>()
         final awsConfig = AwsConfig.config()
         final endpoint = awsConfig.s3Config.endpoint
-        final creds = config.exportAwsAccessKeys() ? awsCreds(awsConfig) : List.<String>of()
+        final creds = config.exportStorageCredentials() ? awsCreds(awsConfig) : List.<String>of()
         if( creds ) {
             result.AWS_ACCESS_KEY_ID = creds[0]
             result.AWS_SECRET_ACCESS_KEY = creds[1]

diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/fusion/AwsFusionEnvTest.groovy
@@ -52,7 +52,7 @@ class AwsFusionEnvTest extends Specification {
         env == [AWS_S3_ENDPOINT:'http://my-host.com']
 
         when:
-        config = Mock(FusionConfig) { exportAwsAccessKeys() >> true }
+        config = Mock(FusionConfig) { exportStorageCredentials() >> true }
         env = new AwsFusionEnv().getEnvironment('s3', config)
         then:
         env == [AWS_ACCESS_KEY_ID: 'x1',