Fix support for S3 custom endpoint with anonymous access

Signed-off-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Sep 18, 2023 · 0375281 · 0375281
1 parent 06843d8
commit 0375281
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 18 deletions.
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/AwsClientFactory.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/AwsClientFactory.groovy
@@ -21,6 +21,7 @@ import com.amazonaws.ClientConfiguration
 import com.amazonaws.auth.AWSCredentialsProvider
 import com.amazonaws.auth.AWSCredentialsProviderChain
 import com.amazonaws.auth.AWSStaticCredentialsProvider
+import com.amazonaws.auth.AnonymousAWSCredentials
 import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper
 import com.amazonaws.auth.EnvironmentVariableCredentialsProvider
@@ -256,9 +257,10 @@ class AwsClientFactory {
         else
             builder.withRegion(region)
 
-        final credentials = new S3CredentialsProvider(getCredentialsProvider0())
-        if( credentials )
-            builder.withCredentials(credentials)
+        final credentials = config.s3Config.anonymous
+                ? new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())
+                : new S3CredentialsProvider(getCredentialsProvider0())
+        builder.withCredentials(credentials)
 
         if( clientConfig )
             builder.withClientConfiguration(clientConfig)

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy
@@ -46,13 +46,16 @@ class AwsS3Config {
 
     private Boolean pathStyleAccess
 
+    private Boolean anonymous
+
     AwsS3Config(Map opts) {
         this.debug = opts.debug as Boolean
         this.endpoint = opts.endpoint ?: SysEnv.get('AWS_S3_ENDPOINT')
         this.storageClass = parseStorageClass((opts.storageClass ?: opts.uploadStorageClass) as String)     // 'uploadStorageClass' is kept for legacy purposes
         this.storageEncryption = parseStorageEncryption(opts.storageEncryption as String)
         this.storageKmsKeyId = opts.storageKmsKeyId
         this.pathStyleAccess = opts.s3PathStyleAccess as Boolean
+        this.anonymous = opts.anonymous as Boolean
         this.s3Acl = parseS3Acl(opts.s3Acl as String)
     }
 
@@ -104,4 +107,8 @@ class AwsS3Config {
     Boolean getPathStyleAccess() {
         return pathStyleAccess
     }
+
+    Boolean getAnonymous() {
+        return anonymous
+    }
 }
diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
@@ -62,9 +62,7 @@
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
-import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.AccessControlList;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
@@ -79,11 +77,11 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+import nextflow.cloud.aws.AwsClientFactory;
+import nextflow.cloud.aws.config.AwsConfig;
 import nextflow.cloud.aws.nio.util.IOUtils;
 import nextflow.cloud.aws.nio.util.S3MultipartOptions;
 import nextflow.cloud.aws.nio.util.S3ObjectSummaryLookup;
-import nextflow.cloud.aws.AwsClientFactory;
-import nextflow.cloud.aws.config.AwsConfig;
 import nextflow.extension.FilesEx;
 import nextflow.file.CopyOptions;
 import nextflow.file.FileHelper;
@@ -833,16 +831,9 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) {
 		ClientConfiguration clientConfig = createClientConfig(props);
 
 		final String bucketName = S3Path.bucketName(uri);
-		final boolean anonymous = "true".equals(props.getProperty("anonymous"));
-		if( anonymous ) {
-			log.debug("Creating AWS S3 client with anonymous credentials");
-			client = new S3Client(new AmazonS3Client(new AnonymousAWSCredentials(), clientConfig));
-		}
-		else {
-			final boolean global = bucketName!=null;
-			final AwsClientFactory factory = new AwsClientFactory(awsConfig, Regions.US_EAST_1.getName());
-			client = new S3Client(factory.getS3Client(clientConfig, global));
-		}
+		final boolean global = bucketName!=null;
+		final AwsClientFactory factory = new AwsClientFactory(awsConfig, Regions.US_EAST_1.getName());
+		client = new S3Client(factory.getS3Client(clientConfig, global));
 
 		// set the client acl
 		client.setCannedAcl(getProp(props, "s_3_acl", "s3_acl", "s3Acl"));

diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy
@@ -38,6 +38,7 @@ class AwsS3ConfigTest extends Specification {
         !client.debug
         !client.s3Acl
         !client.pathStyleAccess
+        !client.anonymous
     }
 
     def 'should set config' () {
@@ -48,7 +49,8 @@ class AwsS3ConfigTest extends Specification {
                 storageKmsKeyId: 'key-1',
                 storageEncryption: 'AES256',
                 s3Acl: 'public-read',
-                s3PathStyleAccess: true
+                s3PathStyleAccess: true,
+                anonymous: true
         ]
 
         when:
@@ -60,6 +62,7 @@ class AwsS3ConfigTest extends Specification {
         client.storageEncryption == 'AES256'
         client.s3Acl == CannedAccessControlList.PublicRead
         client.pathStyleAccess
+        client.anonymous
     }
 
     def 'should use legacy upload storage class' () {