Fix missing SAS token fusion env for Azure

Signed-off-by: Paolo Di Tommaso <[email protected]>

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy

@@ -15,28 +15,32 @@
  */
 package nextflow.cloud.azure.batch
 
-import com.azure.storage.blob.BlobServiceClient
-import com.azure.storage.blob.models.UserDelegationKey
-import com.azure.storage.common.sas.AccountSasPermission
-import com.azure.storage.common.sas.AccountSasResourceType
-import com.azure.storage.common.sas.AccountSasService
-import com.azure.storage.common.sas.AccountSasSignatureValues
-
 import java.nio.file.Path
 import java.time.OffsetDateTime
 
 import com.azure.storage.blob.BlobContainerClient
+import com.azure.storage.blob.BlobServiceClient
+import com.azure.storage.blob.BlobServiceClientBuilder
+import com.azure.storage.blob.models.UserDelegationKey
 import com.azure.storage.blob.sas.BlobContainerSasPermission
 import com.azure.storage.blob.sas.BlobSasPermission
 import com.azure.storage.blob.sas.BlobServiceSasSignatureValues
+import com.azure.storage.common.StorageSharedKeyCredential
+import com.azure.storage.common.sas.AccountSasPermission
+import com.azure.storage.common.sas.AccountSasResourceType
+import com.azure.storage.common.sas.AccountSasService
+import com.azure.storage.common.sas.AccountSasSignatureValues
 import groovy.transform.CompileStatic
+import groovy.transform.Memoized
+import groovy.util.logging.Slf4j
 import nextflow.cloud.azure.nio.AzPath
 import nextflow.util.Duration
 /**
  * Azure helper functions
  *
  * @author Paolo Di Tommaso <[email protected]>
  */
+@Slf4j
 @CompileStatic
 class AzHelper {
 
@@ -126,7 +130,7 @@ class AzHelper {
         return delegationKey
     }
 
-    static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
+        static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
 
         final startTime = OffsetDateTime.now()
         final indicatedExpiryTime = startTime.plusHours(duration.toHours())
@@ -158,4 +162,40 @@ class AzHelper {
 
         return client.generateAccountSas(signature)
     }
+
+    static String generateAccountSas(String accountName, String accountKey, Duration duration) {
+        final client = getOrCreateBlobServiceWithKey(accountName, accountKey)
+        return generateAccountSas(client, duration)
+    }
+
+    @Memoized
+    static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) {
+        log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.."
+
+        final credential = new StorageSharedKeyCredential(accountName, accountKey);
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
+
+        return new BlobServiceClientBuilder()
+                .endpoint(endpoint)
+                .credential(credential)
+                .buildClient()
+    }
+
+    @Memoized
+    static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken) {
+        if( !sasToken )
+            throw new IllegalArgumentException("Missing Azure blob SAS token")
+        if( sasToken.length()<100 )
+            throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken")
+
+        log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.."
+
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
+
+        return new BlobServiceClientBuilder()
+                .endpoint(endpoint)
+                .sasToken(sasToken)
+                .buildClient()
+    }
+
 }

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy

@@ -16,11 +16,10 @@
 
 package nextflow.cloud.azure.config
 
-
 import groovy.transform.CompileStatic
+import nextflow.cloud.azure.batch.AzHelper
 import nextflow.cloud.azure.nio.AzFileSystemProvider
 import nextflow.util.Duration
-
 /**
  * Parse Azure settings from nextflow config file
  *
@@ -64,4 +63,10 @@ class AzStorageOpts {
         }
         return result
     }
+
+    synchronized String getOrCreateSasToken() {
+        if( !sasToken )
+            sasToken = AzHelper.generateAccountSas(accountName, accountKey, tokenDuration)
+        return sasToken
+    }
 }

plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy

@@ -22,7 +22,6 @@ import nextflow.cloud.azure.config.AzConfig
 import nextflow.fusion.FusionConfig
 import nextflow.fusion.FusionEnv
 import org.pf4j.Extension
-
 /**
  * Implement environment provider for Azure specific variables
  * 
@@ -41,11 +40,11 @@ class AzFusionEnv implements FusionEnv {
         final result = new LinkedHashMap(10)
         if( !cfg.accountName )
             throw new IllegalArgumentException("Missing Azure storage account name")
-        if( !cfg.sasToken )
+        if( !cfg.sasToken && !cfg.accountKey )
             throw new IllegalArgumentException("Missing Azure storage SAS token")
 
         result.AZURE_STORAGE_ACCOUNT = cfg.accountName
-        result.AZURE_STORAGE_SAS_TOKEN = cfg.sasToken
+        result.AZURE_STORAGE_SAS_TOKEN = cfg.getOrCreateSasToken()
         return result
     }
 }

plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy

@@ -15,8 +15,6 @@
  */
 package nextflow.cloud.azure.nio
 
-import com.azure.identity.ClientSecretCredentialBuilder
-
 import static java.nio.file.StandardCopyOption.*
 import static java.nio.file.StandardOpenOption.*
 
@@ -40,13 +38,14 @@ import java.nio.file.attribute.FileAttribute
 import java.nio.file.attribute.FileAttributeView
 import java.nio.file.spi.FileSystemProvider
 
+import com.azure.identity.ClientSecretCredentialBuilder
 import com.azure.storage.blob.BlobServiceClient
 import com.azure.storage.blob.BlobServiceClientBuilder
 import com.azure.storage.blob.models.BlobStorageException
-import com.azure.storage.common.StorageSharedKeyCredential
 import groovy.transform.CompileStatic
 import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
+import nextflow.cloud.azure.batch.AzHelper
 /**
  * Implements NIO File system provider for Azure Blob Storage
  *
@@ -111,34 +110,12 @@ class AzFileSystemProvider extends FileSystemProvider {
         return uri.authority.toLowerCase()
     }
 
-    @Memoized
     protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey) {
-        log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.."
-
-        final credential = new StorageSharedKeyCredential(accountName, accountKey);
-        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
-
-        return new BlobServiceClientBuilder()
-                .endpoint(endpoint)
-                .credential(credential)
-                .buildClient()
+        AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey)
     }
 
-    @Memoized
     protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken) {
-        if( !sasToken )
-            throw new IllegalArgumentException("Missing Azure blob SAS token")
-        if( sasToken.length()<100 )
-            throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken")
-
-        log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.."
-
-        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
-
-        return new BlobServiceClientBuilder()
-                .endpoint(endpoint)
-                .sasToken(sasToken)
-                .buildClient()
+        AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken)
     }
 
     @Memoized