From 430150299a57963b140fe2675ce699c4752489db Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 9 May 2023 12:52:50 +0200 Subject: [PATCH] Fix missing SAS token fusion env for Azure Signed-off-by: Paolo Di Tommaso --- .../cloud/azure/batch/AzHelper.groovy | 56 ++++++++++++++++--- .../cloud/azure/config/AzStorageOpts.groovy | 9 ++- .../cloud/azure/fusion/AzFusionEnv.groovy | 5 +- .../azure/nio/AzFileSystemProvider.groovy | 31 ++-------- 4 files changed, 61 insertions(+), 40 deletions(-) diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy index 419cb5d5f3..c1360be582 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy @@ -15,21 +15,24 @@ */ package nextflow.cloud.azure.batch -import com.azure.storage.blob.BlobServiceClient -import com.azure.storage.blob.models.UserDelegationKey -import com.azure.storage.common.sas.AccountSasPermission -import com.azure.storage.common.sas.AccountSasResourceType -import com.azure.storage.common.sas.AccountSasService -import com.azure.storage.common.sas.AccountSasSignatureValues - import java.nio.file.Path import java.time.OffsetDateTime import com.azure.storage.blob.BlobContainerClient +import com.azure.storage.blob.BlobServiceClient +import com.azure.storage.blob.BlobServiceClientBuilder +import com.azure.storage.blob.models.UserDelegationKey import com.azure.storage.blob.sas.BlobContainerSasPermission import com.azure.storage.blob.sas.BlobSasPermission import com.azure.storage.blob.sas.BlobServiceSasSignatureValues +import com.azure.storage.common.StorageSharedKeyCredential +import com.azure.storage.common.sas.AccountSasPermission +import com.azure.storage.common.sas.AccountSasResourceType +import com.azure.storage.common.sas.AccountSasService +import com.azure.storage.common.sas.AccountSasSignatureValues import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.util.logging.Slf4j import nextflow.cloud.azure.nio.AzPath import nextflow.util.Duration /** @@ -37,6 +40,7 @@ import nextflow.util.Duration * * @author Paolo Di Tommaso */ +@Slf4j @CompileStatic class AzHelper { @@ -126,7 +130,7 @@ class AzHelper { return delegationKey } - static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) { + static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) { final startTime = OffsetDateTime.now() final indicatedExpiryTime = startTime.plusHours(duration.toHours()) @@ -158,4 +162,40 @@ class AzHelper { return client.generateAccountSas(signature) } + + static String generateAccountSas(String accountName, String accountKey, Duration duration) { + final client = getOrCreateBlobServiceWithKey(accountName, accountKey) + return generateAccountSas(client, duration) + } + + @Memoized + static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) { + log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.." + + final credential = new StorageSharedKeyCredential(accountName, accountKey); + final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName); + + return new BlobServiceClientBuilder() + .endpoint(endpoint) + .credential(credential) + .buildClient() + } + + @Memoized + static synchronized BlobServiceClient getOrCreateBlobServiceWithToken(String accountName, String sasToken) { + if( !sasToken ) + throw new IllegalArgumentException("Missing Azure blob SAS token") + if( sasToken.length()<100 ) + throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken") + + log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.." + + final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName); + + return new BlobServiceClientBuilder() + .endpoint(endpoint) + .sasToken(sasToken) + .buildClient() + } + } diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy index 83fd4382c3..eeff4bbe3c 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzStorageOpts.groovy @@ -16,11 +16,10 @@ package nextflow.cloud.azure.config - import groovy.transform.CompileStatic +import nextflow.cloud.azure.batch.AzHelper import nextflow.cloud.azure.nio.AzFileSystemProvider import nextflow.util.Duration - /** * Parse Azure settings from nextflow config file * @@ -64,4 +63,10 @@ class AzStorageOpts { } return result } + + synchronized String getOrCreateSasToken() { + if( !sasToken ) + sasToken = AzHelper.generateAccountSas(accountName, accountKey, tokenDuration) + return sasToken + } } diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy index 0a9a2ae17b..111ba2b277 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/fusion/AzFusionEnv.groovy @@ -22,7 +22,6 @@ import nextflow.cloud.azure.config.AzConfig import nextflow.fusion.FusionConfig import nextflow.fusion.FusionEnv import org.pf4j.Extension - /** * Implement environment provider for Azure specific variables * @@ -41,11 +40,11 @@ class AzFusionEnv implements FusionEnv { final result = new LinkedHashMap(10) if( !cfg.accountName ) throw new IllegalArgumentException("Missing Azure storage account name") - if( !cfg.sasToken ) + if( !cfg.sasToken && !cfg.accountKey ) throw new IllegalArgumentException("Missing Azure storage SAS token") result.AZURE_STORAGE_ACCOUNT = cfg.accountName - result.AZURE_STORAGE_SAS_TOKEN = cfg.sasToken + result.AZURE_STORAGE_SAS_TOKEN = cfg.getOrCreateSasToken() return result } } diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy index 0daca93556..53a5e9ed19 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy @@ -15,8 +15,6 @@ */ package nextflow.cloud.azure.nio -import com.azure.identity.ClientSecretCredentialBuilder - import static java.nio.file.StandardCopyOption.* import static java.nio.file.StandardOpenOption.* @@ -40,13 +38,14 @@ import java.nio.file.attribute.FileAttribute import java.nio.file.attribute.FileAttributeView import java.nio.file.spi.FileSystemProvider +import com.azure.identity.ClientSecretCredentialBuilder import com.azure.storage.blob.BlobServiceClient import com.azure.storage.blob.BlobServiceClientBuilder import com.azure.storage.blob.models.BlobStorageException -import com.azure.storage.common.StorageSharedKeyCredential import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.util.logging.Slf4j +import nextflow.cloud.azure.batch.AzHelper /** * Implements NIO File system provider for Azure Blob Storage * @@ -111,34 +110,12 @@ class AzFileSystemProvider extends FileSystemProvider { return uri.authority.toLowerCase() } - @Memoized protected BlobServiceClient createBlobServiceWithKey(String accountName, String accountKey) { - log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.." - - final credential = new StorageSharedKeyCredential(accountName, accountKey); - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName); - - return new BlobServiceClientBuilder() - .endpoint(endpoint) - .credential(credential) - .buildClient() + AzHelper.getOrCreateBlobServiceWithKey(accountName, accountKey) } - @Memoized protected BlobServiceClient createBlobServiceWithToken(String accountName, String sasToken) { - if( !sasToken ) - throw new IllegalArgumentException("Missing Azure blob SAS token") - if( sasToken.length()<100 ) - throw new IllegalArgumentException("Invalid Azure blob SAS token -- offending value: $sasToken") - - log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.." - - final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName); - - return new BlobServiceClientBuilder() - .endpoint(endpoint) - .sasToken(sasToken) - .buildClient() + AzHelper.getOrCreateBlobServiceWithToken(accountName, sasToken) } @Memoized