Merge pull request #138 from rsandell/customendpoint

Add ability to specify custom endpoint url
jenkinsci · Oct 15, 2021 · 7bd1f08 · 7bd1f08
2 parents eaec900 + 622dc58
commit 7bd1f08
Show file tree

Hide file tree

Showing 8 changed files with 284 additions and 40 deletions.
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.jenkins-ci.plugins</groupId>
         <artifactId>plugin</artifactId>
-        <version>4.27</version>
+        <version>4.28</version>
     </parent>
 
     <artifactId>s3</artifactId>
@@ -14,7 +14,7 @@
 
     <properties>
         <java.level>8</java.level>
-        <jenkins.version>2.289.1</jenkins.version>
+        <jenkins.version>2.289.3</jenkins.version>
     </properties>
 
     <developers>
@@ -75,12 +75,11 @@
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>apache-httpcomponents-client-4-api</artifactId>
-            <version>4.5.13-1.0</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>copyartifact</artifactId>
-            <version>1.43</version>
+            <version>1.46</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.main</groupId>
@@ -96,27 +95,42 @@
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>structs</artifactId>
-            <version>1.20</version>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>1.16.0</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-compress</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
 
     <dependencyManagement>
         <dependencies>
-            <!-- fix upper bound-->
+            <dependency>
+                <groupId>io.jenkins.tools.bom</groupId>
+                <artifactId>bom-2.289.x</artifactId>
+                <version>950.v396cb834de1e</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
             <dependency>
                 <groupId>commons-net</groupId>
                 <artifactId>commons-net</artifactId>
                 <version>3.8.0</version>
             </dependency>
-            <dependency>
-                <groupId>org.jenkins-ci.plugins</groupId>
-                <artifactId>display-url-api</artifactId>
-                <version>2.3.5</version>
-            </dependency>
         </dependencies>
     </dependencyManagement>
 
-
     <repositories>
         <repository>
             <id>repo.jenkins-ci.org</id>

diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java
@@ -1,14 +1,18 @@
 package hudson.plugins.s3;
 
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.RegionUtils;
 import com.amazonaws.regions.Regions;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import hudson.ProxyConfiguration;
+import org.apache.commons.lang.StringUtils;
 
+import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
 import javax.annotation.Nonnull;
@@ -18,23 +22,32 @@ public class ClientHelper {
     public final static String DEFAULT_AMAZON_S3_REGION_NAME = System.getProperty(
             "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION",
             com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName());
+    public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("PLUGIN_S3_ENDPOINT"));
 
-    public static AmazonS3Client createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy)
+    public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) {
+        return createClient(accessKey, secretKey, useRole, region, proxy, ENDPOINT);
+    }
+
+    public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy, String customEndpoint)
     {
         Region awsRegion = getRegionFromString(region);
 
         ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion);
 
-        final AmazonS3Client client;
-        if (useRole) {
-            client = new AmazonS3Client(clientConfiguration);
-        } else {
-            client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration);
+        AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfiguration);
+
+        if (!useRole) {
+            builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
         }
 
-        client.setRegion(awsRegion);
+        if (StringUtils.isNotEmpty(customEndpoint)) {
+            builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customEndpoint, awsRegion.getName()))
+                    .withPathStyleAccessEnabled(true);
+        } else {
+            builder = builder.withRegion(awsRegion.getName());
+        }
 
-        return client;
+        return builder.build();
     }
 
     /**
@@ -73,9 +86,13 @@ private static Region getRegionFromString(@Nullable String regionName) {
     @Nonnull
     public static ClientConfiguration getClientConfiguration(@Nonnull ProxyConfiguration proxy, @Nonnull Region region) {
         final ClientConfiguration clientConfiguration = new ClientConfiguration();
-
-        String s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX);
-
+        String s3Endpoint;
+        if (StringUtils.isNotEmpty(ENDPOINT)) {
+            s3Endpoint = ENDPOINT;
+        } else {
+            s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX);
+        }
+        Logger.getLogger(ClientHelper.class.getName()).fine(() -> String.format("ENDPOINT: %s", s3Endpoint));
         if (shouldUseProxy(proxy, s3Endpoint)) {
             clientConfiguration.setProxyHost(proxy.name);
             clientConfiguration.setProxyPort(proxy.port);

diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java
@@ -9,7 +9,7 @@
 
 import javax.servlet.ServletException;
 
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
 import com.amazonaws.services.s3.model.ResponseHeaderOverrides;
 import hudson.Functions;
@@ -25,7 +25,7 @@
 
 @ExportedBean
 public class S3ArtifactsAction implements RunAction2 {
-    private final Run build; // Compatibility for old versions
+    private final Run<?,?> build; // Compatibility for old versions
     private final String profile;
     private final List<FingerprintRecord> artifacts;
 
@@ -89,7 +89,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo
         for (FingerprintRecord record : artifacts) {
             if (record.getArtifact().getName().equals(artifact)) {
                 final S3Profile s3 = S3BucketPublisher.getProfile(profile);
-                final AmazonS3Client client = s3.getClient(record.getArtifact().getRegion());
+                final AmazonS3 client = s3.getClient(record.getArtifact().getRegion());
                 final String url = getDownloadURL(client, s3.getSignedUrlExpirySeconds(), build, record);
                 response.sendRedirect2(url);
                 return;
@@ -106,7 +106,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo
      * download and there's no need for the user to have credentials to
      * access S3.
      */
-    private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) {
+    private String getDownloadURL(AmazonS3 client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) {
         final Destination dest = Destination.newFromRun(run, record.getArtifact());
         final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName);
         request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000));

diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java
@@ -2,10 +2,12 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.regions.Region;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import hudson.AbortException;
 import hudson.Extension;
@@ -201,7 +203,7 @@ public static S3Profile getProfile(String profileName) {
         throw new IllegalArgumentException("Can't find profile: " + profileName);
     }
 
-    @Override
+    @Override @NonNull
     public Collection<? extends Action> getProjectActions(AbstractProject<?, ?> project) {
         return ImmutableList.of(new S3ArtifactsProjectAction(project));
     }
@@ -550,7 +552,7 @@ public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter
             }
 
             final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME;
-            final AmazonS3Client client = ClientHelper.createClient(
+            final AmazonS3 client = ClientHelper.createClient(
                     checkedAccessKey, checkedSecretKey, useRole, defaultRegion, Jenkins.get().proxy);
 
             try {

diff --git a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java
@@ -250,7 +250,7 @@ public void perform(@Nonnull Run<?, ?> dst, @Nonnull FilePath targetDir, @Nonnul
         }
     }
 
-    private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console)
+    private boolean perform(Run<?,?> src, Run<?,?> dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console)
             throws IOException, InterruptedException {
 
         final S3ArtifactsAction action = src.getAction(S3ArtifactsAction.class);
@@ -271,15 +271,15 @@ private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excl
 
         final Map<String, String> fingerprints = Maps.newHashMap();
         for(FingerprintRecord record : records) {
-            final FingerprintMap map = Jenkins.getInstance().getFingerprintMap();
+            final FingerprintMap map = Jenkins.get().getFingerprintMap();
 
             final Fingerprint f = map.getOrCreate(src, record.getName(), record.getFingerprint());
             f.addFor(src);
             f.addFor(dst);
             fingerprints.put(record.getName(), record.getFingerprint());
         }
 
-        for (Run r : new Run[]{src, dst}) {
+        for (Run<?,?> r : new Run<?,?>[]{src, dst}) {
             if (r == null) {
                 continue;
             }
@@ -288,7 +288,7 @@ private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excl
             if (fa != null) {
                 fa.add(fingerprints);
             } else  {
-                r.getActions().add(new FingerprintAction(r, fingerprints));
+                r.addAction(new FingerprintAction(r, fingerprints));
             }
         }
 

diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java
@@ -1,5 +1,6 @@
 package hudson.plugins.s3;
 
+import com.amazonaws.services.s3.AmazonS3;
 import hudson.FilePath;
 
 import java.io.IOException;
@@ -116,7 +117,7 @@ public int getSignedUrlExpirySeconds() {
         return signedUrlExpirySeconds;
     }
 
-    public AmazonS3Client getClient(String region) {
+    public AmazonS3 getClient(String region) {
         return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, getProxy());
     }
 
@@ -202,7 +203,7 @@ private <T> T invoke(boolean uploadFromSlave, FilePath filePath, MasterSlaveCall
     }
 
     public List<String> list(Run build, String bucket) {
-        final AmazonS3Client s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME);
+        final AmazonS3 s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME);
 
         final String buildName = build.getDisplayName();
         final int buildID = build.getNumber();
@@ -230,7 +231,7 @@ public List<String> list(Run build, String bucket) {
       /**
        * Download all artifacts from a given build
        */
-      public List<FingerprintRecord> downloadAll(Run build,
+      public List<FingerprintRecord> downloadAll(Run<?,?> build,
                                                  final List<FingerprintRecord> artifacts,
                                                  final String includeFilter,
                                                  final String excludeFilter,
@@ -286,7 +287,7 @@ private FilePath getFilePath(FilePath targetDir, boolean flatten, String fullNam
       public void delete(Run run, FingerprintRecord record) {
           final Destination dest = Destination.newFromRun(run, record.getArtifact());
           final DeleteObjectRequest req = new DeleteObjectRequest(dest.bucketName, dest.objectName);
-          final AmazonS3Client client = getClient(record.getArtifact().getRegion());
+          final AmazonS3 client = getClient(record.getArtifact().getRegion());
           client.deleteObject(req);
       }
 

diff --git a/src/main/java/hudson/plugins/s3/callable/S3Callable.java b/src/main/java/hudson/plugins/s3/callable/S3Callable.java
@@ -2,12 +2,15 @@
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
 import hudson.FilePath.FileCallable;
 import hudson.ProxyConfiguration;
 import hudson.plugins.s3.ClientHelper;
 import hudson.util.Secret;
+import org.apache.commons.lang.StringUtils;
 import org.jenkinsci.remoting.RoleChecker;
 
+import java.io.ObjectStreamException;
 import java.util.HashMap;
 
 abstract class S3Callable<T> implements FileCallable<T> {
@@ -18,6 +21,7 @@ abstract class S3Callable<T> implements FileCallable<T> {
     private final boolean useRole;
     private final String region;
     private final ProxyConfiguration proxy;
+    private final String customEndpoint;
 
     private static transient HashMap<String, TransferManager> transferManagers = new HashMap<>();
 
@@ -27,13 +31,14 @@ abstract class S3Callable<T> implements FileCallable<T> {
         this.useRole = useRole;
         this.region = region;
         this.proxy = proxy;
+        this.customEndpoint = ClientHelper.ENDPOINT;
     }
 
     protected synchronized TransferManager getTransferManager() {
         final String uniqueKey = getUniqueKey();
         if (transferManagers.get(uniqueKey) == null) {
-            final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy);
-            transferManagers.put(uniqueKey, new TransferManager(client));
+            final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy, customEndpoint);
+            transferManagers.put(uniqueKey, TransferManagerBuilder.standard().withS3Client(client).build());
         }
 
         return transferManagers.get(uniqueKey);