Add support for AWS S3 Glacier Retrieval Tier (#3680)

Signed-off-by: Ben Sherman <[email protected]>

docs/config.rst

@@ -123,6 +123,7 @@ connectionTimeout           The amount of time to wait (in milliseconds) when in
 endpoint                    The AWS S3 API entry point e.g. `s3-us-west-1.amazonaws.com`.
 glacierAutoRetrieval        Enable auto retrieval of S3 objects stored with Glacier class store (EXPERIMENTAL. default: ``false``, requires version ``22.12.0-edge`` or later).
 glacierExpirationDays       The time, in days, between when an object is restored to the bucket and when it expires (EXPERIMENTAL. default: ``7``, requires version ``22.12.0-edge`` or later).
+glacierRetrievalTier        The retrieval tier to use when restoring objects from Glacier, one of [``Expedited``, ``Standard``, ``Bulk``] (EXPERIMENTAL. requires version ``23.03.0-edge`` or later).
 maxConnections              The maximum number of allowed open HTTP connections.
 maxErrorRetry               The maximum number of retry attempts for failed retryable requests.
 protocol                    The protocol (i.e. HTTP or HTTPS) to use when connecting to AWS.

plugins/nf-amazon/src/main/com/upplication/s3fs/AmazonS3Client.java

@@ -76,6 +76,7 @@
 import com.amazonaws.services.s3.model.CopyPartRequest;
 import com.amazonaws.services.s3.model.CopyPartResult;
 import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
+import com.amazonaws.services.s3.model.GlacierJobParameters;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
@@ -139,6 +140,8 @@ public class AmazonS3Client {
 
 	private int glacierExpirationDays = 7;
 
+	private String glacierRetrievalTier;
+
 	public AmazonS3Client(AmazonS3 client) {
 		this.client = client;
 	}
@@ -367,6 +370,11 @@ public void setGlacierExpirationDays(String days) {
 		}
 	}
 
+	public void setGlacierRetrievalTier(String tier) {
+		this.glacierRetrievalTier = tier;
+		log.debug("Setting S3 glacierRetrievalTier={}", glacierRetrievalTier);
+	}
+
 	public AmazonS3 getClient() {
 		return client;
 	}
@@ -596,7 +604,19 @@ protected void restoreFromGlacier(String bucketName, String key) {
 		final long _5_mins = 5 * 60 * 1_000;
 
 		try {
-			client.restoreObjectV2(new RestoreObjectRequest(bucketName, key, glacierExpirationDays));
+			RestoreObjectRequest request = new RestoreObjectRequest(bucketName, key);
+
+			String storageClass = client.getObjectMetadata(bucketName, key).getStorageClass();
+			if( storageClass != "INTELLIGENT_TIERING" )
+				request.setExpirationInDays(glacierExpirationDays);
+
+			if( glacierRetrievalTier != null )
+				request.setGlacierJobParameters(
+					new GlacierJobParameters()
+						.withTier(glacierRetrievalTier)
+				);
+
+			client.restoreObjectV2(request);
 		}
 		catch (AmazonS3Exception e) {
 			if( e.getMessage().contains("RestoreAlreadyInProgress") ) {

plugins/nf-amazon/src/main/com/upplication/s3fs/S3FileSystemProvider.java

@@ -951,6 +951,7 @@ else if (accessKey == null && secretKey == null) {
 		client.setUploadMaxThreads(props.getProperty("upload_max_threads"));
 		client.setGlacierAutoRetrieval(props.getProperty("glacier_auto_retrieval"));
 		client.setGlacierExpirationDays(props.getProperty("glacier_expiration_days"));
+		client.setGlacierRetrievalTier(props.getProperty("glacier_retrieval_tier"));
 
 		if (uri.getHost() != null) {
 			client.setEndpoint(uri.getHost());