Implement copy_if_not_exist for AmazonS3 using DynamoDB (apache#4880

) (apache#4918) * Implement DynamoDBLock (apache#4880) * Cleanup error handling * Clippy * Localstack support * Clippy * Handle integration test concurrency * More docs * Disable request timeout * Fix merge conflicts * Reduce test concurrency * Increase timeouts
doki23 · Dec 26, 2023 · 844b851 · 844b851
1 parent add8f56
commit 844b851
Show file tree

Hide file tree

Showing 6 changed files with 702 additions and 146 deletions.
diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml
@@ -112,6 +112,7 @@ jobs:
       AWS_SECRET_ACCESS_KEY: test
       AWS_ENDPOINT: http://localhost:4566
       AWS_ALLOW_HTTP: true
+      AWS_COPY_IF_NOT_EXISTS: dynamo:test-table:2000
       HTTP_URL: "http://localhost:8080"
       GOOGLE_BUCKET: test-bucket
       GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
@@ -136,6 +137,7 @@ jobs:
           docker run -d -p 4566:4566 localstack/localstack:3.0.1
           docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2
           aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
+          aws --endpoint-url=http://localhost:4566 dynamodb create-table --table-name test-table --key-schema AttributeName=key,KeyType=HASH --attribute-definitions AttributeName=key,AttributeType=S --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5
 
       - name: Configure Azurite (Azure emulation)
         # the magical connection string is from

diff --git a/object_store/src/aws/builder.rs b/object_store/src/aws/builder.rs
@@ -821,27 +821,23 @@ impl AmazonS3Builder {
             )) as _
         };
 
-        let endpoint: String;
-        let bucket_endpoint: String;
-
         // If `endpoint` is provided then its assumed to be consistent with
         // `virtual_hosted_style_request`. i.e. if `virtual_hosted_style_request` is true then
         // `endpoint` should have bucket name included.
-        if self.virtual_hosted_style_request.get()? {
-            endpoint = self
-                .endpoint
-                .unwrap_or_else(|| format!("https://{bucket}.s3.{region}.amazonaws.com"));
-            bucket_endpoint = endpoint.clone();
+        let bucket_endpoint = if self.virtual_hosted_style_request.get()? {
+            self.endpoint
+                .clone()
+                .unwrap_or_else(|| format!("https://{bucket}.s3.{region}.amazonaws.com"))
         } else {
-            endpoint = self
-                .endpoint
-                .unwrap_or_else(|| format!("https://s3.{region}.amazonaws.com"));
-            bucket_endpoint = format!("{endpoint}/{bucket}");
-        }
+            match &self.endpoint {
+                None => format!("https://s3.{region}.amazonaws.com/{bucket}"),
+                Some(endpoint) => format!("{endpoint}/{bucket}"),
+            }
+        };
 
         let config = S3Config {
             region,
-            endpoint,
+            endpoint: self.endpoint,
             bucket,
             bucket_endpoint,
             credentials,