From 35aa301a3cf2e4f0a3bf9bb83bf1ff6e8bb421ac Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Wed, 13 Apr 2022 15:25:53 -0700 Subject: [PATCH 1/3] Add downloadTo to the Storage interface --- .../clirr-ignored-differences.xml | 10 ++ .../java/com/google/cloud/storage/Blob.java | 15 +-- .../com/google/cloud/storage/Storage.java | 18 +++ .../com/google/cloud/storage/StorageImpl.java | 30 +++++ .../com/google/cloud/storage/BlobTest.java | 98 ----------------- .../google/cloud/storage/StorageImplTest.java | 103 ++++++++++++++++++ .../cloud/storage/it/ITStorageTest.java | 9 +- 7 files changed, 165 insertions(+), 118 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 84fba3dc32..1a35ed0f81 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -1,4 +1,14 @@ + + com/google/cloud/storage/Storage* + 7012 + * downloadTo(com.google.cloud.storage.BlobId, java.io.OutputStream, com.google.cloud.storage.Blob$BlobSourceOption[]) + + + com/google/cloud/storage/Storage* + 7012 + * downloadTo(com.google.cloud.storage.BlobId, java.nio.file.Path, com.google.cloud.storage.Blob$BlobSourceOption[]) + diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index a198969953..865c5f71c1 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -245,20 +245,7 @@ public void downloadTo(Path path, BlobSourceOption... options) { * @param options */ public void downloadTo(OutputStream outputStream, BlobSourceOption... options) { - final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); - final StorageRpc storageRpc = this.options.getStorageRpcV1(); - StorageObject pb = getBlobId().toPb(); - final Map requestOptions = StorageImpl.optionMap(getBlobId(), options); - ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForObjectsGet(pb, requestOptions); - Retrying.run( - this.options, - algorithm, - callable( - () -> { - storageRpc.read( - pb, requestOptions, countingOutputStream.getCount(), countingOutputStream); - }), - Function.identity()); + storage.downloadTo(getBlobId(), outputStream, options); } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 634155f244..97fbb61cfc 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -31,6 +31,7 @@ import com.google.cloud.Tuple; import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; +import com.google.cloud.storage.Blob.BlobSourceOption; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.PostPolicyV4.PostConditionsV4; import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; @@ -41,6 +42,7 @@ import com.google.common.io.BaseEncoding; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.Serializable; import java.net.URL; import java.nio.file.Path; @@ -2672,6 +2674,22 @@ Blob createFrom( */ ReadChannel reader(BlobId blob, BlobSourceOption... options); + /** + * Downloads the given blob to the given path using specified blob read options. + * @param blob + * @param path + * @param options + */ + void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options); + + /** + * Downloads the given blob to the given output stream using specified blob read options. + * @param blob + * @param outputStream + * @param options + */ + void downloadTo(BlobId blob, OutputStream outputStream, Blob.BlobSourceOption... options); + /** * Creates a blob and returns a channel for writing its content. By default any MD5 and CRC32C * values in the given {@code blobInfo} are ignored unless requested via the {@code diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 874d997ebd..0e5b2e5828 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -32,6 +32,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.Executors.callable; import com.google.api.gax.paging.Page; import com.google.api.gax.retrying.ResultRetryAlgorithm; @@ -49,6 +50,7 @@ import com.google.cloud.Tuple; import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; +import com.google.cloud.storage.Blob.BlobSourceOption; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.PostPolicyV4.ConditionV4Type; import com.google.cloud.storage.PostPolicyV4.PostConditionsV4; @@ -67,10 +69,12 @@ import com.google.common.collect.Maps; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; +import com.google.common.io.CountingOutputStream; import com.google.common.primitives.Ints; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; @@ -543,6 +547,32 @@ public ReadChannel reader(BlobId blob, BlobSourceOption... options) { return new BlobReadChannel(getOptions(), blob, optionsMap); } + @Override + public void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options) { + try (OutputStream outputStream = Files.newOutputStream(path)) { + downloadTo(blob,outputStream, options); + } catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void downloadTo(BlobId blob, OutputStream outputStream, Blob.BlobSourceOption... options) { + final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); + final StorageObject pb = blob.toPb(); + final Map requestOptions = optionMap(blob, options); + ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForObjectsGet(pb, requestOptions); + Retrying.run( + getOptions(), + algorithm, + callable( + () -> { + storageRpc.read( + pb, requestOptions, countingOutputStream.getCount(), countingOutputStream); + }), + Function.identity()); + } + @Override public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) { Tuple targetOptions = BlobTargetOption.convert(blobInfo, options); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index 555349752c..4802e21b86 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -612,102 +612,4 @@ public void testBuilder() { assertNull(blob.getCustomTime()); assertTrue(blob.isDirectory()); } - - private StorageRpc prepareForDownload() { - StorageRpc mockStorageRpc = createNiceMock(StorageRpc.class); - expect(storage.getOptions()).andReturn(mockOptions).anyTimes(); - replay(storage); - expect(mockOptions.getStorageRpcV1()).andReturn(mockStorageRpc); - expect(mockOptions.getRetrySettings()).andReturn(RETRY_SETTINGS); - expect(mockOptions.getClock()).andReturn(API_CLOCK); - expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes(); - replay(mockOptions); - blob = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO)); - return mockStorageRpc; - } - - @Test - public void testDownloadTo() throws Exception { - final byte[] expected = {1, 2}; - StorageRpc mockStorageRpc = prepareForDownload(); - expect( - mockStorageRpc.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) - .andAnswer( - new IAnswer() { - @Override - public Long answer() throws Throwable { - ((OutputStream) getCurrentArguments()[3]).write(expected); - return 2l; - } - }); - replay(mockStorageRpc); - File file = File.createTempFile("blob", ".tmp"); - blob.downloadTo(file.toPath()); - byte actual[] = Files.readAllBytes(file.toPath()); - assertArrayEquals(expected, actual); - } - - @Test - public void testDownloadToWithRetries() throws Exception { - final byte[] expected = {1, 2}; - StorageRpc mockStorageRpc = prepareForDownload(); - expect( - mockStorageRpc.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) - .andAnswer( - new IAnswer() { - @Override - public Long answer() throws Throwable { - ((OutputStream) getCurrentArguments()[3]).write(expected[0]); - throw new StorageException(504, "error"); - } - }); - expect( - mockStorageRpc.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(1l), - anyObject(OutputStream.class))) - .andAnswer( - new IAnswer() { - @Override - public Long answer() throws Throwable { - ((OutputStream) getCurrentArguments()[3]).write(expected[1]); - return 1l; - } - }); - replay(mockStorageRpc); - File file = File.createTempFile("blob", ".tmp"); - blob.downloadTo(file.toPath()); - byte actual[] = Files.readAllBytes(file.toPath()); - assertArrayEquals(expected, actual); - } - - @Test - public void testDownloadToWithException() throws Exception { - StorageRpc mockStorageRpc = prepareForDownload(); - Exception exception = new IllegalStateException("test"); - expect( - mockStorageRpc.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) - .andThrow(exception); - replay(mockStorageRpc); - File file = File.createTempFile("blob", ".tmp"); - try { - blob.downloadTo(file.toPath()); - fail(); - } catch (StorageException e) { - assertSame(exception, e.getCause()); - } - } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 5619e5f7b9..1aefa13dbd 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -19,6 +19,12 @@ import static com.google.cloud.storage.SignedUrlEncodingHelper.Rfc3986UriEncode; import static com.google.cloud.storage.testing.ApiPolicyMatcher.eqApiPolicy; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.getCurrentArguments; +import static org.easymock.EasyMock.replay; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -27,10 +33,12 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.util.DateTime; import com.google.api.core.ApiClock; +import com.google.api.gax.retrying.RetrySettings; import com.google.api.services.storage.model.Policy.Bindings; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -53,9 +61,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; +import java.io.File; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; +import java.nio.file.Files; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyFactory; @@ -77,6 +88,7 @@ import javax.crypto.spec.SecretKeySpec; import org.easymock.Capture; import org.easymock.EasyMock; +import org.easymock.IAnswer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -2261,4 +2273,95 @@ public void testBucketLifecycleRules() { assertEquals(30, lifecycleRule.getCondition().getDaysSinceCustomTime().intValue()); assertNotNull(lifecycleRule.getCondition().getCustomTimeBefore()); } + + @Test + public void testDownloadTo() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = options.toBuilder().build().getService(); + final byte[] expected = {1, 2}; + EasyMock.expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected); + return 2l; + } + }); + EasyMock.replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + storage.downloadTo(blob, file.toPath()); + byte actual[] = Files.readAllBytes(file.toPath()); + assertArrayEquals(expected, actual); + } + + @Test + public void testDownloadToWithRetries() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = options.toBuilder() + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(2).build()) + .build() + .getService(); + final byte[] expected = {1, 2}; + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected[0]); + throw new StorageException(504, "error"); + } + }); + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(1l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected[1]); + return 1l; + } + }); + replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + storage.downloadTo(blob, file.toPath()); + byte actual[] = Files.readAllBytes(file.toPath()); + assertArrayEquals(expected, actual); + } + + @Test + public void testDownloadToWithException() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = options.toBuilder().build().getService(); + Exception exception = new IllegalStateException("test"); + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andThrow(exception); + replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + try { + storage.downloadTo(blob, file.toPath()); + fail(); + } catch (StorageException e) { + assertSame(exception, e.getCause()); + } + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index d755d38d2a..b81f40fa6d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -883,17 +883,14 @@ public void testGetBlobRawInput() throws IOException { storage.createFrom(blobInfo, gzippedFile.toPath()); Path rawInputGzippedFile = File.createTempFile("rawinputgzippedfile", ".txt").toPath(); - Blob blob = storage.get(blobId); - blob.downloadTo(rawInputGzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(true)); + storage.downloadTo(blobId, rawInputGzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(true)); assertArrayEquals( Files.readAllBytes(gzippedFile.toPath()), Files.readAllBytes(rawInputGzippedFile)); Path unzippedFile = File.createTempFile("unzippedfile", ".txt").toPath(); - storage - .get(blobId) - .downloadTo(unzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(false)); + storage.downloadTo(blobId, unzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(false)); assertArrayEquals("hello world".getBytes(), Files.readAllBytes(unzippedFile)); } @@ -3686,7 +3683,7 @@ public void testUploadFromDownloadTo() throws Exception { assertEquals(BLOB_BYTE_CONTENT.length, (long) blob.getSize()); Path tempFileTo = Files.createTempFile("ITStorageTest_", ".tmp"); - storage.get(blobId).downloadTo(tempFileTo); + storage.downloadTo(blobId, tempFileTo); byte[] readBytes = Files.readAllBytes(tempFileTo); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } From 54805e90fbd4a58e69fe1f13dd8699e2f98e332f Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Fri, 15 Apr 2022 13:40:16 -0700 Subject: [PATCH 2/3] Linter changes --- .../java/com/google/cloud/storage/Blob.java | 4 -- .../com/google/cloud/storage/Storage.java | 4 +- .../com/google/cloud/storage/StorageImpl.java | 2 +- .../com/google/cloud/storage/BlobTest.java | 10 ---- .../google/cloud/storage/StorageImplTest.java | 51 ++++++++++--------- .../cloud/storage/it/ITStorageTest.java | 6 ++- 6 files changed, 34 insertions(+), 43 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 865c5f71c1..6762d42f48 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -19,9 +19,7 @@ import static com.google.cloud.storage.Blob.BlobSourceOption.toGetOptions; import static com.google.cloud.storage.Blob.BlobSourceOption.toSourceOptions; import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.concurrent.Executors.callable; -import com.google.api.gax.retrying.ResultRetryAlgorithm; import com.google.api.services.storage.model.StorageObject; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; @@ -34,7 +32,6 @@ import com.google.cloud.storage.Storage.SignUrlOption; import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.io.BaseEncoding; -import com.google.common.io.CountingOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.OutputStream; @@ -47,7 +44,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.function.Function; /** * An object in Google Cloud Storage. A {@code Blob} object includes the {@code BlobId} instance, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 97fbb61cfc..b6705c74bb 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -2676,6 +2676,7 @@ Blob createFrom( /** * Downloads the given blob to the given path using specified blob read options. + * * @param blob * @param path * @param options @@ -2683,7 +2684,8 @@ Blob createFrom( void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options); /** - * Downloads the given blob to the given output stream using specified blob read options. + * Downloads the given blob to the given output stream using specified blob read options. + * * @param blob * @param outputStream * @param options diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 0e5b2e5828..21f89b91a2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -550,7 +550,7 @@ public ReadChannel reader(BlobId blob, BlobSourceOption... options) { @Override public void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options) { try (OutputStream outputStream = Files.newOutputStream(path)) { - downloadTo(blob,outputStream, options); + downloadTo(blob, outputStream, options); } catch (IOException e) { throw new StorageException(e); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java index 4802e21b86..97a9b7e231 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java @@ -16,14 +16,11 @@ package com.google.cloud.storage; -import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.createStrictMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.getCurrentArguments; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertArrayEquals; @@ -32,11 +29,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import com.google.api.core.ApiClock; import com.google.api.gax.retrying.RetrySettings; -import com.google.api.services.storage.model.StorageObject; import com.google.cloud.ReadChannel; import com.google.cloud.storage.Acl.Project; import com.google.cloud.storage.Acl.Project.ProjectRole; @@ -45,21 +40,16 @@ import com.google.cloud.storage.Blob.BlobSourceOption; import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.Storage.CopyRequest; -import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; -import java.io.File; -import java.io.OutputStream; import java.net.URL; -import java.nio.file.Files; import java.security.Key; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import javax.crypto.spec.SecretKeySpec; import org.easymock.Capture; -import org.easymock.IAnswer; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 1aefa13dbd..ae3682414b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -20,7 +20,6 @@ import static com.google.cloud.storage.testing.ApiPolicyMatcher.eqApiPolicy; import static java.nio.charset.StandardCharsets.UTF_8; import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.getCurrentArguments; @@ -2280,11 +2279,11 @@ public void testDownloadTo() throws Exception { storage = options.toBuilder().build().getService(); final byte[] expected = {1, 2}; EasyMock.expect( - storageRpcMock.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) .andAnswer( new IAnswer() { @Override @@ -2303,17 +2302,19 @@ public Long answer() throws Throwable { @Test public void testDownloadToWithRetries() throws Exception { BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); - storage = options.toBuilder() - .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(2).build()) - .build() - .getService(); + storage = + options + .toBuilder() + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(2).build()) + .build() + .getService(); final byte[] expected = {1, 2}; expect( - storageRpcMock.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) .andAnswer( new IAnswer() { @Override @@ -2323,11 +2324,11 @@ public Long answer() throws Throwable { } }); expect( - storageRpcMock.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(1l), - anyObject(OutputStream.class))) + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(1l), + anyObject(OutputStream.class))) .andAnswer( new IAnswer() { @Override @@ -2349,11 +2350,11 @@ public void testDownloadToWithException() throws Exception { storage = options.toBuilder().build().getService(); Exception exception = new IllegalStateException("test"); expect( - storageRpcMock.read( - anyObject(StorageObject.class), - anyObject(Map.class), - eq(0l), - anyObject(OutputStream.class))) + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) .andThrow(exception); replay(storageRpcMock); File file = File.createTempFile("blob", ".tmp"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index b81f40fa6d..bf55915f0b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -884,13 +884,15 @@ public void testGetBlobRawInput() throws IOException { Path rawInputGzippedFile = File.createTempFile("rawinputgzippedfile", ".txt").toPath(); - storage.downloadTo(blobId, rawInputGzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(true)); + storage.downloadTo( + blobId, rawInputGzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(true)); assertArrayEquals( Files.readAllBytes(gzippedFile.toPath()), Files.readAllBytes(rawInputGzippedFile)); Path unzippedFile = File.createTempFile("unzippedfile", ".txt").toPath(); - storage.downloadTo(blobId, unzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(false)); + storage.downloadTo( + blobId, unzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(false)); assertArrayEquals("hello world".getBytes(), Files.readAllBytes(unzippedFile)); } From 4936105422563443b76f8b51bf8ffb22b2fe5ed6 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Fri, 15 Apr 2022 15:47:10 -0700 Subject: [PATCH 3/3] PR comments --- .../clirr-ignored-differences.xml | 4 +-- .../java/com/google/cloud/storage/Blob.java | 9 ++----- .../com/google/cloud/storage/Storage.java | 25 +++++++++++++++++-- .../com/google/cloud/storage/StorageImpl.java | 4 +-- .../cloud/storage/it/ITStorageTest.java | 4 +-- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 1a35ed0f81..8aabcf10b3 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -4,11 +4,11 @@ com/google/cloud/storage/Storage* 7012 - * downloadTo(com.google.cloud.storage.BlobId, java.io.OutputStream, com.google.cloud.storage.Blob$BlobSourceOption[]) + * downloadTo(com.google.cloud.storage.BlobId, java.io.OutputStream, com.google.cloud.storage.Storage$BlobSourceOption[]) com/google/cloud/storage/Storage* 7012 - * downloadTo(com.google.cloud.storage.BlobId, java.nio.file.Path, com.google.cloud.storage.Blob$BlobSourceOption[]) + * downloadTo(com.google.cloud.storage.BlobId, java.nio.file.Path, com.google.cloud.storage.Storage$BlobSourceOption[]) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 6762d42f48..79167aec88 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -36,7 +36,6 @@ import java.io.ObjectInputStream; import java.io.OutputStream; import java.net.URL; -import java.nio.file.Files; import java.nio.file.Path; import java.security.Key; import java.util.Arrays; @@ -227,11 +226,7 @@ static Storage.BlobGetOption[] toGetOptions(BlobInfo blobInfo, BlobSourceOption. * @throws StorageException upon failure */ public void downloadTo(Path path, BlobSourceOption... options) { - try (OutputStream outputStream = Files.newOutputStream(path)) { - downloadTo(outputStream, options); - } catch (IOException e) { - throw new StorageException(e); - } + storage.downloadTo(getBlobId(), path, BlobSourceOption.toSourceOptions(this, options)); } /** @@ -241,7 +236,7 @@ public void downloadTo(Path path, BlobSourceOption... options) { * @param options */ public void downloadTo(OutputStream outputStream, BlobSourceOption... options) { - storage.downloadTo(getBlobId(), outputStream, options); + storage.downloadTo(getBlobId(), outputStream, BlobSourceOption.toSourceOptions(this, options)); } /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index b6705c74bb..d27de9016b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -2677,20 +2677,41 @@ Blob createFrom( /** * Downloads the given blob to the given path using specified blob read options. * + * 
{@code
+   * String bucketName = "my-unique-bucket";
+   * String blobName = "my-blob-name";
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * Path destination = Paths.get("my-blob-destination.txt");
+   * downloadTo(blobId, destination);
+   * // do stuff with destination
+   * }
+ * * @param blob * @param path * @param options + * @throws StorageException upon failure */ - void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options); + void downloadTo(BlobId blob, Path path, BlobSourceOption... options); /** * Downloads the given blob to the given output stream using specified blob read options. * + * 
{@code
+   * String bucketName = "my-unique-bucket";
+   * String blobName = "my-blob-name";
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * Path destination = Paths.get("my-blob-destination.txt");
+   * try (OutputStream outputStream = Files.newOutputStream(path)) {
+   *  downloadTo(blob, outputStream);
+   *  // do stuff with destination
+   * }
+   * }
+ * * @param blob * @param outputStream * @param options */ - void downloadTo(BlobId blob, OutputStream outputStream, Blob.BlobSourceOption... options); + void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options); /** * Creates a blob and returns a channel for writing its content. By default any MD5 and CRC32C diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 21f89b91a2..bf372c881a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -548,7 +548,7 @@ public ReadChannel reader(BlobId blob, BlobSourceOption... options) { } @Override - public void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options) { + public void downloadTo(BlobId blob, Path path, BlobSourceOption... options) { try (OutputStream outputStream = Files.newOutputStream(path)) { downloadTo(blob, outputStream, options); } catch (IOException e) { @@ -557,7 +557,7 @@ public void downloadTo(BlobId blob, Path path, Blob.BlobSourceOption... options) } @Override - public void downloadTo(BlobId blob, OutputStream outputStream, Blob.BlobSourceOption... options) { + public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options) { final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); final StorageObject pb = blob.toPb(); final Map requestOptions = optionMap(blob, options); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index bf55915f0b..d5aa1b6d45 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -885,14 +885,14 @@ public void testGetBlobRawInput() throws IOException { Path rawInputGzippedFile = File.createTempFile("rawinputgzippedfile", ".txt").toPath(); storage.downloadTo( - blobId, rawInputGzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(true)); + blobId, rawInputGzippedFile, Storage.BlobSourceOption.shouldReturnRawInputStream(true)); assertArrayEquals( Files.readAllBytes(gzippedFile.toPath()), Files.readAllBytes(rawInputGzippedFile)); Path unzippedFile = File.createTempFile("unzippedfile", ".txt").toPath(); storage.downloadTo( - blobId, unzippedFile, Blob.BlobSourceOption.shouldReturnRawInputStream(false)); + blobId, unzippedFile, Storage.BlobSourceOption.shouldReturnRawInputStream(false)); assertArrayEquals("hello world".getBytes(), Files.readAllBytes(unzippedFile)); }