From fbe37f04f6b409ac6d9b4cc23400075ef7338337 Mon Sep 17 00:00:00 2001 From: noak2 <107106049+noak2@users.noreply.github.com> Date: Tue, 28 Jun 2022 15:02:21 +0300 Subject: [PATCH 1/7] Expose row level serialization failures for JsonStreamWriter append --- .../com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 3186c282f4..a683581101 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -105,7 +105,7 @@ public ApiFuture append(JSONArray jsonArr) throws IOException, DescriptorValidationException { return append(jsonArr, -1); } - +//TODO: do /** * Writes a JSONArray that contains JSONObjects to the BigQuery table by first converting the JSON * data to protobuf messages, then using StreamWriter's append() to write the data at the From 86b2b4a6d4bb4818e59e46d96d7198bdeb8880ef Mon Sep 17 00:00:00 2001 From: Noa Katz Date: Tue, 28 Jun 2022 15:14:36 +0300 Subject: [PATCH 2/7] Expose row level serialization failures for JsonStreamWriter append --- .../cloud/bigquery/storage/v1/Exceptions.java | 15 ++++++ .../bigquery/storage/v1/JsonStreamWriter.java | 34 +++++++++---- .../storage/v1/JsonStreamWriterTest.java | 48 ++++++++++++++++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java index 5b02271a58..e827b4ac51 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java @@ -22,6 +22,7 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.protobuf.StatusProto; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -203,5 +204,19 @@ public static StorageException toStorageException(Throwable exception) { return toStorageException(rpcStatus, exception); } + /** Append serializtion error. */ + public static class AppendSerializtionError extends RuntimeException { + private final List rowErrors; + + public AppendSerializtionError(String name, List rowErrors) { + super(String.format("Append serializtion failed for writer: %s", name)); + this.rowErrors = rowErrors; + } + + public List getRowErrors() { + return rowErrors; + } + } + private Exceptions() {} } diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index a683581101..403a6f1b64 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -19,12 +19,16 @@ import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializtionError; +import com.google.cloud.bigquery.storage.v1.RowError.RowErrorCode; import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Message; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -65,7 +69,7 @@ public class JsonStreamWriter implements AutoCloseable { */ private JsonStreamWriter(Builder builder) throws Descriptors.DescriptorValidationException, IllegalArgumentException, IOException, - InterruptedException { + InterruptedException { this.client = builder.client; this.descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(builder.tableSchema); @@ -105,7 +109,7 @@ public ApiFuture append(JSONArray jsonArr) throws IOException, DescriptorValidationException { return append(jsonArr, -1); } -//TODO: do + /** * Writes a JSONArray that contains JSONObjects to the BigQuery table by first converting the JSON * data to protobuf messages, then using StreamWriter's append() to write the data at the @@ -141,13 +145,27 @@ public ApiFuture append(JSONArray jsonArr, long offset) // processing // of JSON data. long currentRequestSize = 0; + List rowErrors = new ArrayList<>(); for (int i = 0; i < jsonArr.length(); i++) { JSONObject json = jsonArr.getJSONObject(i); - Message protoMessage = - JsonToProtoMessage.convertJsonToProtoMessage( - this.descriptor, this.tableSchema, json, ignoreUnknownFields); - rowsBuilder.addSerializedRows(protoMessage.toByteString()); - currentRequestSize += protoMessage.getSerializedSize(); + try { + Message protoMessage = + JsonToProtoMessage.convertJsonToProtoMessage( + this.descriptor, this.tableSchema, json, ignoreUnknownFields); + rowsBuilder.addSerializedRows(protoMessage.toByteString()); + currentRequestSize += protoMessage.getSerializedSize(); + } catch (IllegalArgumentException exception) { + rowErrors.add( + RowError.newBuilder() + .setIndex(i) + .setCode(RowErrorCode.FIELDS_ERROR) + .setMessage(exception.getMessage()) + .build()); + } + } + + if (!rowErrors.isEmpty()) { + throw new AppendSerializtionError(streamName, rowErrors); } final ApiFuture appendResponseFuture = this.streamWriter.append(rowsBuilder.build(), offset); @@ -408,7 +426,7 @@ public Builder setReconnectAfter10M(boolean reconnectAfter10M) { */ public JsonStreamWriter build() throws Descriptors.DescriptorValidationException, IllegalArgumentException, IOException, - InterruptedException { + InterruptedException { return new JsonStreamWriter(this); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index 4a19f17e24..e2a722ad43 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -31,6 +31,8 @@ import com.google.cloud.bigquery.storage.test.JsonTest; import com.google.cloud.bigquery.storage.test.Test.FooType; import com.google.cloud.bigquery.storage.test.Test.UpdatedFooType; +import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializtionError; +import com.google.cloud.bigquery.storage.v1.RowError.RowErrorCode; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; import com.google.protobuf.Timestamp; @@ -38,6 +40,7 @@ import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -518,9 +521,10 @@ public void testWithoutIgnoreUnknownFields() throws Exception { try { ApiFuture appendFuture = writer.append(jsonArr); Assert.fail("expected ExecutionException"); - } catch (Exception ex) { + } catch (AppendSerializtionError ex) { assertEquals( - ex.getMessage(), "JSONObject has fields unknown to BigQuery: root.test_unknown."); + ex.getRowErrors().get(0).getMessage(), + "JSONObject has fields unknown to BigQuery: root.test_unknown."); } } } @@ -603,4 +607,44 @@ public void testFlowControlSettingNoLimitBehavior() throws Exception { appendFuture.get(); } } + + @Test + public void testMultipleAppendSerializtionErrors() + throws DescriptorValidationException, IOException, InterruptedException { + FooType expectedProto = FooType.newBuilder().setFoo("allen").build(); + JSONObject foo = new JSONObject(); + foo.put("not_foo", "allen"); + JSONObject foo1 = new JSONObject(); + foo1.put("foo", "allen"); + JSONObject foo2 = new JSONObject(); + foo2.put("foo", 666); + JSONArray jsonArr = new JSONArray(); + jsonArr.put(foo); + jsonArr.put(foo1); + jsonArr.put(foo2); + RowError columnMismatchError = + RowError.newBuilder() + .setCode(RowErrorCode.FIELDS_ERROR) + .setIndex(0) + .setMessage("JSONObject has fields unknown to BigQuery: root.not_foo.") + .build(); + RowError typeError = + RowError.newBuilder() + .setCode(RowErrorCode.FIELDS_ERROR) + .setIndex(2) + .setMessage("JSONObject does not have a string field at root.foo.") + .build(); + + try (JsonStreamWriter writer = + getTestJsonStreamWriterBuilder(TEST_STREAM, TABLE_SCHEMA).build()) { + try { + ApiFuture appendFuture = writer.append(jsonArr); + } catch (AppendSerializtionError appendSerializtionError) { + List rowErrors = appendSerializtionError.getRowErrors(); + assertEquals(2, rowErrors.size()); + assertEquals(columnMismatchError, rowErrors.get(0)); + assertEquals(typeError, rowErrors.get(1)); + } + } + } } From d6c2df3373aa19ee221fc7358b3938602fe13616 Mon Sep 17 00:00:00 2001 From: Noa Katz Date: Tue, 28 Jun 2022 15:14:36 +0300 Subject: [PATCH 3/7] Expose row level serialization failures for JsonStreamWriter append --- .../cloud/bigquery/storage/v1/Exceptions.java | 18 +++++++++++++++--- .../bigquery/storage/v1/JsonStreamWriter.java | 10 ++++++---- .../storage/v1/JsonStreamWriterTest.java | 1 + 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java index e827b4ac51..947c99d74c 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java @@ -204,18 +204,30 @@ public static StorageException toStorageException(Throwable exception) { return toStorageException(rpcStatus, exception); } - /** Append serializtion error. */ + /** + * This exception is thrown from {@link JsonStreamWriter#append()} when the client side Json to + * Proto serializtion fails. The exception contains a list of {@link RowError} object which + * represent all of the faulty lines. {@link RowError} can be also returned as part of {@link + * AppendRowsResponse}, after the rows were processed on the server side. using the same error + * object should make error hadling easier. + */ public static class AppendSerializtionError extends RuntimeException { private final List rowErrors; + private final String streamName; - public AppendSerializtionError(String name, List rowErrors) { - super(String.format("Append serializtion failed for writer: %s", name)); + public AppendSerializtionError(@Nullable String streamName, List rowErrors) { + super(String.format("Append serializtion failed for writer: %s", streamName)); this.rowErrors = rowErrors; + this.streamName = streamName; } public List getRowErrors() { return rowErrors; } + + public String getStreamName() { + return streamName; + } } private Exceptions() {} diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 403a6f1b64..9998552b21 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -141,9 +141,9 @@ public ApiFuture append(JSONArray jsonArr, long offset) ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder(); // Any error in convertJsonToProtoMessage will throw an - // IllegalArgumentException/IllegalStateException/NullPointerException and will halt - // processing - // of JSON data. + // IllegalArgumentException/IllegalStateException/NullPointerException which will be collected + // into a list of RowErrors. After the coverstion is finished an AppendSerializtionError + // exception that contains all the conversion errors will be thrown. long currentRequestSize = 0; List rowErrors = new ArrayList<>(); for (int i = 0; i < jsonArr.length(); i++) { @@ -154,7 +154,9 @@ public ApiFuture append(JSONArray jsonArr, long offset) this.descriptor, this.tableSchema, json, ignoreUnknownFields); rowsBuilder.addSerializedRows(protoMessage.toByteString()); currentRequestSize += protoMessage.getSerializedSize(); - } catch (IllegalArgumentException exception) { + } catch (IllegalArgumentException + | IllegalStateException + | NullPointerException exception) { rowErrors.add( RowError.newBuilder() .setIndex(i) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index e2a722ad43..44bf473739 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -639,6 +639,7 @@ public void testMultipleAppendSerializtionErrors() getTestJsonStreamWriterBuilder(TEST_STREAM, TABLE_SCHEMA).build()) { try { ApiFuture appendFuture = writer.append(jsonArr); + Assert.fail("expected AppendSerializtionError"); } catch (AppendSerializtionError appendSerializtionError) { List rowErrors = appendSerializtionError.getRowErrors(); assertEquals(2, rowErrors.size()); From 4a39704a6a27cf1a656694ba8aa79151fec74746 Mon Sep 17 00:00:00 2001 From: Noa Katz Date: Tue, 28 Jun 2022 15:14:36 +0300 Subject: [PATCH 4/7] Expose row level serialization failures for JsonStreamWriter append --- .../cloud/bigquery/storage/v1/Exceptions.java | 19 ++++++------ .../bigquery/storage/v1/JsonStreamWriter.java | 22 +++++--------- .../storage/v1/JsonStreamWriterTest.java | 29 +++++++------------ 3 files changed, 27 insertions(+), 43 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java index 947c99d74c..2fecbd784c 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java @@ -22,7 +22,7 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.protobuf.StatusProto; -import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -206,23 +206,22 @@ public static StorageException toStorageException(Throwable exception) { /** * This exception is thrown from {@link JsonStreamWriter#append()} when the client side Json to - * Proto serializtion fails. The exception contains a list of {@link RowError} object which - * represent all of the faulty lines. {@link RowError} can be also returned as part of {@link - * AppendRowsResponse}, after the rows were processed on the server side. using the same error - * object should make error hadling easier. + * Proto serializtion fails. The exception contains a Map of indexes of faulty lines and the + * corresponding error message. */ public static class AppendSerializtionError extends RuntimeException { - private final List rowErrors; + private final Map rowIndexToErrorMessage; private final String streamName; - public AppendSerializtionError(@Nullable String streamName, List rowErrors) { + public AppendSerializtionError( + @Nullable String streamName, Map rowIndexToErrorMessage) { super(String.format("Append serializtion failed for writer: %s", streamName)); - this.rowErrors = rowErrors; + this.rowIndexToErrorMessage = rowIndexToErrorMessage; this.streamName = streamName; } - public List getRowErrors() { - return rowErrors; + public Map getRowIndexToErrorMessage() { + return rowIndexToErrorMessage; } public String getStreamName() { diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 9998552b21..8abaabd17e 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -20,15 +20,14 @@ import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializtionError; -import com.google.cloud.bigquery.storage.v1.RowError.RowErrorCode; import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Message; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -142,10 +141,10 @@ public ApiFuture append(JSONArray jsonArr, long offset) ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder(); // Any error in convertJsonToProtoMessage will throw an // IllegalArgumentException/IllegalStateException/NullPointerException which will be collected - // into a list of RowErrors. After the coverstion is finished an AppendSerializtionError - // exception that contains all the conversion errors will be thrown. + // into a Map of roe indexes to error messages. After the coverstion is finished an + // AppendSerializtionError exception that contains all the conversion errors will be thrown. long currentRequestSize = 0; - List rowErrors = new ArrayList<>(); + Map rowIndexToErrorMessage = new HashMap<>(); for (int i = 0; i < jsonArr.length(); i++) { JSONObject json = jsonArr.getJSONObject(i); try { @@ -157,17 +156,12 @@ public ApiFuture append(JSONArray jsonArr, long offset) } catch (IllegalArgumentException | IllegalStateException | NullPointerException exception) { - rowErrors.add( - RowError.newBuilder() - .setIndex(i) - .setCode(RowErrorCode.FIELDS_ERROR) - .setMessage(exception.getMessage()) - .build()); + rowIndexToErrorMessage.put(i, exception.getMessage()); } } - if (!rowErrors.isEmpty()) { - throw new AppendSerializtionError(streamName, rowErrors); + if (!rowIndexToErrorMessage.isEmpty()) { + throw new AppendSerializtionError(streamName, rowIndexToErrorMessage); } final ApiFuture appendResponseFuture = this.streamWriter.append(rowsBuilder.build(), offset); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index 44bf473739..dc41891b7b 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -32,7 +32,6 @@ import com.google.cloud.bigquery.storage.test.Test.FooType; import com.google.cloud.bigquery.storage.test.Test.UpdatedFooType; import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializtionError; -import com.google.cloud.bigquery.storage.v1.RowError.RowErrorCode; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; import com.google.protobuf.Timestamp; @@ -40,7 +39,7 @@ import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.Arrays; -import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -523,7 +522,7 @@ public void testWithoutIgnoreUnknownFields() throws Exception { Assert.fail("expected ExecutionException"); } catch (AppendSerializtionError ex) { assertEquals( - ex.getRowErrors().get(0).getMessage(), + ex.getRowIndexToErrorMessage().get(1), "JSONObject has fields unknown to BigQuery: root.test_unknown."); } } @@ -622,18 +621,6 @@ public void testMultipleAppendSerializtionErrors() jsonArr.put(foo); jsonArr.put(foo1); jsonArr.put(foo2); - RowError columnMismatchError = - RowError.newBuilder() - .setCode(RowErrorCode.FIELDS_ERROR) - .setIndex(0) - .setMessage("JSONObject has fields unknown to BigQuery: root.not_foo.") - .build(); - RowError typeError = - RowError.newBuilder() - .setCode(RowErrorCode.FIELDS_ERROR) - .setIndex(2) - .setMessage("JSONObject does not have a string field at root.foo.") - .build(); try (JsonStreamWriter writer = getTestJsonStreamWriterBuilder(TEST_STREAM, TABLE_SCHEMA).build()) { @@ -641,10 +628,14 @@ public void testMultipleAppendSerializtionErrors() ApiFuture appendFuture = writer.append(jsonArr); Assert.fail("expected AppendSerializtionError"); } catch (AppendSerializtionError appendSerializtionError) { - List rowErrors = appendSerializtionError.getRowErrors(); - assertEquals(2, rowErrors.size()); - assertEquals(columnMismatchError, rowErrors.get(0)); - assertEquals(typeError, rowErrors.get(1)); + Map rowIndexToErrorMessage = + appendSerializtionError.getRowIndexToErrorMessage(); + assertEquals(2, rowIndexToErrorMessage.size()); + assertEquals( + "JSONObject has fields unknown to BigQuery: root.not_foo.", + rowIndexToErrorMessage.get(0)); + assertEquals( + "JSONObject does not have a string field at root.foo.", rowIndexToErrorMessage.get(2)); } } } From bcef44e06b8633434b24c58cbaa79c8e9707414d Mon Sep 17 00:00:00 2001 From: Noa Katz Date: Tue, 28 Jun 2022 15:14:36 +0300 Subject: [PATCH 5/7] Expose row level serialization failures for JsonStreamWriter append --- .../google/cloud/bigquery/storage/v1/Exceptions.java | 3 +-- .../cloud/bigquery/storage/v1/JsonStreamWriter.java | 11 +++++------ .../bigquery/storage/v1/JsonStreamWriterTest.java | 8 ++++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java index 2fecbd784c..ba1051cd67 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java @@ -213,8 +213,7 @@ public static class AppendSerializtionError extends RuntimeException { private final Map rowIndexToErrorMessage; private final String streamName; - public AppendSerializtionError( - @Nullable String streamName, Map rowIndexToErrorMessage) { + public AppendSerializtionError(String streamName, Map rowIndexToErrorMessage) { super(String.format("Append serializtion failed for writer: %s", streamName)); this.rowIndexToErrorMessage = rowIndexToErrorMessage; this.streamName = streamName; diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 8abaabd17e..249bf73d0d 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -140,9 +140,10 @@ public ApiFuture append(JSONArray jsonArr, long offset) ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder(); // Any error in convertJsonToProtoMessage will throw an - // IllegalArgumentException/IllegalStateException/NullPointerException which will be collected - // into a Map of roe indexes to error messages. After the coverstion is finished an - // AppendSerializtionError exception that contains all the conversion errors will be thrown. + // IllegalArgumentException/IllegalStateException/NullPointerException. + // IllegalArgumentException will be collected into a Map of row indexes to error messages. + // After the conversion is finished an AppendSerializtionError exception that contains all the + // conversion errors will be thrown. long currentRequestSize = 0; Map rowIndexToErrorMessage = new HashMap<>(); for (int i = 0; i < jsonArr.length(); i++) { @@ -153,9 +154,7 @@ public ApiFuture append(JSONArray jsonArr, long offset) this.descriptor, this.tableSchema, json, ignoreUnknownFields); rowsBuilder.addSerializedRows(protoMessage.toByteString()); currentRequestSize += protoMessage.getSerializedSize(); - } catch (IllegalArgumentException - | IllegalStateException - | NullPointerException exception) { + } catch (IllegalArgumentException exception) { rowIndexToErrorMessage.put(i, exception.getMessage()); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index dc41891b7b..434659dee7 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -522,8 +522,9 @@ public void testWithoutIgnoreUnknownFields() throws Exception { Assert.fail("expected ExecutionException"); } catch (AppendSerializtionError ex) { assertEquals( - ex.getRowIndexToErrorMessage().get(1), - "JSONObject has fields unknown to BigQuery: root.test_unknown."); + "JSONObject has fields unknown to BigQuery: root.test_unknown.", + ex.getRowIndexToErrorMessage().get(1)); + assertEquals(TEST_STREAM, ex.getStreamName()); } } } @@ -612,10 +613,13 @@ public void testMultipleAppendSerializtionErrors() throws DescriptorValidationException, IOException, InterruptedException { FooType expectedProto = FooType.newBuilder().setFoo("allen").build(); JSONObject foo = new JSONObject(); + // put a field which is not part of the expected schema foo.put("not_foo", "allen"); JSONObject foo1 = new JSONObject(); + // put a vaild value into the field foo1.put("foo", "allen"); JSONObject foo2 = new JSONObject(); + // put a number into a string field foo2.put("foo", 666); JSONArray jsonArr = new JSONArray(); jsonArr.put(foo); From 9c35cf1c1cf5341f939018534116324296474d41 Mon Sep 17 00:00:00 2001 From: Noa Katz Date: Tue, 28 Jun 2022 15:14:36 +0300 Subject: [PATCH 6/7] Expose row level serialization failures for JsonStreamWriter append --- .../google/cloud/bigquery/storage/v1/JsonStreamWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 249bf73d0d..a7d2a0a589 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -68,7 +68,7 @@ public class JsonStreamWriter implements AutoCloseable { */ private JsonStreamWriter(Builder builder) throws Descriptors.DescriptorValidationException, IllegalArgumentException, IOException, - InterruptedException { + InterruptedException { this.client = builder.client; this.descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(builder.tableSchema); @@ -421,7 +421,7 @@ public Builder setReconnectAfter10M(boolean reconnectAfter10M) { */ public JsonStreamWriter build() throws Descriptors.DescriptorValidationException, IllegalArgumentException, IOException, - InterruptedException { + InterruptedException { return new JsonStreamWriter(this); } } From 99f01504c7ab776f8e6b93065b7512f768ea6cbe Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 7 Jul 2022 16:35:42 +0000 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f39fa1bcb..bf2e730d63 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigquerystorage' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquerystorage:2.14.2' +implementation 'com.google.cloud:google-cloud-bigquerystorage:2.15.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.14.2" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.15.0" ``` ## Authentication