Skip to content

Commit

Permalink
fix: add precision overwritten to 9 digit if the passed in JSON type …
Browse files Browse the repository at this point in the history
…is FLOAT or DOUBLE (#1932)

* feat: Split writer into connection worker and wrapper, this is a
prerequisite for multiplexing client

* feat: add connection worker pool skeleton, used for multiplexing client

* feat: add Load api for connection worker for multiplexing client

* feat: add multiplexing support to connection worker. We will treat every
new stream name as a switch of destinationt

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: port the multiplexing client core algorithm and basic tests
also fixed a tiny bug inside fake bigquery write impl for getting thre
response from offset

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: wire multiplexing connection pool to stream writer

* feat: some fixes for multiplexing client

* feat: fix some todos, and reject the mixed behavior of passed in client or not

* feat: fix the bug that we may peek into the write_stream field but it's
possible the proto schema does not contain this field

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: fix the bug that we may peek into the write_stream field but it's
possible the proto schema does not contain this field

* feat: add getInflightWaitSeconds implementation

* feat: Add schema comparision in connection loop to ensure schema update for
the same stream name can be notified

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: add schema update support to multiplexing

* fix: fix windows build bug: windows Instant resolution is different with
linux

* fix: fix another failing tests for windows build

* fix: fix another test failure for Windows build

* feat: Change new thread for each retry to be a thread pool to avoid
create/tear down too much threads if lots of retries happens

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: add back the background executor provider that's accidentally
removed

* feat: throw error when use connection pool for explicit stream

* fix: Add precision truncation to the passed in value from JSON float and
double type.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* modify the bom version

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
GaoleMeng and gcf-owl-bot[bot] authored Jan 18, 2023
1 parent 2d648cf commit 417bc6c
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 5 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ If you are using Maven without BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:

```Groovy
implementation platform('com.google.cloud:libraries-bom:26.3.0')
implementation platform('com.google.cloud:libraries-bom:26.4.0')
implementation 'com.google.cloud:google-cloud-bigquerystorage'
```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDate;
import java.util.List;
import java.util.logging.Logger;
Expand All @@ -49,6 +50,7 @@
*/
public class JsonToProtoMessage {
private static final Logger LOG = Logger.getLogger(JsonToProtoMessage.class.getName());
private static int NUMERIC_SCALE = 9;
private static ImmutableMap<FieldDescriptor.Type, String> FieldTypeToDebugMessage =
new ImmutableMap.Builder<FieldDescriptor.Type, String>()
.put(FieldDescriptor.Type.BOOL, "boolean")
Expand Down Expand Up @@ -315,10 +317,15 @@ private static void fillField(
new BigDecimal(((Number) val).longValue())));
return;
} else if (val instanceof Float || val instanceof Double) {
// In JSON, the precision passed in is machine dependent. We should round the number
// before passing to backend.
BigDecimal bigDecimal = new BigDecimal(String.valueOf(val));
if (bigDecimal.scale() > 9) {
bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP);
}
protoMsg.setField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(String.valueOf(val))));
BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal));
return;
} else if (val instanceof BigDecimal) {
protoMsg.setField(
Expand Down Expand Up @@ -559,10 +566,13 @@ private static void fillRepeatedField(
new BigDecimal(((Number) val).longValue())));
added = true;
} else if (val instanceof Float || val instanceof Double) {
BigDecimal bigDecimal = new BigDecimal(String.valueOf(val));
if (bigDecimal.scale() > 9) {
bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP);
}
protoMsg.addRepeatedField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(String.valueOf(val))));
BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal));
added = true;
} else if (val instanceof BigDecimal) {
protoMsg.addRepeatedField(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import com.google.cloud.bigquery.storage.test.JsonTest.*;
import com.google.cloud.bigquery.storage.test.SchemaTest.*;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.ByteString;
import com.google.protobuf.Descriptors.Descriptor;
Expand Down Expand Up @@ -685,6 +686,60 @@ public void testDouble() throws Exception {
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDoubleHighPrecision() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(
TableFieldSchema.newBuilder()
.setName("numeric")
.setType(TableFieldSchema.Type.NUMERIC)
.build())
.build();
TestNumeric expectedProto =
TestNumeric.newBuilder()
.setNumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("3.400500513")))
.build();
JSONObject json = new JSONObject();
json.put("numeric", 3.400500512978076);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
TestNumeric.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDoubleHighPrecision_RepeatedField() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(
0,
TableFieldSchema.newBuilder()
.setName("bignumeric")
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.REPEATED)
.build())
.build();
TestBignumeric expectedProto =
TestBignumeric.newBuilder()
.addBignumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("3.400500513")))
.addBignumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.1")))
.addBignumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.12")))
.build();
JSONObject json = new JSONObject();
json.put("bignumeric", ImmutableList.of(3.400500512978076, 0.10000000000055, 0.12));
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
TestBignumeric.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testTimestamp() throws Exception {
TableSchema tableSchema =
Expand Down

0 comments on commit 417bc6c

Please sign in to comment.