Skip to content

Commit

Permalink
fix(floating): floating point numbers as numerics (#1648)
Browse files Browse the repository at this point in the history 
Signed-off-by: dark0dave <[email protected]>
  • Loading branch information
@dark0dave
dark0dave authored Jun 7, 2022
1 parent 3eb8d4f commit ecf8598
Show file tree
Hide file tree
Showing 7 changed files with 218 additions and 114 deletions.
24 changes: 24 additions & 0 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,12 @@ private static void fillField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
return;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.setField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
return;
}
} else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
if (val instanceof String) {
Expand All @@ -310,6 +316,12 @@ private static void fillField(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal(((Number) val).longValue())));
return;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.setField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
return;
}
}
}
Expand Down Expand Up @@ -522,6 +534,12 @@ private static void fillRepeatedField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
added = true;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.addRepeatedField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
added = true;
}
} else if (fieldSchema != null
&& fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
Expand All @@ -537,6 +555,12 @@ private static void fillRepeatedField(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal(((Number) val).longValue())));
added = true;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.addRepeatedField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
added = true;
}
}
if (!added) {
Expand Down
24 changes: 24 additions & 0 deletions ...rystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,12 @@ private static void fillField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
return;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.setField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
return;
}
} else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
if (val instanceof String) {
Expand All @@ -217,6 +223,12 @@ private static void fillField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
return;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.setField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
return;
}
}
}
Expand Down Expand Up @@ -375,6 +387,12 @@ private static void fillRepeatedField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
added = true;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.addRepeatedField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
added = true;
}
} else if (fieldSchema != null
&& fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) {
Expand All @@ -390,6 +408,12 @@ private static void fillRepeatedField(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).longValue())));
added = true;
} else if (val instanceof Float || val instanceof Double) {
protoMsg.addRepeatedField(
fieldDescriptor,
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal(((Number) val).doubleValue())));
added = true;
}
}
if (!added) {
Expand Down
40 changes: 34 additions & 6 deletions ...rc/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_long")
.build();
final TableFieldSchema TEST_NUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_float")
.build();
final TableFieldSchema TEST_NUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_double")
.build();
final TableFieldSchema TEST_BIGNUMERIC =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
Expand All @@ -269,6 +281,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_long")
.build();
final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_float")
.build();
final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_double")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
Expand Down Expand Up @@ -302,12 +326,16 @@ public void testStructComplex() throws Exception {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_NUMERIC_INT)
.addFields(18, TEST_NUMERIC_LONG)
.addFields(19, TEST_BIGNUMERIC)
.addFields(20, TEST_BIGNUMERIC_STR)
.addFields(21, TEST_BIGNUMERIC_INT)
.addFields(22, TEST_BIGNUMERIC_LONG)
.addFields(23, TEST_INTERVAL)
.addFields(24, TEST_JSON)
.addFields(19, TEST_NUMERIC_FLOAT)
.addFields(20, TEST_NUMERIC_DOUBLE)
.addFields(21, TEST_BIGNUMERIC)
.addFields(22, TEST_BIGNUMERIC_STR)
.addFields(23, TEST_BIGNUMERIC_INT)
.addFields(24, TEST_BIGNUMERIC_LONG)
.addFields(25, TEST_BIGNUMERIC_FLOAT)
.addFields(26, TEST_BIGNUMERIC_DOUBLE)
.addFields(27, TEST_INTERVAL)
.addFields(28, TEST_JSON)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
94 changes: 46 additions & 48 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,18 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_long")
.build();
private final TableFieldSchema TEST_NUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_float")
.build();
private final TableFieldSchema TEST_NUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_double")
.build();
private final TableFieldSchema TEST_BIGNUMERIC =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
Expand All @@ -445,6 +457,18 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_long")
.build();
private final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_float")
.build();
private final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.BIGNUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_double")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
Expand Down Expand Up @@ -478,12 +502,16 @@ public class JsonToProtoMessageTest {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_NUMERIC_INT)
.addFields(18, TEST_NUMERIC_LONG)
.addFields(19, TEST_BIGNUMERIC)
.addFields(20, TEST_BIGNUMERIC_STR)
.addFields(21, TEST_BIGNUMERIC_INT)
.addFields(22, TEST_BIGNUMERIC_LONG)
.addFields(23, TEST_INTERVAL)
.addFields(24, TEST_JSON)
.addFields(19, TEST_NUMERIC_FLOAT)
.addFields(20, TEST_NUMERIC_DOUBLE)
.addFields(21, TEST_BIGNUMERIC)
.addFields(22, TEST_BIGNUMERIC_STR)
.addFields(23, TEST_BIGNUMERIC_INT)
.addFields(24, TEST_BIGNUMERIC_LONG)
.addFields(25, TEST_BIGNUMERIC_FLOAT)
.addFields(26, TEST_BIGNUMERIC_DOUBLE)
.addFields(27, TEST_INTERVAL)
.addFields(28, TEST_JSON)
.build();

@Test
Expand Down Expand Up @@ -599,27 +627,6 @@ public void testTimeMismatch() throws Exception {
}
}

@Test
public void testNumericMismatch() throws Exception {
TableFieldSchema field =
TableFieldSchema.newBuilder()
.setName("numeric")
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.build();
TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build();
JSONObject json = new JSONObject();
json.put("numeric", 1.0);
try {
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
TestNumeric.getDescriptor(), tableSchema, json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
assertEquals("JSONObject does not have a bytes field at root.numeric.", e.getMessage());
}
}

@Test
public void testMixedCaseFieldNames() throws Exception {
TableFieldSchema field =
Expand All @@ -638,27 +645,6 @@ public void testMixedCaseFieldNames() throws Exception {
TestMixedCaseFieldNames.getDescriptor(), tableSchema, json);
}

@Test
public void testBigNumericMismatch() throws Exception {
TableFieldSchema field =
TableFieldSchema.newBuilder()
.setName("bignumeric")
.setType(TableFieldSchema.Type.BIGNUMERIC)
.setMode(TableFieldSchema.Mode.REPEATED)
.build();
TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build();
JSONObject json = new JSONObject();
json.put("bignumeric", new JSONArray(new Double[] {1.0}));
try {
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
TestBignumeric.getDescriptor(), tableSchema, json);
Assert.fail("should fail");
} catch (IllegalArgumentException e) {
assertEquals("JSONObject does not have a bytes field at root.bignumeric[0].", e.getMessage());
}
}

@Test
public void testDouble() throws Exception {
TestDouble expectedProto =
Expand Down Expand Up @@ -920,6 +906,10 @@ public void testStructComplex() throws Exception {
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1)))
.setTestNumericLong(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1L)))
.setTestNumericFloat(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1f)))
.setTestNumericDouble(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal(1D)))
.setTestBignumeric(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal("578960446186580977117854925043439539266.3")))
Expand All @@ -929,6 +919,10 @@ public void testStructComplex() throws Exception {
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1)))
.setTestBignumericLong(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1L)))
.setTestBignumericFloat(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1f)))
.setTestBignumericDouble(
BigDecimalByteStringEncoder.encodeToBigNumericByteString(new BigDecimal(1D)))
.setTestInterval("0-0 0 0:0:0.000005")
.addTestJson("{'a':'b'}")
.build();
Expand Down Expand Up @@ -975,13 +969,17 @@ public void testStructComplex() throws Exception {
json.put("test_numeric_str", "12.4");
json.put("test_numeric_int", 1);
json.put("test_numeric_long", 1L);
json.put("test_numeric_float", 1f);
json.put("test_numeric_double", 1D);
json.put(
"test_bignumeric",
BigDecimalByteStringEncoder.encodeToBigNumericByteString(
new BigDecimal("578960446186580977117854925043439539266.3")));
json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"}));
json.put("test_bignumeric_int", 1);
json.put("test_bignumeric_long", 1L);
json.put("test_bignumeric_float", 1f);
json.put("test_bignumeric_double", 1D);
json.put("test_interval", "0-0 0 0:0:0.000005");
json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"}));
DynamicMessage protoMsg =
Expand Down
40 changes: 34 additions & 6 deletions ...st/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_long")
.build();
final TableFieldSchema TEST_NUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_float")
.build();
final TableFieldSchema TEST_NUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric_double")
.build();
final TableFieldSchema TEST_BIGNUMERIC =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
Expand All @@ -269,6 +281,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_long")
.build();
final TableFieldSchema TEST_BIGNUMERIC_FLOAT =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_float")
.build();
final TableFieldSchema TEST_BIGNUMERIC_DOUBLE =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_bignumeric_double")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
Expand Down Expand Up @@ -302,12 +326,16 @@ public void testStructComplex() throws Exception {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_NUMERIC_INT)
.addFields(18, TEST_NUMERIC_LONG)
.addFields(19, TEST_BIGNUMERIC)
.addFields(20, TEST_BIGNUMERIC_STR)
.addFields(21, TEST_BIGNUMERIC_INT)
.addFields(22, TEST_BIGNUMERIC_LONG)
.addFields(23, TEST_INTERVAL)
.addFields(24, TEST_JSON)
.addFields(19, TEST_NUMERIC_FLOAT)
.addFields(20, TEST_NUMERIC_DOUBLE)
.addFields(21, TEST_BIGNUMERIC)
.addFields(22, TEST_BIGNUMERIC_STR)
.addFields(23, TEST_BIGNUMERIC_INT)
.addFields(24, TEST_BIGNUMERIC_FLOAT)
.addFields(25, TEST_BIGNUMERIC_DOUBLE)
.addFields(26, TEST_BIGNUMERIC_LONG)
.addFields(27, TEST_INTERVAL)
.addFields(28, TEST_JSON)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
Loading

0 comments on commit ecf8598

Please sign in to comment.