fix: even better ISO compilance (#1607)

Using docs from google https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#data_types as guide, for how types should be parsed Signed-off-by: dark0dave <[email protected]>
googleapis · Apr 8, 2022 · 5701597 · 5701597
1 parent 3148c3a
commit 5701597
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 10 deletions.
diff --git a/...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
@@ -39,6 +39,7 @@
 import org.threeten.bp.ZoneOffset;
 import org.threeten.bp.format.DateTimeFormatter;
 import org.threeten.bp.format.DateTimeFormatterBuilder;
+import org.threeten.bp.format.TextStyle;
 import org.threeten.bp.temporal.ChronoField;
 import org.threeten.bp.temporal.TemporalAccessor;
 
@@ -61,7 +62,7 @@ public class JsonToProtoMessage {
   private static final DateTimeFormatter timestampFormatter =
       new DateTimeFormatterBuilder()
           .parseLenient()
-          .append(DateTimeFormatter.ISO_LOCAL_DATE)
+          .append(DateTimeFormatter.ofPattern("yyyy[/][-]MM[/][-]dd"))
           .optionalStart()
           .appendLiteral('T')
           .optionalEnd()
@@ -74,10 +75,24 @@ public class JsonToProtoMessage {
           .optionalStart()
           .appendLiteral(':')
           .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+          .optionalEnd()
+          .optionalStart()
+          .appendValue(ChronoField.MILLI_OF_SECOND, 3)
+          .optionalEnd()
+          .optionalStart()
+          .appendFraction(ChronoField.MICRO_OF_SECOND, 3, 6, true)
+          .optionalEnd()
           .optionalStart()
           .appendFraction(ChronoField.NANO_OF_SECOND, 6, 9, true)
+          .optionalEnd()
           .optionalStart()
-          .appendOffset("+HHMM", "+00:00")
+          .appendLiteral(' ')
+          .optionalEnd()
+          .optionalStart()
+          .appendOffset("+HH:MM", "+00:00")
+          .optionalEnd()
+          .optionalStart()
+          .appendZoneText(TextStyle.SHORT)
           .optionalEnd()
           .optionalStart()
           .appendLiteral('Z')
@@ -336,6 +351,11 @@ private static void fillField(
             }
           } else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
             if (val instanceof String) {
+              Double parsed = Doubles.tryParse((String) val);
+              if (parsed != null) {
+                protoMsg.setField(fieldDescriptor, parsed.longValue() * 10000000);
+                return;
+              }
               TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
               protoMsg.setField(
                   fieldDescriptor,
@@ -345,6 +365,9 @@ private static void fillField(
             } else if (val instanceof Long) {
               protoMsg.setField(fieldDescriptor, (Long) val);
               return;
+            } else if (val instanceof Integer) {
+              protoMsg.setField(fieldDescriptor, new Long((Integer) val) * 10000000);
+              return;
             }
           }
         }
@@ -549,13 +572,20 @@ private static void fillRepeatedField(
           } else if (fieldSchema != null
               && fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
             if (val instanceof String) {
-              TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
-              protoMsg.addRepeatedField(
-                  fieldDescriptor,
-                  parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
-                      + parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
+              Double parsed = Doubles.tryParse((String) val);
+              if (parsed != null) {
+                protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue() * 10000000);
+              } else {
+                TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
+                protoMsg.addRepeatedField(
+                    fieldDescriptor,
+                    parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
+                        + parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
+              }
             } else if (val instanceof Long) {
               protoMsg.addRepeatedField(fieldDescriptor, (Long) val);
+            } else if (val instanceof Integer) {
+              protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val) * 10000000);
             } else {
               fail = true;
             }

diff --git a/...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
@@ -664,17 +664,32 @@ public void testTimestamp() throws Exception {
             .addFields(
                 TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_string_T_Z").build())
             .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_long").build())
+            .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_int").build())
+            .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_float").build())
+            .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_offset").build())
+            .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_timezone").build())
+            .addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_saformat").build())
             .build();
     TestTimestamp expectedProto =
         TestTimestamp.newBuilder()
             .setTestString(10L)
-            .setTestStringTZ(1648493279000000L)
+            .setTestStringTZ(1648493279010000L)
             .setTestLong(0L)
+            .setTestInt(1534806950000000L)
+            .setTestFloat(1534680695000000000L)
+            .setTestOffset(1649135171000000L)
+            .setTestTimezone(1649174771000000L)
+            .setTestSaformat(1534680660000000L)
             .build();
     JSONObject json = new JSONObject();
     json.put("test_string", "1970-01-01 00:00:00.000010");
-    json.put("test_string_T_Z", "2022-03-28T18:47:59.00Z");
+    json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z");
     json.put("test_long", 0L);
+    json.put("test_int", 153480695);
+    json.put("test_float", "1.534680695e11");
+    json.put("test_offset", "2022-04-05T09:06:11+04:00");
+    json.put("test_timezone", "2022-04-05 09:06:11 PST");
+    json.put("test_saformat", "2018/08/19 12:11");
     DynamicMessage protoMsg =
         JsonToProtoMessage.convertJsonToProtoMessage(
             TestTimestamp.getDescriptor(), tableSchema, json);
@@ -860,7 +875,7 @@ public void testStructComplex() throws Exception {
             .setTestNumeric(
                 BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")))
             .setTestGeo("POINT(1,1)")
-            .setTestTimestamp(12345678)
+            .setTestTimestamp(123456780000000L)
             .setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
             .setTestTimeStr(89332507144L)
             .addTestNumericRepeated(

diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
@@ -138,6 +138,11 @@ message TestTimestamp {
   optional int64 test_string = 1;
   optional int64 test_string_t_z = 2;
   optional int64 test_long = 3;
+  optional int64 test_int = 4;
+  optional int64 test_float = 5;
+  optional int64 test_offset = 6;
+  optional int64 test_timezone = 7;
+  optional int64 test_saformat = 8;
 }
 
 message TestDate {