Skip to content

Commit

Permalink
Protocol allows future / unknown properties (#2238)
Browse files Browse the repository at this point in the history
* Allow new extra properties in validation
* Create migration script to upgrade all connectors versions
* Bumpversion of all connectors
  • Loading branch information
ChristopheDuong authored Mar 9, 2021
1 parent ac0bcf7 commit 070575f
Show file tree
Hide file tree
Showing 100 changed files with 491 additions and 153 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.core.util.Separators;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
Expand All @@ -45,9 +46,15 @@
public class Jsons {

// Object Mapper is thread-safe
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final ObjectMapper OBJECT_MAPPER = initMapper();
private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writer(new JsonPrettyPrinter());

private static ObjectMapper initMapper() {
final ObjectMapper result = new ObjectMapper();
result.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
return result;
}

public static <T> String serialize(T object) {
try {
return OBJECT_MAPPER.writeValueAsString(object);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertThrows;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.type.TypeReference;
Expand Down Expand Up @@ -97,7 +96,7 @@ void testTryDeserialize() {
Jsons.tryDeserialize("{\"str\":\"abc\", \"num\": 999, \"numLong\": 888}", ToClass.class));

assertEquals(
Optional.empty(),
Optional.of(new ToClass("abc", 999, 0L)),
Jsons.tryDeserialize("{\"str\":\"abc\", \"num\": 999, \"test\": 888}", ToClass.class));
}

Expand Down Expand Up @@ -155,8 +154,9 @@ void testToObject() {
Lists.newArrayList(expected),
Jsons.object(Jsons.jsonNode(Lists.newArrayList(expected)), new TypeReference<List<ToClass>>() {}));

assertThrows(IllegalArgumentException.class,
() -> Jsons.object(Jsons.deserialize("{\"a\":1}"), ToClass.class));
assertEquals(
new ToClass(),
Jsons.object(Jsons.deserialize("{\"a\":1}"), ToClass.class));
}

@Test
Expand All @@ -170,12 +170,13 @@ void testTryToObject() {
Optional.of(expected),
Jsons.tryObject(Jsons.deserialize("{\"str\":\"abc\",\"num\":999,\"numLong\":888}"), new TypeReference<ToClass>() {}));

final ToClass emptyExpected = new ToClass();
assertEquals(
Optional.empty(),
Optional.of(emptyExpected),
Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), ToClass.class));

assertEquals(
Optional.empty(),
Optional.of(emptyExpected),
Jsons.tryObject(Jsons.deserialize("{\"str1\":\"abc\"}"), new TypeReference<ToClass>() {}));

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133",
"name": "BigQuery",
"dockerRepository": "airbyte/destination-bigquery",
"dockerImageTag": "0.1.16",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503",
"name": "Postgres",
"dockerRepository": "airbyte/destination-postgres",
"dockerImageTag": "0.1.16",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba",
"name": "Snowflake",
"dockerRepository": "airbyte/destination-snowflake",
"dockerImageTag": "0.1.19",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/snowflake"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6",
"name": "Local CSV",
"dockerRepository": "airbyte/destination-csv",
"dockerImageTag": "0.1.8",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-csv"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "a625d593-bba5-4a1c-a53d-2d246268a816",
"name": "Local JSON",
"dockerRepository": "airbyte/destination-local-json",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-json"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e",
"name": "MeiliSearch",
"dockerRepository": "airbyte/destination-meilisearch",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc",
"name": "Redshift",
"dockerRepository": "airbyte/destination-redshift",
"dockerImageTag": "0.1.8",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c",
"name": "Looker",
"dockerRepository": "airbyte/source-looker",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-looker"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77",
"name": "Salesforce",
"dockerRepository": "airbyte/source-salesforce-singer",
"dockerImageTag": "0.1.6",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-salesforce-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "2af123bf-0aaf-4e0d-9784-cb497f23741a",
"name": "Appstore",
"dockerRepository": "airbyte/source-appstore-singer",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-appstore-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "396e4ca3-8a97-4b85-aa4e-c9d8c2d5f992",
"name": "Braintree",
"dockerRepository": "airbyte/source-braintree-singer",
"dockerImageTag": "0.1.2",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-braintree-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1",
"name": "Google Analytics",
"dockerRepository": "airbyte/source-googleanalytics-singer",
"dockerImageTag": "0.1.9",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-googleanalytics-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "41375467-61ae-4204-8e38-e2b8b7365f23",
"name": "Slack",
"dockerRepository": "airbyte/source-slack-singer",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-slack-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad",
"name": "MySQL",
"dockerRepository": "airbyte/source-mysql",
"dockerImageTag": "0.1.10",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "445831eb-78db-4b1f-8f1f-0d96ad8739e2",
"name": "Drift",
"dockerRepository": "airbyte/source-drift",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-drift"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "487b930d-7f6a-43ce-8bac-46e6b2de0a55",
"name": "Mongo DB",
"dockerRepository": "airbyte/source-mongodb",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-mongodb"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "57eb1576-8f52-463d-beb6-2e107cdf571d",
"name": "Hubspot",
"dockerRepository": "airbyte/source-hubspot-singer",
"dockerImageTag": "0.1.6",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44",
"name": "Greenhouse",
"dockerRepository": "airbyte/source-greenhouse",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://https://docs.airbyte.io/integrations/sources/greenhouse"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "68e63de2-bb83-4c7e-93fa-a8a9051e3993",
"name": "Jira",
"dockerRepository": "airbyte/source-jira",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-jira"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7",
"name": "Google Sheets",
"dockerRepository": "airbyte/source-google-sheets",
"dockerImageTag": "0.1.7",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-google-sheets"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77",
"name": "File",
"dockerRepository": "airbyte/source-file",
"dockerImageTag": "0.1.10",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-file"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "859e501d-2b67-471f-91bb-1c801414d28f",
"name": "Mixpanel",
"dockerRepository": "airbyte/source-mixpanel-singer",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-mixpanel-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "932e6363-d006-4464-a9f5-102b82e07c06",
"name": "Twilio",
"dockerRepository": "airbyte/source-twilio-singer",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-twilio-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "9845d17a-45f1-4070-8a60-50914b1c8e2b",
"name": "HTTP Request",
"dockerRepository": "airbyte/source-http-request",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-http-request"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348",
"name": "Marketo",
"dockerRepository": "airbyte/source-marketo-singer",
"dockerImageTag": "0.1.5",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-marketo-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "9fed261d-d107-47fd-8c8b-323023db6e20",
"name": "Exchange Rates Api",
"dockerRepository": "airbyte/source-exchangeratesapi-singer",
"dockerImageTag": "0.1.9",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-exchangeratesapi_io-source"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "aea2fd0d-377d-465e-86c0-4fdc4f688e51",
"name": "Zoom",
"dockerRepository": "airbyte/source-zoom-singer",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-zoom-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002",
"name": "Mailchimp",
"dockerRepository": "airbyte/source-mailchimp",
"dockerImageTag": "0.1.6",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-mailchimp"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "b1892b11-788d-44bd-b9ec-3a436f7b54ce",
"name": "Shopify",
"dockerRepository": "airbyte/source-shopify-singer",
"dockerImageTag": "0.1.8",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-shopify-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1",
"name": "Microsoft SQL Server (MSSQL)",
"dockerRepository": "airbyte/source-mssql",
"dockerImageTag": "0.1.11",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-mssql"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01",
"name": "Recurly",
"dockerRepository": "airbyte/source-recurly",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-recurly"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "d29764f8-80d7-4dd7-acbe-1a42005ee5aa",
"name": "Zendesk Support",
"dockerRepository": "airbyte/source-zendesk-support-singer",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-zendesk-support-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a",
"name": "Intercom",
"dockerRepository": "airbyte/source-intercom-singer",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-intercom-singer"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
"name": "Postgres",
"dockerRepository": "airbyte/source-postgres",
"dockerImageTag": "0.1.13",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-postgres"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de",
"name": "Stripe",
"dockerRepository": "airbyte/source-stripe-singer",
"dockerImageTag": "0.1.10",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-stripe-source"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c",
"name": "Facebook Marketing",
"dockerRepository": "airbyte/source-facebook-marketing",
"dockerImageTag": "0.1.4",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6",
"name": "Redshift",
"dockerRepository": "airbyte/source-redshift",
"dockerImageTag": "0.1.6",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-redshift"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227",
"name": "Microsoft teams",
"dockerRepository": "airbyte/source-microsoft-teams",
"dockerImageTag": "0.1.1",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-microsoft-teams"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567",
"name": "Freshdesk",
"dockerRepository": "airbyte/source-freshdesk",
"dockerImageTag": "0.1.3",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-freshdesk"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "ed799e2b-2158-4c66-8da4-b40fe63bc72a",
"name": "Plaid",
"dockerRepository": "airbyte/source-plaid",
"dockerImageTag": "0.1.0",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-plaid"
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
"sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e",
"name": "GitHub",
"dockerRepository": "airbyte/source-github-singer",
"dockerImageTag": "0.1.8",
"dockerImageTag": "0.2.0",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-github-singer"
}
Loading

0 comments on commit 070575f

Please sign in to comment.