only add temporal (#2259)

* only add temporal

* remove unnecessary containers

* add max retention

* fix retention specification

* increase history conns

* remove problematic retention

* fmt

airbyte-commons/src/main/resources/log4j2.xml

@@ -64,6 +64,9 @@
         <Logger name="com.networknt.schema" level="INFO" />
         <Logger name="me.andrz.jackson" level="INFO" />
         <Logger name="com.leansoft.bigqueue" level="INFO" />
+        <Logger name="io.netty" level="INFO" />
+        <Logger name="io.grpc" level="INFO" />
+        <Logger name="io.temporal" level="INFO" />
 
     </Loggers>
 

airbyte-scheduler/build.gradle

@@ -3,6 +3,8 @@ plugins {
 }
 
 dependencies {
+    implementation 'io.temporal:temporal-sdk:1.0.4'
+
     implementation project(':airbyte-analytics')
     implementation project(':airbyte-config:models')
     implementation project(':airbyte-config:persistence')

airbyte-scheduler/src/main/java/io/airbyte/scheduler/TemporalUtils.java

@@ -0,0 +1,41 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.scheduler;
+
+import io.temporal.client.WorkflowClient;
+import io.temporal.serviceclient.WorkflowServiceStubs;
+import io.temporal.serviceclient.WorkflowServiceStubsOptions;
+
+public class TemporalUtils {
+
+  private static final WorkflowServiceStubsOptions TEMPORAL_OPTIONS = WorkflowServiceStubsOptions.newBuilder()
+      .setTarget("temporal:7233")
+      .build();
+
+  public static final WorkflowServiceStubs TEMPORAL_SERVICE = WorkflowServiceStubs.newInstance(TEMPORAL_OPTIONS);
+
+  public static final WorkflowClient TEMPORAL_CLIENT = WorkflowClient.newInstance(TEMPORAL_SERVICE);
+
+}

docker-compose.yaml

@@ -82,6 +82,20 @@ services:
       - IS_DEMO=${IS_DEMO:-}
       - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-}
       - TRACKING_STRATEGY=${TRACKING_STRATEGY}
+  temporal:
+    image: temporalio/auto-setup:1.7.0
+    container_name: temporal
+    ports:
+      - 7233:7233
+    environment:
+      - DB=postgresql
+      - DB_PORT=5432
+      - POSTGRES_USER=${DATABASE_USER}
+      - POSTGRES_PWD=${DATABASE_PASSWORD}
+      - POSTGRES_SEEDS=db
+      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
+    volumes:
+      - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
 volumes:
   workspace:
     name: ${WORKSPACE_DOCKER_MOUNT}

temporal/dynamicconfig/development.yaml

@@ -0,0 +1,33 @@
+frontend.enableClientVersionCheck:
+  - value: true
+    constraints: {}
+history.persistenceMaxQPS:
+  - value: 3000
+    constraints: {}
+frontend.persistenceMaxQPS:
+  - value: 3000
+    constraints: {}
+frontend.historyMgrNumConns:
+  - value: 30
+    constraints: {}
+frontend.throttledLogRPS:
+  - value: 20
+    constraints: {}
+history.historyMgrNumConns:
+  - value: 50
+    constraints: {}
+system.advancedVisibilityWritingMode:
+  - value: "off"
+    constraints: {}
+history.defaultActivityRetryPolicy:
+  - value:
+      InitialIntervalInSeconds: 1
+      MaximumIntervalCoefficient: 100.0
+      BackoffCoefficient: 2.0
+      MaximumAttempts: 0
+history.defaultWorkflowRetryPolicy:
+  - value:
+      InitialIntervalInSeconds: 1
+      MaximumIntervalCoefficient: 100.0
+      BackoffCoefficient: 2.0
+      MaximumAttempts: 0