Merge branch 'master' into fleetio-source-connector-cursor-pagination

.github/labeler.yml

@@ -11,10 +11,6 @@ area/documentation:
   - docs/*
   - docs/**/*
 
-area/octavia-cli:
-  - octavia-cli/*
-  - octavia-cli/**/*
-
 CDK:
   - airbyte-cdk/*
   - airbyte-cdk/**/*

.github/workflows/gradle.yml

@@ -1,4 +1,4 @@
-name: Airbyte CI - Repository Health Check
+name: Connector Ops CI - Gradle Check
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -20,47 +20,18 @@ on:
       - synchronize
 
 jobs:
-  # In case of self-hosted EC2 errors, remove this block.
-  start-check-runner:
-    name: Start EC2 Runner
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Start AWS Runner
-        id: start-ec2-runner
-        uses: ./.github/actions/start-aws-runner
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          # Use a beefier instance type than the default c5.2xlarge, but with the same per-core cost.
-          # When gradle runs on this instance, it will use up all the available cores anyway.
-          # There should be little to no difference in total cost, however the job latency will be improved.
-          # At the time of this writing, the latency doesn't improve much beyond this instance size (approx 5 minutes).
-          # This is largely thanks to the gradle cache.
-          ec2-instance-type: "c5.4xlarge"
-          github-token: ${{ env.PAT }}
-
   run-check:
-    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-check-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner
+    # The gradle check task which we will run is embarrassingly parallelizable.
+    # We therefore run this on a machine with a maximum number of cores.
+    # We pay per time and per core, so there should be little difference in total cost.
+    # The latency overhead of setting up gradle prior to running the actual task adds up to about a minute.
+    runs-on: connector-test-xxlarge
     name: Gradle Check
     timeout-minutes: 30
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
-
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
+      # IMPORTANT! This is necessary to make sure that a status is reported on the PR
       # even if the workflow is skipped. If we used github actions filters, the workflow
       # would not be reported as skipped, but instead would be forever pending.
       #
@@ -106,39 +77,6 @@ jobs:
           # TODO: be able to remove the skipSlowTests property
           arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true
 
-  # In case of self-hosted EC2 errors, remove this block.
-  stop-check-runner:
-    name: Stop EC2 Runner
-    timeout-minutes: 10
-    needs:
-      - start-check-runner # required to get output from the start-runner job
-      - run-check # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
-    # Thus, we check for skipped here.
-    if: ${{ always() && needs.start-check-runner.result != 'skipped'}}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Checkout Airbyte
-        uses: actions/checkout@v3
-      - name: Check PAT rate limits
-        run: |
-          ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
-      - name: Stop EC2 runner
-        uses: supertopher/[email protected]
-        with:
-          mode: stop
-          github-token: ${{ env.PAT }}
-          label: ${{ needs.start-check-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-check-runner.outputs.ec2-instance-id }}
-
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure
     runs-on: ubuntu-latest

.github/workflows/legacy-test-command.yml

@@ -113,9 +113,6 @@ jobs:
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
-      - name: Build Java CDK Snapshot if Needed
-        # If a snapshot version is specified for the Java CDK, build publish locally. Otherwise, do nothing.
-        run: ./gradlew :airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded
       - name: Test ${{ github.event.inputs.connector }}
         id: test
         env:

.github/workflows/publish-cdk-command-manually.yml

@@ -73,7 +73,7 @@ jobs:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
       - name: Build CDK Package
-        run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: (cd airbyte-cdk/python; ./gradlew --no-daemon --no-build-cache :build)
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/[email protected]

.github/workflows/publish-java-cdk-command.yml

@@ -122,9 +122,6 @@ jobs:
         run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished
       - name: Build Java CDK
         run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build
-      - name: Publish Java Modules to MavenLocal (Dry-Run)
-        if: ${{ !(env.DRY_RUN == 'false') }}
-        run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publishToMavenLocal
       - name: Upload jars as artifacts
         if: ${{ !(env.DRY_RUN == 'false') }}
         uses: actions/upload-artifact@v2

.github/workflows/release-airbyte-os.yml

@@ -198,29 +198,6 @@ jobs:
           PART_TO_BUMP: ${{ github.event.inputs.partToBump }}
         run: ./tools/bin/release_version_octavia.sh
 
-      - name: Publish Python Package to test.pypi.org
-        if: github.event.inputs.skip-publish-test != 'true'
-        uses: mariamrf/[email protected]
-        with:
-          # specify the same version as in ~/.python-version
-          python_version: "3.10"
-          pip_version: "23.2"
-          subdir: "octavia-cli/"
-        env:
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-          TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/"
-      - name: Publish Python Package
-        uses: mariamrf/[email protected]
-        with:
-          # specify the same version as in ~/.python-version
-          python_version: "3.10"
-          pip_version: "23.2"
-          subdir: "octavia-cli/"
-        env:
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-
   # In case of self-hosted EC2 errors, remove this block.
   stop-release-airbyte-runner:
     name: "Release Airbyte: Stop EC2 Runner"

README.md

@@ -66,7 +66,7 @@ See the [LICENSE](docs/project-overview/licenses/) file for licensing informatio
 
 ### Thank You
 
-Airbyte would not be possible without the support and assistance of other open-source tools and companies. Visit our [thank you page](THANK-YOU.md) to learn more about how we build Airbyte.
+Airbyte would not be possible without the support and assistance of other open-source tools and companies! Visit our [thank you page](THANK-YOU.md) to learn more about how we build Airbyte.
 
 <a href="https://github.com/airbytehq/airbyte/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=airbytehq/airbyte"/>

airbyte-cdk/java/airbyte-cdk/README.md

@@ -166,6 +166,8 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.21.1  | 2024-02-20 | [\#35199](https://github.com/airbytehq/airbyte/pull/35199) | Add thread names to the logs.                                                                                                                                  |
+| 0.21.0  | 2024-02-16 | [\#35314](https://github.com/airbytehq/airbyte/pull/35314) | Delete S3StreamCopier classes. These have been superseded by the async destinations framework.                                                                 |
 | 0.20.9  | 2024-02-15 | [\#35240](https://github.com/airbytehq/airbyte/pull/35240) | Make state emission to platform inside state manager itself.                                                                                                   |
 | 0.20.8  | 2024-02-15 | [\#35285](https://github.com/airbytehq/airbyte/pull/35285) | Improve blobstore module structure.                                                                                                                            |
 | 0.20.7  | 2024-02-13 | [\#35236](https://github.com/airbytehq/airbyte/pull/35236) | output logs to files in addition to stdout when running tests                                                                                                  |

airbyte-cdk/java/airbyte-cdk/build.gradle

@@ -1,17 +1,11 @@
 allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
-    apply plugin: 'airbyte-java-cdk'
-    apply plugin: 'airbyte-integration-test-java'
-    apply plugin: 'airbyte-performance-test-java'
     apply plugin: 'java-test-fixtures'
 
     group 'io.airbyte.cdk'
-    version = getCdkTargetVersion()
-}
 
-subprojects { subproject ->
-    def artifactBaseName = 'airbyte-cdk-' + subproject.name
+    def artifactBaseName = 'airbyte-cdk-' + project.name
     // E.g. airbyte-cdk-core, airbyte-cdk-db-sources, airbyte-cdk-db-destinations, etc.
 
     publishing {
@@ -25,7 +19,7 @@ subprojects { subproject ->
                 groupId = 'io.airbyte.cdk'
                 artifactId = artifactBaseName + '-test-fixtures'
                 version = project.version
-                artifact subproject.tasks.testFixturesJar
+                artifact project.tasks.testFixturesJar
             }
         }
         // This repository is only defined and used in the context of an artifact publishing
@@ -52,69 +46,30 @@ subprojects { subproject ->
 
 description = "Airbyte Connector Development Kit (CDK) for Java."
 
-def recursiveTasks = [
-    'assemble',
-    'build',
-    'integrationTestJava',
-    'publish',
-    'publishToMavenLocal',
-    'test',
-]
-recursiveTasks.each { taskName ->
-    tasks.named(taskName).configure {
-        dependsOn subprojects.collect { it.tasks.named(taskName) }
-    }
-}
-
-// The `publishSnapshotIfNeeded` task will be a no-op if CDK version does not end with '-SNAPSHOT'.
-task publishSnapshotIfNeeded {}
-
-if (version.endsWith("-SNAPSHOT")) {
-    logger.lifecycle("Version ${version} ends with '-SNAPSHOT'. Enqueing 'publishToMavenLocal'...")
-    publishSnapshotIfNeeded.dependsOn publishToMavenLocal
-} else {
-    // Uncomment as needed for debugging:
-    // logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.")
-}
-
-task assertCdkVersionNotPublished {
+tasks.register('assertCdkVersionNotPublished') {
     doLast {
-        def checkGroupId = "io.airbyte.cdk"
-        def checkArtifactId = "airbyte-cdk-core"
-        def checkVersion = getCdkTargetVersion()
-        def repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
-        def groupIdUrl = checkGroupId.replace('.', '/')
-        def artifactUrl = "${repoUrl}/${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom"
+        var props = new Properties()
+        file("core/src/main/resources/version.properties").withInputStream(props::load)
 
-        def connection = artifactUrl.toURL().openConnection() as HttpURLConnection
+        var checkGroupId = "io.airbyte.cdk"
+        var checkArtifactId = "airbyte-cdk-core"
+        var checkVersion = props.getProperty('version') ?: 'undefined'
+        var repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars"
+        var groupIdUrl = "${repoUrl}/${checkGroupId.replace('.', '/')}"
+        var artifactUrl = "${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom"
+
+        var connection = artifactUrl.toURL().openConnection() as HttpURLConnection
         connection.setRequestMethod("HEAD")
         connection.connect()
 
-        def responseCode = connection.getResponseCode()
+        var responseCode = connection.getResponseCode()
 
         if (responseCode == 200) {
-            throw new GradleException("Assert failed. Java CDK '${checkVersion}' already published at: ${artifactUrl}")
+            throw new GradleException("Java CDK '${checkVersion}' already published at ${groupIdUrl}")
         } else if (responseCode == 404) {
-            logger.lifecycle(
-                "Assert succeeded. Version ${checkVersion} of ${checkArtifactId} has not been published. " +
-                "Checked: ${artifactUrl}"
-            )
+            logger.lifecycle("Java CDK '${checkVersion}' not yet published at ${groupIdUrl}.")
         } else {
-            logger.error("Received unexpected HTTP response code ${responseCode}. Ensure the repository is accessible.")
-            throw new GradleException("Error during assertion. Received unexpected HTTP response code ${responseCode}.")
+            throw new GradleException("Unexpected HTTP response code ${responseCode} from ${artifactUrl}, expected either 200 or 404.")
         }
     }
 }
-
-def cleanLocalCache = tasks.register('cleanLocalCache') {
-    def userHome = System.getProperty("user.home")
-    doLast {
-        delete '.gradle'
-        delete '${userHome}/.m2/repository/io/airbyte/'
-        delete '${userHome}/.gradle/caches/modules-2/files-2.1/io.airbyte.cdk/'
-    }
-}
-cleanLocalCache.configure {
-    dependsOn tasks.named('clean')
-    dependsOn subprojects.collect { it.tasks.named('clean') }
-}

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json

@@ -7,7 +7,7 @@
     },
     "message": {
       "$resolver": "pattern",
-      "pattern": "%level %C{1.}(%M):%L %m",
+      "pattern": "%level %thread %C{1.}(%M):%L %m",
       "stringified": true
     },
     "stack_trace": {

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

@@ -1 +1 @@
-version=0.20.9
+version=0.21.1

airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java

@@ -88,7 +88,8 @@ public void testAirbyteLogMessageFormat() throws java.io.IOException {
     // validate that the message inside AirbyteLogMessage matches the pattern.
     // pattern to check for is: LOG_LEVEL className(methodName):LineNumber logMessage
     final String connectorLogMessageRegex =
-        "^INFO [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$";
+        String.format("^INFO %s [\\w+.]*.AirbyteLogMessageTemplateTest\\(testAirbyteLogMessageFormat\\):\\d+ hello$",
+            Pattern.compile(Thread.currentThread().getName()));
     final Pattern pattern = Pattern.compile(connectorLogMessageRegex);
 
     final Matcher matcher = pattern.matcher(connectorLogMessage);

airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml

@@ -2,7 +2,7 @@
 <Configuration status="INFO">
     <Properties>
         <!-- Set the LOG_SCRUB_PATTERN env var to a regex pattern to scrub the log messages of secrets or any other unwanted content. -->
-        <Property name="jvm-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%highlight{%p}`%C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
+        <Property name="jvm-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0} %thread `%highlight{%p}`%C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
         <!--Logs the timestamp and log_source/application name in the beginning of the line if it exists with a > separator, and then always the rest of the line.-->
         <Property name="container-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
         <!-- Always log INFO by default. -->