Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bulk Core/Load CDK: Support for connector-type-specific non-config sp… #45463

Merged
merged 1 commit into from
Sep 16, 2024
Merged

Bulk Core/Load CDK: Support for connector-type-specific non-config sp… #45463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/spec/SpecOperation.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import io.airbyte.cdk.Operation
import io.airbyte.cdk.command.ConfigurationJsonObjectSupplier
import io.airbyte.cdk.output.OutputConsumer
import io.airbyte.protocol.models.v0.ConnectorSpecification
import io.micronaut.context.annotation.DefaultImplementation
import io.micronaut.context.annotation.Requires
import io.micronaut.context.annotation.Value
import jakarta.inject.Singleton
Expand All @@ -15,13 +16,24 @@ import java.net.URI
class SpecOperation(
@Value("\${airbyte.connector.metadata.documentation-url}") val documentationUrl: String,
val configJsonObjectSupplier: ConfigurationJsonObjectSupplier<*>,
val extendSpecification: SpecificationExtender,
val outputConsumer: OutputConsumer,
) : Operation {
override fun execute() {
outputConsumer.accept(
val spec =
ConnectorSpecification()
.withDocumentationUrl(URI.create(documentationUrl))
.withConnectionSpecification(configJsonObjectSupplier.jsonSchema),
)
.withConnectionSpecification(configJsonObjectSupplier.jsonSchema)
outputConsumer.accept(extendSpecification(spec))
}
}

interface SpecificationExtender : (ConnectorSpecification) -> ConnectorSpecification
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL you can do this. that's cool!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes @postamar suggested that (in a different context). It's cleaner than using functions as arguments and allows you to extend/inject as needed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if this may or should be a fun interface. I don't know. I'm not sure it matters either.


@Singleton
@DefaultImplementation
class IdentitySpecificationExtender : SpecificationExtender {
override fun invoke(specification: ConnectorSpecification): ConnectorSpecification {
return specification
}
}
28 changes: 28 additions & 0 deletions airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/spec/DestinationSpecification.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.cdk.spec

import io.airbyte.protocol.models.v0.ConnectorSpecification
import io.airbyte.protocol.models.v0.DestinationSyncMode
import io.micronaut.context.annotation.Replaces
import io.micronaut.context.annotation.Requires
import jakarta.inject.Singleton

@Singleton
@Replaces(IdentitySpecificationExtender::class)
@Requires(env = ["destination"])
class DestinationSpecificationExtender(private val spec: DestinationSpecification) :
SpecificationExtender {
override fun invoke(specification: ConnectorSpecification): ConnectorSpecification {
return specification
.withSupportedDestinationSyncModes(spec.supportedSyncModes)
.withSupportsIncremental(spec.supportsIncremental)
}
}

interface DestinationSpecification {
val supportedSyncModes: List<DestinationSyncMode>
val supportsIncremental: Boolean
}
Loading