Feat: implement change stream reader from GCP SpannerChangeStream V2

docs/modules/components/pages/inputs/gcp_spanner_cdc.adoc

@@ -0,0 +1,148 @@
+= gcp_spanner_cdc
+:type: input
+:status: beta
+:categories: ["Services","GCP"]
+
+
+
+////
+     THIS FILE IS AUTOGENERATED!
+
+     To make changes, edit the corresponding source file under:
+
+     https://github.com/redpanda-data/connect/tree/main/internal/impl/<provider>.
+
+     And:
+
+     https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl
+////
+
+// © 2024 Redpanda Data Inc.
+
+
+component_type_dropdown::[]
+
+
+Creates an input that consumes from a spanner change stream.
+
+Introduced in version 3.43.0.
+
+
+[tabs]
+======
+Common::
++
+--
+
+```yml
+# Common config fields, showing default values
+input:
+  label: ""
+  gcp_spanner_cdc:
+    stream_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id> # No default (required)
+    stream_id: ""
+    partition_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id> # No default (optional)
+    partition_table: "" # No default (optional)
+    use_in_memory_partition: false
+```
+
+--
+Advanced::
++
+--
+
+```yml
+# All config fields, showing default values
+input:
+  label: ""
+  gcp_spanner_cdc:
+    stream_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id> # No default (required)
+    stream_id: ""
+    start_time_epoch: 0
+    partition_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id> # No default (optional)
+    partition_table: "" # No default (optional)
+    use_in_memory_partition: false
+    allowed_mod_types:
+      - INSERT
+      - UPDATE
+      - DELETE
+```
+
+--
+======
+
+== Fields
+
+=== `stream_dsn`
+
+Required field to use to connect to spanner for the change stream.
+
+
+*Type*: `string`
+
+
+```yml
+# Examples
+
+stream_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id>
+```
+
+=== `stream_id`
+
+Required name of the change stream to track.
+
+
+*Type*: `string`
+
+*Default*: `""`
+
+=== `start_time_epoch`
+
+Optional microsecond accurate epoch timestamp to start reading from. If empty time.Now() will be used.
+
+
+*Type*: `int`
+
+*Default*: `0`
+
+=== `partition_dsn`
+
+Field used to set the DSN for the metadata partition table, can be the same as stream_dsn.
+
+
+*Type*: `string`
+
+
+```yml
+# Examples
+
+partition_dsn: projects/<project_id>/instances/<instance_id>/databases/<database_id>
+```
+
+=== `partition_table`
+
+Name of the table to create/use in spanner to track change stream partition metadata.
+
+
+*Type*: `string`
+
+
+=== `use_in_memory_partition`
+
+use an in memory partition table for tracking the partitions.
+
+
+*Type*: `bool`
+
+*Default*: `false`
+
+=== `allowed_mod_types`
+
+Mod types to allow through when reading the change stream, default all.
+
+
+*Type*: `array`
+
+*Default*: `["INSERT","UPDATE","DELETE"]`
+
+

go.mod

@@ -3,9 +3,10 @@ module github.com/redpanda-data/connect/v4
 replace github.com/99designs/keyring => github.com/Jeffail/keyring v1.2.3
 
 require (
-	cloud.google.com/go/aiplatform v1.68.0
+	cloud.google.com/go/aiplatform v1.69.0
 	cloud.google.com/go/bigquery v1.64.0
 	cloud.google.com/go/pubsub v1.45.1
+	cloud.google.com/go/spanner v1.73.0
 	cloud.google.com/go/storage v1.43.0
 	cloud.google.com/go/vertexai v0.12.0
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
@@ -61,7 +62,7 @@ require (
 	github.com/eclipse/paho.mqtt.golang v1.5.0
 	github.com/generikvault/gvalstrings v0.0.0-20180926130504-471f38f0112a
 	github.com/getsentry/sentry-go v0.28.1
-	github.com/go-faker/faker/v4 v4.4.2
+	github.com/go-faker/faker/v4 v4.5.0
 	github.com/go-jose/go-jose/v3 v3.0.3
 	github.com/go-resty/resty/v2 v2.15.3
 	github.com/go-sql-driver/mysql v1.8.1
@@ -115,7 +116,7 @@ require (
 	github.com/smira/go-statsd v1.3.3
 	github.com/snowflakedb/gosnowflake v1.11.0
 	github.com/sourcegraph/conc v0.3.0
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/testcontainers/testcontainers-go/modules/ollama v0.32.0
 	github.com/testcontainers/testcontainers-go/modules/qdrant v0.32.0
 	github.com/tetratelabs/wazero v1.7.3
@@ -141,13 +142,13 @@ require (
 	go.opentelemetry.io/otel/sdk v1.29.0
 	go.opentelemetry.io/otel/trace v1.29.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/crypto v0.28.0
+	golang.org/x/crypto v0.30.0
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
-	golang.org/x/net v0.30.0
-	golang.org/x/sync v0.8.0
-	golang.org/x/text v0.19.0
-	google.golang.org/api v0.205.0
-	google.golang.org/protobuf v1.35.1
+	golang.org/x/net v0.32.0
+	golang.org/x/sync v0.10.0
+	golang.org/x/text v0.21.0
+	google.golang.org/api v0.211.0
+	google.golang.org/protobuf v1.35.2
 	modernc.org/sqlite v1.32.0
 )
 
@@ -156,7 +157,6 @@ require (
 	cloud.google.com/go/longrunning v0.6.2 // indirect
 	cloud.google.com/go/monitoring v1.21.2 // indirect
 	cloud.google.com/go/secretmanager v1.14.2 // indirect
-	cloud.google.com/go/spanner v1.73.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
 	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
@@ -186,8 +186,8 @@ require (
 
 require (
 	cloud.google.com/go v0.116.0 // indirect
-	cloud.google.com/go/auth v0.10.1 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
+	cloud.google.com/go/auth v0.12.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/iam v1.2.2 // indirect
 	cloud.google.com/go/trace v1.11.2 // indirect
@@ -252,8 +252,8 @@ require (
 	github.com/couchbase/gocbcoreps v0.1.3 // indirect
 	github.com/couchbase/goprotostellar v1.0.2 // indirect
 	github.com/couchbaselabs/gocbconnstr/v2 v2.0.0-20240607131231-fb385523de28 // indirect
-	github.com/cpuguy83/dockercfg v0.3.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/danieljoos/wincred v1.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -292,7 +292,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.0 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/handlers v1.5.2 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
@@ -385,11 +385,11 @@ require (
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/testcontainers/testcontainers-go v0.33.0
+	github.com/testcontainers/testcontainers-go v0.34.0
 	github.com/tilinna/z85 v1.0.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.13 // indirect
 	github.com/tklauser/numcpus v0.7.0 // indirect
-	github.com/urfave/cli/v2 v2.27.4
+	github.com/urfave/cli/v2 v2.27.5
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
@@ -406,16 +406,16 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/term v0.25.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/oauth2 v0.24.0
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect
-	google.golang.org/grpc v1.68.0
+	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect
+	google.golang.org/grpc v1.68.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
 	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
@@ -432,6 +432,4 @@ require (
 	modernc.org/token v1.1.0 // indirect
 )
 
-go 1.22.7
-
-toolchain go1.23.0
+go 1.23.2