Add to public components and generate docs

Signed-off-by: Mihai Todor <[email protected]>

docs/modules/components/pages/inputs/mysql_cdc.adoc

@@ -0,0 +1,281 @@
+= mysql_cdc
+:type: input
+:status: beta
+:categories: ["Services"]
+
+
+
+////
+     THIS FILE IS AUTOGENERATED!
+
+     To make changes, edit the corresponding source file under:
+
+     https://github.com/redpanda-data/connect/tree/main/internal/impl/<provider>.
+
+     And:
+
+     https://github.com/redpanda-data/connect/tree/main/cmd/tools/docs_gen/templates/plugin.adoc.tmpl
+////
+
+// © 2024 Redpanda Data Inc.
+
+
+component_type_dropdown::[]
+
+
+Enables MySQL streaming for RedPanda Connect.
+
+Introduced in version 4.45.0.
+
+
+[tabs]
+======
+Common::
++
+--
+
+```yml
+# Common config fields, showing default values
+input:
+  label: ""
+  mysql_cdc:
+    dsn: user:password@tcp(localhost:3306)/database # No default (required)
+    tables: [] # No default (required)
+    checkpoint_cache: "" # No default (required)
+    checkpoint_key: mysql_binlog_position
+    snapshot_max_batch_size: 1000
+    stream_snapshot: false # No default (required)
+    auto_replay_nacks: true
+    checkpoint_limit: 1024
+    batching:
+      count: 0
+      byte_size: 0
+      period: ""
+      check: ""
+```
+
+--
+Advanced::
++
+--
+
+```yml
+# All config fields, showing default values
+input:
+  label: ""
+  mysql_cdc:
+    dsn: user:password@tcp(localhost:3306)/database # No default (required)
+    tables: [] # No default (required)
+    checkpoint_cache: "" # No default (required)
+    checkpoint_key: mysql_binlog_position
+    snapshot_max_batch_size: 1000
+    stream_snapshot: false # No default (required)
+    auto_replay_nacks: true
+    checkpoint_limit: 1024
+    batching:
+      count: 0
+      byte_size: 0
+      period: ""
+      check: ""
+      processors: [] # No default (optional)
+```
+
+--
+======
+
+== Metadata
+
+This input adds the following metadata fields to each message:
+
+- operation
+- table
+- binlog_position
+
+
+== Fields
+
+=== `dsn`
+
+The DSN of the MySQL database to connect to.
+
+
+*Type*: `string`
+
+
+```yml
+# Examples
+
+dsn: user:password@tcp(localhost:3306)/database
+```
+
+=== `tables`
+
+A list of tables to stream from the database.
+
+
+*Type*: `array`
+
+
+```yml
+# Examples
+
+tables:
+  - table1
+  - table2
+```
+
+=== `checkpoint_cache`
+
+A https://www.docs.redpanda.com/redpanda-connect/components/caches/about[cache resource^] to use for storing the current latest BinLog Position that has been successfully delivered, this allows Redpanda Connect to continue from that BinLog Position upon restart, rather than consume the entire state of the table.
+
+
+*Type*: `string`
+
+
+=== `checkpoint_key`
+
+The key to use to store the snapshot position in `checkpoint_cache`. An alternative key can be provided if multiple CDC inputs share the same cache.
+
+
+*Type*: `string`
+
+*Default*: `"mysql_binlog_position"`
+
+=== `snapshot_max_batch_size`
+
+The maximum number of rows to be streamed in a single batch when taking a snapshot.
+
+
+*Type*: `int`
+
+*Default*: `1000`
+
+=== `stream_snapshot`
+
+If set to true, the connector will query all the existing data as a part of snapshot process. Otherwise, it will start from the current binlog position.
+
+
+*Type*: `bool`
+
+
+=== `auto_replay_nacks`
+
+Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.
+
+
+*Type*: `bool`
+
+*Default*: `true`
+
+=== `checkpoint_limit`
+
+The maximum number of messages that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level. Any given BinLog Position will not be acknowledged unless all messages under that offset are delivered in order to preserve at least once delivery guarantees.
+
+
+*Type*: `int`
+
+*Default*: `1024`
+
+=== `batching`
+
+Allows you to configure a xref:configuration:batching.adoc[batching policy].
+
+
+*Type*: `object`
+
+
+```yml
+# Examples
+
+batching:
+  byte_size: 5000
+  count: 0
+  period: 1s
+
+batching:
+  count: 10
+  period: 1s
+
+batching:
+  check: this.contains("END BATCH")
+  count: 0
+  period: 1m
+```
+
+=== `batching.count`
+
+A number of messages at which the batch should be flushed. If `0` disables count based batching.
+
+
+*Type*: `int`
+
+*Default*: `0`
+
+=== `batching.byte_size`
+
+An amount of bytes at which the batch should be flushed. If `0` disables size based batching.
+
+
+*Type*: `int`
+
+*Default*: `0`
+
+=== `batching.period`
+
+A period in which an incomplete batch should be flushed regardless of its size.
+
+
+*Type*: `string`
+
+*Default*: `""`
+
+```yml
+# Examples
+
+period: 1s
+
+period: 1m
+
+period: 500ms
+```
+
+=== `batching.check`
+
+A xref:guides:bloblang/about.adoc[Bloblang query] that should return a boolean value indicating whether a message should end a batch.
+
+
+*Type*: `string`
+
+*Default*: `""`
+
+```yml
+# Examples
+
+check: this.type == "end_of_transaction"
+```
+
+=== `batching.processors`
+
+A list of xref:components:processors/about.adoc[processors] to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.
+
+
+*Type*: `array`
+
+
+```yml
+# Examples
+
+processors:
+  - archive:
+      format: concatenate
+
+processors:
+  - archive:
+      format: lines
+
+processors:
+  - archive:
+      format: json_array
+```
+
+

internal/impl/mysql/input_mysql_stream.go

@@ -43,6 +43,9 @@ const (
 )
 
 var mysqlStreamConfigSpec = service.NewConfigSpec().
+	Beta().
+	Categories("Services").
+	Version("4.45.0").
 	Summary("Enables MySQL streaming for RedPanda Connect.").
 	Description(`
 == Metadata

internal/plugins/info.csv

@@ -131,6 +131,7 @@ mqtt                      ,output    ,mqtt                      ,4.37.0  ,certif
 msgpack                   ,processor ,msgpack                   ,3.59.0  ,community  ,n          ,n     ,n
 multilevel                ,cache     ,Multilevel                ,0.0.0   ,certified  ,n          ,y     ,y
 mutation                  ,processor ,mutation                  ,4.5.0   ,certified  ,n          ,y     ,y
+mysql_cdc                 ,input     ,mysql_cdc                 ,4.45.0  ,enterprise ,n          ,y     ,y
 nanomsg                   ,input     ,nanomsg                   ,0.0.0   ,community  ,n          ,n     ,n
 nanomsg                   ,output    ,nanomsg                   ,0.0.0   ,community  ,n          ,n     ,n
 nats                      ,input     ,NATS                      ,0.0.0   ,certified  ,n          ,y     ,y
@@ -170,7 +171,7 @@ parquet                   ,processor ,parquet                   ,3.62.0  ,commun
 parquet_decode            ,processor ,parquet_decode            ,4.4.0   ,certified  ,n          ,y     ,y
 parquet_encode            ,processor ,parquet_encode            ,4.4.0   ,certified  ,n          ,y     ,y
 parse_log                 ,processor ,parse_log                 ,0.0.0   ,community  ,n          ,y     ,y
-pg_stream                 ,input     ,pg_stream                 ,0.0.0   ,enterprise ,y          ,y     ,y
+pg_stream                 ,input     ,pg_stream                 ,4.43.0  ,enterprise ,y          ,y     ,y
 pinecone                  ,output    ,pinecone                  ,4.31.0  ,certified  ,n          ,y     ,y
 postgres_cdc              ,input     ,postgres_cdc              ,4.43.0  ,enterprise ,n          ,y     ,y
 processors                ,processor ,processors                ,0.0.0   ,certified  ,n          ,y     ,y

public/components/all/package.go

@@ -21,6 +21,7 @@ import (
 	_ "github.com/redpanda-data/connect/v4/public/components/cohere"
 	_ "github.com/redpanda-data/connect/v4/public/components/gcp/enterprise"
 	_ "github.com/redpanda-data/connect/v4/public/components/kafka/enterprise"
+	_ "github.com/redpanda-data/connect/v4/public/components/mysql"
 	_ "github.com/redpanda-data/connect/v4/public/components/ollama"
 	_ "github.com/redpanda-data/connect/v4/public/components/openai"
 	_ "github.com/redpanda-data/connect/v4/public/components/postgresql"

public/components/cloud/package.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/redpanda-data/connect/v4/public/components/memcached"
 	_ "github.com/redpanda-data/connect/v4/public/components/mqtt"
 	_ "github.com/redpanda-data/connect/v4/public/components/msgpack"
+	_ "github.com/redpanda-data/connect/v4/public/components/mysql"
 	_ "github.com/redpanda-data/connect/v4/public/components/nats"
 	_ "github.com/redpanda-data/connect/v4/public/components/openai"
 	_ "github.com/redpanda-data/connect/v4/public/components/opensearch"

public/components/mysql/package.go

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2024 Redpanda Data, Inc.
+ *
+ * Licensed as a Redpanda Enterprise file under the Redpanda Community
+ * License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md
+ */
+
+package mysql
+
+import (
+	// Bring in the internal plugin definitions.
+	_ "github.com/redpanda-data/connect/v4/internal/impl/mysql"
+)