Merge branch 'main' into kwannoel/reserve-buffer

Makefile.toml

@@ -801,9 +801,8 @@ script = """
 
 echo "Running $(tput setaf 4)cargo hakari$(tput sgr0) checks and attempting to fix"
 
-# cargo hakari will generate new Cargo.toml regardless whether the original Cargo.toml is correct or not.
-# So we always verify before generate, so as not to change the modified time of Cargo.toml.
-(cargo hakari verify > /dev/null) || cargo hakari generate
+cargo hakari generate --diff --quiet || cargo hakari generate
+cargo hakari verify > /dev/null
 test $? -eq 0 || exit 1
 """
 

ci/scripts/integration-tests.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+while getopts 'c:f:' opt; do
+    case ${opt} in
+        c )
+            case=$OPTARG
+            ;;
+        f )
+            format=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+echo "--- install postgresql"
+sudo yum install -y postgresql
+
+cd integration_tests/scripts
+
+echo "--- case: ${case}, format: ${format}"
+
+echo "--- Rewrite docker compose for protobuf"
+if [ "${format}" == "protobuf" ]; then
+  python3 gen_pb_compose.py ${case} ${format}
+fi
+
+echo "--- Run Demos"
+python3 run_demos.py --case ${case} --format ${format}
+
+echo "--- Check if the ingestion is successful"
+python3 check_data.py ${case}

ci/workflows/integration-tests.yml

@@ -0,0 +1,68 @@
+steps:
+  - label: "Run Demos {{matrix.testcase}} {{matrix.format}}"
+    command: "ci/scripts/integration-tests.sh -c {{matrix.testcase}} -f {{matrix.format}}"
+    timeout_in_minutes: 30
+    matrix:
+      setup:
+        testcase:
+          - "ad-click"
+          - "ad-ctr"
+          - "cdn-metrics"
+          - "clickstream"
+          - "livestream"
+          - "twitter"
+          - "prometheus"
+          - "schema-registry"
+          - "mysql-cdc"
+          - "postgres-cdc"
+          - "mysql-sink"
+          - "postgres-sink"
+          - "iceberg-sink"
+        format:
+          - "json"
+          - "protobuf"
+      adjustments:
+        - with:
+            testcase: "ad-click"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "ad-ctr"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "cdn-metrics"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "clickstream"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "prometheus"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "schema-registry"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "mysql-cdc"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "postgres-cdc"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "mysql-sink"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "postgres-sink"
+            format: "protobuf"
+          skip: true
+        - with:
+            testcase: "iceberg-sink"
+            format: "protobuf"
+          skip: true

ci/workflows/pull-request.yml

@@ -178,7 +178,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
+    timeout_in_minutes: 6
 
   - label: "regress test"
     command: "ci/scripts/regress-test.sh -p ci-dev"

docker/docker-compose.yml

@@ -99,6 +99,8 @@ services:
       - risedev-meta
       - "--max-txn-ops"
       - "999999"
+      - "--max-request-bytes"
+      - "10485760"
       - "--auto-compaction-mode"
       - periodic
       - "--auto-compaction-retention"

e2e_test/batch/basic/logical_view.slt.part

@@ -40,8 +40,7 @@ SELECT * FROM v3;
 statement error
 DROP TABLE t;
 
-# Currently it is allowed to drop a view even it is depended by another view.
-statement ok
+statement error other relation\(s\) depend on it
 DROP VIEW v2;
 
 statement error
@@ -50,6 +49,9 @@ SELECT * from v3;
 statement ok
 DROP VIEW v3;
 
+statement ok
+DROP VIEW v2;
+
 statement error
 SELECT * FROM v3;
 

e2e_test/ddl/alter_rename_relation.slt

@@ -37,6 +37,9 @@ CREATE VIEW v3 AS (SELECT MAX((t.v2).v1) FROM t AS t);
 statement ok
 CREATE VIEW v4 AS (SELECT * FROM t join t as t2 on (t.v1 = t2.v1) ORDER BY t.v1, t2.v1);
 
+statement ok
+CREATE MATERIALIZED VIEW mv_on_v1 AS SELECT * FROM v1;
+
 statement ok
 CREATE index idx ON t(v1);
 
@@ -102,6 +105,11 @@ SHOW CREATE VIEW v5;
 ----
 public.v5 CREATE VIEW v5 AS (SELECT * FROM t_as_1 AS t_as WHERE id = 1)
 
+query TT
+SHOW CREATE MATERIALIZED VIEW mv_on_v1;
+----
+public.mv_on_v1 CREATE MATERIALIZED VIEW mv_on_v1 AS SELECT * FROM v5 AS v1
+
 statement ok
 ALTER INDEX idx RENAME TO idx1;
 
@@ -147,6 +155,12 @@ public.mv4 CREATE MATERIALIZED VIEW mv4 AS SELECT * FROM src1 AS src
 statement ok
 DROP SINK sink1;
 
+statement error other relation\(s\) depend on it
+DROP VIEW v5;
+
+statement ok
+DROP MATERIALIZED VIEW mv_on_v1;
+
 statement ok
 DROP VIEW v5;