feat(storage): limit hummock write based on LSM tree stats (risingwav…

…elabs#8383)
Little-Wallace · Mar 12, 2023 · b7c46d4 · b7c46d4
1 parent 4b49428
commit b7c46d4
Show file tree

Hide file tree

Showing 21 changed files with 665 additions and 50 deletions.
diff --git a/dashboard/proto/gen/hummock.ts b/dashboard/proto/gen/hummock.ts
diff --git a/dashboard/proto/gen/meta.ts b/dashboard/proto/gen/meta.ts
diff --git a/proto/hummock.proto b/proto/hummock.proto
@@ -519,6 +519,7 @@ message RiseCtlUpdateCompactionConfigRequest {
       uint64 target_file_size_base = 7;
       uint32 compaction_filter_mask = 8;
       uint32 max_sub_compaction = 9;
+      uint64 level0_stop_write_threshold_sub_level_number = 10;
     }
   }
   repeated uint64 compaction_group_ids = 1;
@@ -602,6 +603,8 @@ message CompactionConfig {
   uint32 max_sub_compaction = 12;
   uint64 max_space_reclaim_bytes = 13;
   bool split_by_state_table = 14;
+  // soft limit for max number of sub level number
+  uint64 level0_stop_write_threshold_sub_level_number = 15;
 }
 
 message TableStats {
@@ -614,3 +617,12 @@ message HummockVersionStats {
   uint64 hummock_version_id = 1;
   map<uint32, TableStats> table_stats = 2;
 }
+
+message WriteLimits {
+  message WriteLimit {
+    repeated uint32 table_ids = 1;
+    string reason = 2;
+  }
+  // < compaction group id, write limit info >
+  map<uint64, WriteLimit> write_limits = 1;
+}