Skip to content

Commit

Permalink
perf(streaming): add the missed read prefix hint on state table (risi…
Browse files Browse the repository at this point in the history 
…ngwavelabs#8545)
  • Loading branch information
@st1page
st1page authored Mar 16, 2023
1 parent e61af5a commit 777e836
Show file tree
Hide file tree
Showing 15 changed files with 128 additions and 96 deletions.
4 changes: 4 additions & 0 deletions dashboard/proto/gen/catalog.ts
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions proto/catalog.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ message Table {
repeated int32 value_indices = 20;
string definition = 21;
HandleConflictBehavior handle_pk_conflict_behavior = 22;
// Anticipated read prefix pattern (number of fields) for the table, which can be utilized
// for implementing the table's bloom filter or other storage optimization techniques.
uint32 read_prefix_len_hint = 23;
repeated int32 watermark_indices = 24;
repeated int32 dist_key_in_pk = 25;
Expand Down
10 changes: 5 additions & 5 deletions src/frontend/planner_test/tests/testdata/nexmark.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -784,8 +784,8 @@
BatchPlanNode
Table 0 { columns: [$expr1, count, count filter((bid_price < 10000:Int32)), count filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count filter((bid_price >= 1000000:Int32)), count(distinct bid_bidder), count(distinct bid_bidder) filter((bid_price < 10000:Int32)), count(distinct bid_bidder) filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count(distinct bid_bidder) filter((bid_price >= 1000000:Int32)), count(distinct bid_auction), count(distinct bid_auction) filter((bid_price < 10000:Int32)), count(distinct bid_auction) filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count(distinct bid_auction) filter((bid_price >= 1000000:Int32))], primary key: [$0 ASC], value indices: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], distribution key: [0], read pk prefix len hint: 1 }
Table 1 { columns: [$expr1, bid_bidder, count_for_agg_call_4, count_for_agg_call_5, count_for_agg_call_6, count_for_agg_call_7], primary key: [$0 ASC, $1 ASC], value indices: [2, 3, 4, 5], distribution key: [0], read pk prefix len hint: 0 }
Table 2 { columns: [$expr1, bid_auction, count_for_agg_call_8, count_for_agg_call_9, count_for_agg_call_10, count_for_agg_call_11], primary key: [$0 ASC, $1 ASC], value indices: [2, 3, 4, 5], distribution key: [0], read pk prefix len hint: 0 }
Table 1 { columns: [$expr1, bid_bidder, count_for_agg_call_4, count_for_agg_call_5, count_for_agg_call_6, count_for_agg_call_7], primary key: [$0 ASC, $1 ASC], value indices: [2, 3, 4, 5], distribution key: [0], read pk prefix len hint: 2 }
Table 2 { columns: [$expr1, bid_auction, count_for_agg_call_8, count_for_agg_call_9, count_for_agg_call_10, count_for_agg_call_11], primary key: [$0 ASC, $1 ASC], value indices: [2, 3, 4, 5], distribution key: [0], read pk prefix len hint: 2 }
Table 4294967294 { columns: [day, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], primary key: [$0 ASC], value indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], distribution key: [0], read pk prefix len hint: 1 }
- id: nexmark_q16
before:
Expand Down Expand Up @@ -840,8 +840,8 @@
BatchPlanNode
Table 0 { columns: [bid_channel, $expr1, max($expr2), count, count filter((bid_price < 10000:Int32)), count filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count filter((bid_price >= 1000000:Int32)), count(distinct bid_bidder), count(distinct bid_bidder) filter((bid_price < 10000:Int32)), count(distinct bid_bidder) filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count(distinct bid_bidder) filter((bid_price >= 1000000:Int32)), count(distinct bid_auction), count(distinct bid_auction) filter((bid_price < 10000:Int32)), count(distinct bid_auction) filter((bid_price >= 10000:Int32) AND (bid_price < 1000000:Int32)), count(distinct bid_auction) filter((bid_price >= 1000000:Int32))], primary key: [$0 ASC, $1 ASC], value indices: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], distribution key: [0, 1], read pk prefix len hint: 2 }
Table 1 { columns: [bid_channel, $expr1, bid_bidder, count_for_agg_call_5, count_for_agg_call_6, count_for_agg_call_7, count_for_agg_call_8], primary key: [$0 ASC, $1 ASC, $2 ASC], value indices: [3, 4, 5, 6], distribution key: [0, 1], read pk prefix len hint: 0 }
Table 2 { columns: [bid_channel, $expr1, bid_auction, count_for_agg_call_9, count_for_agg_call_10, count_for_agg_call_11, count_for_agg_call_12], primary key: [$0 ASC, $1 ASC, $2 ASC], value indices: [3, 4, 5, 6], distribution key: [0, 1], read pk prefix len hint: 0 }
Table 1 { columns: [bid_channel, $expr1, bid_bidder, count_for_agg_call_5, count_for_agg_call_6, count_for_agg_call_7, count_for_agg_call_8], primary key: [$0 ASC, $1 ASC, $2 ASC], value indices: [3, 4, 5, 6], distribution key: [0, 1], read pk prefix len hint: 3 }
Table 2 { columns: [bid_channel, $expr1, bid_auction, count_for_agg_call_9, count_for_agg_call_10, count_for_agg_call_11, count_for_agg_call_12], primary key: [$0 ASC, $1 ASC, $2 ASC], value indices: [3, 4, 5, 6], distribution key: [0, 1], read pk prefix len hint: 3 }
Table 4294967294 { columns: [channel, day, minute, total_bids, rank1_bids, rank2_bids, rank3_bids, total_bidders, rank1_bidders, rank2_bidders, rank3_bidders, total_auctions, rank1_auctions, rank2_auctions, rank3_auctions], primary key: [$0 ASC, $1 ASC], value indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], distribution key: [0, 1], read pk prefix len hint: 2 }
- id: nexmark_q17
before:
Expand Down Expand Up @@ -1206,7 +1206,7 @@
Upstream
BatchPlanNode
Table 0 { columns: [auction_id, auction_item_name, count(bid_auction)], primary key: [$2 ASC, $0 ASC, $1 ASC], value indices: [0, 1, 2], distribution key: [0], read pk prefix len hint: 0 }
Table 0 { columns: [auction_id, auction_item_name, count(bid_auction)], primary key: [$2 ASC, $0 ASC, $1 ASC], value indices: [0, 1, 2], distribution key: [0], read pk prefix len hint: 1 }
Table 1 { columns: [$expr1], primary key: [], value indices: [0], distribution key: [], read pk prefix len hint: 0 }
Table 2 { columns: [auction_id, auction_item_name, count(bid_auction), count], primary key: [$0 ASC, $1 ASC], value indices: [2, 3], distribution key: [0], read pk prefix len hint: 2 }
Table 3 { columns: [auction_id, auction_item_name], primary key: [$0 ASC], value indices: [0, 1], distribution key: [0], read pk prefix len hint: 1 }
Expand Down
Loading

0 comments on commit 777e836

Please sign in to comment.