Merge pull request #4484 from EthanYuan/fix-rich-indexer-partial-query

Fix rich indexer `partial` query by args performance issue
nervosnetwork · Jun 19, 2024 · 7db9023 · 7db9023
2 parents a02f2db + 0750c26
commit 7db9023
Show file tree

Hide file tree

Showing 7 changed files with 125 additions and 23 deletions.
diff --git a/util/indexer-sync/src/lib.rs b/util/indexer-sync/src/lib.rs
@@ -161,7 +161,9 @@ impl IndexerSyncService {
                                     block.number(),
                                     block.hash()
                                 );
-                                indexer.append(&block).expect("append block should be OK");
+                                if let Err(e) = indexer.append(&block) {
+                                    error!("Failed to append block: {}. Will attempt to retry.", e);
+                                }
                             } else {
                                 info!(
                                     "{} rollback {}, {}",
@@ -178,7 +180,11 @@ impl IndexerSyncService {
                     }
                 }
                 Ok(None) => match self.get_block_by_number(0) {
-                    Some(block) => indexer.append(&block).expect("append block should be OK"),
+                    Some(block) => {
+                        if let Err(e) = indexer.append(&block) {
+                            error!("Failed to append block: {}. Will attempt to retry.", e);
+                        }
+                    }
                     None => {
                         error!("CKB node returns an empty genesis block");
                         break;

diff --git a/util/rich-indexer/README.md b/util/rich-indexer/README.md
@@ -13,6 +13,7 @@ In order to run a CKB node with the Rich-Indexer enabled, it is recommended to a
 
 - Processor: 4 core
 - RAM: 8 GB
+- Disk: Solid State Drive (SSD) to ensure performance
 
 ## Quick Start
 
@@ -57,7 +58,7 @@ Note that CKB starting options `--indexer` and `--rich-indexer` can only be used
 
 ## Enabling Rich Indexer with PostgreSQL
 
-To enable PostgreSQL, you must first set up a functional PostgreSQL service on your own. Please refer to [Server Administration](https://www.postgresql.org/docs/16/admin.html) for guidance. It is recommended to install version 12 or above.
+To enable PostgreSQL, you must first set up a functional PostgreSQL service on your own. Please refer to [Server Administration](https://www.postgresql.org/docs/16/admin.html) for guidance. It is recommended to install version 16 or above.
 
 For hardware with 4 cores and 8 GB of RAM, it is recommended to make the following two configuration parameter adjustments in PostgreSQL to achieve optimal query performance.
 

diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs
@@ -169,9 +169,18 @@ impl AsyncRichIndexerHandle {
                     .bind(search_key.script.args.as_bytes())
                     .bind(get_binary_upper_boundary(search_key.script.args.as_bytes()));
             }
-            Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+            Some(IndexerSearchMode::Exact) => {
                 query = query.bind(search_key.script.args.as_bytes());
             }
+            Some(IndexerSearchMode::Partial) => match self.store.db_driver {
+                DBDriver::Postgres => {
+                    let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args);
+                    query = query.bind(new_args);
+                }
+                DBDriver::Sqlite => {
+                    query = query.bind(search_key.script.args.as_bytes());
+                }
+            },
         }
         if let Some(filter) = search_key.filter.as_ref() {
             if let Some(script) = filter.script.as_ref() {
@@ -190,9 +199,18 @@ impl AsyncRichIndexerHandle {
                             .bind(data.as_bytes())
                             .bind(get_binary_upper_boundary(data.as_bytes()));
                     }
-                    Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+                    Some(IndexerSearchMode::Exact) => {
                         query = query.bind(data.as_bytes());
                     }
+                    Some(IndexerSearchMode::Partial) => match self.store.db_driver {
+                        DBDriver::Postgres => {
+                            let new_data = escape_and_wrap_for_postgres_like(data);
+                            query = query.bind(new_data);
+                        }
+                        DBDriver::Sqlite => {
+                            query = query.bind(data.as_bytes());
+                        }
+                    },
                 }
             }
         }

diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs
@@ -129,9 +129,18 @@ impl AsyncRichIndexerHandle {
                     .bind(search_key.script.args.as_bytes())
                     .bind(get_binary_upper_boundary(search_key.script.args.as_bytes()));
             }
-            Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+            Some(IndexerSearchMode::Exact) => {
                 query = query.bind(search_key.script.args.as_bytes());
             }
+            Some(IndexerSearchMode::Partial) => match self.store.db_driver {
+                DBDriver::Postgres => {
+                    let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args);
+                    query = query.bind(new_args);
+                }
+                DBDriver::Sqlite => {
+                    query = query.bind(search_key.script.args.as_bytes());
+                }
+            },
         }
         if let Some(filter) = search_key.filter.as_ref() {
             if let Some(script) = filter.script.as_ref() {
@@ -150,9 +159,18 @@ impl AsyncRichIndexerHandle {
                             .bind(data.as_bytes())
                             .bind(get_binary_upper_boundary(data.as_bytes()));
                     }
-                    Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+                    Some(IndexerSearchMode::Exact) => {
                         query = query.bind(data.as_bytes());
                     }
+                    Some(IndexerSearchMode::Partial) => match self.store.db_driver {
+                        DBDriver::Postgres => {
+                            let new_data = escape_and_wrap_for_postgres_like(data);
+                            query = query.bind(new_data);
+                        }
+                        DBDriver::Sqlite => {
+                            query = query.bind(data.as_bytes());
+                        }
+                    },
                 }
             }
         }

diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs
@@ -194,9 +194,18 @@ pub async fn get_tx_with_cell(
                     .bind(search_key.script.args.as_bytes())
                     .bind(get_binary_upper_boundary(search_key.script.args.as_bytes()));
             }
-            Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+            Some(IndexerSearchMode::Exact) => {
                 query = query.bind(search_key.script.args.as_bytes());
             }
+            Some(IndexerSearchMode::Partial) => match db_driver {
+                DBDriver::Postgres => {
+                    let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args);
+                    query = query.bind(new_args);
+                }
+                DBDriver::Sqlite => {
+                    query = query.bind(search_key.script.args.as_bytes());
+                }
+            },
         }
         if let Some(filter) = search_key.filter.as_ref() {
             if let Some(script) = filter.script.as_ref() {
@@ -215,9 +224,18 @@ pub async fn get_tx_with_cell(
                             .bind(data.as_bytes())
                             .bind(get_binary_upper_boundary(data.as_bytes()));
                     }
-                    Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+                    Some(IndexerSearchMode::Exact) => {
                         query = query.bind(data.as_bytes());
                     }
+                    Some(IndexerSearchMode::Partial) => match db_driver {
+                        DBDriver::Postgres => {
+                            let new_data = escape_and_wrap_for_postgres_like(data);
+                            query = query.bind(new_data);
+                        }
+                        DBDriver::Sqlite => {
+                            query = query.bind(data.as_bytes());
+                        }
+                    },
                 }
             }
         }
@@ -326,9 +344,18 @@ pub async fn get_tx_with_cells(
                     .bind(search_key.script.args.as_bytes())
                     .bind(get_binary_upper_boundary(search_key.script.args.as_bytes()));
             }
-            Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+            Some(IndexerSearchMode::Exact) => {
                 query = query.bind(search_key.script.args.as_bytes());
             }
+            Some(IndexerSearchMode::Partial) => match db_driver {
+                DBDriver::Postgres => {
+                    let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args);
+                    query = query.bind(new_args);
+                }
+                DBDriver::Sqlite => {
+                    query = query.bind(search_key.script.args.as_bytes());
+                }
+            },
         }
         if let Some(filter) = search_key.filter.as_ref() {
             if let Some(script) = filter.script.as_ref() {
@@ -347,9 +374,18 @@ pub async fn get_tx_with_cells(
                             .bind(data.as_bytes())
                             .bind(get_binary_upper_boundary(data.as_bytes()));
                     }
-                    Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => {
+                    Some(IndexerSearchMode::Exact) => {
                         query = query.bind(data.as_bytes());
                     }
+                    Some(IndexerSearchMode::Partial) => match db_driver {
+                        DBDriver::Postgres => {
+                            let new_data = escape_and_wrap_for_postgres_like(data);
+                            query = query.bind(new_data);
+                        }
+                        DBDriver::Sqlite => {
+                            query = query.bind(data.as_bytes());
+                        }
+                    },
                 }
             }
         }
@@ -566,10 +602,7 @@ fn build_filter(
                 Some(IndexerSearchMode::Partial) => {
                     match db_driver {
                         DBDriver::Postgres => {
-                            query_builder.and_where(format!(
-                                "position(${} in output.data) > 0",
-                                param_index
-                            ));
+                            query_builder.and_where(format!("output.data LIKE ${}", param_index));
                         }
                         DBDriver::Sqlite => {
                             query_builder

diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs
@@ -9,7 +9,7 @@ use ckb_app_config::DBDriver;
 use ckb_indexer_sync::{Error, Pool};
 use ckb_jsonrpc_types::{
     IndexerRange, IndexerScriptType, IndexerSearchKey, IndexerSearchKeyFilter, IndexerSearchMode,
-    IndexerTip,
+    IndexerTip, JsonBytes,
 };
 use ckb_types::H256;
 use num_bigint::BigUint;
@@ -99,7 +99,7 @@ fn build_query_script_sql(
         Some(IndexerSearchMode::Partial) => {
             match db_driver {
                 DBDriver::Postgres => {
-                    query_builder.and_where(format!("position(${} in args) > 0", param_index));
+                    query_builder.and_where(format!("args LIKE ${}", param_index));
                 }
                 DBDriver::Sqlite => {
                     query_builder.and_where(format!("instr(args, ${}) > 0", param_index));
@@ -140,7 +140,7 @@ fn build_query_script_id_sql(
         Some(IndexerSearchMode::Partial) => {
             match db_driver {
                 DBDriver::Postgres => {
-                    query_builder.and_where(format!("position(${} in args) > 0", param_index));
+                    query_builder.and_where(format!("args LIKE ${}", param_index));
                 }
                 DBDriver::Sqlite => {
                     query_builder.and_where(format!("instr(args, ${}) > 0", param_index));
@@ -229,10 +229,7 @@ fn build_cell_filter(
                 Some(IndexerSearchMode::Partial) => {
                     match db_driver {
                         DBDriver::Postgres => {
-                            query_builder.and_where(format!(
-                                "position(${} in output.data) > 0",
-                                param_index
-                            ));
+                            query_builder.and_where(format!("output.data LIKE ${}", param_index));
                         }
                         DBDriver::Sqlite => {
                             query_builder
@@ -316,6 +313,35 @@ pub(crate) fn convert_max_values_in_search_filter(
     })
 }
 
+/// Escapes special characters and wraps data with '%' for PostgreSQL LIKE queries.
+///
+/// This function escapes the characters '%', '\' and '_' in the input `JsonBytes` by prefixing them with '\'.
+/// It then wraps the processed data with '%' at both the start and end for use in PostgreSQL LIKE queries.
+/// Note: This function is not suitable for SQLite queries if the data contains NUL characters (0x00),
+/// as SQLite treats NUL as the end of the string.
+fn escape_and_wrap_for_postgres_like(data: &JsonBytes) -> Vec<u8> {
+    // 0x5c is the default escape character '\'
+    // 0x25 is the '%' wildcard
+    // 0x5f is the '_' wildcard
+
+    let mut new_data: Vec<u8> = data
+        .as_bytes()
+        .iter()
+        .flat_map(|&b| {
+            if b == 0x25 || b == 0x5c || b == 0x5f {
+                vec![0x5c, b]
+            } else {
+                vec![b]
+            }
+        })
+        .collect();
+
+    new_data.insert(0, 0x25); // Start with %
+    new_data.push(0x25); // End with %
+
+    new_data
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

diff --git a/util/rich-indexer/src/tests/query.rs b/util/rich-indexer/src/tests/query.rs
@@ -1566,7 +1566,7 @@ async fn output_data_filter_mode_rpc() {
         .unwrap();
     assert_eq!(1, cells.objects.len(),);
 
-    // test get_cells_capacity rpc with output_data Partial search mode
+    // test get_cells_capacity rpc with output_data Prefix search mode
     let cells = rpc
         .get_cells_capacity(IndexerSearchKey {
             script: lock_script11.clone().into(),