diff --git a/docs/metrics.md b/docs/metrics.md index c0bb08a1a0c..61c223f8256 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -57,6 +57,9 @@ the **maximum size of a query result** (in CacheWeight) the **size of the result of successful GraphQL queries** (in CacheWeight) - `query_semaphore_wait_ms` Moving **average of time spent on waiting for postgres query semaphore** +- `query_blocks_behind` +A histogram for how many blocks behind the subgraph head queries are being made at. +This helps inform pruning decisions. - `query_kill_rate` The rate at which the load manager kills queries - `registered_metrics` @@ -66,4 +69,4 @@ The **number of Postgres connections** currently **checked out** - `store_connection_error_count` The **number of Postgres connections errors** - `store_connection_wait_time_ms` -**Average connection wait time** +**Average connection wait time** \ No newline at end of file diff --git a/graph/src/components/graphql.rs b/graph/src/components/graphql.rs index 2e160706989..4a6fcbe5638 100644 --- a/graph/src/components/graphql.rs +++ b/graph/src/components/graphql.rs @@ -49,4 +49,5 @@ pub trait GraphQLMetrics: Send + Sync + 'static { fn observe_query_parsing(&self, duration: Duration, results: &QueryResults); fn observe_query_validation(&self, duration: Duration, id: &DeploymentHash); fn observe_query_validation_error(&self, error_codes: Vec<&str>, id: &DeploymentHash); + fn observe_query_blocks_behind(&self, blocks_behind: i32, id: &DeploymentHash); } diff --git a/graphql/src/metrics.rs b/graphql/src/metrics.rs index 5427a56f383..549945bdeec 100644 --- a/graphql/src/metrics.rs +++ b/graphql/src/metrics.rs @@ -14,6 +14,7 @@ pub struct GraphQLMetrics { query_result_size: Box, query_result_size_max: Box, query_validation_error_counter: Box, + query_blocks_behind: Box, } impl fmt::Debug for GraphQLMetrics { @@ -73,6 +74,12 @@ impl GraphQLMetricsTrait for GraphQLMetrics { .inc(); } } + + fn observe_query_blocks_behind(&self, blocks_behind: i32, id: &DeploymentHash) { + self.query_blocks_behind + .with_label_values(&[id.as_str()]) + .observe(blocks_behind as f64); + } } impl GraphQLMetrics { @@ -128,6 +135,18 @@ impl GraphQLMetrics { ) .unwrap(); + let query_blocks_behind = registry + .new_histogram_vec( + "query_blocks_behind", + "How many blocks the query block is behind the subgraph head", + vec![String::from("deployment")], + vec![ + 0.0, 5.0, 10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 10000.0, + 100000.0, 1000000.0, 10000000.0, + ], + ) + .unwrap(); + Self { query_execution_time, query_parsing_time, @@ -135,6 +154,7 @@ impl GraphQLMetrics { query_result_size, query_result_size_max, query_validation_error_counter, + query_blocks_behind, } } diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs index d76a31813f9..ed6f02dc955 100644 --- a/graphql/src/store/resolver.rs +++ b/graphql/src/store/resolver.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::result; use std::sync::Arc; +use graph::components::graphql::GraphQLMetrics as _; use graph::components::store::{QueryPermit, SubscriptionManager, UnitStream}; use graph::data::graphql::load_manager::LoadManager; use graph::data::graphql::{object, ObjectOrInterface}; @@ -105,6 +106,9 @@ impl StoreResolver { let store_clone = store.cheap_clone(); let block_ptr = Self::locate_block(store_clone.as_ref(), bc, state).await?; + let blocks_behind = state.latest_block.number - block_ptr.ptr.number; + graphql_metrics.observe_query_blocks_behind(blocks_behind, &deployment); + let has_non_fatal_errors = store .has_deterministic_errors(block_ptr.ptr.block_number()) .await?;