Add prometheus metrics to monitor watcher progress and GQL queries (#364

)

* Add prometheus metrics to monitor GQL queries

* Update metrics to monitor watcher progress

* Update metrics ports for uni-watcher

* Fix metric help comment

packages/erc20-watcher/environments/local.toml

@@ -4,6 +4,12 @@
   mode = "eth_call"
   kind = "lazy"
 
+[metrics]
+  host = "127.0.0.1"
+  port = 9000
+    [metrics.gql]
+    port = 9001
+
 [database]
   type = "postgres"
   host = "localhost"

packages/erc20-watcher/src/job-runner.ts

@@ -18,7 +18,8 @@ import {
   QUEUE_EVENT_PROCESSING,
   JobQueueConfig,
   DEFAULT_CONFIG_PATH,
-  getCustomProvider
+  getCustomProvider,
+  startMetricsServer
 } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
@@ -103,6 +104,8 @@ export const main = async (): Promise<any> => {
 
   const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue);
   await jobRunner.start();
+
+  await startMetricsServer(config, indexer);
 };
 
 main().then(() => {

packages/erc20-watcher/src/resolvers.ts

@@ -6,7 +6,7 @@ import assert from 'assert';
 import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
 
-import { ValueResult } from '@vulcanize/util';
+import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -46,36 +46,56 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       totalSupply: (_: any, { blockHash, token }: { blockHash: string, token: string }): Promise<ValueResult> => {
         log('totalSupply', blockHash, token);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('totalSupply').inc(1);
+
         return indexer.totalSupply(blockHash, token);
       },
 
       balanceOf: async (_: any, { blockHash, token, owner }: { blockHash: string, token: string, owner: string }) => {
         log('balanceOf', blockHash, token, owner);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('balanceOf').inc(1);
+
         return indexer.balanceOf(blockHash, token, owner);
       },
 
       allowance: async (_: any, { blockHash, token, owner, spender }: { blockHash: string, token: string, owner: string, spender: string }) => {
         log('allowance', blockHash, token, owner, spender);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('allowance').inc(1);
+
         return indexer.allowance(blockHash, token, owner, spender);
       },
 
       name: (_: any, { blockHash, token }: { blockHash: string, token: string }) => {
         log('name', blockHash, token);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('name').inc(1);
+
         return indexer.name(blockHash, token);
       },
 
       symbol: (_: any, { blockHash, token }: { blockHash: string, token: string }) => {
         log('symbol', blockHash, token);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('symbol').inc(1);
+
         return indexer.symbol(blockHash, token);
       },
 
       decimals: (_: any, { blockHash, token }: { blockHash: string, token: string }) => {
         log('decimals', blockHash, token);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('decimals').inc(1);
+
         return indexer.decimals(blockHash, token);
       },
 
       events: async (_: any, { blockHash, token, name }: { blockHash: string, token: string, name: string }) => {
         log('events', blockHash, token, name || '');
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('events').inc(1);
 
         const block = await indexer.getBlockProgress(blockHash);
         if (!block || !block.isComplete) {
@@ -88,6 +108,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
         log('eventsInRange', fromBlockNumber, toBlockNumber);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('eventsInRange').inc(1);
 
         const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
         if (expected !== actual) {

packages/erc20-watcher/src/server.ts

@@ -14,7 +14,7 @@ import { createServer } from 'http';
 
 import { getCache } from '@vulcanize/cache';
 import { EthClient } from '@vulcanize/ipld-eth-client';
-import { DEFAULT_CONFIG_PATH, getConfig, getCustomProvider, JobQueue, KIND_ACTIVE } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, getConfig, getCustomProvider, JobQueue, KIND_ACTIVE, startGQLMetricsServer } from '@vulcanize/util';
 
 import typeDefs from './schema';
 
@@ -100,6 +100,8 @@ export const main = async (): Promise<any> => {
     log(`Server is listening on host ${host} port ${port}`);
   });
 
+  await startGQLMetricsServer(config);
+
   return { app, server };
 };
 

packages/uni-info-watcher/environments/local.toml

@@ -7,7 +7,9 @@
 
 [metrics]
   host = "127.0.0.1"
-  port = 8084
+  port = 9004
+    [metrics.gql]
+    port = 9005
 
 [database]
   type = "postgres"

packages/uni-info-watcher/src/job-runner.ts

@@ -134,7 +134,7 @@ export const main = async (): Promise<any> => {
   const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue);
   await jobRunner.start();
 
-  await startMetricsServer(metrics);
+  await startMetricsServer(config, indexer);
 };
 
 main().then(() => {

packages/uni-info-watcher/src/resolvers.ts

@@ -7,7 +7,7 @@ import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
 import { GraphQLScalarType } from 'graphql';
 
-import { BlockHeight, GraphDecimal, OrderDirection } from '@vulcanize/util';
+import { BlockHeight, gqlQueryCount, gqlTotalQueryCount, GraphDecimal, OrderDirection } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
 import { Burn } from './entity/Burn';
@@ -64,18 +64,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
     Query: {
       bundle: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
         log('bundle', id, block);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('bundle').inc(1);
 
         return indexer.getBundle(id, block);
       },
 
       bundles: async (_: any, { block = {}, first }: { first: number, block: BlockHeight }) => {
         log('bundles', block, first);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('bundles').inc(1);
 
         return indexer.getEntities(Bundle, block, {}, { limit: first });
       },
 
       burns: async (_: any, { block = {}, first, orderBy, orderDirection, where }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('burns', first, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('burns').inc(1);
 
         return indexer.getEntities(
           Burn,
@@ -111,12 +117,16 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       factories: async (_: any, { block = {}, first }: { first: number, block: BlockHeight }) => {
         log('factories', block, first);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('factories').inc(1);
 
         return indexer.getEntities(Factory, block, {}, { limit: first });
       },
 
       mints: async (_: any, { block = {}, first, orderBy, orderDirection, where }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('mints', first, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('mints').inc(1);
 
         return indexer.getEntities(
           Mint,
@@ -152,18 +162,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       pool: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
         log('pool', id, block);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('pool').inc(1);
 
         return indexer.getPool(id, block);
       },
 
       poolDayDatas: async (_: any, { block = {}, first, skip, orderBy, orderDirection, where }: { block: BlockHeight, first: number, skip: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('poolDayDatas', first, skip, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('poolDayDatas').inc(1);
 
         return indexer.getEntities(PoolDayData, block, where, { limit: first, skip, orderBy, orderDirection });
       },
 
       pools: async (_: any, { block = {}, first, orderBy, orderDirection, where = {} }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('pools', block, first, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('pools').inc(1);
 
         return indexer.getEntities(
           Pool,
@@ -187,6 +203,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       swaps: async (_: any, { block = {}, first, orderBy, orderDirection, where }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('swaps', first, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('swaps').inc(1);
 
         return indexer.getEntities(
           Swap,
@@ -222,18 +240,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       ticks: async (_: any, { block = {}, first, skip, where = {} }: { block: BlockHeight, first: number, skip: number, where: { [key: string]: any } }) => {
         log('ticks', block, first, skip, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('ticks').inc(1);
 
         return indexer.getEntities(Tick, block, where, { limit: first, skip });
       },
 
       token: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
         log('token', id, block);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('token').inc(1);
 
         return indexer.getToken(id, block);
       },
 
       tokens: async (_: any, { block = {}, first, orderBy, orderDirection, where }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('tokens', orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('tokens').inc(1);
 
         return indexer.getEntities(
           Token,
@@ -252,18 +276,24 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       tokenDayDatas: async (_: any, { block = {}, first, skip, orderBy, orderDirection, where }: { block: BlockHeight, first: number, skip: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('tokenDayDatas', first, skip, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('tokenDayDatas').inc(1);
 
         return indexer.getEntities(TokenDayData, block, where, { limit: first, skip, orderBy, orderDirection });
       },
 
       tokenHourDatas: async (_: any, { block = {}, first, skip, orderBy, orderDirection, where }: { block: BlockHeight, first: number, skip: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('tokenHourDatas', first, skip, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('tokenHourDatas').inc(1);
 
         return indexer.getEntities(TokenHourData, block, where, { limit: first, skip, orderBy, orderDirection });
       },
 
       transactions: async (_: any, { block = {}, first, orderBy, orderDirection, where }: { block: BlockHeight, first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('transactions', first, orderBy, orderDirection);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('transactions').inc(1);
 
         return indexer.getEntities(
           Transaction,
@@ -367,12 +397,16 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       uniswapDayDatas: async (_: any, { block = {}, first, skip, orderBy, orderDirection, where }: { block: BlockHeight, first: number, skip: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('uniswapDayDatas', first, skip, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('uniswapDayDatas').inc(1);
 
         return indexer.getEntities(UniswapDayData, block, where, { limit: first, skip, orderBy, orderDirection });
       },
 
       positions: async (_: any, { block = {}, first, where }: { block: BlockHeight, first: number, where: { [key: string]: any } }) => {
         log('positions', first, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('positions').inc(1);
 
         return indexer.getEntities(
           Position,
@@ -416,12 +450,16 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       blocks: async (_: any, { first, orderBy, orderDirection, where }: { first: number, orderBy: string, orderDirection: OrderDirection, where: { [key: string]: any } }) => {
         log('blocks', first, orderBy, orderDirection, where);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('blocks').inc(1);
 
         return indexer.getBlockEntities(where, { limit: first, orderBy, orderDirection });
       },
 
       indexingStatusForCurrentVersion: async (_: any, { subgraphName }: { subgraphName: string }) => {
         log('health', subgraphName);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('indexingStatusForCurrentVersion').inc(1);
 
         return indexer.getIndexingStatus();
       }

packages/uni-info-watcher/src/server.ts

@@ -15,7 +15,7 @@ import { createServer } from 'http';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 import { EthClient } from '@vulcanize/ipld-eth-client';
-import { DEFAULT_CONFIG_PATH, getConfig, getCustomProvider, JobQueue } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, getConfig, getCustomProvider, JobQueue, startGQLMetricsServer } from '@vulcanize/util';
 import { getCache } from '@vulcanize/cache';
 
 import typeDefs from './schema';
@@ -107,6 +107,8 @@ export const main = async (): Promise<any> => {
     log(`Server is listening on host ${host} port ${port}`);
   });
 
+  await startGQLMetricsServer(config);
+
   return { app, server };
 };
 

packages/uni-watcher/environments/local.toml

@@ -4,7 +4,9 @@
 
 [metrics]
   host = "127.0.0.1"
-  port = 8083
+  port = 9002
+    [metrics.gql]
+    port = 9003
 
 [database]
   type = "postgres"

packages/uni-watcher/src/job-runner.ts

@@ -73,7 +73,7 @@ export const main = async (): Promise<any> => {
 
   assert(config.server, 'Missing server config');
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, metrics } = config;
+  const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config;
 
   assert(dbConfig, 'Missing database config');
 
@@ -106,7 +106,7 @@ export const main = async (): Promise<any> => {
   const jobRunner = new JobRunner(jobQueueConfig, indexer, jobQueue);
   await jobRunner.start();
 
-  startMetricsServer(metrics);
+  await startMetricsServer(config, indexer);
 };
 
 main().then(() => {