Reuse util code from watcher-ts (#399)

* Use event watcher from watcher-ts * Use job queue from watcher-ts * Use graph decimal implementation from watcher-ts * Use misc code from watcher-ts * Use gql-metrics util code from watcher-ts * Reuse config util code from watcher-ts * Reuse types from watcher-ts * Reuse fill util code from watcher-ts * Remove unused method from indexer interface * Update dependencies
vulcanize · Nov 18, 2022 · 6ffc248 · 6ffc248
1 parent 84c3854
commit 6ffc248
Show file tree

Hide file tree

Showing 93 changed files with 251 additions and 2,049 deletions.
diff --git a/packages/erc20-watcher/package.json b/packages/erc20-watcher/package.json
@@ -42,7 +42,6 @@
   "homepage": "https://github.com/vulcanize/watcher-ts#readme",
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@ethersproject/providers": "5.3.0",
     "@types/lodash": "^4.14.168",
     "@vulcanize/util": "^0.1.0",
     "apollo-type-bigint": "^0.1.3",

diff --git a/packages/erc20-watcher/src/cli/reset-cmds/job-queue.ts b/packages/erc20-watcher/src/cli/reset-cmds/job-queue.ts
@@ -4,7 +4,8 @@
 
 import debug from 'debug';
 
-import { getConfig, resetJobs } from '@vulcanize/util';
+import { resetJobs, getConfig } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 const log = debug('vulcanize:reset-job-queue');
 
@@ -15,7 +16,7 @@ export const desc = 'Reset job queue';
 export const builder = {};
 
 export const handler = async (argv: any): Promise<void> => {
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
   await resetJobs(config);
 
   log('Job queue reset successfully');

diff --git a/packages/erc20-watcher/src/cli/reset-cmds/watcher.ts b/packages/erc20-watcher/src/cli/reset-cmds/watcher.ts
@@ -3,17 +3,13 @@
 //
 
 import debug from 'debug';
-import { MoreThan } from 'typeorm';
 import assert from 'assert';
 
-import { getConfig, getResetConfig, JobQueue, resetJobs } from '@vulcanize/util';
+import { JobQueue, resetJobs, getConfig, initClients } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 import { Database } from '../../database';
 import { Indexer } from '../../indexer';
-import { BlockProgress } from '../../entity/BlockProgress';
-import { Allowance } from '../../entity/Allowance';
-import { Balance } from '../../entity/Balance';
-import { Contract } from '../../entity/Contract';
 
 const log = debug('vulcanize:reset-watcher');
 
@@ -28,13 +24,13 @@ export const builder = {
 };
 
 export const handler = async (argv: any): Promise<void> => {
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
   await resetJobs(config);
   const { jobQueue: jobQueueConfig } = config;
-  const { dbConfig, serverConfig, ethClient, ethProvider } = await getResetConfig(config);
+  const { ethClient, ethProvider } = await initClients(config);
 
   // Initialize database.
-  const db = new Database(dbConfig);
+  const db = new Database(config.database);
   await db.init();
 
   assert(jobQueueConfig, 'Missing job queue config');
@@ -44,7 +40,7 @@ export const handler = async (argv: any): Promise<void> => {
 
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
 
-  const indexer = new Indexer(serverConfig, db, ethClient, ethProvider, jobQueue);
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
 
   const syncStatus = await indexer.getSyncStatus();
   assert(syncStatus, 'Missing syncStatus');

diff --git a/packages/erc20-watcher/src/cli/reset.ts b/packages/erc20-watcher/src/cli/reset.ts
@@ -5,7 +5,7 @@
 import 'reflect-metadata';
 import debug from 'debug';
 
-import { getResetYargs } from '@vulcanize/util';
+import { getResetYargs } from '@cerc-io/util';
 
 const log = debug('vulcanize:reset');
 

diff --git a/packages/erc20-watcher/src/cli/watch-contract.ts b/packages/erc20-watcher/src/cli/watch-contract.ts
@@ -6,8 +6,8 @@ import assert from 'assert';
 import yargs from 'yargs';
 import 'reflect-metadata';
 
-import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
-import { Config, getConfig, getResetConfig, JobQueue } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, JobQueue, initClients, getConfig } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { CONTRACT_KIND, Indexer } from '../indexer';
@@ -45,7 +45,7 @@ import { CONTRACT_KIND, Indexer } from '../indexer';
 
   const config: Config = await getConfig(argv.configFile);
   const { database: dbConfig, jobQueue: jobQueueConfig } = config;
-  const { ethClient, ethProvider } = await getResetConfig(config);
+  const { ethClient, ethProvider } = await initClients(config);
 
   assert(dbConfig);
 

diff --git a/packages/erc20-watcher/src/entity/Allowance.ts b/packages/erc20-watcher/src/entity/Allowance.ts
@@ -3,7 +3,8 @@
 //
 
 import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
-import { bigintTransformer } from '@vulcanize/util';
+
+import { bigintTransformer } from '@cerc-io/util';
 
 @Entity()
 @Index(['blockHash', 'token', 'owner', 'spender'], { unique: true })

diff --git a/packages/erc20-watcher/src/entity/Balance.ts b/packages/erc20-watcher/src/entity/Balance.ts
@@ -3,7 +3,8 @@
 //
 
 import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
-import { bigintTransformer } from '@vulcanize/util';
+
+import { bigintTransformer } from '@cerc-io/util';
 
 @Entity()
 @Index(['blockHash', 'token', 'owner'], { unique: true })

diff --git a/packages/erc20-watcher/src/entity/BlockProgress.ts b/packages/erc20-watcher/src/entity/BlockProgress.ts
@@ -4,7 +4,7 @@
 
 import { Entity, PrimaryGeneratedColumn, Column, Index, CreateDateColumn } from 'typeorm';
 
-import { BlockProgressInterface } from '@vulcanize/util';
+import { BlockProgressInterface } from '@cerc-io/util';
 
 @Entity()
 @Index(['blockHash'], { unique: true })

diff --git a/packages/erc20-watcher/src/entity/SyncStatus.ts b/packages/erc20-watcher/src/entity/SyncStatus.ts
@@ -4,7 +4,7 @@
 
 import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
 
-import { SyncStatusInterface } from '@vulcanize/util';
+import { SyncStatusInterface } from '@cerc-io/util';
 
 @Entity()
 export class SyncStatus implements SyncStatusInterface {

diff --git a/packages/erc20-watcher/src/events.ts b/packages/erc20-watcher/src/events.ts
@@ -7,14 +7,14 @@ import debug from 'debug';
 import { PubSub } from 'graphql-subscriptions';
 
 import {
+  EventWatcher as BaseEventWatcher,
   QUEUE_BLOCK_PROCESSING,
   QUEUE_EVENT_PROCESSING,
-  UNKNOWN_EVENT_NAME
+  UNKNOWN_EVENT_NAME,
+  JobQueue
 } from '@cerc-io/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 import {
-  JobQueue,
-  EventWatcher as BaseEventWatcher,
   UpstreamConfig
 } from '@vulcanize/util';
 

diff --git a/packages/erc20-watcher/src/fill.ts b/packages/erc20-watcher/src/fill.ts
@@ -9,10 +9,10 @@ import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
 import { PubSub } from 'graphql-subscriptions';
 
-import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH, JobQueue, getCustomProvider, getConfig, fillBlocks, DEFAULT_PREFETCH_BATCH_SIZE } from '@cerc-io/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 import { getCache } from '@cerc-io/cache';
-import { getConfig, fillBlocks, JobQueue, getCustomProvider, DEFAULT_PREFETCH_BATCH_SIZE } from '@vulcanize/util';
+import { Config } from '@vulcanize/util';
 
 import { Database } from './database';
 import { Indexer } from './indexer';
@@ -63,7 +63,7 @@ export const main = async (): Promise<any> => {
     }
   }).argv;
 
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
 
   assert(config.server, 'Missing server config');
 

diff --git a/packages/erc20-watcher/src/indexer.ts b/packages/erc20-watcher/src/indexer.ts
@@ -9,7 +9,7 @@ import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import JSONbig from 'json-bigint';
 import { ethers } from 'ethers';
 
-import { JobQueue, IndexerInterface } from '@vulcanize/util';
+import { IndexerInterface } from '@vulcanize/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 import {
   Indexer as BaseIndexer,
@@ -18,7 +18,8 @@ import {
   ValueResult,
   Where,
   QueryOptions,
-  UNKNOWN_EVENT_NAME
+  UNKNOWN_EVENT_NAME,
+  JobQueue
 } from '@cerc-io/util';
 import { StorageLayout, MappingKey } from '@cerc-io/solidity-mapper';
 

diff --git a/packages/erc20-watcher/src/job-runner.ts b/packages/erc20-watcher/src/job-runner.ts
@@ -14,15 +14,13 @@ import {
   startMetricsServer,
   QUEUE_BLOCK_PROCESSING,
   QUEUE_EVENT_PROCESSING,
-  DEFAULT_CONFIG_PATH
+  DEFAULT_CONFIG_PATH,
+  JobQueue,
+  getCustomProvider,
+  getConfig
 } from '@cerc-io/util';
 import { getCache } from '@cerc-io/cache';
-import {
-  getConfig,
-  JobQueue,
-  JobRunner as BaseJobRunner,
-  getCustomProvider
-} from '@vulcanize/util';
+import { Config, JobRunner as BaseJobRunner } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
 import { Database } from './database';
@@ -73,7 +71,7 @@ export const main = async (): Promise<any> => {
     })
     .argv;
 
-  const config = await getConfig(argv.f);
+  const config: Config = await getConfig(argv.f);
 
   assert(config.server, 'Missing server config');
 

diff --git a/packages/erc20-watcher/src/resolvers.ts b/packages/erc20-watcher/src/resolvers.ts
@@ -6,8 +6,7 @@ import assert from 'assert';
 import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
 
-import { gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
-import { ValueResult } from '@cerc-io/util';
+import { gqlTotalQueryCount, gqlQueryCount, ValueResult } from '@cerc-io/util';
 
 import { CONTRACT_KIND, Indexer } from './indexer';
 import { EventWatcher } from './events';

diff --git a/packages/erc20-watcher/src/server.ts b/packages/erc20-watcher/src/server.ts
@@ -15,9 +15,13 @@ import { getCache } from '@cerc-io/cache';
 import {
   KIND_ACTIVE,
   DEFAULT_CONFIG_PATH,
-  createAndStartServer
+  createAndStartServer,
+  JobQueue,
+  getCustomProvider,
+  startGQLMetricsServer,
+  getConfig
 } from '@cerc-io/util';
-import { getConfig, getCustomProvider, JobQueue, startGQLMetricsServer } from '@vulcanize/util';
+import { Config } from '@vulcanize/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 
 import typeDefs from './schema';
@@ -41,7 +45,7 @@ export const main = async (): Promise<any> => {
     })
     .argv;
 
-  const config = await getConfig(argv.f);
+  const config: Config = await getConfig(argv.f);
 
   assert(config.server, 'Missing server config');
 

diff --git a/packages/uni-info-watcher/package.json b/packages/uni-info-watcher/package.json
@@ -77,7 +77,6 @@
     "graphql-schema-linter": "^2.0.1",
     "mocha": "^8.4.0",
     "nodemon": "^2.0.7",
-    "pprof": "^3.2.0",
     "ts-node": "^10.0.0",
     "typescript": "^4.3.2"
   }

diff --git a/packages/uni-info-watcher/src/cli/checkpoint-cmds/create.ts b/packages/uni-info-watcher/src/cli/checkpoint-cmds/create.ts
@@ -5,7 +5,8 @@
 import debug from 'debug';
 import assert from 'assert';
 
-import { getConfig, initClients, JobQueue, Config } from '@cerc-io/util';
+import { getConfig, initClients, JobQueue } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 

diff --git a/packages/uni-info-watcher/src/cli/export-state.ts b/packages/uni-info-watcher/src/cli/export-state.ts
@@ -9,7 +9,8 @@ import debug from 'debug';
 import fs from 'fs';
 import path from 'path';
 
-import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util';
+import { DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 import * as codec from '@ipld/dag-cbor';

diff --git a/packages/uni-info-watcher/src/cli/import-state.ts b/packages/uni-info-watcher/src/cli/import-state.ts
@@ -11,9 +11,9 @@ import { PubSub } from 'graphql-subscriptions';
 import fs from 'fs';
 import path from 'path';
 
-import { getConfig, JobQueue, DEFAULT_CONFIG_PATH, Config, initClients, StateKind } from '@cerc-io/util';
+import { getConfig, JobQueue, DEFAULT_CONFIG_PATH, initClients, StateKind, fillBlocks } from '@cerc-io/util';
 import { updateEntitiesFromState } from '@cerc-io/graph-node';
-import { fillBlocks } from '@vulcanize/util';
+import { Config } from '@vulcanize/util';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 import * as codec from '@ipld/dag-cbor';

diff --git a/packages/uni-info-watcher/src/cli/inspect-cid.ts b/packages/uni-info-watcher/src/cli/inspect-cid.ts
@@ -10,8 +10,8 @@ import util from 'util';
 
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
-import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
-import { Config, getConfig, getResetConfig, JobQueue } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, JobQueue, getConfig, initClients } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 import { Database } from '../database';
 import { Indexer } from '../indexer';
@@ -39,7 +39,7 @@ const main = async (): Promise<void> => {
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
-  const { ethClient, ethProvider } = await getResetConfig(config);
+  const { ethClient, ethProvider } = await initClients(config);
 
   const db = new Database(config.database, config.server);
   await db.init();

diff --git a/packages/uni-info-watcher/src/cli/reset-cmds/job-queue.ts b/packages/uni-info-watcher/src/cli/reset-cmds/job-queue.ts
@@ -4,7 +4,8 @@
 
 import debug from 'debug';
 
-import { getConfig, resetJobs } from '@vulcanize/util';
+import { resetJobs, getConfig } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 const log = debug('vulcanize:reset-job-queue');
 
@@ -15,7 +16,7 @@ export const desc = 'Reset job queue';
 export const builder = {};
 
 export const handler = async (argv: any): Promise<void> => {
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
   await resetJobs(config);
 
   log('Job queue reset successfully');

diff --git a/packages/uni-info-watcher/src/cli/reset-cmds/state.ts b/packages/uni-info-watcher/src/cli/reset-cmds/state.ts
@@ -5,6 +5,7 @@
 import debug from 'debug';
 
 import { getConfig } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 
 import { Database } from '../../database';
 
@@ -22,7 +23,7 @@ export const builder = {
 
 export const handler = async (argv: any): Promise<void> => {
   const { blockNumber } = argv;
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
 
   // Initialize database
   const db = new Database(config.database, config.server);

diff --git a/packages/uni-info-watcher/src/cli/reset-cmds/watcher.ts b/packages/uni-info-watcher/src/cli/reset-cmds/watcher.ts
@@ -5,7 +5,8 @@
 import debug from 'debug';
 import assert from 'assert';
 
-import { getConfig, getResetConfig, JobQueue, resetJobs } from '@vulcanize/util';
+import { JobQueue, resetJobs, getConfig, initClients } from '@cerc-io/util';
+import { Config } from '@vulcanize/util';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 
@@ -25,19 +26,19 @@ export const builder = {
 };
 
 export const handler = async (argv: any): Promise<void> => {
-  const config = await getConfig(argv.configFile);
+  const config: Config = await getConfig(argv.configFile);
   await resetJobs(config);
   const { jobQueue: jobQueueConfig } = config;
-  const { dbConfig, upstreamConfig, ethClient, ethProvider } = await getResetConfig(config);
+  const { ethClient, ethProvider } = await initClients(config);
 
   // Initialize database.
-  const db = new Database(dbConfig, config.server);
+  const db = new Database(config.database, config.server);
   await db.init();
 
   const {
     uniWatcher,
     tokenWatcher
-  } = upstreamConfig;
+  } = config.upstream;
 
   const uniClient = new UniClient(uniWatcher);
   const erc20Client = new ERC20Client(tokenWatcher);