Fix for miscellaneous mismatches and errors in watcher (#363)

* Fix block number parsing in eth-client * Avoid loading relations unnecessarily in mapping code * Check that endBlock is greater than chainHeadBlock in fill blocks CLI * Make index on block and event index unique in event table * Fix GQL query related fields order by id * Avoid loading relation in Swap event mapping code Co-authored-by: nabarun <[email protected]> Co-authored-by: nikugogoi <[email protected]>
vulcanize · Sep 22, 2022 · d5023ac · d5023ac
1 parent 1407257
commit d5023ac
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 19 deletions.
diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts
@@ -83,11 +83,15 @@ export class EthClient {
   }
 
   async getBlockByHash (blockHash?: string): Promise<any> {
-    const { block } = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash });
-    block.number = parseInt(block.number, 16);
-    block.timestamp = parseInt(block.timestamp, 16);
+    const result = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash });
 
-    return { block };
+    return {
+      block: {
+        ...result.block,
+        number: parseInt(result.block.number, 16),
+        timestamp: parseInt(result.block.timestamp, 16)
+      }
+    };
   }
 
   async getLogs (vars: Vars): Promise<any> {

diff --git a/packages/uni-info-watcher/src/entity/Event.ts b/packages/uni-info-watcher/src/entity/Event.ts
@@ -7,7 +7,7 @@ import { BlockProgress } from './BlockProgress';
 
 @Entity()
 // Index to query events by block and event index.
-@Index(['block', 'index'])
+@Index(['block', 'index'], { unique: true })
 export class Event {
   @PrimaryGeneratedColumn()
   id!: number;

diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts
@@ -11,7 +11,7 @@ import { providers, utils, BigNumber } from 'ethers';
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { EthClient } from '@vulcanize/ipld-eth-client';
-import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal, JobQueue, Where } from '@vulcanize/util';
+import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal, JobQueue, Where, DEFAULT_LIMIT } from '@vulcanize/util';
 
 import { findEthPerToken, getEthPriceInUSD, getTrackedAmountUSD, sqrtPriceX96ToTokenPrices, WHITELIST_TOKENS } from './utils/pricing';
 import { updatePoolDayData, updatePoolHourData, updateTickDayData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates';
@@ -36,7 +36,6 @@ import { Tick } from './entity/Tick';
 import { Contract, KIND_POOL } from './entity/Contract';
 
 const SYNC_DELTA = 5;
-const DEFAULT_LIMIT = 100;
 
 const log = debug('vulcanize:indexer');
 
@@ -876,8 +875,8 @@ export class Indexer implements IndexerInterface {
       // Tick entities.
       const lowerTickId = poolAddress + '#' + (burnEvent.tickLower).toString();
       const upperTickId = poolAddress + '#' + (burnEvent.tickUpper).toString();
-      const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }, true);
-      const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }, true);
+      const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }, false);
+      const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }, false);
       assert(lowerTick && upperTick);
       const amount = BigInt(burnEvent.amount);
       lowerTick.liquidityGross = BigInt(lowerTick.liquidityGross) - amount;
@@ -1367,7 +1366,7 @@ export class Indexer implements IndexerInterface {
         id: poolAddress.concat('#').concat(tickId.toString()),
         blockHash: block.hash
       },
-      true
+      false
     );
 
     if (tick) {

diff --git a/packages/uni-watcher/src/entity/Event.ts b/packages/uni-watcher/src/entity/Event.ts
@@ -9,7 +9,7 @@ export const UNKNOWN_EVENT_NAME = '__unknown__';
 
 @Entity()
 // Index to query events by block and event index.
-@Index(['block', 'index'])
+@Index(['block', 'index'], { unique: true })
 export class Event {
   @PrimaryGeneratedColumn()
   id!: number;

diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts
@@ -36,6 +36,7 @@ const OPERATOR_MAP = {
 };
 
 const INSERT_EVENTS_BATCH = 100;
+export const DEFAULT_LIMIT = 100;
 
 export interface BlockHeight {
   number?: number;
@@ -352,8 +353,6 @@ export class Database {
       .andWhere('blockProgress.is_pruned = :isPruned', { isPruned: false })
       .groupBy('subTable.id');
 
-    subQuery = this._buildQuery(repo, subQuery, where);
-
     if (block.hash) {
       const { canonicalBlockNumber, blockHashes } = await this.getFrothyRegion(queryRunner, block.hash);
 
@@ -376,6 +375,8 @@ export class Database {
       )
       .setParameters(subQuery.getParameters());
 
+    selectQueryBuilder = this._buildQuery(repo, selectQueryBuilder, where);
+
     if (queryOptions.orderBy) {
       selectQueryBuilder = this._orderQuery(repo, selectQueryBuilder, queryOptions);
     }
@@ -434,7 +435,9 @@ export class Database {
               acc[parentEntityId] = [];
             }
 
-            acc[parentEntityId].push(entity);
+            if (acc[parentEntityId].length < DEFAULT_LIMIT) {
+              acc[parentEntityId].push(entity);
+            }
 
             return acc;
           }, {});
@@ -481,10 +484,18 @@ export class Database {
           }, {});
 
           entities.forEach((entity: any) => {
-            const relatedField = entity[field] as any[];
+            const relatedEntityIds: Set<string> = entity[field].reduce((acc: Set<string>, id: string) => {
+              acc.add(id);
+
+              return acc;
+            }, new Set());
+
+            entity[field] = [];
 
-            relatedField.forEach((relatedEntityId, index) => {
-              relatedField[index] = relatedEntitiesMap[relatedEntityId];
+            relatedEntities.forEach((relatedEntity: any) => {
+              if (relatedEntityIds.has(relatedEntity.id) && entity[field].length < DEFAULT_LIMIT) {
+                entity[field].push(relatedEntity);
+              }
             });
           });
 

diff --git a/packages/util/src/fill.ts b/packages/util/src/fill.ts
@@ -2,7 +2,6 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import assert from 'assert';
 import debug from 'debug';
 
 import { JobQueue } from './job-queue';
@@ -25,7 +24,10 @@ export const fillBlocks = async (
   }
 ): Promise<any> => {
   let { startBlock, endBlock, prefetch, batchBlocks } = argv;
-  assert(startBlock < endBlock, 'endBlock should be greater than startBlock');
+
+  if (startBlock > endBlock) {
+    throw new Error(`endBlock ${endBlock} should be greater than or equal to startBlock ${startBlock}`);
+  }
 
   const syncStatus = await indexer.getSyncStatus();
 
@@ -34,6 +36,10 @@ export const fillBlocks = async (
       throw new Error(`Missing blocks between startBlock ${startBlock} and chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`);
     }
 
+    if (endBlock <= syncStatus.chainHeadBlockNumber) {
+      throw new Error(`endBlock ${endBlock} should be greater than chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`);
+    }
+
     startBlock = syncStatus.chainHeadBlockNumber + 1;
   }