Changes for backward compatibility with old watcher DB data (#378)

* Make cid column in uni-watcher BlockProgress table nullable * Make new fields in uni-info-watcher nullable to be compatible with old data * Add skipStateFieldsUpdate flag to avoid eth-server calls Co-authored-by: prathamesh0 <[email protected]>
vulcanize · Nov 4, 2022 · 49d5100 · 49d5100
1 parent aec8e7f
commit 49d5100
Show file tree

Hide file tree

Showing 12 changed files with 99 additions and 66 deletions.
diff --git a/packages/uni-info-watcher/environments/local.toml b/packages/uni-info-watcher/environments/local.toml
@@ -18,6 +18,9 @@
   # Interval in number of blocks at which to clear entities cache.
   clearEntitiesCacheInterval = 1000
 
+  # Boolean to skip updating entity fields required in state creation and not required in the frontend.
+  skipStateFieldsUpdate = false
+
 [metrics]
   host = "127.0.0.1"
   port = 9004

diff --git a/packages/uni-info-watcher/src/entity/Burn.ts b/packages/uni-info-watcher/src/entity/Burn.ts
@@ -58,6 +58,7 @@ export class Burn {
   @Column('numeric', { transformer: bigintTransformer })
   tickUpper!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   logIndex!: bigint
 }
diff --git a/packages/uni-info-watcher/src/entity/Mint.ts b/packages/uni-info-watcher/src/entity/Mint.ts
@@ -63,6 +63,7 @@ export class Mint {
   @Column('numeric', { transformer: bigintTransformer })
   tickUpper!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   logIndex!: bigint
 }
diff --git a/packages/uni-info-watcher/src/entity/Pool.ts b/packages/uni-info-watcher/src/entity/Pool.ts
@@ -72,10 +72,12 @@ export class Pool {
   @Column('numeric', { default: 0, transformer: graphDecimalTransformer })
   feesUSD!: GraphDecimal
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   createdAtTimestamp!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   createdAtBlockNumber!: bigint
 
   @Column('numeric', { default: BigInt(0), transformer: bigintTransformer })

diff --git a/packages/uni-info-watcher/src/entity/PositionSnapshot.ts b/packages/uni-info-watcher/src/entity/PositionSnapshot.ts
@@ -59,7 +59,8 @@ export class PositionSnapshot {
   @Column('varchar')
   position!: string
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   _blockNumber!: bigint;
 
   @Column('varchar')

diff --git a/packages/uni-info-watcher/src/entity/Swap.ts b/packages/uni-info-watcher/src/entity/Swap.ts
@@ -58,6 +58,7 @@ export class Swap {
   @Column('numeric', { transformer: bigintTransformer })
   sqrtPriceX96!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   logIndex!: bigint
 }
diff --git a/packages/uni-info-watcher/src/entity/Tick.ts b/packages/uni-info-watcher/src/entity/Tick.ts
@@ -63,10 +63,12 @@ export class Tick {
   @Column('numeric', { default: 0, transformer: graphDecimalTransformer })
   collectedFeesUSD!: GraphDecimal
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   createdAtTimestamp!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   createdAtBlockNumber!: bigint
 
   @Column('numeric', { default: 0, transformer: bigintTransformer })

diff --git a/packages/uni-info-watcher/src/entity/TickDayData.ts b/packages/uni-info-watcher/src/entity/TickDayData.ts
@@ -33,16 +33,20 @@ export class TickDayData {
   @Column('numeric', { transformer: bigintTransformer })
   liquidityNet!: bigint;
 
-  @Column('numeric', { transformer: graphDecimalTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: graphDecimalTransformer })
   volumeToken0!: GraphDecimal
 
-  @Column('numeric', { transformer: graphDecimalTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: graphDecimalTransformer })
   volumeToken1!: GraphDecimal
 
-  @Column('numeric', { transformer: graphDecimalTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: graphDecimalTransformer })
   volumeUSD!: GraphDecimal
 
-  @Column('numeric', { transformer: graphDecimalTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: graphDecimalTransformer })
   feesUSD!: GraphDecimal
 
   @Column('numeric', { default: 0, transformer: bigintTransformer })

diff --git a/packages/uni-info-watcher/src/entity/Transaction.ts b/packages/uni-info-watcher/src/entity/Transaction.ts
@@ -21,12 +21,15 @@ export class Transaction {
   @Column('numeric', { transformer: bigintTransformer })
   timestamp!: bigint;
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   _blockNumber!: bigint;
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   gasUsed!: bigint
 
-  @Column('numeric', { transformer: bigintTransformer })
+  // Field is nullable to work with old DB schema.
+  @Column('numeric', { nullable: true, transformer: bigintTransformer })
   gasPrice!: bigint
 }
diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts
@@ -618,16 +618,19 @@ export class Indexer implements IndexerInterface {
     const transactionsMap = new Map();
 
     const txHashSet = new Set();
-    events.forEach((event: any) => {
-      txHashSet.add(event.tx.hash);
-    });
-
-    await Promise.all(
-      Array.from(txHashSet).map(async (txHash: any) => {
-        const transaction = await getFullTransaction(this._ethClient, txHash, blockNumber);
-        transactionsMap.set(txHash, transaction);
-      })
-    );
+
+    if (!this._serverConfig.skipStateFieldsUpdate) {
+      events.forEach((event: any) => {
+        txHashSet.add(event.tx.hash);
+      });
+
+      await Promise.all(
+        Array.from(txHashSet).map(async (txHash: any) => {
+          const transaction = await getFullTransaction(this._ethClient, txHash, blockNumber);
+          transactionsMap.set(txHash, transaction);
+        })
+      );
+    }
 
     for (let i = 0; i < events.length; i++) {
       const {
@@ -638,15 +641,16 @@ export class Indexer implements IndexerInterface {
         proof
       } = events[i];
 
+      const { __typename: eventName, ...eventInfo } = event;
+
+      const extraInfo = { tx, eventIndex };
+
       // Get full transaction for extra params like gasUsed and gasPrice.
       const transaction = transactionsMap.get(tx.hash);
 
-      const { __typename: eventName, ...eventInfo } = event;
-
-      const extraInfo = {
-        tx: transaction,
-        eventIndex
-      };
+      if (transaction) {
+        extraInfo.tx = transaction;
+      }
 
       dbEvents.push({
         index: i,
@@ -685,14 +689,20 @@ export class Indexer implements IndexerInterface {
     const { __typename: eventName } = event;
 
     if (!this._fullBlock || (this._fullBlock.hash !== block.hash)) {
-      const { blockHash, blockNumber, timestamp, ...blockData } = await getFullBlock(this._ethClient, this._ethProvider, block.hash, block.number);
-
-      this._fullBlock = {
-        hash: blockHash,
-        number: blockNumber,
-        timestamp: Number(timestamp),
-        ...blockData
-      };
+      // resultEvent.block does not contain all properties of a full block.
+      // It is fetched using getFullBlock below only if required for updating entity fields required in state creation.
+      this._fullBlock = block as Block;
+
+      if (!this._serverConfig.skipStateFieldsUpdate) {
+        const { blockHash, blockNumber, timestamp, ...blockData } = await getFullBlock(this._ethClient, this._ethProvider, block.hash, block.number);
+
+        this._fullBlock = {
+          hash: blockHash,
+          number: blockNumber,
+          timestamp: Number(timestamp),
+          ...blockData
+        };
+      }
 
       assert(this._fullBlock);
     }
@@ -1014,7 +1024,7 @@ export class Indexer implements IndexerInterface {
       factory.totalValueLockedETH = factory.totalValueLockedETH.plus(pool.totalValueLockedETH);
       factory.totalValueLockedUSD = factory.totalValueLockedETH.times(bundle.ethPriceUSD);
 
-      const transaction = await loadTransaction(this._db, dbTx, { block, tx });
+      const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate);
 
       const mint = new Mint();
       mint.id = transaction.id + '#' + pool.txCount.toString();
@@ -1178,7 +1188,7 @@ export class Indexer implements IndexerInterface {
       factory.totalValueLockedUSD = factory.totalValueLockedETH.times(bundle.ethPriceUSD);
 
       // Burn entity.
-      const transaction = await loadTransaction(this._db, dbTx, { block, tx });
+      const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate);
 
       const burn = new Burn();
       burn.id = transaction.id + '#' + pool.txCount.toString();
@@ -1378,7 +1388,7 @@ export class Indexer implements IndexerInterface {
       token1.totalValueLockedUSD = token1.totalValueLocked.times(token1.derivedETH).times(bundle.ethPriceUSD);
 
       // Create Swap event
-      const transaction = await loadTransaction(this._db, dbTx, { block, tx });
+      const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate);
 
       const swap = new Swap();
       swap.id = transaction.id + '#' + pool.txCount.toString();
@@ -1730,17 +1740,19 @@ export class Indexer implements IndexerInterface {
   async _updateTickFeeVarsAndSave (dbTx: QueryRunner, tick: Tick, block: Block, contractAddress: string): Promise<void> {
     const poolAddress = contractAddress;
 
-    // Not all ticks are initialized so obtaining null is expected behavior.
-    console.time('time:indexer#_getPosition-eth_call_for_ticks');
-    const endTimer = eventProcessingEthCallDuration.startTimer();
-    const { value: tickResult } = await this._uniClient.ticks(block.hash, poolAddress, Number(tick.tickIdx));
-    endTimer();
-    console.timeEnd('time:indexer#_getPosition-eth_call_for_ticks');
+    if (!this._serverConfig.skipStateFieldsUpdate) {
+      // Not all ticks are initialized so obtaining null is expected behavior.
+      console.time('time:indexer#_getPosition-eth_call_for_ticks');
+      const endTimer = eventProcessingEthCallDuration.startTimer();
+      const { value: tickResult } = await this._uniClient.ticks(block.hash, poolAddress, Number(tick.tickIdx));
+      endTimer();
+      console.timeEnd('time:indexer#_getPosition-eth_call_for_ticks');
 
-    tick.feeGrowthOutside0X128 = tickResult.feeGrowthOutside0X128;
-    tick.feeGrowthOutside1X128 = tickResult.feeGrowthOutside1X128;
+      tick.feeGrowthOutside0X128 = tickResult.feeGrowthOutside0X128;
+      tick.feeGrowthOutside1X128 = tickResult.feeGrowthOutside1X128;
 
-    await this._db.saveTick(dbTx, tick, block);
+      await this._db.saveTick(dbTx, tick, block);
+    }
 
     await updateTickDayData(this._db, dbTx, tick, { block });
   }
@@ -1809,7 +1821,7 @@ export class Indexer implements IndexerInterface {
         const dbTx = await this._db.createTransactionRunner();
 
         try {
-          const transaction = await loadTransaction(this._db, dbTx, { block, tx });
+          const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate);
           position.transaction = transaction.id;
 
           await dbTx.commitTransaction();
@@ -1874,7 +1886,7 @@ export class Indexer implements IndexerInterface {
     positionSnapshot.withdrawnToken1 = position.withdrawnToken1;
     positionSnapshot.collectedFeesToken0 = position.collectedFeesToken0;
     positionSnapshot.collectedFeesToken1 = position.collectedFeesToken1;
-    const transaction = await loadTransaction(this._db, dbTx, { block, tx });
+    const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate);
     positionSnapshot.transaction = transaction.id;
     positionSnapshot.feeGrowthInside0LastX128 = position.feeGrowthInside0LastX128;
     positionSnapshot.feeGrowthInside1LastX128 = position.feeGrowthInside1LastX128;

diff --git a/packages/uni-info-watcher/src/utils/index.ts b/packages/uni-info-watcher/src/utils/index.ts
@@ -51,7 +51,7 @@ export const convertTokenToDecimal = (tokenAmount: bigint, exchangeDecimals: big
   return (new GraphDecimal(tokenAmount.toString())).div(exponentToBigDecimal(exchangeDecimals));
 };
 
-export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: { block: Block, tx: Transaction }): Promise<TransactionEntity> => {
+export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: { block: Block, tx: Transaction }, skipStateFieldsUpdate: boolean): Promise<TransactionEntity> => {
   const { tx, block } = event;
   // Get the txHash in lowercase.
   const txHash = utils.hexlify(tx.hash);
@@ -64,20 +64,23 @@ export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: {
 
   transaction._blockNumber = BigInt(block.number);
   transaction.timestamp = BigInt(block.timestamp);
-  transaction.gasUsed = BigInt(tx.gasLimit);
 
-  let gasPrice = tx.gasPrice;
+  if (!skipStateFieldsUpdate) {
+    transaction.gasUsed = BigInt(tx.gasLimit);
 
-  if (!gasPrice) {
-    // Compute gasPrice for EIP-1559 transaction
-    // https://ethereum.stackexchange.com/questions/122090/what-does-tx-gasprice-represent-after-eip-1559
-    const feeDifference = BigNumber.from(tx.maxFeePerGas).sub(BigNumber.from(block.baseFee));
-    const maxPriorityFeePerGas = BigNumber.from(tx.maxPriorityFeePerGas);
-    const priorityFeePerGas = maxPriorityFeePerGas.lt(feeDifference) ? maxPriorityFeePerGas : feeDifference;
-    gasPrice = BigNumber.from(block.baseFee).add(priorityFeePerGas).toString();
-  }
+    let gasPrice = tx.gasPrice;
+
+    if (!gasPrice) {
+      // Compute gasPrice for EIP-1559 transaction
+      // https://ethereum.stackexchange.com/questions/122090/what-does-tx-gasprice-represent-after-eip-1559
+      const feeDifference = BigNumber.from(tx.maxFeePerGas).sub(BigNumber.from(block.baseFee));
+      const maxPriorityFeePerGas = BigNumber.from(tx.maxPriorityFeePerGas);
+      const priorityFeePerGas = maxPriorityFeePerGas.lt(feeDifference) ? maxPriorityFeePerGas : feeDifference;
+      gasPrice = BigNumber.from(block.baseFee).add(priorityFeePerGas).toString();
+    }
 
-  transaction.gasPrice = BigInt(gasPrice);
+    transaction.gasPrice = BigInt(gasPrice);
+  }
 
   return db.saveTransaction(dbTx, transaction, block);
 };

diff --git a/packages/uni-watcher/src/entity/BlockProgress.ts b/packages/uni-watcher/src/entity/BlockProgress.ts
@@ -14,7 +14,7 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
   id!: number;
 
-  @Column('varchar')
+  @Column('varchar', { nullable: true })
   cid!: string;
 
   @Column('varchar', { length: 66 })