From e00f6e498f4b6905ef7254413d38bc3014602f12 Mon Sep 17 00:00:00 2001 From: prathamesh0 Date: Wed, 12 Oct 2022 17:11:44 +0530 Subject: [PATCH 1/3] Make cid column in uni-watcher BlockProgress table nullable --- packages/uni-watcher/src/entity/BlockProgress.ts | 2 +- packages/uni-watcher/src/entity/SyncStatus.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/uni-watcher/src/entity/BlockProgress.ts b/packages/uni-watcher/src/entity/BlockProgress.ts index 59f1a6d4..a6ca5522 100644 --- a/packages/uni-watcher/src/entity/BlockProgress.ts +++ b/packages/uni-watcher/src/entity/BlockProgress.ts @@ -14,7 +14,7 @@ export class BlockProgress implements BlockProgressInterface { @PrimaryGeneratedColumn() id!: number; - @Column('varchar') + @Column('varchar', { nullable: true }) cid!: string; @Column('varchar', { length: 66 }) diff --git a/packages/uni-watcher/src/entity/SyncStatus.ts b/packages/uni-watcher/src/entity/SyncStatus.ts index 56bd3a2a..689f9605 100644 --- a/packages/uni-watcher/src/entity/SyncStatus.ts +++ b/packages/uni-watcher/src/entity/SyncStatus.ts @@ -35,9 +35,9 @@ export class SyncStatus implements SyncStatusInterface { @Column('integer') latestCanonicalBlockNumber!: number; - @Column('varchar', { length: 66 }) + @Column('varchar', { length: 66, default: '0xf1156cb7e1a2acaa348c66fdc45b02f2547024ded63350a96a46ed12c62b4ff4' }) initialIndexedBlockHash!: string; - @Column('integer') + @Column('integer', { default: 12369621 }) initialIndexedBlockNumber!: number; } From 6fd35f1366ea590f640adb0707f04b0650bc286e Mon Sep 17 00:00:00 2001 From: nabarun Date: Wed, 2 Nov 2022 22:04:27 +0530 Subject: [PATCH 2/3] Make new fields in uni-info-watcher nullable to be compatible with old data --- packages/uni-info-watcher/src/entity/Burn.ts | 3 ++- packages/uni-info-watcher/src/entity/Mint.ts | 3 ++- packages/uni-info-watcher/src/entity/Pool.ts | 6 ++++-- .../uni-info-watcher/src/entity/PositionSnapshot.ts | 3 ++- packages/uni-info-watcher/src/entity/Swap.ts | 3 ++- packages/uni-info-watcher/src/entity/Tick.ts | 6 ++++-- packages/uni-info-watcher/src/entity/TickDayData.ts | 12 ++++++++---- packages/uni-info-watcher/src/entity/Transaction.ts | 9 ++++++--- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/uni-info-watcher/src/entity/Burn.ts b/packages/uni-info-watcher/src/entity/Burn.ts index 2c137732..648a56cd 100644 --- a/packages/uni-info-watcher/src/entity/Burn.ts +++ b/packages/uni-info-watcher/src/entity/Burn.ts @@ -58,6 +58,7 @@ export class Burn { @Column('numeric', { transformer: bigintTransformer }) tickUpper!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) logIndex!: bigint } diff --git a/packages/uni-info-watcher/src/entity/Mint.ts b/packages/uni-info-watcher/src/entity/Mint.ts index 18d93538..27039c98 100644 --- a/packages/uni-info-watcher/src/entity/Mint.ts +++ b/packages/uni-info-watcher/src/entity/Mint.ts @@ -63,6 +63,7 @@ export class Mint { @Column('numeric', { transformer: bigintTransformer }) tickUpper!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) logIndex!: bigint } diff --git a/packages/uni-info-watcher/src/entity/Pool.ts b/packages/uni-info-watcher/src/entity/Pool.ts index 6a07ea0e..71ed42ce 100644 --- a/packages/uni-info-watcher/src/entity/Pool.ts +++ b/packages/uni-info-watcher/src/entity/Pool.ts @@ -72,10 +72,12 @@ export class Pool { @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) feesUSD!: GraphDecimal - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) createdAtTimestamp!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) createdAtBlockNumber!: bigint @Column('numeric', { default: BigInt(0), transformer: bigintTransformer }) diff --git a/packages/uni-info-watcher/src/entity/PositionSnapshot.ts b/packages/uni-info-watcher/src/entity/PositionSnapshot.ts index 63311b8c..ce908288 100644 --- a/packages/uni-info-watcher/src/entity/PositionSnapshot.ts +++ b/packages/uni-info-watcher/src/entity/PositionSnapshot.ts @@ -59,7 +59,8 @@ export class PositionSnapshot { @Column('varchar') position!: string - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) _blockNumber!: bigint; @Column('varchar') diff --git a/packages/uni-info-watcher/src/entity/Swap.ts b/packages/uni-info-watcher/src/entity/Swap.ts index 725fc289..f2daa8dc 100644 --- a/packages/uni-info-watcher/src/entity/Swap.ts +++ b/packages/uni-info-watcher/src/entity/Swap.ts @@ -58,6 +58,7 @@ export class Swap { @Column('numeric', { transformer: bigintTransformer }) sqrtPriceX96!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) logIndex!: bigint } diff --git a/packages/uni-info-watcher/src/entity/Tick.ts b/packages/uni-info-watcher/src/entity/Tick.ts index c6e9ddcf..a4620a2f 100644 --- a/packages/uni-info-watcher/src/entity/Tick.ts +++ b/packages/uni-info-watcher/src/entity/Tick.ts @@ -63,10 +63,12 @@ export class Tick { @Column('numeric', { default: 0, transformer: graphDecimalTransformer }) collectedFeesUSD!: GraphDecimal - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) createdAtTimestamp!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) createdAtBlockNumber!: bigint @Column('numeric', { default: 0, transformer: bigintTransformer }) diff --git a/packages/uni-info-watcher/src/entity/TickDayData.ts b/packages/uni-info-watcher/src/entity/TickDayData.ts index 3827d92d..bbcaff0f 100644 --- a/packages/uni-info-watcher/src/entity/TickDayData.ts +++ b/packages/uni-info-watcher/src/entity/TickDayData.ts @@ -33,16 +33,20 @@ export class TickDayData { @Column('numeric', { transformer: bigintTransformer }) liquidityNet!: bigint; - @Column('numeric', { transformer: graphDecimalTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: graphDecimalTransformer }) volumeToken0!: GraphDecimal - @Column('numeric', { transformer: graphDecimalTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: graphDecimalTransformer }) volumeToken1!: GraphDecimal - @Column('numeric', { transformer: graphDecimalTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: graphDecimalTransformer }) volumeUSD!: GraphDecimal - @Column('numeric', { transformer: graphDecimalTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: graphDecimalTransformer }) feesUSD!: GraphDecimal @Column('numeric', { default: 0, transformer: bigintTransformer }) diff --git a/packages/uni-info-watcher/src/entity/Transaction.ts b/packages/uni-info-watcher/src/entity/Transaction.ts index 6c484a5c..164b04a9 100644 --- a/packages/uni-info-watcher/src/entity/Transaction.ts +++ b/packages/uni-info-watcher/src/entity/Transaction.ts @@ -21,12 +21,15 @@ export class Transaction { @Column('numeric', { transformer: bigintTransformer }) timestamp!: bigint; - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) _blockNumber!: bigint; - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) gasUsed!: bigint - @Column('numeric', { transformer: bigintTransformer }) + // Field is nullable to work with old DB schema. + @Column('numeric', { nullable: true, transformer: bigintTransformer }) gasPrice!: bigint } From 5a46ac6e4ffd408a27f0d81bbeadd2e7663e06fe Mon Sep 17 00:00:00 2001 From: nabarun Date: Thu, 3 Nov 2022 16:30:28 +0530 Subject: [PATCH 3/3] Add skipStateFieldsUpdate flag to avoid eth-server calls --- .../uni-info-watcher/environments/local.toml | 3 + packages/uni-info-watcher/src/indexer.ts | 88 +++++++++++-------- packages/uni-info-watcher/src/utils/index.ts | 27 +++--- packages/uni-watcher/src/entity/SyncStatus.ts | 4 +- 4 files changed, 70 insertions(+), 52 deletions(-) diff --git a/packages/uni-info-watcher/environments/local.toml b/packages/uni-info-watcher/environments/local.toml index d0a54987..c15ee00e 100644 --- a/packages/uni-info-watcher/environments/local.toml +++ b/packages/uni-info-watcher/environments/local.toml @@ -18,6 +18,9 @@ # Interval in number of blocks at which to clear entities cache. clearEntitiesCacheInterval = 1000 + # Boolean to skip updating entity fields required in state creation and not required in the frontend. + skipStateFieldsUpdate = false + [metrics] host = "127.0.0.1" port = 9004 diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 81f28a23..288e49bc 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -618,16 +618,19 @@ export class Indexer implements IndexerInterface { const transactionsMap = new Map(); const txHashSet = new Set(); - events.forEach((event: any) => { - txHashSet.add(event.tx.hash); - }); - - await Promise.all( - Array.from(txHashSet).map(async (txHash: any) => { - const transaction = await getFullTransaction(this._ethClient, txHash, blockNumber); - transactionsMap.set(txHash, transaction); - }) - ); + + if (!this._serverConfig.skipStateFieldsUpdate) { + events.forEach((event: any) => { + txHashSet.add(event.tx.hash); + }); + + await Promise.all( + Array.from(txHashSet).map(async (txHash: any) => { + const transaction = await getFullTransaction(this._ethClient, txHash, blockNumber); + transactionsMap.set(txHash, transaction); + }) + ); + } for (let i = 0; i < events.length; i++) { const { @@ -638,15 +641,16 @@ export class Indexer implements IndexerInterface { proof } = events[i]; + const { __typename: eventName, ...eventInfo } = event; + + const extraInfo = { tx, eventIndex }; + // Get full transaction for extra params like gasUsed and gasPrice. const transaction = transactionsMap.get(tx.hash); - const { __typename: eventName, ...eventInfo } = event; - - const extraInfo = { - tx: transaction, - eventIndex - }; + if (transaction) { + extraInfo.tx = transaction; + } dbEvents.push({ index: i, @@ -685,14 +689,20 @@ export class Indexer implements IndexerInterface { const { __typename: eventName } = event; if (!this._fullBlock || (this._fullBlock.hash !== block.hash)) { - const { blockHash, blockNumber, timestamp, ...blockData } = await getFullBlock(this._ethClient, this._ethProvider, block.hash, block.number); - - this._fullBlock = { - hash: blockHash, - number: blockNumber, - timestamp: Number(timestamp), - ...blockData - }; + // resultEvent.block does not contain all properties of a full block. + // It is fetched using getFullBlock below only if required for updating entity fields required in state creation. + this._fullBlock = block as Block; + + if (!this._serverConfig.skipStateFieldsUpdate) { + const { blockHash, blockNumber, timestamp, ...blockData } = await getFullBlock(this._ethClient, this._ethProvider, block.hash, block.number); + + this._fullBlock = { + hash: blockHash, + number: blockNumber, + timestamp: Number(timestamp), + ...blockData + }; + } assert(this._fullBlock); } @@ -1014,7 +1024,7 @@ export class Indexer implements IndexerInterface { factory.totalValueLockedETH = factory.totalValueLockedETH.plus(pool.totalValueLockedETH); factory.totalValueLockedUSD = factory.totalValueLockedETH.times(bundle.ethPriceUSD); - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate); const mint = new Mint(); mint.id = transaction.id + '#' + pool.txCount.toString(); @@ -1178,7 +1188,7 @@ export class Indexer implements IndexerInterface { factory.totalValueLockedUSD = factory.totalValueLockedETH.times(bundle.ethPriceUSD); // Burn entity. - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate); const burn = new Burn(); burn.id = transaction.id + '#' + pool.txCount.toString(); @@ -1378,7 +1388,7 @@ export class Indexer implements IndexerInterface { token1.totalValueLockedUSD = token1.totalValueLocked.times(token1.derivedETH).times(bundle.ethPriceUSD); // Create Swap event - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate); const swap = new Swap(); swap.id = transaction.id + '#' + pool.txCount.toString(); @@ -1730,17 +1740,19 @@ export class Indexer implements IndexerInterface { async _updateTickFeeVarsAndSave (dbTx: QueryRunner, tick: Tick, block: Block, contractAddress: string): Promise { const poolAddress = contractAddress; - // Not all ticks are initialized so obtaining null is expected behavior. - console.time('time:indexer#_getPosition-eth_call_for_ticks'); - const endTimer = eventProcessingEthCallDuration.startTimer(); - const { value: tickResult } = await this._uniClient.ticks(block.hash, poolAddress, Number(tick.tickIdx)); - endTimer(); - console.timeEnd('time:indexer#_getPosition-eth_call_for_ticks'); + if (!this._serverConfig.skipStateFieldsUpdate) { + // Not all ticks are initialized so obtaining null is expected behavior. + console.time('time:indexer#_getPosition-eth_call_for_ticks'); + const endTimer = eventProcessingEthCallDuration.startTimer(); + const { value: tickResult } = await this._uniClient.ticks(block.hash, poolAddress, Number(tick.tickIdx)); + endTimer(); + console.timeEnd('time:indexer#_getPosition-eth_call_for_ticks'); - tick.feeGrowthOutside0X128 = tickResult.feeGrowthOutside0X128; - tick.feeGrowthOutside1X128 = tickResult.feeGrowthOutside1X128; + tick.feeGrowthOutside0X128 = tickResult.feeGrowthOutside0X128; + tick.feeGrowthOutside1X128 = tickResult.feeGrowthOutside1X128; - await this._db.saveTick(dbTx, tick, block); + await this._db.saveTick(dbTx, tick, block); + } await updateTickDayData(this._db, dbTx, tick, { block }); } @@ -1809,7 +1821,7 @@ export class Indexer implements IndexerInterface { const dbTx = await this._db.createTransactionRunner(); try { - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate); position.transaction = transaction.id; await dbTx.commitTransaction(); @@ -1874,7 +1886,7 @@ export class Indexer implements IndexerInterface { positionSnapshot.withdrawnToken1 = position.withdrawnToken1; positionSnapshot.collectedFeesToken0 = position.collectedFeesToken0; positionSnapshot.collectedFeesToken1 = position.collectedFeesToken1; - const transaction = await loadTransaction(this._db, dbTx, { block, tx }); + const transaction = await loadTransaction(this._db, dbTx, { block, tx }, this._serverConfig.skipStateFieldsUpdate); positionSnapshot.transaction = transaction.id; positionSnapshot.feeGrowthInside0LastX128 = position.feeGrowthInside0LastX128; positionSnapshot.feeGrowthInside1LastX128 = position.feeGrowthInside1LastX128; diff --git a/packages/uni-info-watcher/src/utils/index.ts b/packages/uni-info-watcher/src/utils/index.ts index a5d19679..8bbc9e89 100644 --- a/packages/uni-info-watcher/src/utils/index.ts +++ b/packages/uni-info-watcher/src/utils/index.ts @@ -51,7 +51,7 @@ export const convertTokenToDecimal = (tokenAmount: bigint, exchangeDecimals: big return (new GraphDecimal(tokenAmount.toString())).div(exponentToBigDecimal(exchangeDecimals)); }; -export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: { block: Block, tx: Transaction }): Promise => { +export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: { block: Block, tx: Transaction }, skipStateFieldsUpdate: boolean): Promise => { const { tx, block } = event; // Get the txHash in lowercase. const txHash = utils.hexlify(tx.hash); @@ -64,20 +64,23 @@ export const loadTransaction = async (db: Database, dbTx: QueryRunner, event: { transaction._blockNumber = BigInt(block.number); transaction.timestamp = BigInt(block.timestamp); - transaction.gasUsed = BigInt(tx.gasLimit); - let gasPrice = tx.gasPrice; + if (!skipStateFieldsUpdate) { + transaction.gasUsed = BigInt(tx.gasLimit); - if (!gasPrice) { - // Compute gasPrice for EIP-1559 transaction - // https://ethereum.stackexchange.com/questions/122090/what-does-tx-gasprice-represent-after-eip-1559 - const feeDifference = BigNumber.from(tx.maxFeePerGas).sub(BigNumber.from(block.baseFee)); - const maxPriorityFeePerGas = BigNumber.from(tx.maxPriorityFeePerGas); - const priorityFeePerGas = maxPriorityFeePerGas.lt(feeDifference) ? maxPriorityFeePerGas : feeDifference; - gasPrice = BigNumber.from(block.baseFee).add(priorityFeePerGas).toString(); - } + let gasPrice = tx.gasPrice; + + if (!gasPrice) { + // Compute gasPrice for EIP-1559 transaction + // https://ethereum.stackexchange.com/questions/122090/what-does-tx-gasprice-represent-after-eip-1559 + const feeDifference = BigNumber.from(tx.maxFeePerGas).sub(BigNumber.from(block.baseFee)); + const maxPriorityFeePerGas = BigNumber.from(tx.maxPriorityFeePerGas); + const priorityFeePerGas = maxPriorityFeePerGas.lt(feeDifference) ? maxPriorityFeePerGas : feeDifference; + gasPrice = BigNumber.from(block.baseFee).add(priorityFeePerGas).toString(); + } - transaction.gasPrice = BigInt(gasPrice); + transaction.gasPrice = BigInt(gasPrice); + } return db.saveTransaction(dbTx, transaction, block); }; diff --git a/packages/uni-watcher/src/entity/SyncStatus.ts b/packages/uni-watcher/src/entity/SyncStatus.ts index 689f9605..56bd3a2a 100644 --- a/packages/uni-watcher/src/entity/SyncStatus.ts +++ b/packages/uni-watcher/src/entity/SyncStatus.ts @@ -35,9 +35,9 @@ export class SyncStatus implements SyncStatusInterface { @Column('integer') latestCanonicalBlockNumber!: number; - @Column('varchar', { length: 66, default: '0xf1156cb7e1a2acaa348c66fdc45b02f2547024ded63350a96a46ed12c62b4ff4' }) + @Column('varchar', { length: 66 }) initialIndexedBlockHash!: string; - @Column('integer', { default: 12369621 }) + @Column('integer') initialIndexedBlockNumber!: number; }