diff --git a/packages/ipld-eth-client/src/eth-client.ts b/packages/ipld-eth-client/src/eth-client.ts index b574bf96..b996a756 100644 --- a/packages/ipld-eth-client/src/eth-client.ts +++ b/packages/ipld-eth-client/src/eth-client.ts @@ -83,11 +83,15 @@ export class EthClient { } async getBlockByHash (blockHash?: string): Promise { - const { block } = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash }); - block.number = parseInt(block.number, 16); - block.timestamp = parseInt(block.timestamp, 16); + const result = await this._graphqlClient.query(ethQueries.getBlockByHash, { blockHash }); - return { block }; + return { + block: { + ...result.block, + number: parseInt(result.block.number, 16), + timestamp: parseInt(result.block.timestamp, 16) + } + }; } async getLogs (vars: Vars): Promise { diff --git a/packages/uni-info-watcher/src/entity/Event.ts b/packages/uni-info-watcher/src/entity/Event.ts index a854c405..8bb47f6f 100644 --- a/packages/uni-info-watcher/src/entity/Event.ts +++ b/packages/uni-info-watcher/src/entity/Event.ts @@ -7,7 +7,7 @@ import { BlockProgress } from './BlockProgress'; @Entity() // Index to query events by block and event index. -@Index(['block', 'index']) +@Index(['block', 'index'], { unique: true }) export class Event { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index 2126e2ee..098ab4fd 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -11,7 +11,7 @@ import { providers, utils, BigNumber } from 'ethers'; import { Client as UniClient } from '@vulcanize/uni-watcher'; import { Client as ERC20Client } from '@vulcanize/erc20-watcher'; import { EthClient } from '@vulcanize/ipld-eth-client'; -import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal, JobQueue, Where } from '@vulcanize/util'; +import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal, JobQueue, Where, DEFAULT_LIMIT } from '@vulcanize/util'; import { findEthPerToken, getEthPriceInUSD, getTrackedAmountUSD, sqrtPriceX96ToTokenPrices, WHITELIST_TOKENS } from './utils/pricing'; import { updatePoolDayData, updatePoolHourData, updateTickDayData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates'; @@ -36,7 +36,6 @@ import { Tick } from './entity/Tick'; import { Contract, KIND_POOL } from './entity/Contract'; const SYNC_DELTA = 5; -const DEFAULT_LIMIT = 100; const log = debug('vulcanize:indexer'); @@ -876,8 +875,8 @@ export class Indexer implements IndexerInterface { // Tick entities. const lowerTickId = poolAddress + '#' + (burnEvent.tickLower).toString(); const upperTickId = poolAddress + '#' + (burnEvent.tickUpper).toString(); - const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }, true); - const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }, true); + const lowerTick = await this._db.getTick(dbTx, { id: lowerTickId, blockHash: block.hash }, false); + const upperTick = await this._db.getTick(dbTx, { id: upperTickId, blockHash: block.hash }, false); assert(lowerTick && upperTick); const amount = BigInt(burnEvent.amount); lowerTick.liquidityGross = BigInt(lowerTick.liquidityGross) - amount; @@ -1367,7 +1366,7 @@ export class Indexer implements IndexerInterface { id: poolAddress.concat('#').concat(tickId.toString()), blockHash: block.hash }, - true + false ); if (tick) { diff --git a/packages/uni-watcher/src/entity/Event.ts b/packages/uni-watcher/src/entity/Event.ts index 9a2689ad..d2692cc5 100644 --- a/packages/uni-watcher/src/entity/Event.ts +++ b/packages/uni-watcher/src/entity/Event.ts @@ -9,7 +9,7 @@ export const UNKNOWN_EVENT_NAME = '__unknown__'; @Entity() // Index to query events by block and event index. -@Index(['block', 'index']) +@Index(['block', 'index'], { unique: true }) export class Event { @PrimaryGeneratedColumn() id!: number; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index a32128ce..f5cd1bd6 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -36,6 +36,7 @@ const OPERATOR_MAP = { }; const INSERT_EVENTS_BATCH = 100; +export const DEFAULT_LIMIT = 100; export interface BlockHeight { number?: number; @@ -352,8 +353,6 @@ export class Database { .andWhere('blockProgress.is_pruned = :isPruned', { isPruned: false }) .groupBy('subTable.id'); - subQuery = this._buildQuery(repo, subQuery, where); - if (block.hash) { const { canonicalBlockNumber, blockHashes } = await this.getFrothyRegion(queryRunner, block.hash); @@ -376,6 +375,8 @@ export class Database { ) .setParameters(subQuery.getParameters()); + selectQueryBuilder = this._buildQuery(repo, selectQueryBuilder, where); + if (queryOptions.orderBy) { selectQueryBuilder = this._orderQuery(repo, selectQueryBuilder, queryOptions); } @@ -434,7 +435,9 @@ export class Database { acc[parentEntityId] = []; } - acc[parentEntityId].push(entity); + if (acc[parentEntityId].length < DEFAULT_LIMIT) { + acc[parentEntityId].push(entity); + } return acc; }, {}); @@ -481,10 +484,18 @@ export class Database { }, {}); entities.forEach((entity: any) => { - const relatedField = entity[field] as any[]; + const relatedEntityIds: Set = entity[field].reduce((acc: Set, id: string) => { + acc.add(id); + + return acc; + }, new Set()); + + entity[field] = []; - relatedField.forEach((relatedEntityId, index) => { - relatedField[index] = relatedEntitiesMap[relatedEntityId]; + relatedEntities.forEach((relatedEntity: any) => { + if (relatedEntityIds.has(relatedEntity.id) && entity[field].length < DEFAULT_LIMIT) { + entity[field].push(relatedEntity); + } }); }); diff --git a/packages/util/src/fill.ts b/packages/util/src/fill.ts index 9e7ba7bd..5415aae3 100644 --- a/packages/util/src/fill.ts +++ b/packages/util/src/fill.ts @@ -2,7 +2,6 @@ // Copyright 2021 Vulcanize, Inc. // -import assert from 'assert'; import debug from 'debug'; import { JobQueue } from './job-queue'; @@ -25,7 +24,10 @@ export const fillBlocks = async ( } ): Promise => { let { startBlock, endBlock, prefetch, batchBlocks } = argv; - assert(startBlock < endBlock, 'endBlock should be greater than startBlock'); + + if (startBlock > endBlock) { + throw new Error(`endBlock ${endBlock} should be greater than or equal to startBlock ${startBlock}`); + } const syncStatus = await indexer.getSyncStatus(); @@ -34,6 +36,10 @@ export const fillBlocks = async ( throw new Error(`Missing blocks between startBlock ${startBlock} and chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`); } + if (endBlock <= syncStatus.chainHeadBlockNumber) { + throw new Error(`endBlock ${endBlock} should be greater than chainHeadBlockNumber ${syncStatus.chainHeadBlockNumber}`); + } + startBlock = syncStatus.chainHeadBlockNumber + 1; }