From e3933ccde8a8fe13883204aa7fec1c17ccd3d239 Mon Sep 17 00:00:00 2001 From: nabarun Date: Tue, 13 Dec 2022 11:39:27 +0530 Subject: [PATCH 1/2] Fix job-runner to process missing parent block --- packages/util/src/job-runner.ts | 38 ++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/packages/util/src/job-runner.ts b/packages/util/src/job-runner.ts index dd673ac..9fec484 100644 --- a/packages/util/src/job-runner.ts +++ b/packages/util/src/job-runner.ts @@ -77,14 +77,30 @@ export class JobRunner { switch (kind) { case JOB_KIND_INDEX: { - const blocksToBeIndexed = await fetchBlocksAtHeight( - job, - this._indexer, - this._jobQueueConfig, - this._blockAndEventsMap - ); - const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed)); - await Promise.all(indexBlockPromises); + const { data: { cid, blockHash, blockNumber, parentHash, timestamp } } = job; + + // Check if blockHash present in job. + if (blockHash) { + // If blockHash is present it is a job for indexing missing parent block. + await this._indexBlock(job, { + blockTimestamp: timestamp, + cid, + blockHash, + blockNumber, + parentHash + }); + } else { + // If blockHash is not present, it is a job to index the next consecutive blockNumber. + const blocksToBeIndexed = await fetchBlocksAtHeight( + blockNumber, + this._indexer, + this._jobQueueConfig, + this._blockAndEventsMap + ); + const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed)); + await Promise.all(indexBlockPromises); + } + break; } @@ -277,7 +293,8 @@ export class JobRunner { const message = `Parent block number ${parentBlockNumber} hash ${parentHash} of block number ${blockNumber} hash ${blockHash} not fetched yet, aborting`; log(message); - throw new Error(message); + // Do not throw error and complete the job as block will be processed after parent block processing. + return; } if (!parentBlock.isComplete) { @@ -295,7 +312,8 @@ export class JobRunner { priority: newPriority }, { priority: newPriority }); - throw new Error(message); + // Do not throw error and complete the job as block will be processed after parent block processing. + return; } else { // Remove the unknown events of the parent block if it is marked complete. await this._indexer.removeUnknownEvents(parentBlock); From 6b8d2e62786da1a6438b66a3b0cb9171aff9680d Mon Sep 17 00:00:00 2001 From: nabarun Date: Tue, 13 Dec 2022 12:25:09 +0530 Subject: [PATCH 2/2] Update watcher-ts packages --- packages/erc20-watcher/package.json | 8 ++--- packages/uni-info-watcher/package.json | 8 ++--- packages/uni-watcher/package.json | 10 +++--- packages/util/package.json | 4 +-- yarn.lock | 46 +++++++++++++------------- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/erc20-watcher/package.json b/packages/erc20-watcher/package.json index cec1689..d39c4a9 100644 --- a/packages/erc20-watcher/package.json +++ b/packages/erc20-watcher/package.json @@ -40,10 +40,10 @@ "homepage": "https://github.com/vulcanize/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.17", - "@cerc-io/ipld-eth-client": "^0.2.17", - "@cerc-io/solidity-mapper": "^0.2.17", - "@cerc-io/util": "^0.2.17", + "@cerc-io/cli": "^0.2.18", + "@cerc-io/ipld-eth-client": "^0.2.18", + "@cerc-io/solidity-mapper": "^0.2.18", + "@cerc-io/util": "^0.2.18", "@vulcanize/util": "^0.1.0", "apollo-type-bigint": "^0.1.3", "debug": "^4.3.1", diff --git a/packages/uni-info-watcher/package.json b/packages/uni-info-watcher/package.json index 680c9d7..0e1b180 100644 --- a/packages/uni-info-watcher/package.json +++ b/packages/uni-info-watcher/package.json @@ -6,10 +6,10 @@ "private": true, "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.17", - "@cerc-io/ipld-eth-client": "^0.2.17", - "@cerc-io/solidity-mapper": "^0.2.17", - "@cerc-io/util": "^0.2.17", + "@cerc-io/cli": "^0.2.18", + "@cerc-io/ipld-eth-client": "^0.2.18", + "@cerc-io/solidity-mapper": "^0.2.18", + "@cerc-io/util": "^0.2.18", "@types/lodash": "^4.14.168", "@vulcanize/erc20-watcher": "^0.1.0", "@vulcanize/uni-watcher": "^0.1.0", diff --git a/packages/uni-watcher/package.json b/packages/uni-watcher/package.json index 36e5e61..7c3841d 100644 --- a/packages/uni-watcher/package.json +++ b/packages/uni-watcher/package.json @@ -37,11 +37,11 @@ "homepage": "https://github.com/vulcanize/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.17", - "@cerc-io/ipld-eth-client": "^0.2.17", - "@cerc-io/solidity-mapper": "^0.2.17", - "@cerc-io/util": "^0.2.17", - "@cerc-io/cache": "^0.2.17", + "@cerc-io/cli": "^0.2.18", + "@cerc-io/ipld-eth-client": "^0.2.18", + "@cerc-io/solidity-mapper": "^0.2.18", + "@cerc-io/util": "^0.2.18", + "@cerc-io/cache": "^0.2.18", "@types/lodash": "^4.14.168", "@vulcanize/util": "^0.1.0", "apollo-type-bigint": "^0.1.3", diff --git a/packages/util/package.json b/packages/util/package.json index 17583a5..de0f7f0 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -4,8 +4,8 @@ "main": "dist/index.js", "license": "AGPL-3.0", "dependencies": { - "@cerc-io/ipld-eth-client": "^0.2.17", - "@cerc-io/util": "^0.2.17", + "@cerc-io/ipld-eth-client": "^0.2.18", + "@cerc-io/util": "^0.2.18", "csv-writer": "^1.6.0", "debug": "^4.3.1", "ethers": "^5.2.0", diff --git a/yarn.lock b/yarn.lock index 7af0b3c..4f76ac5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -273,10 +273,10 @@ "@babel/helper-validator-identifier" "^7.14.9" to-fast-properties "^2.0.0" -"@cerc-io/cache@^0.2.17": - version "0.2.17" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.17/cache-0.2.17.tgz#158dca80ea080157300c13cfc89ab745341d3a5e" - integrity sha512-81DFKM2BDw8kz279PLeUIqz8SSbzzHU3HjdLPgU/PAbRKcPDuQ63QyiiYYYkIzDw//029heBCagmQczdOYG5zA== +"@cerc-io/cache@^0.2.18": + version "0.2.18" + resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.18/cache-0.2.18.tgz#c672aa4c6592266c25cdc8f2a17bafd5654e70c3" + integrity sha512-lL3Jxv3N6L1hlOE4zbhYQyP7dXCVQOqBcic1NLn8zA3tWUq7v2YzkC0mvUujRQXtk+kHsCVoIjWdKCbNedOM7Q== dependencies: canonical-json "^0.0.4" debug "^4.3.1" @@ -284,12 +284,12 @@ fs-extra "^10.0.0" level "^7.0.0" -"@cerc-io/cli@^0.2.17": - version "0.2.17" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.17/cli-0.2.17.tgz#384a63eb327db82fb6e76072a07a06652fa51924" - integrity sha512-NkmxKO6ycyfLMvr4TOSCetYu3nnTy3a5FeMskkQ+arXpIgUvPitASHPhwcKytlD57L2Y9/224Q8gVND5v+Sa2g== +"@cerc-io/cli@^0.2.18": + version "0.2.18" + resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.18/cli-0.2.18.tgz#819edbd0a57572b9c0177dde307ed591b362649d" + integrity sha512-ggwTcUERiq9s6VoHfWcrVWsSyS5vfXPfTGQmtWGsQB3wBtICKfUqvyC+spDupi6ToMDe6mbEiY7G20Mmo0P/iQ== dependencies: - "@cerc-io/util" "^0.2.17" + "@cerc-io/util" "^0.2.18" "@ethersproject/providers" "^5.4.4" "@graphql-tools/utils" "^9.1.1" "@ipld/dag-cbor" "^6.0.12" @@ -301,13 +301,13 @@ typeorm "^0.2.32" yargs "^17.0.1" -"@cerc-io/ipld-eth-client@^0.2.17": - version "0.2.17" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.17/ipld-eth-client-0.2.17.tgz#11d6ba6bb016e73df192e1b473003439857da3f4" - integrity sha512-S7KnJhycIpynQ2Gaqxh4YD7u5OQUaF0p9yKNN9E1DkkUtWoCNyYBqtIz19y9fo7AdmvFHsmMKKosBnKptVCVBw== +"@cerc-io/ipld-eth-client@^0.2.18": + version "0.2.18" + resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.18/ipld-eth-client-0.2.18.tgz#d4bbdeb5f8726ec2d3a96b82ae027f2c9617a79a" + integrity sha512-xnvy1mctZxLgbzgbk8WyN+SAxGhQizpuxt/RJpj9B93QFsZT/keVsGhEWmrsCo/7d9Jnbjv6iz0iT0j096dNYA== dependencies: "@apollo/client" "^3.7.1" - "@cerc-io/cache" "^0.2.17" + "@cerc-io/cache" "^0.2.18" cross-fetch "^3.1.4" debug "^4.3.1" ethers "^5.4.4" @@ -317,20 +317,20 @@ ws "^8.11.0" zen-observable-ts "^1.1.0" -"@cerc-io/solidity-mapper@^0.2.17": - version "0.2.17" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.17/solidity-mapper-0.2.17.tgz#14bc5dbf44c5b77f1bb69458e2d545d823c9adda" - integrity sha512-P6GSukqCQW95GheQUA2VrD0l5XNfwCDbPXjjPINevo2h6/c5/euLMoCOrVLQFgGPUPPzgV43G5RaZYaGUNzaZg== +"@cerc-io/solidity-mapper@^0.2.18": + version "0.2.18" + resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.18/solidity-mapper-0.2.18.tgz#cf05698d4f5a61085c6fd1fb6493382f84d656db" + integrity sha512-Jzid/RYpxt3KjkuHmYMSEY9bSwKOxsYcULDBtZ5d2nizB5sYqq0tD4GRep7Nz8JlhqQ7TlmAED/y++SlpRYZWw== dependencies: dotenv "^10.0.0" -"@cerc-io/util@^0.2.17": - version "0.2.17" - resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.17/util-0.2.17.tgz#a72deaaefb6782dfb7e56f5e9f328a2697d50fa7" - integrity sha512-cbqZQsKMVkHD4ILXTAqv/WFvAKt/+Y2ra6SnqzOUa+h5TdY6sdZODvxGMzezFCFUeWBxFNc3dSxcJqN9qpSE8A== +"@cerc-io/util@^0.2.18": + version "0.2.18" + resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.18/util-0.2.18.tgz#049502d804e01cb4a8799d5eb004d304a2a406fe" + integrity sha512-qolIr4H1zjqMHR2yV8znGZMCQzbjGaGFplGiMNEPnhrHa+fCe2A2LL1nN97djFoR47jA8th0Ep3VI2PLG7HtDA== dependencies: "@apollo/utils.keyvaluecache" "^1.0.1" - "@cerc-io/solidity-mapper" "^0.2.17" + "@cerc-io/solidity-mapper" "^0.2.18" "@ethersproject/providers" "^5.4.4" "@graphql-tools/schema" "^9.0.10" "@graphql-tools/utils" "^9.1.1"