Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix job-runner to process missing parent block #421

Merged
merged 2 commits into from
Dec 13, 2022
Merged

Fix job-runner to process missing parent block #421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e3933cc
Fix job-runner to process missing parent block
nikugogoi Dec 13, 2022
6b8d2e6
Update watcher-ts packages
nikugogoi Dec 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions packages/erc20-watcher/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@vulcanize/util": "^0.1.0",
"apollo-type-bigint": "^0.1.3",
"debug": "^4.3.1",
Expand Down
8 changes: 4 additions & 4 deletions packages/uni-info-watcher/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
"private": true,
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@types/lodash": "^4.14.168",
"@vulcanize/erc20-watcher": "^0.1.0",
"@vulcanize/uni-watcher": "^0.1.0",
Expand Down
10 changes: 5 additions & 5 deletions packages/uni-watcher/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@
"homepage": "https://github.com/vulcanize/watcher-ts#readme",
"dependencies": {
"@apollo/client": "^3.3.19",
"@cerc-io/cli": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/solidity-mapper": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/cache": "^0.2.17",
"@cerc-io/cli": "^0.2.18",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/solidity-mapper": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"@cerc-io/cache": "^0.2.18",
"@types/lodash": "^4.14.168",
"@vulcanize/util": "^0.1.0",
"apollo-type-bigint": "^0.1.3",
Expand Down
4 changes: 2 additions & 2 deletions packages/util/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
"main": "dist/index.js",
"license": "AGPL-3.0",
"dependencies": {
"@cerc-io/ipld-eth-client": "^0.2.17",
"@cerc-io/util": "^0.2.17",
"@cerc-io/ipld-eth-client": "^0.2.18",
"@cerc-io/util": "^0.2.18",
"csv-writer": "^1.6.0",
"debug": "^4.3.1",
"ethers": "^5.2.0",
Expand Down
38 changes: 28 additions & 10 deletions packages/util/src/job-runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,30 @@ export class JobRunner {

switch (kind) {
case JOB_KIND_INDEX: {
const blocksToBeIndexed = await fetchBlocksAtHeight(
job,
this._indexer,
this._jobQueueConfig,
this._blockAndEventsMap
);
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed));
await Promise.all(indexBlockPromises);
const { data: { cid, blockHash, blockNumber, parentHash, timestamp } } = job;

// Check if blockHash present in job.
if (blockHash) {
// If blockHash is present it is a job for indexing missing parent block.
await this._indexBlock(job, {
blockTimestamp: timestamp,
cid,
blockHash,
blockNumber,
parentHash
});
} else {
// If blockHash is not present, it is a job to index the next consecutive blockNumber.
const blocksToBeIndexed = await fetchBlocksAtHeight(
blockNumber,
this._indexer,
this._jobQueueConfig,
this._blockAndEventsMap
);
const indexBlockPromises = blocksToBeIndexed.map(blockToBeIndexed => this._indexBlock(job, blockToBeIndexed));
await Promise.all(indexBlockPromises);
}

break;
}

Expand Down Expand Up @@ -277,7 +293,8 @@ export class JobRunner {
const message = `Parent block number ${parentBlockNumber} hash ${parentHash} of block number ${blockNumber} hash ${blockHash} not fetched yet, aborting`;
log(message);

throw new Error(message);
// Do not throw error and complete the job as block will be processed after parent block processing.
return;
}

if (!parentBlock.isComplete) {
Expand All @@ -295,7 +312,8 @@ export class JobRunner {
priority: newPriority
}, { priority: newPriority });

throw new Error(message);
// Do not throw error and complete the job as block will be processed after parent block processing.
return;
} else {
// Remove the unknown events of the parent block if it is marked complete.
await this._indexer.removeUnknownEvents(parentBlock);
Expand Down
46 changes: 23 additions & 23 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,23 +273,23 @@
"@babel/helper-validator-identifier" "^7.14.9"
to-fast-properties "^2.0.0"

"@cerc-io/cache@^0.2.17":
version "0.2.17"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.17/cache-0.2.17.tgz#158dca80ea080157300c13cfc89ab745341d3a5e"
integrity sha512-81DFKM2BDw8kz279PLeUIqz8SSbzzHU3HjdLPgU/PAbRKcPDuQ63QyiiYYYkIzDw//029heBCagmQczdOYG5zA==
"@cerc-io/cache@^0.2.18":
version "0.2.18"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcache/-/0.2.18/cache-0.2.18.tgz#c672aa4c6592266c25cdc8f2a17bafd5654e70c3"
integrity sha512-lL3Jxv3N6L1hlOE4zbhYQyP7dXCVQOqBcic1NLn8zA3tWUq7v2YzkC0mvUujRQXtk+kHsCVoIjWdKCbNedOM7Q==
dependencies:
canonical-json "^0.0.4"
debug "^4.3.1"
ethers "^5.4.4"
fs-extra "^10.0.0"
level "^7.0.0"

"@cerc-io/cli@^0.2.17":
version "0.2.17"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.17/cli-0.2.17.tgz#384a63eb327db82fb6e76072a07a06652fa51924"
integrity sha512-NkmxKO6ycyfLMvr4TOSCetYu3nnTy3a5FeMskkQ+arXpIgUvPitASHPhwcKytlD57L2Y9/224Q8gVND5v+Sa2g==
"@cerc-io/cli@^0.2.18":
version "0.2.18"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fcli/-/0.2.18/cli-0.2.18.tgz#819edbd0a57572b9c0177dde307ed591b362649d"
integrity sha512-ggwTcUERiq9s6VoHfWcrVWsSyS5vfXPfTGQmtWGsQB3wBtICKfUqvyC+spDupi6ToMDe6mbEiY7G20Mmo0P/iQ==
dependencies:
"@cerc-io/util" "^0.2.17"
"@cerc-io/util" "^0.2.18"
"@ethersproject/providers" "^5.4.4"
"@graphql-tools/utils" "^9.1.1"
"@ipld/dag-cbor" "^6.0.12"
Expand All @@ -301,13 +301,13 @@
typeorm "^0.2.32"
yargs "^17.0.1"

"@cerc-io/ipld-eth-client@^0.2.17":
version "0.2.17"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.17/ipld-eth-client-0.2.17.tgz#11d6ba6bb016e73df192e1b473003439857da3f4"
integrity sha512-S7KnJhycIpynQ2Gaqxh4YD7u5OQUaF0p9yKNN9E1DkkUtWoCNyYBqtIz19y9fo7AdmvFHsmMKKosBnKptVCVBw==
"@cerc-io/ipld-eth-client@^0.2.18":
version "0.2.18"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fipld-eth-client/-/0.2.18/ipld-eth-client-0.2.18.tgz#d4bbdeb5f8726ec2d3a96b82ae027f2c9617a79a"
integrity sha512-xnvy1mctZxLgbzgbk8WyN+SAxGhQizpuxt/RJpj9B93QFsZT/keVsGhEWmrsCo/7d9Jnbjv6iz0iT0j096dNYA==
dependencies:
"@apollo/client" "^3.7.1"
"@cerc-io/cache" "^0.2.17"
"@cerc-io/cache" "^0.2.18"
cross-fetch "^3.1.4"
debug "^4.3.1"
ethers "^5.4.4"
Expand All @@ -317,20 +317,20 @@
ws "^8.11.0"
zen-observable-ts "^1.1.0"

"@cerc-io/solidity-mapper@^0.2.17":
version "0.2.17"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.17/solidity-mapper-0.2.17.tgz#14bc5dbf44c5b77f1bb69458e2d545d823c9adda"
integrity sha512-P6GSukqCQW95GheQUA2VrD0l5XNfwCDbPXjjPINevo2h6/c5/euLMoCOrVLQFgGPUPPzgV43G5RaZYaGUNzaZg==
"@cerc-io/solidity-mapper@^0.2.18":
version "0.2.18"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Fsolidity-mapper/-/0.2.18/solidity-mapper-0.2.18.tgz#cf05698d4f5a61085c6fd1fb6493382f84d656db"
integrity sha512-Jzid/RYpxt3KjkuHmYMSEY9bSwKOxsYcULDBtZ5d2nizB5sYqq0tD4GRep7Nz8JlhqQ7TlmAED/y++SlpRYZWw==
dependencies:
dotenv "^10.0.0"

"@cerc-io/util@^0.2.17":
version "0.2.17"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.17/util-0.2.17.tgz#a72deaaefb6782dfb7e56f5e9f328a2697d50fa7"
integrity sha512-cbqZQsKMVkHD4ILXTAqv/WFvAKt/+Y2ra6SnqzOUa+h5TdY6sdZODvxGMzezFCFUeWBxFNc3dSxcJqN9qpSE8A==
"@cerc-io/util@^0.2.18":
version "0.2.18"
resolved "https://git.vdb.to/api/packages/cerc-io/npm/%40cerc-io%2Futil/-/0.2.18/util-0.2.18.tgz#049502d804e01cb4a8799d5eb004d304a2a406fe"
integrity sha512-qolIr4H1zjqMHR2yV8znGZMCQzbjGaGFplGiMNEPnhrHa+fCe2A2LL1nN97djFoR47jA8th0Ep3VI2PLG7HtDA==
dependencies:
"@apollo/utils.keyvaluecache" "^1.0.1"
"@cerc-io/solidity-mapper" "^0.2.17"
"@cerc-io/solidity-mapper" "^0.2.18"
"@ethersproject/providers" "^5.4.4"
"@graphql-tools/schema" "^9.0.10"
"@graphql-tools/utils" "^9.1.1"
Expand Down