Generate IPLD blocks table and related GQL API (#260)

* Add ipld-blocks entity generation

* Populate ipld-blocks table

* Rename ipld-block entity and update after each event

* Move ipld-hook to hooks.ts

* Change IPLD block structure

* Add cid field in blocks

* Fetch prev. IPLDBlock for a contract

* GQL API to query IPLDBlock by CID

* Save cid in blocks in existing watchers

* Update codegen docs

* GQL API for getting last derived state (#3)

* GQL API for getting last derived state

* Rename query to getState

* Change query names to getState and getStateByCid

* Save BigInt as string

* Move function to prepare IPLDBlock to indexer

* Refactor IPLDBlock hook

* Add genesis hook

* Call post-block hook after a block is marked as complete

* Add IPLDBlock checkpointing

* Use queryRunner instead of a new repo for queries

* Add a query to get block in ipld-eth-client

* Get latest checkpoints for all contracts for checkpointing.

* Call post-block hook in a queue

* Pass server config to Indexer in watch-contract cli

Co-authored-by: nikugogoi <[email protected]>

packages/codegen/README.md

@@ -53,13 +53,13 @@
   yarn codegen --input-file ./test/examples/contracts/ERC721.sol --contract-name ERC721 --output-folder ../my-erc721-watcher --mode storage --kind lazy
   ```
 
-  Generate code for `ERC721` contract in both `eth_call` and `storage` mode, `active` kind:
+  Generate code for `ERC20` contract in both `eth_call` and `storage` mode, `active` kind:
 
   ```bash
-  yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC721/ERC721.sol --contract-name ERC721 --output-folder ../demo-erc721-watcher --mode all --kind active
+  yarn codegen --input-file ../../node_modules/@openzeppelin/contracts/token/ERC20/ERC20.sol --contract-name ERC20 --output-folder ../demo-erc20-watcher --mode all --kind active
   ```
 
-  This will create a folder called `demo-erc721-watcher` containing the generated code at the specified path. Follow the steps in [Run Generated Watcher](#run-generated-watcher) to setup and run the generated watcher.
+  This will create a folder called `demo-erc20-watcher` containing the generated code at the specified path. Follow the steps in [Run Generated Watcher](#run-generated-watcher) to setup and run the generated watcher.
 
 ## Run Generated Watcher
 
@@ -79,7 +79,9 @@
 
   * Edit the custom hook function `handleEvent` (triggered on an event) in `src/hooks.ts` to perform corresponding indexing using the `Indexer` object.
 
-  * Refer to `src/hooks.example.ts` for an example hook function for events in an ERC20 contract.
+  * Edit the custom hook function `handleBlock` (triggered on a block) in `src/hooks.ts` to save `IPLDBlock`s using the `Indexer` object.
+
+  * The existing example hooks in `src/hooks.ts` are for an `ERC20` contract.
 
 ### Run
 
@@ -106,7 +108,7 @@
   * To watch a contract:
 
     ```bash
-    yarn watch:contract --address <contract-address> --kind ERC721 --starting-block [block-number]
+    yarn watch:contract --address <contract-address> --kind <contract-kind> --starting-block [block-number]
     ```
 
   * To fill a block range:

packages/codegen/package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git",
     "@solidity-parser/parser": "^0.13.2",
+    "@vulcanize/util": "^0.1.0",
     "gql-generator": "https://github.com/vulcanize/gql-generator.git",
     "graphql": "^15.5.0",
     "graphql-compose": "^9.0.3",

packages/codegen/src/data/entities/BlockProgress.yaml

@@ -9,6 +9,10 @@ indexOn:
   - columns:
       - parentHash
 columns:
+  - name: cid
+    pgType: varchar
+    tsType: string
+    columnType: Column
   - name: blockHash
     pgType: varchar
     tsType: string

packages/codegen/src/data/entities/IPLDBlock.yaml

@@ -0,0 +1,41 @@
+className: IPLDBlock
+indexOn:
+  - columns:
+      - block
+      - contractAddress
+columns:
+  - name: block
+    tsType: BlockProgress
+    columnType: ManyToOne
+    lhs: ()
+    rhs: BlockProgress
+  - name: contractAddress
+    pgType: varchar
+    tsType: string
+    columnType: Column
+    columnOptions:
+      - option: length
+        value: 42
+  - name: cid
+    pgType: varchar
+    tsType: string
+    columnType: Column
+  - name: kind
+    pgType: varchar
+    tsType: string
+    columnType: Column
+  - name: data
+    pgType: text
+    tsType: string
+    columnType: Column
+imports:
+  - toImport:
+      - Entity
+      - PrimaryGeneratedColumn
+      - Column
+      - Index
+      - ManyToOne
+    from: typeorm
+  - toImport:
+      - BlockProgress
+    from: ./BlockProgress

packages/codegen/src/entity.ts

@@ -37,7 +37,6 @@ export class Entity {
     }
 
     const entityObject: any = {
-      // Capitalize the first letter of name.
       className: '',
       indexOn: [],
       columns: [],
@@ -188,6 +187,7 @@ export class Entity {
     this._addSyncStatusEntity();
     this._addContractEntity();
     this._addBlockProgressEntity();
+    this._addIPLDBlockEntity();
 
     const template = Handlebars.compile(this._templateString);
     this._entities.forEach(entityObj => {
@@ -218,4 +218,9 @@ export class Entity {
     const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'BlockProgress.yaml'), 'utf8'));
     this._entities.push(entity);
   }
+
+  _addIPLDBlockEntity (): void {
+    const entity = yaml.load(fs.readFileSync(path.resolve(__dirname, TABLES_DIR, 'IPLDBlock.yaml'), 'utf8'));
+    this._entities.push(entity);
+  }
 }

packages/codegen/src/generate-code.ts

@@ -7,8 +7,8 @@ import fetch from 'node-fetch';
 import path from 'path';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import { flatten } from '@poanet/solidity-flattener';
 
+import { flatten } from '@poanet/solidity-flattener';
 import { parse, visit } from '@solidity-parser/parser';
 import { KIND_ACTIVE, KIND_LAZY } from '@vulcanize/util';
 
@@ -209,15 +209,12 @@ function generateWatcher (data: string, visitor: Visitor, argv: any) {
   exportWatchContract(outStream);
 
   let hooksOutStream;
-  let exampleOutStream;
   if (outputDir) {
     hooksOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.ts'));
-    exampleOutStream = fs.createWriteStream(path.join(outputDir, 'src/hooks.example.ts'));
   } else {
     hooksOutStream = process.stdout;
-    exampleOutStream = process.stdout;
   }
-  exportHooks(hooksOutStream, exampleOutStream);
+  exportHooks(hooksOutStream);
 
   outStream = outputDir
     ? fs.createWriteStream(path.join(outputDir, 'src/fill.ts'))

packages/codegen/src/hooks.ts

@@ -8,23 +8,14 @@ import Handlebars from 'handlebars';
 import { Writable } from 'stream';
 
 const HOOKS_TEMPLATE_FILE = './templates/hooks-template.handlebars';
-const EXAMPLE_TEMPLATE_FILE = './templates/hooks-example-template.handlebars';
 
 /**
- * Writes the hooks and hooks.example files generated from templates to a stream.
+ * Writes the hooks file generated from template to a stream.
  * @param outStream A writable output stream to write the hooks file to.
- * @param exampleOutStream A writable output stream to write the hooks.example file to.
  */
-export function exportHooks (hooksOutStream: Writable, exampleOutStream: Writable): void {
+export function exportHooks (hooksOutStream: Writable): void {
   const hooksTemplateString = fs.readFileSync(path.resolve(__dirname, HOOKS_TEMPLATE_FILE)).toString();
-  const exampleTemplateString = fs.readFileSync(path.resolve(__dirname, EXAMPLE_TEMPLATE_FILE)).toString();
-
   const hooksTemplate = Handlebars.compile(hooksTemplateString);
-  const exampleTemplate = Handlebars.compile(exampleTemplateString);
-
   const hooks = hooksTemplate({});
-  const example = exampleTemplate({});
-
   hooksOutStream.write(hooks);
-  exampleOutStream.write(example);
 }

packages/codegen/src/schema.ts

@@ -97,6 +97,9 @@ export class Schema {
     // Add a mutation for watching a contract.
     this._addWatchContractMutation();
 
+    this._addIPLDType();
+    this._addIPLDQuery();
+
     return this._composer.buildSchema();
   }
 
@@ -173,6 +176,7 @@ export class Schema {
     this._composer.createObjectTC({
       name: blockName,
       fields: {
+        cid: 'String!',
         hash: 'String!',
         number: 'Int!',
         timestamp: 'Int!',
@@ -234,6 +238,40 @@ export class Schema {
     });
   }
 
+  _addIPLDType (): void {
+    this._composer.createObjectTC({
+      name: 'ResultIPLDBlock',
+      fields: {
+        block: () => this._composer.getOTC('Block').NonNull,
+        contractAddress: 'String!',
+        cid: 'String!',
+        kind: 'String!',
+        data: 'String!'
+      }
+    });
+  }
+
+  _addIPLDQuery (): void {
+    this._composer.Query.addFields({
+      getStateByCID: {
+        type: this._composer.getOTC('ResultIPLDBlock'),
+        args: {
+          cid: 'String!'
+        }
+      }
+    });
+
+    this._composer.Query.addFields({
+      getState: {
+        type: this._composer.getOTC('ResultIPLDBlock'),
+        args: {
+          blockHash: 'String!',
+          contractAddress: 'String!'
+        }
+      }
+    });
+  }
+
   /**
    * Adds an event subscription to the schema.
    */
@@ -254,6 +292,7 @@ export class Schema {
         type: 'Boolean!',
         args: {
           contractAddress: 'String!',
+          kind: 'String!',
           startingBlock: 'Int'
         }
       }

packages/codegen/src/templates/client-template.handlebars

@@ -3,7 +3,6 @@
 //
 
 import { gql } from '@apollo/client/core';
-
 import { GraphQLClient, GraphQLConfig } from '@vulcanize/ipld-eth-client';
 
 import { queries, mutations, subscriptions } from './gql';

packages/codegen/src/templates/config-template.handlebars

@@ -3,6 +3,12 @@
   port = 3008
   kind = "{{watcherKind}}"
 
+  # Checkpointing derived state.
+  checkpointing = true
+
+  # Checkpoint interval in number of blocks.
+  checkpointInterval = 2000
+
 [database]
   type = "postgres"
   host = "localhost"