Implement fill state to create state in uni-info-watcher (#369)

* Implement fill state to create state in uni-info-watcher

* Add inspect-cid and watch-contract CLIs

* Use util indexer and database from watcher-ts

* Implement CLI to fetch and update block CIDs

* Expose entity fields in GQL query for verifying state in comapre CLI

* Use watcher-ts packages from gitea registry

* Remove cerc-io packages to run locally with yarn link

.npmrc

@@ -0,0 +1 @@
+@cerc-io:registry=https://git.vdb.to/api/packages/cerc-io/npm/

README.md

@@ -1,16 +1,48 @@
-# watcher-ts
+# uniswap-watcher-ts
 
 ## Setup
 
 This project uses [yarn workspaces](https://classic.yarnpkg.com/en/docs/workspaces/).
 
-Install packages (Node.JS v16.13.1):
-
-```bash
-yarn
-
-yarn build
-```
+- Install packages (Node.JS v16.15.1):
+
+  ```bash
+  yarn
+  ```
+
+- Link [@cerc-io/watcher-ts](https://github.com/cerc-io/watcher-ts) packages:
+
+  - In `@cerc-io/watcher-ts` repo, build and link the packages to use from uniswap-watcher-ts
+
+    ```bash
+    # Build packages
+    yarn && yarn build
+
+    # Link packages
+    cd packages/util && yarn link && cd ../..
+    cd packages/ipld-eth-client && yarn link && cd ../..
+    cd packages/solidity-mapper && yarn link && cd ../..
+    # Workaround for typeorm dependency issue when using yarn link
+    cd node_modules/typeorm && yarn link && cd ../..
+    ```
+
+  - In `uniswap-watcher-ts`:
+
+    ```bash
+    yarn link "@cerc-io/util"
+    yarn link "@cerc-io/ipld-eth-client"
+    yarn link "@cerc-io/solidity-mapper"
+    yarn link "typeorm"
+    ```
+
+- Build packages:
+
+  ```bash
+  yarn build
+
+  # For running tests
+  yarn build:contracts
+  ```
 
 ### Services
 

packages/erc20-watcher/package.json

@@ -46,7 +46,6 @@
     "@types/lodash": "^4.14.168",
     "@vulcanize/cache": "^0.1.0",
     "@vulcanize/ipld-eth-client": "^0.1.0",
-    "@vulcanize/solidity-mapper": "^0.1.0",
     "@vulcanize/util": "^0.1.0",
     "apollo-server-express": "^2.25.0",
     "apollo-type-bigint": "^0.1.3",

packages/erc20-watcher/src/cli/reset-cmds/state.ts

@@ -44,7 +44,7 @@ export const handler = async (argv: any): Promise<void> => {
 
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
 
-  const indexer = new Indexer(db, ethClient, ethProvider, jobQueue, serverConfig.mode);
+  const indexer = new Indexer(serverConfig, db, ethClient, ethProvider, jobQueue);
 
   const syncStatus = await indexer.getSyncStatus();
   assert(syncStatus, 'Missing syncStatus');

packages/erc20-watcher/src/cli/watch-contract.ts

@@ -9,7 +9,7 @@ import 'reflect-metadata';
 import { Config, DEFAULT_CONFIG_PATH, getConfig, getResetConfig, JobQueue } from '@vulcanize/util';
 
 import { Database } from '../database';
-import { Indexer } from '../indexer';
+import { CONTRACT_KIND, Indexer } from '../indexer';
 
 (async () => {
   const argv = await yargs.parserConfiguration({
@@ -29,6 +29,12 @@ import { Indexer } from '../indexer';
       demandOption: true,
       describe: 'Address of the deployed contract'
     },
+    checkpoint: {
+      type: 'boolean',
+      require: true,
+      demandOption: true,
+      describe: 'Turn checkpointing on'
+    },
     startingBlock: {
       type: 'number',
       default: 1,
@@ -37,7 +43,7 @@ import { Indexer } from '../indexer';
   }).argv;
 
   const config: Config = await getConfig(argv.configFile);
-  const { database: dbConfig, server: { mode }, jobQueue: jobQueueConfig } = config;
+  const { database: dbConfig, jobQueue: jobQueueConfig } = config;
   const { ethClient, ethProvider } = await getResetConfig(config);
 
   assert(dbConfig);
@@ -53,9 +59,9 @@ import { Indexer } from '../indexer';
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
   await jobQueue.start();
 
-  const indexer = new Indexer(db, ethClient, ethProvider, jobQueue, mode);
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
 
-  await indexer.watchContract(argv.address, argv.startingBlock);
+  await indexer.watchContract(argv.address, CONTRACT_KIND, argv.checkpoint, argv.startingBlock);
 
   await db.close();
   await jobQueue.stop();

packages/erc20-watcher/src/database.ts

@@ -6,7 +6,7 @@ import assert from 'assert';
 import { Connection, ConnectionOptions, DeepPartial, FindConditions, FindManyOptions, QueryRunner } from 'typeorm';
 import path from 'path';
 
-import { Database as BaseDatabase, QueryOptions, Where } from '@vulcanize/util';
+import { Database as BaseDatabase, DatabaseInterface, StateKind, QueryOptions, Where } from '@cerc-io/util';
 
 import { Allowance } from './entity/Allowance';
 import { Balance } from './entity/Balance';
@@ -17,8 +17,10 @@ import { Contract } from './entity/Contract';
 import { Event } from './entity/Event';
 import { SyncStatus } from './entity/SyncStatus';
 import { BlockProgress } from './entity/BlockProgress';
+import { IPLDBlock } from './entity/IPLDBlock';
+import { IpldStatus } from './entity/IpldStatus';
 
-export class Database {
+export class Database implements DatabaseInterface {
   _config: ConnectionOptions
   _conn!: Connection
   _baseDatabase: BaseDatabase;
@@ -42,6 +44,47 @@ export class Database {
     return this._baseDatabase.close();
   }
 
+  getNewIPLDBlock (): IPLDBlock {
+    return new IPLDBlock();
+  }
+
+  async getIPLDBlocks (where: FindConditions<IPLDBlock>): Promise<IPLDBlock[]> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getIPLDBlocks(repo, where);
+  }
+
+  async getLatestIPLDBlock (contractAddress: string, kind: StateKind | null, blockNumber?: number): Promise<IPLDBlock | undefined> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getLatestIPLDBlock(repo, contractAddress, kind, blockNumber);
+  }
+
+  // Fetch all diff IPLDBlocks after the specified block number.
+  async getDiffIPLDBlocksInRange (contractAddress: string, startblock: number, endBlock: number): Promise<IPLDBlock[]> {
+    const repo = this._conn.getRepository(IPLDBlock);
+
+    return this._baseDatabase.getDiffIPLDBlocksInRange(repo, contractAddress, startblock, endBlock);
+  }
+
+  async saveOrUpdateIPLDBlock (dbTx: QueryRunner, ipldBlock: IPLDBlock): Promise<IPLDBlock> {
+    const repo = dbTx.manager.getRepository(IPLDBlock);
+
+    return this._baseDatabase.saveOrUpdateIPLDBlock(repo, ipldBlock);
+  }
+
+  async removeIPLDBlocks (dbTx: QueryRunner, blockNumber: number, kind: string): Promise<void> {
+    const repo = dbTx.manager.getRepository(IPLDBlock);
+
+    await this._baseDatabase.removeIPLDBlocks(repo, blockNumber, kind);
+  }
+
+  async getIPLDStatus (): Promise<IpldStatus | undefined> {
+    const repo = this._conn.getRepository(IpldStatus);
+
+    return this._baseDatabase.getIPLDStatus(repo);
+  }
+
   async getBalance ({ blockHash, token, owner }: { blockHash: string, token: string, owner: string }): Promise<Balance | undefined> {
     return this._conn.getRepository(Balance)
       .createQueryBuilder('balance')
@@ -154,16 +197,23 @@ export class Database {
     return this._baseDatabase.getBlockEvents(repo, blockHash, where, queryOptions);
   }
 
+  async saveBlockWithEvents (queryRunner: QueryRunner, block: DeepPartial<BlockProgress>, events: DeepPartial<Event>[]): Promise<BlockProgress> {
+    const blockRepo = queryRunner.manager.getRepository(BlockProgress);
+    const eventRepo = queryRunner.manager.getRepository(Event);
+
+    return this._baseDatabase.saveBlockWithEvents(blockRepo, eventRepo, block, events);
+  }
+
   async saveEvents (queryRunner: QueryRunner, events: Event[]): Promise<void> {
     const eventRepo = queryRunner.manager.getRepository(Event);
 
     return this._baseDatabase.saveEvents(eventRepo, events);
   }
 
-  async saveContract (queryRunner: QueryRunner, address: string, kind: string, startingBlock: number): Promise<Contract> {
+  async saveContract (queryRunner: QueryRunner, address: string, kind: string, checkpoint: boolean, startingBlock: number): Promise<Contract> {
     const repo = queryRunner.manager.getRepository(Contract);
 
-    return this._baseDatabase.saveContract(repo, address, startingBlock, kind);
+    return this._baseDatabase.saveContract(repo, address, kind, checkpoint, startingBlock);
   }
 
   async updateSyncStatusIndexedBlock (queryRunner: QueryRunner, blockHash: string, blockNumber: number, force = false): Promise<SyncStatus> {

packages/erc20-watcher/src/entity/BlockProgress.ts

@@ -14,6 +14,9 @@ export class BlockProgress implements BlockProgressInterface {
   @PrimaryGeneratedColumn()
   id!: number;
 
+  @Column('varchar')
+  cid!: string;
+
   @Column('varchar', { length: 66 })
   blockHash!: string;
 

packages/erc20-watcher/src/entity/Contract.ts

@@ -16,6 +16,9 @@ export class Contract {
   @Column('varchar', { length: 8 })
   kind!: string;
 
+  @Column('boolean', { default: false })
+  checkpoint!: boolean;
+
   @Column('integer')
   startingBlock!: number;
 }

packages/erc20-watcher/src/entity/IPLDBlock.ts

@@ -0,0 +1,36 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column, Index, ManyToOne } from 'typeorm';
+
+import { StateKind } from '@cerc-io/util';
+
+import { BlockProgress } from './BlockProgress';
+
+@Entity()
+@Index(['cid'], { unique: true })
+@Index(['block', 'contractAddress'])
+@Index(['block', 'contractAddress', 'kind'], { unique: true })
+export class IPLDBlock {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @ManyToOne(() => BlockProgress, { onDelete: 'CASCADE' })
+  block!: BlockProgress;
+
+  @Column('varchar', { length: 42 })
+  contractAddress!: string;
+
+  @Column('varchar')
+  cid!: string;
+
+  @Column({
+    type: 'enum',
+    enum: StateKind
+  })
+  kind!: StateKind;
+
+  @Column('bytea')
+  data!: Buffer;
+}

packages/erc20-watcher/src/entity/IpldStatus.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2022 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
+
+@Entity()
+export class IpldStatus {
+  @PrimaryGeneratedColumn()
+  id!: number;
+
+  @Column('integer')
+  latestHooksBlockNumber!: number;
+
+  @Column('integer', { nullable: true })
+  latestCheckpointBlockNumber!: number;
+
+  @Column('integer', { nullable: true })
+  latestIPFSBlockNumber!: number;
+}

packages/erc20-watcher/src/entity/SyncStatus.ts

@@ -34,4 +34,10 @@ export class SyncStatus implements SyncStatusInterface {
 
   @Column('integer')
   latestCanonicalBlockNumber!: number;
+
+  @Column('varchar', { length: 66 })
+  initialIndexedBlockHash!: string;
+
+  @Column('integer')
+  initialIndexedBlockNumber!: number;
 }

packages/erc20-watcher/src/fill.ts

@@ -10,8 +10,8 @@ import debug from 'debug';
 import { PubSub } from 'apollo-server-express';
 
 import { getCache } from '@vulcanize/cache';
-import { EthClient } from '@vulcanize/ipld-eth-client';
 import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, getCustomProvider } from '@vulcanize/util';
+import { EthClient } from '@cerc-io/ipld-eth-client';
 
 import { Database } from './database';
 import { Indexer } from './indexer';
@@ -54,14 +54,19 @@ export const main = async (): Promise<any> => {
       type: 'number',
       default: 10,
       describe: 'Number of blocks prefetched in batch'
+    },
+    blockCid: {
+      type: 'boolean',
+      default: false,
+      describe: 'Only fetch and update block CIDs'
     }
   }).argv;
 
   const config = await getConfig(argv.configFile);
 
   assert(config.server, 'Missing server config');
 
-  const { upstream, database: dbConfig, jobQueue: jobQueueConfig, server: { mode } } = config;
+  const { upstream, database: dbConfig, jobQueue: jobQueueConfig } = config;
 
   assert(dbConfig, 'Missing database config');
 
@@ -89,7 +94,7 @@ export const main = async (): Promise<any> => {
   const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
   await jobQueue.start();
 
-  const indexer = new Indexer(db, ethClient, ethProvider, jobQueue, mode);
+  const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue);
 
   const eventWatcher = new EventWatcher(upstream, ethClient, indexer, pubsub, jobQueue);