make db its own package (#460)

* make db its own package

.github/workflows/publish.yml

@@ -46,8 +46,8 @@ jobs:
     - run: |
         echo npmAuthToken: "\${NPM_AUTH_TOKEN}" >> ./.yarnrc.yml
 
-    - name: Publish @acala-network/chopsticks
-      run: yarn workspace @acala-network/chopsticks npm publish --tolerate-republish --access public ${{ env.NPM_TAG }}
+    - name: Publish @acala-network/chopsticks-executor
+      run: yarn workspace @acala-network/chopsticks-executor npm publish --tolerate-republish --access public ${{ env.NPM_TAG }}
       env:
         NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
@@ -56,8 +56,13 @@ jobs:
       env:
         NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
-    - name: Publish @acala-network/chopsticks-executor
-      run: yarn workspace @acala-network/chopsticks-executor npm publish --tolerate-republish --access public ${{ env.NPM_TAG }}
+    - name: Publish @acala-network/chopsticks-db
+      run: yarn workspace @acala-network/chopsticks-db npm publish --tolerate-republish --access public ${{ env.NPM_TAG }}
+      env:
+        NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
+
+    - name: Publish @acala-network/chopsticks
+      run: yarn workspace @acala-network/chopsticks npm publish --tolerate-republish --access public ${{ env.NPM_TAG }}
       env:
         NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 

packages/chopsticks/package.json

@@ -18,6 +18,7 @@
 	},
 	"dependencies": {
 		"@acala-network/chopsticks-core": "workspace:*",
+		"@acala-network/chopsticks-db": "workspace:*",
 		"@pnpm/npm-conf": "^2.2.2",
 		"@polkadot/api": "^10.9.1",
 		"axios": "^1.5.1",

packages/chopsticks/src/context.ts

@@ -1,8 +1,8 @@
 import './utils/tunnel'
-import { BlockEntity } from '@acala-network/chopsticks-core/db/entities'
+import { BlockEntry, defaultLogger, setup, timeTravel } from '@acala-network/chopsticks-core'
 import { Config } from './schema'
 import { HexString } from '@polkadot/util/types'
-import { defaultLogger, setup, timeTravel } from '@acala-network/chopsticks-core'
+import { SqliteDatabase } from '@acala-network/chopsticks-db'
 import { overrideStorage, overrideWasm } from './utils/override'
 
 const logger = defaultLogger.child({ name: 'setup-context' })
@@ -13,7 +13,7 @@ export const setupContext = async (argv: Config, overrideParent = false) => {
     block: argv.block,
     genesis: argv.genesis,
     buildBlockMode: argv['build-block-mode'],
-    db: argv.db,
+    db: argv.db ? new SqliteDatabase(argv.db) : undefined,
     mockSignatureHost: argv['mock-signature-host'],
     allowUnresolvedImports: argv['allow-unresolved-imports'],
     runtimeLogLevel: argv['runtime-log-level'],
@@ -25,20 +25,21 @@ export const setupContext = async (argv: Config, overrideParent = false) => {
   // load block from db
   if (chain.db) {
     if (argv.resume) {
-      const where: Record<string, string | number> = {}
+      let blockData: BlockEntry | null = null
+
       switch (typeof argv.resume) {
         case 'string':
-          where.hash = argv.resume
+          blockData = await chain.db.queryBlock(argv.resume as HexString)
           break
         case 'number':
-          where.number = argv.resume
+          blockData = await chain.db.queryBlockByNumber(argv.resume)
           break
         default:
+          blockData = await chain.db.queryHighestBlock()
           break
       }
-      const blockData = await chain.db.getRepository(BlockEntity).findOne({ where, order: { number: 'desc' } })
       if (blockData) {
-        const block = await chain.loadBlockFromDB(blockData?.number)
+        const block = await chain.loadBlockFromDB(blockData.number)
         block && (await chain.setHead(block))
         logger.info(`Resume from block ${blockData.number}, hash: ${blockData.hash}`)
       } else {

packages/chopsticks/tsconfig.json

@@ -6,7 +6,7 @@
 	},
 	"include": ["src/**/*"],
 	"exclude": ["src/**/*.test.ts"],
-	"references": [{ "path": "../core" }],
+	"references": [{ "path": "../core" }, { "path": "../db" }],
 	"typedocOptions": {
 		"entryPoints": ["src/types.ts"],
 		"out": "../../docs-src/chopsticks",

packages/core/package.json

@@ -5,8 +5,7 @@
 	"license": "Apache-2.0",
 	"scripts": {
 		"clean": "rm -rf lib tsconfig.tsbuildinfo",
-		"pack-wasm": "scripts/pack-wasm.cjs",
-		"build": "yarn pack-wasm; tsc -p ./tsconfig.json; yarn copyfiles",
+		"build": "tsc -p ./tsconfig.json; yarn copyfiles",
 		"copyfiles": "cp -r src/wasm-executor/*.mjs lib/wasm-executor/",
 		"docs:prep": "typedoc"
 	},
@@ -17,19 +16,13 @@
 		"axios": "^1.5.1",
 		"comlink": "^4.4.1",
 		"eventemitter3": "^5.0.1",
-		"localforage": "^1.10.0",
 		"lodash": "^4.17.21",
 		"pino": "^8.15.0",
 		"pino-pretty": "^10.2.0",
-		"sql.js": "^1.8.0",
-		"sqlite3": "^5.1.6",
-		"typeorm": "^0.3.17",
 		"zod": "^3.22.3"
 	},
 	"devDependencies": {
 		"@types/lodash": "^4.14.199",
-		"@types/sql.js": "^1.4.4",
-		"fflate": "^0.8.0",
 		"typescript": "^5.1.6"
 	},
 	"files": [

packages/core/src/blockchain/index.ts

@@ -1,5 +1,4 @@
 import { ApplyExtrinsicResult, Header } from '@polkadot/types/interfaces'
-import { DataSource } from 'typeorm'
 import { HexString } from '@polkadot/util/types'
 import { RegisteredTypes } from '@polkadot/types/types'
 import { blake2AsHex } from '@polkadot/util-crypto'
@@ -8,8 +7,8 @@ import type { TransactionValidity } from '@polkadot/types/interfaces/txqueue'
 
 import { Api } from '../api'
 import { Block } from './block'
-import { BlockEntity } from '../db/entities'
 import { BuildBlockMode, BuildBlockParams, DownwardMessage, HorizontalMessage, TxPool } from './txpool'
+import { Database } from '../database'
 import { HeadState } from './head-state'
 import { InherentProvider } from './inherent'
 import { OffchainWorker } from '../offchain'
@@ -29,7 +28,7 @@ export interface Options {
   /** Inherent provider, for creating inherents. */
   inherentProvider: InherentProvider
   /** Datasource for caching storage and blocks data. */
-  db?: DataSource
+  db?: Database
   /** Used to create the initial head. */
   header: { number: number; hash: HexString }
   /** Whether to enable mock signature. Any signature starts with 0xdeadbeef and filled by 0xcd is considered valid */
@@ -74,7 +73,7 @@ export class Blockchain {
   /** API instance, for getting on-chain data. */
   readonly api: Api
   /** Datasource for caching storage and blocks data. */
-  readonly db: DataSource | undefined
+  readonly db: Database | undefined
   /** Enable mock signature. Any signature starts with 0xdeadbeef and filled by 0xcd is considered valid */
   readonly mockSignatureHost: boolean
   /** Allow wasm unresolved imports. */
@@ -166,19 +165,13 @@ export class Blockchain {
     if (this.db) {
       const { hash, number, header, extrinsics } = block
       // delete old ones with the same block number if any, keep the latest one
-      await this.db.transaction(async (transactionalEntityManager) => {
-        await transactionalEntityManager.getRepository(BlockEntity).delete({ number })
-        await transactionalEntityManager.getRepository(BlockEntity).upsert(
-          {
-            hash,
-            number,
-            header: await header,
-            extrinsics: await extrinsics,
-            parentHash: (await block.parentBlock)?.hash,
-            storageDiff: await block.storageDiff(),
-          },
-          ['hash'],
-        )
+      await this.db.saveBlock({
+        hash,
+        number,
+        header: (await header).toHex(),
+        extrinsics: await extrinsics,
+        parentHash: (await block.parentBlock)?.hash || null,
+        storageDiff: await block.storageDiff(),
       })
     }
   }
@@ -187,11 +180,12 @@ export class Blockchain {
    * Try to load block from db and register it.
    * If pass in number, get block by number, else get block by hash.
    */
-  async loadBlockFromDB(key: number | HexString): Promise<Block | undefined> {
+  async loadBlockFromDB(hashOrNumber: number | HexString): Promise<Block | undefined> {
     if (this.db) {
-      const blockData = await this.db
-        .getRepository(BlockEntity)
-        .findOne({ where: { [typeof key === 'number' ? 'number' : 'hash']: key } })
+      const blockData =
+        typeof hashOrNumber === 'number'
+          ? await this.db.queryBlockByNumber(hashOrNumber)
+          : await this.db.queryBlock(hashOrNumber)
       if (blockData) {
         const { hash, number, header, extrinsics } = blockData
         const parentHash = blockData.parentHash || undefined
@@ -295,7 +289,7 @@ export class Blockchain {
     delete this.#blocksByHash[block.hash]
     // delete from db
     if (this.db) {
-      await this.db.getRepository(BlockEntity).delete({ hash: block.hash })
+      await this.db.deleteBlock(block.hash)
     }
   }
 
@@ -516,6 +510,6 @@ export class Blockchain {
   async close() {
     await releaseWorker()
     await this.api.disconnect()
-    await this.db?.destroy()
+    await this.db?.close()
   }
 }

packages/core/src/blockchain/storage-layer.ts

@@ -1,8 +1,8 @@
-import { DataSource } from 'typeorm'
+import { HexString } from '@polkadot/util/types'
 import _ from 'lodash'
 
 import { Api } from '../api'
-import { KeyValuePair } from '../db/entities'
+import { Database } from '../database'
 import { defaultLogger } from '../logger'
 import KeyCache, { PREFIX_LENGTH } from '../utils/key-cache'
 
@@ -39,26 +39,25 @@ export interface StorageLayerProvider {
 export class RemoteStorageLayer implements StorageLayerProvider {
   readonly #api: Api
   readonly #at: string
-  readonly #db: DataSource | undefined
+  readonly #db: Database | undefined
   readonly #keyCache = new KeyCache()
 
-  constructor(api: Api, at: string, db: DataSource | undefined) {
+  constructor(api: Api, at: string, db: Database | undefined) {
     this.#api = api
     this.#at = at
     this.#db = db
   }
 
   async get(key: string, _cache: boolean): Promise<StorageValue> {
-    const keyValuePair = this.#db?.getRepository(KeyValuePair)
     if (this.#db) {
-      const res = await keyValuePair?.findOne({ where: { key, blockHash: this.#at } })
+      const res = await this.#db.queryStorage(this.#at as HexString, key as HexString)
       if (res) {
         return res.value ?? undefined
       }
     }
     logger.trace({ at: this.#at, key }, 'RemoteStorageLayer get')
     const data = await this.#api.getStorage(key, this.#at)
-    keyValuePair?.upsert({ key, blockHash: this.#at, value: data }, ['key', 'blockHash'])
+    this.#db?.saveStorage(this.#at as HexString, key as HexString, data)
     return data ?? undefined
   }
 

packages/core/src/chopsticks-provider.ts

@@ -8,6 +8,7 @@ import {
 
 import { Blockchain } from './blockchain'
 import { Context, Handlers, allHandlers } from './rpc'
+import { Database } from './database'
 import { defaultLogger } from './logger'
 import { setup } from './setup'
 
@@ -52,13 +53,13 @@ export class ChopsticksProvider implements ProviderInterface {
     })
   }
 
-  static fromEndpoint = async (endpoint: string, block?: number | string | null, cache?: string) => {
+  static fromEndpoint = async (endpoint: string, block?: number | string | null, db?: Database) => {
     return new ChopsticksProvider(
       await setup({
         endpoint,
         mockSignatureHost: true,
         block,
-        db: cache,
+        db,
       }),
     )
   }

packages/core/src/database.ts

@@ -0,0 +1,29 @@
+import { HexString } from '@polkadot/util/types'
+
+export interface BlockEntry {
+  hash: HexString
+  number: number
+  header: HexString
+  parentHash: HexString | null
+  extrinsics: HexString[]
+  storageDiff: Record<HexString, HexString | null> | null
+}
+
+export interface KeyValueEntry {
+  blockHash: string
+  key: string
+  value: string | null
+}
+
+export declare class Database {
+  constructor(location: string)
+  close: () => Promise<void>
+  saveBlock: (block: BlockEntry) => Promise<void>
+  queryBlock: (hash: HexString) => Promise<BlockEntry | null>
+  queryBlockByNumber: (number: number) => Promise<BlockEntry | null>
+  queryHighestBlock: () => Promise<BlockEntry | null>
+  deleteBlock: (hash: HexString) => Promise<void>
+  blocksCount: () => Promise<number>
+  saveStorage: (blockHash: HexString, key: HexString, value: HexString | null) => Promise<void>
+  queryStorage: (blockHash: HexString, key: HexString) => Promise<KeyValueEntry | null>
+}

packages/core/src/index.ts

@@ -19,6 +19,7 @@ export * from './wasm-executor'
 export * from './schema'
 export * from './xcm'
 export * from './setup'
+export * from './database'
 export * from './blockchain/inherent'
 export * from './logger'
 export * from './offchain'

packages/core/src/setup.ts

@@ -1,5 +1,4 @@
 import '@polkadot/types-codec'
-import { DataSource } from 'typeorm'
 import { HexString } from '@polkadot/util/types'
 import { ProviderInterface } from '@polkadot/rpc-provider/types'
 import { RegisteredTypes } from '@polkadot/types/types'
@@ -8,6 +7,7 @@ import { WsProvider } from '@polkadot/api'
 import { Api } from './api'
 import { Blockchain } from './blockchain'
 import { BuildBlockMode } from './blockchain/txpool'
+import { Database } from './database'
 import { Genesis } from './schema'
 import { GenesisProvider } from './genesis-provider'
 import {
@@ -19,14 +19,13 @@ import {
   SetValidationData,
 } from './blockchain/inherent'
 import { defaultLogger } from './logger'
-import { openDb } from './db'
 
 export type SetupOptions = {
   endpoint?: string
   block?: string | number | null
   genesis?: string | Genesis
   buildBlockMode?: BuildBlockMode
-  db?: string
+  db?: Database
   mockSignatureHost?: boolean
   allowUnresolvedImports?: boolean
   runtimeLogLevel?: number
@@ -73,11 +72,6 @@ export const setup = async (options: SetupOptions) => {
 
   defaultLogger.debug({ ...options, blockHash }, 'Args')
 
-  let db: DataSource | undefined
-  if (options.db) {
-    db = await openDb(options.db)
-  }
-
   const header = await api.getHeader(blockHash)
   if (!header) {
     throw new Error(`Cannot find header for ${blockHash}`)
@@ -94,7 +88,7 @@ export const setup = async (options: SetupOptions) => {
     api,
     buildBlockMode: options.buildBlockMode,
     inherentProvider: inherents,
-    db,
+    db: options.db,
     header: {
       hash: blockHash as HexString,
       number: Number(header.number),