use indexedDB instead of sqljs

packages/db/package.json

@@ -5,21 +5,16 @@
 	"license": "Apache-2.0",
 	"scripts": {
 		"clean": "rm -rf lib tsconfig.tsbuildinfo",
-		"pack-wasm": "scripts/pack-wasm.cjs",
-		"build": "yarn pack-wasm; tsc -p ./tsconfig.json"
+		"build": "tsc -p ./tsconfig.json"
 	},
 	"dependencies": {
 		"@acala-network/chopsticks-core": "workspace:*",
-		"@polkadot/util": "^12.5.1",
-		"@polkadot/wasm-util": "^7.2.2",
+		"idb": "^7.1.1",
 		"localforage": "^1.10.0",
-		"sql.js": "^1.8.0",
 		"sqlite3": "^5.1.6",
 		"typeorm": "^0.3.17"
 	},
 	"devDependencies": {
-		"@types/sql.js": "^1.4.4",
-		"fflate": "^0.8.0",
 		"typescript": "^5.1.6"
 	},
 	"files": [

packages/db/src/browser.ts

@@ -1,13 +1,84 @@
-import { DataSource } from 'typeorm'
+import { BlockEntry, Database, KeyValueEntry } from '@acala-network/chopsticks-core'
+import { DBSchema, IDBPDatabase, openDB } from 'idb'
 
-import { BaseSqlDatabase } from './base-sql'
-import { openDb } from './db/browser'
-
-export class SqljsDatabase extends BaseSqlDatabase {
-  datasource: Promise<DataSource>
+interface Schema extends DBSchema {
+  keyValue: {
+    key: string
+    value: KeyValueEntry
+  }
+  block: {
+    key: string
+    value: BlockEntry
+    indexes: { byNumber: number }
+  }
+}
+export class IdbDatabase implements Database {
+  datasource: Promise<IDBPDatabase<Schema>>
 
   constructor(location: string) {
-    super()
-    this.datasource = openDb(location)
+    this.datasource = openDB<Schema>(location, 1, {
+      upgrade(db) {
+        db.createObjectStore('keyValue', { keyPath: 'key' })
+        const blockStore = db.createObjectStore('block', { keyPath: 'hash' })
+        blockStore.createIndex('byNumber', 'number')
+      },
+    })
+  }
+
+  async close(): Promise<void> {
+    const db = await this.datasource
+    db.close()
+  }
+
+  async saveBlock(block: BlockEntry): Promise<void> {
+    const db = await this.datasource
+    const tx = db.transaction(['block'], 'readwrite')
+    const store = tx.objectStore('block')
+    store.delete(block.hash)
+    store.put(block)
+    await tx.done
+  }
+
+  async queryBlock(hash: `0x${string}`): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const block = await db.get('block', hash)
+    return block ?? null
+  }
+
+  async queryBlockByNumber(number: number): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const block = await db.getFromIndex('block', 'byNumber', number)
+    return block ?? null
+  }
+
+  async queryHighestBlock(): Promise<BlockEntry | null> {
+    const db = await this.datasource
+    const block = await db.getAllFromIndex('block', 'byNumber')
+    return block[0] ?? null
+  }
+
+  async deleteBlock(hash: `0x${string}`): Promise<void> {
+    const db = await this.datasource
+    await db.delete('block', hash)
+  }
+
+  async blocksCount(): Promise<number> {
+    const db = await this.datasource
+    return db.count('block')
+  }
+
+  async saveStorage(blockHash: `0x${string}`, key: `0x${string}`, value: `0x${string}` | null): Promise<void> {
+    const db = await this.datasource
+    await db.put('keyValue', {
+      blockHash,
+      key,
+      value,
+    })
+  }
+
+  async queryStorage(_blockHash: `0x${string}`, key: `0x${string}`): Promise<KeyValueEntry | null> {
+    const db = await this.datasource
+    const keyValue = await db.get('keyValue', key)
+    return keyValue ?? null
   }
 }

packages/web-test/src/App.tsx

@@ -15,11 +15,11 @@ import { ApiPromise } from '@polkadot/api'
 import { Buffer } from 'buffer'
 import { ChopsticksProvider, setStorage, setup } from '@acala-network/chopsticks-core'
 import { HexString } from '@polkadot/util/types'
+import { IdbDatabase } from '@acala-network/chopsticks-db/browser'
 import { createTestPairs } from '@polkadot/keyring'
 import { styled } from '@mui/system'
 import { useEffect, useState } from 'react'
 import type { SetupOptions } from '@acala-network/chopsticks-core'
-// import { SqljsDatabase } from '@acala-network/chopsticks-db/browser'
 
 window.Buffer = Buffer
 
@@ -109,8 +109,7 @@ function App() {
 			endpoint: config.endpoint,
 			block: config.block,
 			mockSignatureHost: true,
-			// disable cache, this makes it slower
-			// db: new SqljsDatabase('cache'),
+			db: new IdbDatabase('cache'),
 		})
 		globalThis.chain = chain
 

packages/web-test/tests/index.spec.ts

@@ -20,6 +20,21 @@ test.describe('index', () => {
     await page.getByText(/build block/i).click()
     // wait for new block
     await expect(page.locator('#blocks-section')).toHaveText(/4000001/, { timeout: 200_000 })
+    // check indexedDB
+    const savedHash = await page.evaluate(async () => {
+      const db = window.indexedDB.open('cache', 1)
+      await new Promise((resolve) => {
+        db.onsuccess = resolve
+      })
+      const tx = db.result.transaction('block', 'readonly')
+      const store = tx.objectStore('block')
+      const block = store.index('byNumber').get(4000001)
+      await new Promise((resolve) => {
+        block.onsuccess = resolve
+      })
+      return block.result.hash
+    })
+    expect(savedHash).toBe('0x6b81a9a7fabbe32c1e685b944c8f1afd06be7e58ae48bb8d5ac50cc761d9bb77')
   })
 
   test('dry run extrinsic', async ({ page }) => {

yarn.lock

@@ -36,12 +36,8 @@ __metadata:
   resolution: "@acala-network/chopsticks-db@workspace:packages/db"
   dependencies:
     "@acala-network/chopsticks-core": "workspace:*"
-    "@polkadot/util": ^12.5.1
-    "@polkadot/wasm-util": ^7.2.2
-    "@types/sql.js": ^1.4.4
-    fflate: ^0.8.0
+    idb: ^7.1.1
     localforage: ^1.10.0
-    sql.js: ^1.8.0
     sqlite3: ^5.1.6
     typeorm: ^0.3.17
     typescript: ^5.1.6
@@ -2274,13 +2270,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/emscripten@npm:*":
-  version: 1.39.7
-  resolution: "@types/emscripten@npm:1.39.7"
-  checksum: 9871e4495358cc06cc45b2798022cd097d8ac2eb5b2fae7c276c6c5cadea05507150fad053c73ed346d4cbd844c50a3438604e5d7c3c2a7446b703cacb1ce172
-  languageName: node
-  linkType: hard
-
 "@types/estree@npm:^1.0.0":
   version: 1.0.1
   resolution: "@types/estree@npm:1.0.1"
@@ -2420,16 +2409,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/sql.js@npm:^1.4.4":
-  version: 1.4.4
-  resolution: "@types/sql.js@npm:1.4.4"
-  dependencies:
-    "@types/emscripten": "*"
-    "@types/node": "*"
-  checksum: c71928d76b59e4899da4b137a4f1bbf8cc0cf198928efa183bba90880ae69331da23a99175b88a7f5eaa34ce9ebef8f94dd1dcca8e4abdac2abb560d860e6636
-  languageName: node
-  linkType: hard
-
 "@types/strip-bom@npm:^3.0.0":
   version: 3.0.0
   resolution: "@types/strip-bom@npm:3.0.0"
@@ -5427,6 +5406,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"idb@npm:^7.1.1":
+  version: 7.1.1
+  resolution: "idb@npm:7.1.1"
+  checksum: 1973c28d53c784b177bdef9f527ec89ec239ec7cf5fcbd987dae75a16c03f5b7dfcc8c6d3285716fd0309dd57739805390bd9f98ce23b1b7d8849a3b52de8d56
+  languageName: node
+  linkType: hard
+
 "ieee754@npm:^1.2.1":
   version: 1.2.1
   resolution: "ieee754@npm:1.2.1"
@@ -7600,13 +7586,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"sql.js@npm:^1.8.0":
-  version: 1.8.0
-  resolution: "sql.js@npm:1.8.0"
-  checksum: 4c16846815664ef95c4dc2ffbbef4cba5236ead0798c9f1fac11f66c194a0ac51862fe26980be55cef8738b12ebe9d12efe71262953773cdcb41fbef034b2320
-  languageName: node
-  linkType: hard
-
 "sqlite3@npm:^5.1.6":
   version: 5.1.6
   resolution: "sqlite3@npm:5.1.6"