feat(gatsby): node persistence

.circleci/config.yml

@@ -58,11 +58,20 @@ aliases:
           - /blog.+/
 
   test_template: &test_template
+    parameters:
+      npm_rebuild:
+        type: boolean
+        default: false
     steps:
       - checkout
       - run: ./scripts/assert-changed-files.sh "packages/*|.circleci/*"
       - <<: *attach_to_bootstrap
+      - when:
+          condition: << parameters.npm_rebuild >>
+          steps:
+            - run: npm rebuild
       - run: yarn list react
+      - run: yarn why lmdb-store
       - run:
           command: node --max-old-space-size=2048 ./node_modules/.bin/jest -w 1 --ci
           environment:
@@ -230,6 +239,14 @@ jobs:
       image: "14"
     <<: *test_template
 
+  unit_tests_node14_strict_mode:
+    executor:
+      name: node
+      image: "14"
+    environment:
+      GATSBY_EXPERIMENTAL_STRICT_MODE: 1
+    <<: *test_template
+
   integration_tests_gatsby_source_wordpress:
     machine: true
     steps:
@@ -626,6 +643,14 @@ workflows:
             - lint
             - typecheck
             - bootstrap
+      - unit_tests_node14_strict_mode:
+          <<: *ignore_docs
+          # rebuild to get correct version of lmdb-store (compiled for node14 not node12)
+          npm_rebuild: true
+          requires:
+            - lint
+            - typecheck
+            - bootstrap
       - integration_tests_gatsby_source_wordpress:
           <<: *e2e-test-workflow
       - integration_tests_long_term_caching:

packages/gatsby/package.json

@@ -180,6 +180,7 @@
     "cross-env": "^7.0.3",
     "documentation": "^13.1.0",
     "enhanced-resolve": "^4.2.0",
+    "lmdb-store": "^1.5.0",
     "react": "^16.12.0",
     "react-dom": "^16.12.0",
     "rimraf": "^3.0.2",

packages/gatsby/src/datastore/datastore.ts

@@ -0,0 +1,51 @@
+import { IDataStore } from "./types"
+import { emitter } from "../redux"
+
+let dataStore: IDataStore
+let isLmdb = isLmdbStoreFlagSet()
+
+export function getDataStore(): IDataStore {
+  if (!dataStore) {
+    if (isLmdb) {
+      const { setupLmdbStore } = require(`./lmdb/lmdb-datastore`)
+      dataStore = setupLmdbStore()
+    } else {
+      const { setupInMemoryStore } = require(`./in-memory/in-memory-datastore`)
+      dataStore = setupInMemoryStore()
+    }
+  }
+  return dataStore
+}
+
+export function isLmdbStore(): boolean {
+  return isLmdb
+}
+
+export function detectLmdbStore(): boolean {
+  const flagIsSet = isLmdbStoreFlagSet()
+
+  if (dataStore && isLmdb !== flagIsSet) {
+    throw new Error(
+      `GATSBY_EXPERIMENTAL_LMDB_STORE flag had changed after the data store was initialized.` +
+        `(original value: ${isLmdb ? `true` : `false`}, ` +
+        `new value: ${flagIsSet ? `true` : `false`})`
+    )
+  }
+  isLmdb = flagIsSet
+  return flagIsSet
+}
+
+function isLmdbStoreFlagSet(): boolean {
+  return (
+    Boolean(process.env.GATSBY_EXPERIMENTAL_LMDB_STORE) &&
+    process.env.GATSBY_EXPERIMENTAL_LMDB_STORE !== `false` &&
+    process.env.GATSBY_EXPERIMENTAL_LMDB_STORE !== `0`
+  )
+}
+
+// It is possible that the store is not initialized yet when calling `DELETE_CACHE`.
+//  The code below ensures we wipe cache from the proper store
+//  (mostly relevant for tests)
+emitter.on(`DELETE_CACHE`, () => {
+  getDataStore()
+})

packages/gatsby/src/datastore/in-memory/in-memory-datastore.ts

@@ -0,0 +1,67 @@
+import { IDataStore } from "../types"
+import { store } from "../../redux"
+import { IGatsbyNode } from "../../redux/types"
+
+/**
+ * @deprecated
+ */
+function getNodes(): Array<IGatsbyNode> {
+  const nodes = store.getState().nodes
+  if (nodes) {
+    return Array.from(nodes.values())
+  } else {
+    return []
+  }
+}
+
+/**
+ * @deprecated
+ */
+function getNodesByType(type: string): Array<IGatsbyNode> {
+  const nodes = store.getState().nodesByType.get(type)
+  if (nodes) {
+    return Array.from(nodes.values())
+  } else {
+    return []
+  }
+}
+
+function getNode(id: string): IGatsbyNode | undefined {
+  return store.getState().nodes.get(id)
+}
+
+function getTypes(): Array<string> {
+  // Note: sorting to match the output of the LMDB version (where keys are sorted by default)
+  return Array.from(store.getState().nodesByType.keys()).sort()
+}
+
+function countNodes(typeName?: string): number {
+  if (!typeName) {
+    return store.getState().nodes.size
+  }
+  const nodes = store.getState().nodesByType.get(typeName)
+  return nodes ? nodes.size : 0
+}
+
+const readyPromise = Promise.resolve(undefined)
+
+/**
+ * Returns promise that resolves when the store is ready for reads
+ * (the in-memory store is always ready)
+ */
+function ready(): Promise<void> {
+  return readyPromise
+}
+
+export function setupInMemoryStore(): IDataStore {
+  return {
+    getNode,
+    getTypes,
+    countNodes,
+    ready,
+
+    // deprecated:
+    getNodes,
+    getNodesByType,
+  }
+}

packages/gatsby/src/datastore/index.ts

@@ -0,0 +1 @@
+export { getDataStore, isLmdbStore, detectLmdbStore } from "./datastore"

packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts

@@ -0,0 +1,173 @@
+import { ArrayLikeIterable, RootDatabase, open } from "lmdb-store"
+// import { performance } from "perf_hooks"
+import { ActionsUnion, IGatsbyNode } from "../../redux/types"
+import { updateNodes } from "./updates/nodes"
+import { updateNodesByType } from "./updates/nodes-by-type"
+import { IDataStore, ILmdbDatabases } from "../types"
+import { emitter, replaceReducer } from "../../redux"
+
+const rootDbFile =
+  process.env.NODE_ENV === `test`
+    ? `test-datastore-${process.env.JEST_WORKER_ID}`
+    : `datastore`
+
+let rootDb
+let databases
+
+function getRootDb(): RootDatabase {
+  if (!rootDb) {
+    rootDb = open({
+      name: `root`,
+      path: process.cwd() + `/.cache/data/` + rootDbFile,
+      sharedStructuresKey: Symbol.for(`structures`),
+      compression: true,
+    })
+  }
+  return rootDb
+}
+
+function getDatabases(): ILmdbDatabases {
+  if (!databases) {
+    const rootDb = getRootDb()
+    databases = {
+      nodes: rootDb.openDB({
+        name: `nodes`,
+        cache: true,
+      }),
+      nodesByType: rootDb.openDB({
+        name: `nodesByType`,
+        dupSort: true,
+      }),
+    }
+  }
+  return databases
+}
+
+/**
+ * @deprecated
+ */
+function getNodes(): Array<IGatsbyNode> {
+  // const start = performance.now()
+  const result = Array.from<IGatsbyNode>(iterateNodes())
+  // const timeTotal = performance.now() - start
+  // console.warn(
+  //   `getNodes() is deprecated, use iterateNodes() instead; ` +
+  //     `array length: ${result.length}; time(ms): ${timeTotal}`
+  // )
+  return result ?? []
+}
+
+/**
+ * @deprecated
+ */
+function getNodesByType(type: string): Array<IGatsbyNode> {
+  // const start = performance.now()
+  const result = Array.from<IGatsbyNode>(iterateNodesByType(type))
+  // const timeTotal = performance.now() - start
+  // console.warn(
+  //   `getNodesByType() is deprecated, use iterateNodesByType() instead; ` +
+  //     `array length: ${result.length}; time(ms): ${timeTotal}`
+  // )
+  return result ?? []
+}
+
+function iterateNodes(): ArrayLikeIterable<IGatsbyNode> {
+  // Additionally fetching items by id to leverage lmdb-store cache
+  const nodesDb = getDatabases().nodes
+  return nodesDb
+    .getKeys({ snapshot: false })
+    .map(nodeId => getNode(nodeId)!)
+    .filter(Boolean)
+}
+
+function iterateNodesByType(type: string): ArrayLikeIterable<IGatsbyNode> {
+  const nodesByType = getDatabases().nodesByType
+  return nodesByType
+    .getValues(type)
+    .map(nodeId => getNode(nodeId)!)
+    .filter(Boolean)
+}
+
+function getNode(id: string): IGatsbyNode | undefined {
+  if (!id) return undefined
+  const { nodes } = getDatabases()
+  return nodes.get(id)
+}
+
+function getTypes(): Array<string> {
+  return getDatabases().nodesByType.getKeys({}).asArray
+}
+
+function countNodes(typeName?: string): number {
+  if (!typeName) {
+    const stats = getDatabases().nodes.getStats()
+    // @ts-ignore
+    return Number(stats.entryCount || 0)
+  }
+
+  const { nodesByType } = getDatabases()
+  let count = 0
+  nodesByType.getValues(typeName).forEach(() => {
+    count++
+  })
+  return count
+}
+
+let lastOperationPromise: Promise<any> = Promise.resolve()
+
+function updateDataStore(action: ActionsUnion): void {
+  switch (action.type) {
+    case `DELETE_CACHE`: {
+      const dbs = getDatabases()
+      // Force sync commit
+      dbs.nodes.transactionSync(() => {
+        dbs.nodes.clear()
+        dbs.nodesByType.clear()
+      })
+      break
+    }
+    case `CREATE_NODE`:
+    case `ADD_FIELD_TO_NODE`:
+    case `ADD_CHILD_NODE_TO_PARENT_NODE`:
+    case `DELETE_NODE`: {
+      const dbs = getDatabases()
+      lastOperationPromise = Promise.all([
+        updateNodes(dbs.nodes, action),
+        updateNodesByType(dbs.nodesByType, action),
+      ])
+    }
+  }
+}
+
+/**
+ * Resolves when all the data is synced
+ */
+async function ready(): Promise<void> {
+  await lastOperationPromise
+}
+
+export function setupLmdbStore(): IDataStore {
+  const lmdbDatastore = {
+    getNode,
+    getTypes,
+    countNodes,
+    iterateNodes,
+    iterateNodesByType,
+    updateDataStore,
+    ready,
+
+    // deprecated:
+    getNodes,
+    getNodesByType,
+  }
+  replaceReducer({
+    nodes: (state = new Map(), _) => state,
+    nodesByType: (state = new Map(), _) => state,
+  })
+  emitter.on(`*`, action => {
+    if (action) {
+      updateDataStore(action)
+    }
+  })
+  return lmdbDatastore
+}

packages/gatsby/src/datastore/lmdb/updates/nodes-by-type.ts

@@ -0,0 +1,22 @@
+import { ActionsUnion } from "../../../redux/types"
+import { ILmdbDatabases } from "../../types"
+
+export function updateNodesByType(
+  nodesByTypeDb: ILmdbDatabases["nodesByType"],
+  action: ActionsUnion
+): Promise<boolean> | boolean {
+  switch (action.type) {
+    case `CREATE_NODE`:
+    case `ADD_FIELD_TO_NODE`:
+    case `ADD_CHILD_NODE_TO_PARENT_NODE`: {
+      // nodesByType db uses dupSort, so `put` will effectively append an id
+      return nodesByTypeDb.put(action.payload.internal.type, action.payload.id)
+    }
+    case `DELETE_NODE`: {
+      return action.payload
+        ? nodesByTypeDb.remove(action.payload.internal.type, action.payload.id)
+        : false
+    }
+  }
+  return false
+}