Implement caching for GQL requests (#393)

* Use response cache plugin with apollo server to cache GQL responses

* Upgrade apollo server to v3

* Add cache control settings for pool queries

* Update server setup in uni-info-watcher

* Use common code from watcher-ts util

* Use in memory GQL cache with configurable max size

* Add cache control settings for remaining queries

packages/erc20-watcher/package.json

@@ -54,6 +54,7 @@
     "express": "^4.17.1",
     "graphql-import-node": "^0.0.4",
     "graphql-request": "^3.4.0",
+    "graphql-subscriptions": "^2.0.0",
     "json-bigint": "^1.0.0",
     "lodash": "^4.17.21",
     "reflect-metadata": "^0.1.13",

packages/erc20-watcher/src/events.ts

@@ -4,7 +4,7 @@
 
 import assert from 'assert';
 import debug from 'debug';
-import { PubSub } from 'apollo-server-express';
+import { PubSub } from 'graphql-subscriptions';
 
 import {
   QUEUE_BLOCK_PROCESSING,

packages/erc20-watcher/src/fill.ts

@@ -7,7 +7,7 @@ import 'reflect-metadata';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
-import { PubSub } from 'apollo-server-express';
+import { PubSub } from 'graphql-subscriptions';
 
 import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
 import { EthClient } from '@cerc-io/ipld-eth-client';

packages/erc20-watcher/src/server.ts

@@ -5,7 +5,8 @@
 import assert from 'assert';
 import 'reflect-metadata';
 import express, { Application } from 'express';
-import { ApolloServer, PubSub } from 'apollo-server-express';
+import { ApolloServer } from 'apollo-server-express';
+import { PubSub } from 'graphql-subscriptions';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';

packages/uni-info-watcher/environments/local.toml

@@ -27,6 +27,20 @@
   # Boolean to load GQL query nested entity relations sequentially.
   loadRelationsSequential = false
 
+  # Max GQL API requests to process simultaneously (defaults to 1).
+  maxSimultaneousRequests = 50
+
+  # GQL cache settings
+  [server.gqlCache]
+    enabled = true
+
+    # Max in-memory cache size (in bytes) (default 8 MB)
+    # maxCacheSize
+
+    # GQL cache-control max-age settings (in seconds)
+    maxAge = 15
+    timeTravelMaxAge = 86400 # 1 day
+
 [metrics]
   host = "127.0.0.1"
   port = 9004
@@ -44,6 +58,10 @@
   logging = false
   maxQueryExecutionTime = 100
 
+  [database.extra]
+    # maximum number of clients the pool should contain
+    max = 20
+
 [upstream]
   [upstream.ethServer]
     gqlApiEndpoint = "http://127.0.0.1:8082/graphql"

packages/uni-info-watcher/package.json

@@ -6,25 +6,32 @@
   "private": true,
   "dependencies": {
     "@apollo/client": "^3.3.19",
-    "@types/lodash": "^4.14.168",
+    "@apollo/utils.keyvaluecache": "^1.0.1",
+    "@graphql-tools/schema": "^9.0.10",
     "@ipld/dag-cbor": "^6.0.12",
+    "@types/lodash": "^4.14.168",
     "@vulcanize/cache": "^0.1.0",
     "@vulcanize/erc20-watcher": "^0.1.0",
     "@vulcanize/ipld-eth-client": "^0.1.0",
     "@vulcanize/uni-watcher": "^0.1.0",
     "@vulcanize/util": "^0.1.0",
-    "apollo-server-express": "^2.25.0",
+    "apollo-server-core": "^3.11.1",
+    "apollo-server-express": "^3.11.1",
+    "apollo-server-plugin-response-cache": "^3.8.1",
     "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
     "express": "^4.17.1",
     "express-queue": "^0.0.13",
     "graphql": "^15.5.0",
     "graphql-import-node": "^0.0.4",
     "graphql-request": "^3.4.0",
+    "graphql-subscriptions": "^2.0.0",
+    "graphql-ws": "^5.11.2",
     "json-bigint": "^1.0.0",
+    "lodash": "^4.17.21",
     "reflect-metadata": "^0.1.13",
     "typeorm": "^0.2.32",
-    "lodash": "^4.17.21",
+    "ws": "^7.4.6",
     "yargs": "^17.0.1"
   },
   "scripts": {
@@ -62,6 +69,7 @@
   "devDependencies": {
     "@types/chance": "^1.1.2",
     "@types/express": "^4.17.11",
+    "@types/ws": "^7.4.4",
     "@typescript-eslint/eslint-plugin": "^4.25.0",
     "@typescript-eslint/parser": "^4.25.0",
     "@uniswap/v3-core": "1.0.0",

packages/uni-info-watcher/src/cli/import-state.ts

@@ -7,7 +7,7 @@ import 'reflect-metadata';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
-import { PubSub } from 'apollo-server-express';
+import { PubSub } from 'graphql-subscriptions';
 import fs from 'fs';
 import path from 'path';
 

packages/uni-info-watcher/src/database.ts

@@ -18,7 +18,6 @@ import {
 import path from 'path';
 import { SelectionNode } from 'graphql';
 import debug from 'debug';
-import _ from 'lodash';
 
 import {
   StateKind,

packages/uni-info-watcher/src/events.ts

@@ -4,7 +4,7 @@
 
 import assert from 'assert';
 import debug from 'debug';
-import { PubSub } from 'apollo-server-express';
+import { PubSub } from 'graphql-subscriptions';
 
 import { EthClient } from '@vulcanize/ipld-eth-client';
 import { QUEUE_BLOCK_PROCESSING, QUEUE_EVENT_PROCESSING } from '@cerc-io/util';

packages/uni-info-watcher/src/fill.ts

@@ -7,6 +7,7 @@ import 'reflect-metadata';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import debug from 'debug';
+import { PubSub } from 'graphql-subscriptions';
 
 import { getCache } from '@vulcanize/cache';
 import { DEFAULT_CONFIG_PATH } from '@cerc-io/util';
@@ -16,7 +17,6 @@ import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
 import { EthClient } from '@cerc-io/ipld-eth-client';
 
 import { Database } from './database';
-import { PubSub } from 'apollo-server-express';
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
 import { fillState } from './cli/fill-state';