Skip to content

Commit

Permalink
Problem: there's no compact historical state storage
Browse files Browse the repository at this point in the history
Closes: crypto-org-chain#704
Solution:
- Integration version store and streaming service.

multiple backends

db size benchmark

support mdbx backend through tm-db

store latest version

support GetLatestVersion

query multistore

test versiondb streamer

fix lint

fix state listener

temp

fix state listening

optimize common case

fix lint

try to fix mdbx build

update cosmos-sdk

fix clone append

add test case

fix subkey problem in history index

revert chunking, hard to work with variable length key

support iterator

check future height

fix lint

new state listener

fix latest state in query

fix integration test

fix prune node test

update dependency

add utility to read from file streamer

Update versiondb/multistore.go

Signed-off-by: yihuang <[email protected]>

add unit test

create common backend test cases

update dependency

update with new file streamer format

Problem: python3.10 is not used in integration tests

Solution:
- start using python3.10, prepare to later PRs which need the new features
- update nixpkgs nesserary for the nix stuff to work.

python-roaring64

remove debug log

add test cases, improve coverage
  • Loading branch information
yihuang committed Nov 27, 2022
1 parent 7dc3c4c commit 3bcb198
Show file tree
Hide file tree
Showing 29 changed files with 1,944 additions and 64 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS)))
BUILD_TAGS += boltdb
endif

ifeq (mdbx,$(findstring mdbx,$(COSMOS_BUILD_OPTIONS)))
BUILD_TAGS += mdbx
endif

ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS)))
ldflags += -w -s
endif
Expand Down
45 changes: 42 additions & 3 deletions app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"sync"

"github.com/crypto-org-chain/cronos/x/cronos"
"github.com/crypto-org-chain/cronos/x/cronos/middleware"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/codec/types"
"github.com/cosmos/cosmos-sdk/server"
"github.com/gorilla/mux"
"github.com/rakyll/statik/fs"
"github.com/spf13/cast"
Expand Down Expand Up @@ -122,7 +125,8 @@ import (

// this line is used by starport scaffolding # stargate/app/moduleImport
cronosappclient "github.com/crypto-org-chain/cronos/client"
"github.com/crypto-org-chain/cronos/x/cronos"
"github.com/crypto-org-chain/cronos/versiondb"
"github.com/crypto-org-chain/cronos/versiondb/tmdb"
cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client"
cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper"
evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers"
Expand Down Expand Up @@ -350,7 +354,8 @@ func New(
// configure state listening capabilities using AppOptions
// we are doing nothing with the returned streamingServices and waitGroup in this case
// Only support file streamer right now.
if cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) == "file" {
streamers := cast.ToString(appOpts.Get(cronosappclient.FlagStreamers))
if strings.Contains(streamers, "file") {
streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory)
if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil {
panic(err)
Expand All @@ -361,7 +366,7 @@ func New(
for _, storeKey := range keys {
exposeStoreKeys = append(exposeStoreKeys, storeKey)
}
service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec)
service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec, false)
if err != nil {
panic(err)
}
Expand All @@ -373,6 +378,40 @@ func New(
}
}

if strings.Contains(streamers, "versiondb") {
rootDir := cast.ToString(appOpts.Get(flags.FlagHome))
dataDir := filepath.Join(rootDir, "data", "versiondb")
if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
panic(err)
}
backendType := server.GetAppDBBackend(appOpts)
plainDB, err := dbm.NewDB("plain", backendType, dataDir)
if err != nil {
panic(err)
}
historyDB, err := dbm.NewDB("history", backendType, dataDir)
if err != nil {
panic(err)
}
changesetDB, err := dbm.NewDB("changeset", backendType, dataDir)
if err != nil {
panic(err)
}
versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB)

// default to exposing all
exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys))
for _, storeKey := range keys {
exposeStoreKeys = append(exposeStoreKeys, storeKey)
}
service := versiondb.NewStreamingService(versionDB, exposeStoreKeys)
bApp.SetStreamingService(service)
qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys)
qms.MountTransientStores(tkeys)
qms.MountMemoryStores(memKeys)
bApp.SetQueryMultiStore(qms)
}

app := &App{
BaseApp: bApp,
cdc: cdc,
Expand Down
3 changes: 2 additions & 1 deletion default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
let
version = "v0.9.0";
pname = "cronosd";
tags = [ "ledger" "netgo" network ]
tags = [ "ledger" "netgo" network "mdbx" ]
++ lib.lists.optionals (rocksdb != null) [ "rocksdb" "rocksdb_build" ];
ldflags = lib.concatStringsSep "\n" ([
"-X github.com/cosmos/cosmos-sdk/version.Name=cronos"
Expand All @@ -27,6 +27,7 @@ buildGoApplication rec {
"!/app/"
"!/cmd/"
"!/client/"
"!/versiondb/"
"!go.mod"
"!go.sum"
"!gomod2nix.toml"
Expand Down
19 changes: 15 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.18

require (
cosmossdk.io/math v1.0.0-beta.3
github.com/RoaringBitmap/roaring v1.2.1
github.com/armon/go-metrics v0.4.1
github.com/cosmos/cosmos-sdk v0.46.3
github.com/cosmos/ibc-go/v5 v5.0.0
Expand Down Expand Up @@ -45,10 +46,12 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
Expand All @@ -69,9 +72,8 @@ require (
github.com/deckarep/golang-set v1.8.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
github.com/dgraph-io/badger/v3 v3.2103.2 // indirect
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
Expand All @@ -91,7 +93,8 @@ require (
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/flatbuffers v2.0.0+incompatible // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/orderedcode v0.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
Expand Down Expand Up @@ -135,7 +138,9 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/natefinch/atomic v1.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
Expand Down Expand Up @@ -165,6 +170,7 @@ require (
github.com/tendermint/go-amino v0.16.0 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/torquem-ch/mdbx-go v0.26.0 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
github.com/ulikunitz/xz v0.5.8 // indirect
github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect
Expand All @@ -189,7 +195,7 @@ require (
)

replace (
github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2
github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221014023203-2c6b9d06b12d
github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19

// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
Expand All @@ -199,6 +205,11 @@ replace (

github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20221027085649-2107c6bd6bc4

// https://github.com/tendermint/tm-db/pull/297
github.com/tendermint/tm-db => github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d

github.com/torquem-ch/mdbx-go => github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88

// TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364
github.com/zondax/hid => github.com/zondax/hid v0.9.0
)
51 changes: 38 additions & 13 deletions gomod2nix.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ schema = 3
[mod."github.com/ChainSafe/go-schnorrkel"]
version = "v0.0.0-20200405005733-88cbf1b4c40d"
hash = "sha256-i8RXZemJGlSjBT35oPm0SawFiBoIU5Pkq5xp4n/rzCY="
[mod."github.com/RoaringBitmap/roaring"]
version = "v1.2.1"
hash = "sha256-0/R956wrCW71eOE36CbxGJJRuQjKwvvIQ/D8QTn2A6w="
[mod."github.com/StackExchange/wmi"]
version = "v1.2.1"
hash = "sha256-1BoEeWAWyebH+1mMuyPhWZut8nWHb6r73MgcqlGuUEY="
Expand Down Expand Up @@ -58,6 +61,9 @@ schema = 3
[mod."github.com/bgentry/speakeasy"]
version = "v0.1.0"
hash = "sha256-Gt1vj6CFovLnO6wX5u2O4UfecY9V2J9WGw1ez4HMrgk="
[mod."github.com/bits-and-blooms/bitset"]
version = "v1.2.0"
hash = "sha256-IxNmtELycM+XVzg4qBv04hAJUT3nSWuyP9R+8zc9LmU="
[mod."github.com/btcsuite/btcd"]
version = "v0.22.1"
hash = "sha256-hBU+roIELcmbW2Gz7eGZzL9qNA1bakq5wNxqCgs4TKc="
Expand All @@ -70,6 +76,9 @@ schema = 3
[mod."github.com/btcsuite/btcutil"]
version = "v1.0.3-0.20201208143702-a53e38424cce"
hash = "sha256-4kasJReFcj25JRHx9dJMct3yDkHqVoHGUx5cu45Msfo="
[mod."github.com/c2h5oh/datasize"]
version = "v0.0.0-20220606134207-859f65c6625b"
hash = "sha256-1uH+D3w0Y/B3poXm545XGrT4S4c+msTbj7gKgu9pbPM="
[mod."github.com/cenkalti/backoff/v4"]
version = "v4.1.3"
hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU="
Expand Down Expand Up @@ -98,12 +107,15 @@ schema = 3
version = "v1.0.0-alpha7"
hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0="
[mod."github.com/cosmos/cosmos-sdk"]
version = "v0.46.2"
hash = "sha256-Lgn4+Vd5PUUkfHc+lTdK2G6/nymZekFVTe1FxWRqh2w="
replaced = "github.com/cosmos/cosmos-sdk"
version = "v0.43.0-beta1.0.20221014023203-2c6b9d06b12d"
hash = "sha256-t/QEOJgATr82KFXes+AzNlNL6w2tGHfiwZAE4PdO5GE="
replaced = "github.com/yihuang/cosmos-sdk"
[mod."github.com/cosmos/go-bip39"]
version = "v1.0.0"
hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="
[mod."github.com/cosmos/gogoproto"]
version = "v1.4.2"
hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY="
[mod."github.com/cosmos/gorocksdb"]
version = "v1.2.0"
hash = "sha256-209TcVuXc5s/TcOvNlaQ1HEJAUDTEK3nxPhs+d8TEcY="
Expand Down Expand Up @@ -137,15 +149,12 @@ schema = 3
[mod."github.com/desertbit/timer"]
version = "v0.0.0-20180107155436-c41aec40b27f"
hash = "sha256-abLOtEcomAqCWLphd2X6WkD/ED764w6sa6unox4BXss="
[mod."github.com/dgraph-io/badger/v2"]
version = "v2.2007.4"
hash = "sha256-+KwqZJZpViv8S3TqUVvPXrFoMgWFyS3NoLsi4RR5fGk="
[mod."github.com/dgraph-io/badger/v3"]
version = "v3.2103.2"
hash = "sha256-F6pvsaSKwXOl9RfnUQFqAl6xpCVu9+rthQgOxhKVk1g="
[mod."github.com/dgraph-io/ristretto"]
version = "v0.1.0"
hash = "sha256-01jneg1+1x8tTfUTBZ+6mHkQaqXVnPYxLJyJhJQcvt4="
[mod."github.com/dgryski/go-farm"]
version = "v0.0.0-20200201041132-a6ae2369ad13"
hash = "sha256-aOMlPwFY36bLiiIx4HonbCYRAhagk5N6HAWN7Ygif+E="
[mod."github.com/dlclark/regexp2"]
version = "v1.4.1-0.20201116162257-a2a8dda75c91"
hash = "sha256-VNNMZIc7NkDg3DVLnqeJNM/KZqkkaZu2/HTLBL8X2xE="
Expand All @@ -164,6 +173,7 @@ schema = 3
[mod."github.com/ethereum/go-ethereum"]
version = "v1.10.19"
hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8="
replaced = "github.com/ethereum/go-ethereum"
[mod."github.com/evmos/ethermint"]
version = "v0.6.1-0.20221003153722-491c3da7ebd7"
hash = "sha256-vnfjk57gYa+F8nn0LByX/B1LV8PY2Jvm8vXV6be4ufc="
Expand Down Expand Up @@ -217,8 +227,11 @@ schema = 3
version = "v0.0.4"
hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA="
[mod."github.com/google/btree"]
version = "v1.0.1"
hash = "sha256-1PIeFGgUL4BK/StL/D12pg9bEQ5HfMT/fMLdus4pZTs="
version = "v1.1.2"
hash = "sha256-K7V2obq3pLM71Mg0vhhHtZ+gtaubwXPQx3xcIyZDCjM="
[mod."github.com/google/flatbuffers"]
version = "v2.0.0+incompatible"
hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA="
[mod."github.com/google/go-cmp"]
version = "v0.5.8"
hash = "sha256-8zkIo+Sr1NXMnj3PNmvjX2sZKnAKWXOFvmnX7D9bwxQ="
Expand Down Expand Up @@ -354,9 +367,15 @@ schema = 3
[mod."github.com/mitchellh/mapstructure"]
version = "v1.5.0"
hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE="
[mod."github.com/mschoch/smat"]
version = "v0.2.0"
hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E="
[mod."github.com/mtibben/percent"]
version = "v0.2.1"
hash = "sha256-Zj1lpCP6mKQ0UUTMs2By4LC414ou+iJzKkK+eBHfEcc="
[mod."github.com/natefinch/atomic"]
version = "v1.0.1"
hash = "sha256-fbOVHCwRNI8PFjC4o0YXpKZO0JU2aWTfH5c7WXXKMHg="
[mod."github.com/olekukonko/tablewriter"]
version = "v0.0.5"
hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4="
Expand Down Expand Up @@ -461,14 +480,19 @@ schema = 3
version = "v0.34.22"
hash = "sha256-4p4cpyCWjBbNQUpYN2gDJvnyj+Pov9hw5uRjHrrO++Y="
[mod."github.com/tendermint/tm-db"]
version = "v0.6.7"
hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo="
version = "v0.0.0-20221006023748-f6214ae9454d"
hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk="
replaced = "github.com/yihuang/tm-db"
[mod."github.com/tklauser/go-sysconf"]
version = "v0.3.10"
hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4="
[mod."github.com/tklauser/numcpus"]
version = "v0.4.0"
hash = "sha256-ndE82nOb3agubhEV7aRzEqqTlN4DPbKFHEm2+XZLn8k="
[mod."github.com/torquem-ch/mdbx-go"]
version = "v0.0.0-20221010042614-b72b4f091d88"
hash = "sha256-HWsrhzSGoYgEUUziQPHEtQGwBQ/upg3/f1fY2NGiC0g="
replaced = "github.com/yihuang/mdbx-go"
[mod."github.com/tyler-smith/go-bip39"]
version = "v1.1.0"
hash = "sha256-3YhWBtSwRLGwm7vNwqumphZG3uLBW1vwT9QkQ8JuSjU="
Expand All @@ -478,6 +502,7 @@ schema = 3
[mod."github.com/zondax/hid"]
version = "v0.9.0"
hash = "sha256-PvXtxXo/3C+DS9ZeGBlr4zXbIpaYNtMqLzxYhusFXNY="
replaced = "github.com/zondax/hid"
[mod."go.etcd.io/bbolt"]
version = "v1.3.6"
hash = "sha256-DenVAmyN22xUiivk6fdJp4C9ZnUJXCMDUf8E0goRRV4="
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/configs/default.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
dotenv: '../../scripts/.env',
'cronos_777-1': {
cmd: 'cronosd',
'start-flags': '--trace --streamers file',
'start-flags': '--trace --streamers versiondb,file',
config: {
mempool: {
version: 'v1',
Expand Down
2 changes: 2 additions & 0 deletions integration_tests/configs/pruned-node.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ local config = import 'default.jsonnet';

config {
'cronos_777-1'+: {
// don't enable versiondb, since it don't do pruning right now
'start-flags': '--trace --streamers file',
'app-config'+: {
pruning: 'everything',
'state-sync'+: {
Expand Down
36 changes: 36 additions & 0 deletions integration_tests/configs/state_benchmark.jsonnet
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
local config = import 'default.jsonnet';

config {
'cronos_777-1'+: {
'start-flags': '--trace --streamers file,versiondb',
'app-config'+: {
'app-db-backend': 'rocksdb',
'state-sync'+: {
'snapshot-interval': 0,
},
},
validators: [
super.validators[0],
super.validators[1] {
'app-config'+: {
pruning: 'everything',
},
},
] + super.validators[2:],
genesis+: {
consensus_params+: {
block+: {
max_gas: '163000000',
},
},
app_state+: {
feemarket+: {
params+: {
no_base_fee: true,
min_gas_multiplier: '0',
},
},
},
},
},
}
Loading

0 comments on commit 3bcb198

Please sign in to comment.