v5.0.6 random panic: fatal error: concurrent map read and map write

[ghost]

after update to v5.0.6, coin node offen panic

fatal error: concurrent map read and map write

goroutine 27419948 [running]:
runtime.throw(0x1d579c1, 0x21)
        runtime/panic.go:1116 +0x72 fp=0xc005d8ebd8 sp=0xc005d8eba8 pc=0x43d0f2
runtime.mapaccess1_fast64(0x1ad3660, 0xc0040a4600, 0x7586a4, 0x7)
        runtime/map_fast64.go:21 +0x198 fp=0xc005d8ec00 sp=0xc005d8ebd8 pc=0x418338
github.com/cosmos/iavl.(*MutableTree).VersionExists(0xc0040a45a0, 0x7586a4, 0xc003243db0)
        github.com/cosmos/[email protected]/mutable_tree.go:63 +0xfa fp=0xc005d8ec40 sp=0xc005d8ec00 pc=0xf59ada
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).VersionExists(...)
        github.com/cosmos/[email protected]/store/iavl/store.go:144
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).GetImmutable(0xc000d08380, 0x7586a4, 0xc003243db0, 0x21cbb20, 0xc000d08380)
        github.com/cosmos/[email protected]/store/iavl/store.go:92 +0x44 fp=0xc005d8ec88 sp=0xc005d8ec40 pc=0xf7d624
github.com/cosmos/cosmos-sdk/store/rootmulti.(*Store).CacheMultiStoreWithVersion(0xc0001503c0, 0x7586a4, 0x131a303030303030, 0x3030303030303031, 0x3030303030303030, 0xdedd080c12303030)
        github.com/cosmos/[email protected]/store/rootmulti/store.go:433 +0x289 fp=0xc005d8ef90 sp=0xc005d8ec88 pc=0xf87729
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).createQueryContext(0xc0001d1520, 0x7586a4, 0x78a5636f748f8200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        github.com/cosmos/[email protected]/baseapp/abci.go:635 +0xdd fp=0xc005d8f718 sp=0xc005d8ef90 pc=0x11f29dd
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).handleQueryGRPC(0xc0001d1520, 0xc02ac760c0, 0xc01d07daa0, 0x13, 0x13, 0x1d634d5, 0x26, 0x7586a4, 0x0, 0x0, ...)
        github.com/cosmos/[email protected]/baseapp/abci.go:569 +0x7d fp=0xc005d8fdc0 sp=0xc005d8f718 pc=0x11f22fd
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).Query(0xc0001d1520, 0xc01d07daa0, 0x13, 0x13, 0x1d634d5, 0x26, 0x7586a4, 0x0, 0x0, 0x0, ...)
        github.com/cosmos/[email protected]/baseapp/abci.go:418 +0x8ef fp=0xc005d90028 sp=0xc005d8fdc0 pc=0x11f0ccf
github.com/tendermint/tendermint/abci/client.(*localClient).QuerySync(0xc0001558c0, 0xc01d07daa0, 0x13, 0x13, 0x1d634d5, 0x26, 0x0, 0x0, 0x0, 0x0, ...)
        github.com/tendermint/[email protected]/abci/client/local_client.go:258 +0x10c fp=0xc005d90198 sp=0xc005d90028 pc=0xe9f82c
github.com/tendermint/tendermint/proxy.(*appConnQuery).QuerySync(0xc0035c7530, 0xc01d07daa0, 0x13, 0x13, 0x1d634d5, 0x26, 0x0, 0x0, 0x0, 0x97, ...)
        github.com/tendermint/[email protected]/proxy/app_conn.go:159 +0x68 fp=0xc005d90200 sp=0xc005d90198 pc=0xea2b88
github.com/tendermint/tendermint/rpc/core.ABCIQuery(0xc004460020, 0x1d634d5, 0x26, 0xc01d07daa0, 0x13, 0x13, 0x0, 0xc52900, 0xc0000a13b0, 0x216ff40, ...)
        github.com/tendermint/[email protected]/rpc/core/abci.go:20 +0xfc fp=0xc005d902b0 sp=0xc005d90200 pc=0x1173bbc
github.com/tendermint/tendermint/rpc/client/local.(*Local).ABCIQueryWithOptions(0xc004460000, 0x21b1f20, 0xc00020c010, 0x1d634d5, 0x26, 0xc01d07daa0, 0x13, 0x13, 0x0, 0x1d25400, ...)
        github.com/tendermint/[email protected]/rpc/client/local/local.go:87 +0x8d fp=0xc005d90318 sp=0xc005d902b0 pc=0x11d2ead
github.com/cosmos/cosmos-sdk/client.Context.queryABCI(0x0, 0x0, 0x0, 0x21d5760, 0xc004460000, 0xc026a20170, 0xb, 0x21bb600, 0xc000df5ad0, 0x21beba0, ...)
        github.com/cosmos/[email protected]/client/query.go:79 +0x13c fp=0xc005d90618 sp=0xc005d90318 pc=0xfd489c
github.com/cosmos/cosmos-sdk/client.Context.QueryABCI(...)
        github.com/cosmos/[email protected]/client/query.go:55
github.com/cosmos/cosmos-sdk/client.RunGRPCQuery(0x0, 0x0, 0x0, 0x21d5760, 0xc004460000, 0xc026a20170, 0xb, 0x21bb600, 0xc000df5ad0, 0x21beba0, ...)
        github.com/cosmos/[email protected]/client/grpc_query.go:119 +0x278 fp=0xc005d90c90 sp=0xc005d90618 pc=0xfd3cb8
github.com/cosmos/cosmos-sdk/client.Context.Invoke(0x0, 0x0, 0x0, 0x21d5760, 0xc004460000, 0xc026a20170, 0xb, 0x21bb600, 0xc000df5ad0, 0x21beba0, ...)
        github.com/cosmos/[email protected]/client/grpc_query.go:58 +0x3a5 fp=0xc005d90f90 sp=0xc005d90c90 pc=0xfd3605
github.com/cosmos/cosmos-sdk/client.(*Context).Invoke(0xc016ae2420, 0x21b1fa0, 0xc049e551d0, 0x1d634d5, 0x26, 0x1c8ebc0, 0xc049e55200, 0x1c8ed20, 0xc01e24b9c0, 0xc03b664300, ...)
        <autogenerated>:1 +0x168 fp=0xc005d91298 sp=0xc005d90f90 pc=0xfd61c8
github.com/cosmos/cosmos-sdk/types/tx.(*serviceClient).GetTxsEvent(0xc01d8f5c80, 0x21b1fa0, 0xc049e551d0, 0xc049e55200, 0xc03b664300, 0x2, 0x2, 0x0, 0x21b1ee0, 0xc01e24b940)
        github.com/cosmos/[email protected]/types/tx/service.pb.go:686 +0xd4 fp=0xc005d91320 sp=0xc005d91298 pc=0xf00ff4
github.com/cosmos/cosmos-sdk/types/tx.request_Service_GetTxsEvent_0(0x21b1fa0, 0xc049e551d0, 0x21b54a0, 0xc01dfda420, 0x21b20e0, 0xc01d8f5c80, 0xc004e1d000, 0xc049e55140, 0x4, 0x3, ...)
        github.com/cosmos/[email protected]/types/tx/service.pb.gw.go:171 +0x2c4 fp=0xc005d913a0 sp=0xc005d91320 pc=0xf0a0c4
github.com/cosmos/cosmos-sdk/types/tx.RegisterServiceHandlerClient.func4(0x21ab1e0, 0xc03b664280, 0xc004e1d000, 0xc049e55140)
        github.com/cosmos/[email protected]/types/tx/service.pb.gw.go:388 +0x1d5 fp=0xc005d914a8 sp=0xc005d913a0 pc=0xf1da95
github.com/grpc-ecosystem/grpc-gateway/runtime.(*ServeMux).ServeHTTP(0xc019c7ccb0, 0x21ab1e0, 0xc03b664280, 0xc004e1d000)
        github.com/grpc-ecosystem/[email protected]/runtime/mux.go:240 +0xc34 fp=0xc005d918d0 sp=0xc005d914a8 pc=0xef33d4
github.com/gorilla/mux.(*Router).ServeHTTP(0xc01697c000, 0x21ab1e0, 0xc03b664280, 0xc004e1ce00)
        github.com/gorilla/[email protected]/mux.go:210 +0xd3 fp=0xc005d91a08 sp=0xc005d918d0 pc=0xfddb33
github.com/tendermint/tendermint/rpc/jsonrpc/server.maxBytesHandler.ServeHTTP(0x2170980, 0xc01697c000, 0xf4240, 0x21ab1e0, 0xc03b664280, 0xc004e1ce00)
        github.com/tendermint/[email protected]/rpc/jsonrpc/server/http_server.go:236 +0xd4 fp=0xc005d91a48 sp=0xc005d91a08 pc=0xfea3f4
github.com/tendermint/tendermint/rpc/jsonrpc/server.(*maxBytesHandler).ServeHTTP(0xc01efae8e0, 0x21ab1e0, 0xc03b664280, 0xc004e1ce00)
        <autogenerated>:1 +0x6c fp=0xc005d91a88 sp=0xc005d91a48 pc=0xff43ec
github.com/tendermint/tendermint/rpc/jsonrpc/server.RecoverAndLogHandler.func1(0x21aba60, 0xc003e64380, 0xc004e1ce00)
        github.com/tendermint/[email protected]/rpc/jsonrpc/server/http_server.go:209 +0x39a fp=0xc005d91bc8 sp=0xc005d91a88 pc=0xff263a
net/http.HandlerFunc.ServeHTTP(0xc01e0d3ce0, 0x21aba60, 0xc003e64380, 0xc004e1ce00)
        net/http/server.go:2042 +0x44 fp=0xc005d91bf0 sp=0xc005d91bc8 pc=0xa073c4
net/http.serverHandler.ServeHTTP(0xc01640c0e0, 0x21aba60, 0xc003e64380, 0xc004e1ce00)
        net/http/server.go:2843 +0xa3 fp=0xc005d91c20 sp=0xc005d91bf0 pc=0xa0aac3
net/http.(*conn).serve(0xc026cc6280, 0x21b1ee0, 0xc013cdf640)
        net/http/server.go:1925 +0x8ad fp=0xc005d91fc8 sp=0xc005d91c20 pc=0xa061cd
runtime.goexit()
        runtime/asm_amd64.s:1374 +0x1 fp=0xc005d91fd0 sp=0xc005d91fc8 pc=0x4758e1
created by net/http.(*Server).Serve
        net/http/server.go:2969 +0x36c

goroutine 1 [chan receive, 790 minutes]:
github.com/cosmos/cosmos-sdk/server.WaitForQuitSignals(0x0)
        github.com/cosmos/[email protected]/server/util.go:335 +0xbf
github.com/cosmos/cosmos-sdk/server.startInProcess(0xc000f724c0, 0x0, 0x0, 0x0, 0x21d5760, 0xc004460000, 0x0, 0x0, 0x21bb600, 0xc000df5ad0, ...)
        github.com/cosmos/[email protected]/server/start.go:334 +0x877
github.com/cosmos/cosmos-sdk/server.StartCmd.func2(0xc000efaa00, 0xc000f5ffe0, 0x0, 0x1, 0x0, 0x0)
        github.com/cosmos/[email protected]/server/start.go:120 +0x189
github.com/spf13/cobra.(*Command).execute(0xc000efaa00, 0xc000f5ffd0, 0x1, 0x1, 0xc000efaa00, 0xc000f5ffd0)
        github.com/spf13/[email protected]/command.go:852 +0x47c
github.com/spf13/cobra.(*Command).ExecuteC(0xc000493680, 0x0, 0x0, 0xc000e94a80)
        github.com/spf13/[email protected]/command.go:960 +0x375
github.com/spf13/cobra.(*Command).Execute(...)
        github.com/spf13/[email protected]/command.go:897
github.com/spf13/cobra.(*Command).ExecuteContext(...)
        github.com/spf13/[email protected]/command.go:890
github.com/cosmos/cosmos-sdk/server/cmd.Execute(0xc000493680, 0xc000e94a80, 0xb, 0x21cede0, 0xc000df5ad0)
        github.com/cosmos/[email protected]/server/cmd/execute.go:36 +0x265
main.main()
        github.com/cosmos/gaia/v5/cmd/gaiad/main.go:16 +0x45

.... and other goroutine log

[yaruwangway]

Hi @jingniao , thank you for reporting this issue. This is probably because Tendermint introduces multi reader mutex, but the cosmos sdk that gaia v5.0.6 depending on does not include latest iavl fixes yet.

This fix will be included in the next release of gaia using the upcoming version of cosmos-sdk.

[njmurarka]

Does this happen anymore? I am on Cosmos 0.44.x and got this error:

9:09PM INF executed block height=185035 module=state num_invalid_txs=0 num_valid_txs=0
fatal error: concurrent map read and map write

goroutine 4267938119 [running]:
runtime.throw({0x2654adf, 0x7f3a3e2bf108})
	/usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc031a842d8 sp=0xc031a842a8 pc=0x435a91
runtime.mapaccess2_fast64(0xc001cbb790, 0x21f82a0, 0x2d2c9)```