Skip to content

Commit

Permalink
feat: add prometheus metrics for caches reverting telemetry metrics (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
Woosang Son authored May 11, 2021
1 parent 9717c49 commit f9c3e64
Show file tree
Hide file tree
Showing 15 changed files with 300 additions and 40 deletions.
13 changes: 11 additions & 2 deletions baseapp/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"

"github.com/line/lbm-sdk/v2/store/cache"
"github.com/line/lbm-sdk/v2/store/iavl"
tmdb "github.com/line/tm-db/v2"

Expand Down Expand Up @@ -65,12 +66,12 @@ func SetInterBlockCache(cache sdk.MultiStorePersistentCache) func(*BaseApp) {
}

// SetIAVLCacheManager provides a BaseApp option function that sets the iavl CacheManager
func SetIAVLCacheManager(size int) func(*BaseApp) {
func SetIAVLCacheManager(size int, provider iavl.MetricsProvider) func(*BaseApp) {
return func(app *BaseApp) {
if size == 0 {
app.cms.SetIAVLCacheManager(iavl.NewCacheManagerNoCache())
} else {
app.cms.SetIAVLCacheManager(iavl.NewCacheManagerSingleton(size))
app.cms.SetIAVLCacheManager(iavl.NewCacheManagerSingleton(size, provider))
}
}
}
Expand Down Expand Up @@ -245,3 +246,11 @@ func (app *BaseApp) SetInterfaceRegistry(registry types.InterfaceRegistry) {
app.grpcQueryRouter.SetInterfaceRegistry(registry)
app.msgServiceRouter.SetInterfaceRegistry(registry)
}

func MetricsProvider(prometheus bool) (cache.MetricsProvider, iavl.MetricsProvider) {
namespace := "app"
if prometheus {
return cache.PrometheusMetricsProvider(namespace), iavl.PrometheusMetricsProvider(namespace)
}
return cache.NopMetricsProvider(), iavl.NopMetricsProvider()
}
22 changes: 22 additions & 0 deletions baseapp/options_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package baseapp

import (
"testing"

"github.com/go-kit/kit/metrics/discard"
"github.com/stretchr/testify/require"
)

func TestMetricsProvider(t *testing.T) {
p1, p2 := MetricsProvider(true)
c1 := p1()
c2 := p2()
require.NotEqual(t, c1.InterBlockCacheHits, discard.NewCounter())
require.NotEqual(t, c2.IAVLCacheHits, discard.NewGauge())

p1, p2 = MetricsProvider(false)
c1 = p1()
c2 = p2()
require.Equal(t, c1.InterBlockCacheHits, discard.NewCounter())
require.Equal(t, c2.IAVLCacheHits, discard.NewGauge())
}
4 changes: 4 additions & 0 deletions server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ type BaseConfig struct {

// IAVL cache size; bytes size unit
IAVLCacheSize int `mapstructure:"iavl-cache-size"`

// When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
// It works when tendermint's prometheus option (config.toml) is set to true.
Prometheus bool `mapstructure:"prometheus"`
}

// APIConfig defines the API listener configuration.
Expand Down
4 changes: 4 additions & 0 deletions server/config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ iavl-cache-size = {{ .BaseConfig.IAVLCacheSize }}
# ["message.sender", "message.recipient"]
index-events = {{ .BaseConfig.IndexEvents }}
# When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
# It works when tendermint's prometheus option (config.toml) is set to true.
prometheus = {{ .BaseConfig.Prometheus }}
###############################################################################
### Telemetry Configuration ###
###############################################################################
Expand Down
2 changes: 2 additions & 0 deletions server/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,8 @@ which accepts a path for the resulting pprof file.
cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval")
cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep")

cmd.Flags().Bool(FlagPrometheus, false, "Enable prometheus metric for app")

// add support for all Ostracon-specific command line options
ostcmd.AddNodeFlags(cmd)
return cmd
Expand Down
3 changes: 2 additions & 1 deletion simapp/sim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
// interBlockCacheOpt returns a BaseApp option function that sets the persistent
// inter-block write-through cache.
func interBlockCacheOpt() func(*baseapp.BaseApp) {
return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize))
return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(
cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider()))
}

func TestFullAppSimulation(t *testing.T) {
Expand Down
20 changes: 12 additions & 8 deletions simapp/simd/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ import (
"os"
"path/filepath"

ostcli "github.com/line/ostracon/libs/cli"
"github.com/line/ostracon/libs/log"
tmdb "github.com/line/tm-db/v2"
"github.com/spf13/cast"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/line/lbm-sdk/v2/baseapp"
"github.com/line/lbm-sdk/v2/client"
"github.com/line/lbm-sdk/v2/client/debug"
Expand All @@ -26,11 +33,6 @@ import (
banktypes "github.com/line/lbm-sdk/v2/x/bank/types"
"github.com/line/lbm-sdk/v2/x/crisis"
genutilcli "github.com/line/lbm-sdk/v2/x/genutil/client/cli"
ostcli "github.com/line/ostracon/libs/cli"
"github.com/line/ostracon/libs/log"
tmdb "github.com/line/tm-db/v2"
"github.com/spf13/cast"
"github.com/spf13/cobra"
)

// NewRootCmd creates a new root command for simd. It is called once in the
Expand Down Expand Up @@ -156,9 +158,11 @@ type appCreator struct {
func (a appCreator) newApp(logger log.Logger, db tmdb.DB, traceStore io.Writer, appOpts servertypes.AppOptions) servertypes.Application {
var cache sdk.MultiStorePersistentCache

ibCacheMetricsProvider, iavlCacheMetricsProvider :=
baseapp.MetricsProvider(cast.ToBool(viper.GetBool(server.FlagPrometheus)))
if cast.ToBool(appOpts.Get(server.FlagInterBlockCache)) {
cache = store.NewCommitKVStoreCacheManager(
cast.ToInt(appOpts.Get(server.FlagInterBlockCacheSize)))
cast.ToInt(appOpts.Get(server.FlagInterBlockCacheSize)), ibCacheMetricsProvider)
}

skipUpgradeHeights := make(map[int64]bool)
Expand Down Expand Up @@ -192,10 +196,10 @@ func (a appCreator) newApp(logger log.Logger, db tmdb.DB, traceStore io.Writer,
baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),
baseapp.SetHaltTime(cast.ToUint64(appOpts.Get(server.FlagHaltTime))),
baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))),
baseapp.SetIAVLCacheManager(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize))),
baseapp.SetInterBlockCache(cache),
baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
baseapp.SetIAVLCacheManager(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize)), iavlCacheMetricsProvider),
baseapp.SetTrace(cast.ToBool(appOpts.Get(server.FlagTrace))),
baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
baseapp.SetSnapshotStore(snapshotStore),
baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))),
baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))),
Expand Down
30 changes: 17 additions & 13 deletions store/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"github.com/VictoriaMetrics/fastcache"
"github.com/line/lbm-sdk/v2/store/cachekv"
"github.com/line/lbm-sdk/v2/store/types"
"github.com/line/lbm-sdk/v2/telemetry"
)

const (
Expand All @@ -27,18 +26,20 @@ type (
// CommitKVStore and below is completely irrelevant to this layer.
CommitKVStoreCache struct {
types.CommitKVStore
cache *fastcache.Cache
prefix []byte
cache *fastcache.Cache
prefix []byte
metrics *Metrics
}

// CommitKVStoreCacheManager maintains a mapping from a StoreKey to a
// CommitKVStoreCache. Each CommitKVStore, per StoreKey, is meant to be used
// in an inter-block (persistent) manner and typically provided by a
// CommitMultiStore.
CommitKVStoreCacheManager struct {
mutex sync.Mutex
cache *fastcache.Cache
caches map[string]types.CommitKVStore
mutex sync.Mutex
cache *fastcache.Cache
caches map[string]types.CommitKVStore
metrics *Metrics

// All cache stores use the unique prefix that has one byte length
// Contract: The number of all cache stores cannot exceed 127(max byte)
Expand All @@ -47,22 +48,25 @@ type (
}
)

func NewCommitKVStoreCache(store types.CommitKVStore, prefix []byte, cache *fastcache.Cache) *CommitKVStoreCache {
func NewCommitKVStoreCache(store types.CommitKVStore, prefix []byte, cache *fastcache.Cache,
metrics *Metrics) *CommitKVStoreCache {
return &CommitKVStoreCache{
CommitKVStore: store,
prefix: prefix,
cache: cache,
metrics: metrics,
}
}

func NewCommitKVStoreCacheManager(cacheSize int) *CommitKVStoreCacheManager {
func NewCommitKVStoreCacheManager(cacheSize int, provider MetricsProvider) *CommitKVStoreCacheManager {
if cacheSize <= 0 {
// This function was called because it intended to use the inter block cache, creating a cache of minimal size.
cacheSize = DefaultCommitKVStoreCacheSize
}
return &CommitKVStoreCacheManager{
cache: fastcache.New(cacheSize),
caches: make(map[string]types.CommitKVStore),
metrics: provider(),
prefixMap: make(map[string][]byte),
prefixOrder: 0,
}
Expand All @@ -81,7 +85,7 @@ func (cmgr *CommitKVStoreCacheManager) GetStoreCache(key types.StoreKey, store t
if cmgr.prefixOrder <= 0 {
panic("The number of cache stores exceed the maximum(127)")
}
cmgr.caches[key.Name()] = NewCommitKVStoreCache(store, cmgr.prefixMap[key.Name()], cmgr.cache)
cmgr.caches[key.Name()] = NewCommitKVStoreCache(store, cmgr.prefixMap[key.Name()], cmgr.cache, cmgr.metrics)
}
cmgr.mutex.Unlock()
}
Expand Down Expand Up @@ -123,18 +127,18 @@ func (ckv *CommitKVStoreCache) Get(key []byte) []byte {
valueI := ckv.cache.Get(nil, prefixedKey)
if valueI != nil {
// cache hit
telemetry.IncrCounter(1, "store", "inter-block-cache", "hits")
ckv.metrics.InterBlockCacheHits.Add(1)
return valueI
}

// cache miss; write to cache
telemetry.IncrCounter(1, "store", "inter-block-cache", "misses")
ckv.metrics.InterBlockCacheMisses.Add(1)
value := ckv.CommitKVStore.Get(key)
ckv.cache.Set(prefixedKey, value)
stats := fastcache.Stats{}
ckv.cache.UpdateStats(&stats)
telemetry.SetGauge(float32(stats.EntriesCount), "store", "inter-block-cache", "entries")
telemetry.SetGauge(float32(stats.BytesSize), "store", "inter-block-cache", "bytes")
ckv.metrics.InterBlockCacheEntries.Set(float64(stats.EntriesCount))
ckv.metrics.InterBlockCacheBytes.Set(float64(stats.BytesSize))
return value
}

Expand Down
6 changes: 3 additions & 3 deletions store/cache/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (

func TestGetOrSetStoreCache(t *testing.T) {
db := memdb.NewDB()
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())

sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
Expand All @@ -30,7 +30,7 @@ func TestGetOrSetStoreCache(t *testing.T) {

func TestUnwrap(t *testing.T) {
db := memdb.NewDB()
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())

sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
Expand All @@ -44,7 +44,7 @@ func TestUnwrap(t *testing.T) {

func TestStoreCache(t *testing.T) {
db := memdb.NewDB()
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())

sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
Expand Down
84 changes: 84 additions & 0 deletions store/cache/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package cache

import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)

const (
// MetricsSubsystem is a subsystem shared by all metrics exposed by this
// package.
MetricsSubsystem = "inter_block_cache"
)

// Metrics contains metrics exposed by this package.
type Metrics struct {
InterBlockCacheHits metrics.Counter
InterBlockCacheMisses metrics.Counter
InterBlockCacheEntries metrics.Gauge
InterBlockCacheBytes metrics.Gauge
}

// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
InterBlockCacheHits: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "hits",
Help: "Cache hits of the inter block cache",
}, labels).With(labelsAndValues...),
InterBlockCacheMisses: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "misses",
Help: "Cache misses of the inter block cache",
}, labels).With(labelsAndValues...),
InterBlockCacheEntries: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "entries",
Help: "Cache entry count of the inter block cache",
}, labels).With(labelsAndValues...),
InterBlockCacheBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "bytes_size",
Help: "Cache bytes size of the inter block cache",
}, labels).With(labelsAndValues...),
}
}

// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
InterBlockCacheHits: discard.NewCounter(),
InterBlockCacheMisses: discard.NewCounter(),
InterBlockCacheEntries: discard.NewGauge(),
InterBlockCacheBytes: discard.NewGauge(),
}
}

type MetricsProvider func() *Metrics

// PrometheusMetricsProvider returns PrometheusMetrics for each store
func PrometheusMetricsProvider(namespace string, labelsAndValues ...string) func() *Metrics {
return func() *Metrics {
return PrometheusMetrics(namespace, labelsAndValues...)
}
}

// NopMetricsProvider returns NopMetrics for each store
func NopMetricsProvider() func() *Metrics {
return func() *Metrics {
return NopMetrics()
}
}
16 changes: 16 additions & 0 deletions store/cache/metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package cache

import (
"testing"

"github.com/go-kit/kit/metrics/discard"
"github.com/stretchr/testify/require"
)

func TestPrometheusMetrics(t *testing.T) {
metrics := PrometheusMetrics("test")
require.NotEqual(t, metrics.InterBlockCacheHits, discard.NewCounter())
require.NotEqual(t, metrics.InterBlockCacheMisses, discard.NewCounter())
require.NotEqual(t, metrics.InterBlockCacheEntries, discard.NewGauge())
require.NotEqual(t, metrics.InterBlockCacheBytes, discard.NewGauge())
}
Loading

0 comments on commit f9c3e64

Please sign in to comment.