Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(telemetry): enable statsd and dogstatsd telemetry sinks #18646

Merged
merged 21 commits into from
Dec 9, 2023
Merged

feat(telemetry): enable statsd and dogstatsd telemetry sinks #18646

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
c9c05d3
init
itsdevbear Dec 6, 2023
da47c25
fix template
itsdevbear Dec 6, 2023
a8cc689
rename
itsdevbear Dec 6, 2023
52d7b96
Update telemetry/metrics.go
itsdevbear Dec 6, 2023
ed19439
Merge branch 'main' into datadog-metrics
itsdevbear Dec 6, 2023
7503b66
Revert "Update telemetry/metrics.go"
itsdevbear Dec 6, 2023
d306750
nits from bez
itsdevbear Dec 6, 2023
74f1869
Merge branch 'main' into datadog-metrics
itsdevbear Dec 6, 2023
9cdaabe
changelog
itsdevbear Dec 6, 2023
7a11ee2
Update telemetry/metrics.go
itsdevbear Dec 6, 2023
513d92c
update 51 app config
itsdevbear Dec 6, 2023
8f47698
v50 too
itsdevbear Dec 7, 2023
5fc488d
fix
itsdevbear Dec 7, 2023
4731792
lint and gomod tidy
tac0turtle Dec 7, 2023
812dee4
Merge branch 'main' into datadog-metrics
tac0turtle Dec 7, 2023
8664b7d
fix build
tac0turtle Dec 7, 2023
5ad56f5
revert script change
tac0turtle Dec 7, 2023
040268f
somemore
tac0turtle Dec 8, 2023
9ef410e
set default metric sink to inmem
tac0turtle Dec 8, 2023
150a5dc
use blackhole sink if no sink is set
tac0turtle Dec 8, 2023
4905617
went back to inmem sink being the default
tac0turtle Dec 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ require (
require (
filippo.io/edwards25519 v1.0.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
github.com/bufbuild/protocompile v0.6.0 // indirect
Expand Down Expand Up @@ -151,10 +153,12 @@ require (
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.0 // indirect
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q=
github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
Expand Down
11 changes: 11 additions & 0 deletions server/config/toml.go
View file
Open in desktop
itsdevbear marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,17 @@ global-labels = [{{ range $k, $v := .Telemetry.GlobalLabels }}
["{{index $v 0 }}", "{{ index $v 1}}"],{{ end }}
]

# MetricsBackend defines the type of metrics backend to use.
metrics-backend = "{{ .Telemetry.MetricsBackend }}"

# StatsdAddr defines the address of a statsd server to send metrics to.
# Only utilized if MetricsBackend is set to "statsd" or "dogstatsd".
statsd-addr = "{{ .Telemetry.StatsdAddr }}"

# DatadogHostname defines the hostname to use when emitting metrics to
# Datadog. Only utilized if MetricsBackend is set to "dogstatsd".
datadog-hostname = "{{ .Telemetry.DatadogHostname }}"

###############################################################################
### API Configuration ###
###############################################################################
Expand Down
4 changes: 4 additions & 0 deletions simapp/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ require (
filippo.io/edwards25519 v1.0.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/aws/aws-sdk-go v1.45.25 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
Expand Down Expand Up @@ -189,12 +191,14 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.16.0 // indirect
golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/api v0.149.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
Expand Down
2 changes: 2 additions & 0 deletions simapp/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q=
github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
Expand Down
61 changes: 50 additions & 11 deletions telemetry/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"time"

"github.com/hashicorp/go-metrics"
"github.com/hashicorp/go-metrics/datadog"
metricsprom "github.com/hashicorp/go-metrics/prometheus"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/expfmt"
Expand Down Expand Up @@ -52,6 +54,17 @@ type Config struct {
// Example:
// [["chain_id", "cosmoshub-1"]]
GlobalLabels [][]string `mapstructure:"global-labels"`

// MetricsBackend defines the type of metrics backend to use.
MetricsBackend string `mapstructure:"type" default:"mem"`

// StatsdAddr defines the address of a statsd server to send metrics to.
// Only utilized if MetricsBackend is set to "statsd" or "dogstatsd".
StatsdAddr string `mapstructure:"statsd-addr"`

// DatadogHostname defines the hostname to use when emitting metrics to
// Datadog. Only utilized if MetricsBackend is set to "dogstatsd".
DatadogHostname string `mapstructure:"datadog-hostname"`
}

// Metrics defines a wrapper around application telemetry functionality. It allows
Expand All @@ -60,7 +73,7 @@ type Config struct {
// by the operator. In addition to the sinks, when a process gets a SIGUSR1, a
// dump of formatted recent metrics will be sent to STDERR.
type Metrics struct {
memSink *metrics.InmemSink
memSink metrics.MetricSink
itsdevbear marked this conversation as resolved.
Show resolved Hide resolved
prometheusEnabled bool
}

Expand Down Expand Up @@ -88,17 +101,34 @@ func New(cfg Config) (_ *Metrics, rerr error) {
metricsConf := metrics.DefaultConfig(cfg.ServiceName)
metricsConf.EnableHostname = cfg.EnableHostname
metricsConf.EnableHostnameLabel = cfg.EnableHostnameLabel

memSink := metrics.NewInmemSink(10*time.Second, time.Minute)
inMemSig := metrics.DefaultInmemSignal(memSink)
defer func() {
if rerr != nil {
inMemSig.Stop()
var sink metrics.MetricSink

switch cfg.MetricsBackend {
case "mem":
itsdevbear marked this conversation as resolved.
Show resolved Hide resolved
memSink := metrics.NewInmemSink(10*time.Second, time.Minute)
sink = memSink
inMemSig := metrics.DefaultInmemSignal(memSink)
defer func() {
if rerr != nil {
inMemSig.Stop()
}
itsdevbear marked this conversation as resolved.
Show resolved Hide resolved
}()
tac0turtle marked this conversation as resolved.
Show resolved Hide resolved
case "statsd":
var err error
sink, err = metrics.NewStatsdSink(cfg.StatsdAddr)
if err != nil {
return nil, err
}
case "dogstatsd":
var err error
sink, err = datadog.NewDogStatsdSink(cfg.StatsdAddr, cfg.DatadogHostname)
if err != nil {
return nil, err
}
}()
}

m := &Metrics{memSink: memSink}
fanout := metrics.FanoutSink{memSink}
m := &Metrics{memSink: sink}
fanout := metrics.FanoutSink{sink}
itsdevbear marked this conversation as resolved.
Show resolved Hide resolved

if cfg.PrometheusRetentionTime > 0 {
m.prometheusEnabled = true
Expand All @@ -121,6 +151,10 @@ func New(cfg Config) (_ *Metrics, rerr error) {
return m, nil
}

type DisplayableMetrics interface {
DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error)
}

// Gather collects all registered metrics and returns a GatherResponse where the
// metrics are encoded depending on the type. Metrics are either encoded via
// Prometheus or JSON if in-memory.
Expand Down Expand Up @@ -164,7 +198,12 @@ func (m *Metrics) gatherPrometheus() (GatherResponse, error) {
}

func (m *Metrics) gatherGeneric() (GatherResponse, error) {
summary, err := m.memSink.DisplayMetrics(nil, nil)
gm, ok := m.memSink.(DisplayableMetrics)
if !ok {
return GatherResponse{}, fmt.Errorf("non in-memory metrics sink does not support generic format")
}

summary, err := gm.DisplayMetrics(nil, nil)
if err != nil {
return GatherResponse{}, fmt.Errorf("failed to gather in-memory metrics: %w", err)
}
Expand Down
Loading