Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add a flag to aggr ipvs metrics to avoid high cardinality metrics #1709

Merged
merged 1 commit into from
Jun 2, 2020
Merged

add a flag to aggr ipvs metrics to avoid high cardinality metrics #1709

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions collector/fixtures/ip_vs_result_lbs_local_address_local_port.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
# TYPE node_ipvs_backend_connections_active gauge
node_ipvs_backend_connections_active{local_address="",local_port="0"} 385
node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306"} 744
node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306"} 0
node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306"} 2997
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
# TYPE node_ipvs_backend_connections_inactive gauge
node_ipvs_backend_connections_inactive{local_address="",local_port="0"} 6
node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306"} 5
node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306"} 0
node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306"} 0
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
# TYPE node_ipvs_backend_weight gauge
node_ipvs_backend_weight{local_address="",local_port="0"} 120
node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306"} 300
node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306"} 100
node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306"} 200
# HELP node_ipvs_connections_total The total number of connections made.
# TYPE node_ipvs_connections_total counter
node_ipvs_connections_total 2.3765872e+07
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
# TYPE node_ipvs_incoming_bytes_total counter
node_ipvs_incoming_bytes_total 8.9991519156915e+13
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
# TYPE node_ipvs_incoming_packets_total counter
node_ipvs_incoming_packets_total 3.811989221e+09
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
# TYPE node_ipvs_outgoing_bytes_total counter
node_ipvs_outgoing_bytes_total 0
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
# TYPE node_ipvs_outgoing_packets_total counter
node_ipvs_outgoing_packets_total 0
27 changes: 27 additions & 0 deletions collector/fixtures/ip_vs_result_lbs_local_port.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
# TYPE node_ipvs_backend_connections_active gauge
node_ipvs_backend_connections_active{local_port="0"} 385
node_ipvs_backend_connections_active{local_port="3306"} 3741
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
# TYPE node_ipvs_backend_connections_inactive gauge
node_ipvs_backend_connections_inactive{local_port="0"} 6
node_ipvs_backend_connections_inactive{local_port="3306"} 5
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
# TYPE node_ipvs_backend_weight gauge
node_ipvs_backend_weight{local_port="0"} 120
node_ipvs_backend_weight{local_port="3306"} 600
# HELP node_ipvs_connections_total The total number of connections made.
# TYPE node_ipvs_connections_total counter
node_ipvs_connections_total 2.3765872e+07
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
# TYPE node_ipvs_incoming_bytes_total counter
node_ipvs_incoming_bytes_total 8.9991519156915e+13
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
# TYPE node_ipvs_incoming_packets_total counter
node_ipvs_incoming_packets_total 3.811989221e+09
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
# TYPE node_ipvs_outgoing_bytes_total counter
node_ipvs_outgoing_bytes_total 0
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
# TYPE node_ipvs_outgoing_packets_total counter
node_ipvs_outgoing_packets_total 0
24 changes: 24 additions & 0 deletions collector/fixtures/ip_vs_result_lbs_none.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
# TYPE node_ipvs_backend_connections_active gauge
node_ipvs_backend_connections_active 4126
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
# TYPE node_ipvs_backend_connections_inactive gauge
node_ipvs_backend_connections_inactive 11
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
# TYPE node_ipvs_backend_weight gauge
node_ipvs_backend_weight 720
# HELP node_ipvs_connections_total The total number of connections made.
# TYPE node_ipvs_connections_total counter
node_ipvs_connections_total 2.3765872e+07
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
# TYPE node_ipvs_incoming_bytes_total counter
node_ipvs_incoming_bytes_total 8.9991519156915e+13
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
# TYPE node_ipvs_incoming_packets_total counter
node_ipvs_incoming_packets_total 3.811989221e+09
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
# TYPE node_ipvs_outgoing_bytes_total counter
node_ipvs_outgoing_bytes_total 0
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
# TYPE node_ipvs_outgoing_packets_total counter
node_ipvs_outgoing_packets_total 0
121 changes: 100 additions & 21 deletions collector/ipvs_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,53 @@ package collector
import (
"fmt"
"os"
"sort"
"strconv"
"strings"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/procfs"
kingpin "gopkg.in/alecthomas/kingpin.v2"
)

type ipvsCollector struct {
Collector
fs procfs.FS
backendLabels []string
backendConnectionsActive, backendConnectionsInact, backendWeight typedDesc
connections, incomingPackets, outgoingPackets, incomingBytes, outgoingBytes typedDesc
logger log.Logger
}

type ipvsBackendStatus struct {
ActiveConn uint64
InactConn uint64
Weight uint64
}

const (
ipvsLabelLocalAddress = "local_address"
ipvsLabelLocalPort = "local_port"
ipvsLabelRemoteAddress = "remote_address"
ipvsLabelRemotePort = "remote_port"
ipvsLabelProto = "proto"
ipvsLabelLocalMark = "local_mark"
)

var (
fullIpvsBackendLabels = []string{
ipvsLabelLocalAddress,
ipvsLabelLocalPort,
ipvsLabelRemoteAddress,
ipvsLabelRemotePort,
ipvsLabelProto,
ipvsLabelLocalMark,
}
ipvsLabels = kingpin.Flag("collector.ipvs.backend-labels", "Comma separated list for IPVS backend stats labels.").Default(strings.Join(fullIpvsBackendLabels, ",")).String()
)

func init() {
registerCollector("ipvs", defaultEnabled, NewIPVSCollector)
}
Expand All @@ -46,19 +77,15 @@ func NewIPVSCollector(logger log.Logger) (Collector, error) {

func newIPVSCollector(logger log.Logger) (*ipvsCollector, error) {
var (
ipvsBackendLabelNames = []string{
"local_address",
"local_port",
"remote_address",
"remote_port",
"proto",
"local_mark",
}
c ipvsCollector
err error
subsystem = "ipvs"
)

if c.backendLabels, err = c.parseIpvsLabels(*ipvsLabels); err != nil {
return nil, err
}

c.logger = logger
c.fs, err = procfs.NewFS(*procPath)
if err != nil {
Expand Down Expand Up @@ -93,17 +120,17 @@ func newIPVSCollector(logger log.Logger) (*ipvsCollector, error) {
c.backendConnectionsActive = typedDesc{prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "backend_connections_active"),
"The current active connections by local and remote address.",
ipvsBackendLabelNames, nil,
c.backendLabels, nil,
), prometheus.GaugeValue}
c.backendConnectionsInact = typedDesc{prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "backend_connections_inactive"),
"The current inactive connections by local and remote address.",
ipvsBackendLabelNames, nil,
c.backendLabels, nil,
), prometheus.GaugeValue}
c.backendWeight = typedDesc{prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "backend_weight"),
"The current backend weight by local and remote address.",
ipvsBackendLabelNames, nil,
c.backendLabels, nil,
), prometheus.GaugeValue}

return &c, nil
Expand All @@ -130,22 +157,74 @@ func (c *ipvsCollector) Update(ch chan<- prometheus.Metric) error {
return fmt.Errorf("could not get backend status: %s", err)
}

sums := map[string]ipvsBackendStatus{}
labelValues := map[string][]string{}
for _, backend := range backendStats {
localAddress := ""
if backend.LocalAddress.String() != "<nil>" {
localAddress = backend.LocalAddress.String()
}
labelValues := []string{
localAddress,
strconv.FormatUint(uint64(backend.LocalPort), 10),
backend.RemoteAddress.String(),
strconv.FormatUint(uint64(backend.RemotePort), 10),
backend.Proto,
backend.LocalMark,
kv := make([]string, len(c.backendLabels))
for i, label := range c.backendLabels {
var labelValue string
switch label {
case ipvsLabelLocalAddress:
labelValue = localAddress
case ipvsLabelLocalPort:
labelValue = strconv.FormatUint(uint64(backend.LocalPort), 10)
case ipvsLabelRemoteAddress:
labelValue = backend.RemoteAddress.String()
case ipvsLabelRemotePort:
labelValue = strconv.FormatUint(uint64(backend.RemotePort), 10)
case ipvsLabelProto:
labelValue = backend.Proto
case ipvsLabelLocalMark:
labelValue = backend.LocalMark
}
kv[i] = labelValue
}
ch <- c.backendConnectionsActive.mustNewConstMetric(float64(backend.ActiveConn), labelValues...)
ch <- c.backendConnectionsInact.mustNewConstMetric(float64(backend.InactConn), labelValues...)
ch <- c.backendWeight.mustNewConstMetric(float64(backend.Weight), labelValues...)
key := strings.Join(kv, "-")
status := sums[key]
status.ActiveConn += backend.ActiveConn
status.InactConn += backend.InactConn
status.Weight += backend.Weight
sums[key] = status
labelValues[key] = kv
}
for key, status := range sums {
kv := labelValues[key]
ch <- c.backendConnectionsActive.mustNewConstMetric(float64(status.ActiveConn), kv...)
ch <- c.backendConnectionsInact.mustNewConstMetric(float64(status.InactConn), kv...)
ch <- c.backendWeight.mustNewConstMetric(float64(status.Weight), kv...)
}
return nil
}

func (c *ipvsCollector) parseIpvsLabels(labelString string) ([]string, error) {
labels := strings.Split(labelString, ",")
labelSet := make(map[string]bool, len(labels))
results := make([]string, 0, len(labels))
for _, label := range labels {
if label != "" {
labelSet[label] = true
}
}

for _, label := range fullIpvsBackendLabels {
if labelSet[label] {
results = append(results, label)
}
delete(labelSet, label)
}

if len(labelSet) > 0 {
keys := make([]string, 0, len(labelSet))
for label := range labelSet {
keys = append(keys, label)
}
sort.Strings(keys)
return nil, fmt.Errorf("unknown IPVS backend labels: %q", strings.Join(keys, ", "))
}

return results, nil
}
Loading