Skip to content

Commit ce154fe

Browse files
Wing924oblitorum
Wing924
authored and
oblitorum
committed
Add flag to aggr ipvs metrics to avoid high cardinality metrics (prometheus#1709)
Fixes prometheus#1708 Signed-off-by: Wing924 <[email protected]>
1 parent 5fe49dc commit ce154fe

File tree

5 files changed

+345
-81
lines changed

5 files changed

+345
-81
lines changed

‎collector/fixtures/ip_vs_result_lbs_local_address_local_port.txt

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
2+
# TYPE node_ipvs_backend_connections_active gauge
3+
node_ipvs_backend_connections_active{local_address="",local_port="0"} 385
4+
node_ipvs_backend_connections_active{local_address="192.168.0.22",local_port="3306"} 744
5+
node_ipvs_backend_connections_active{local_address="192.168.0.55",local_port="3306"} 0
6+
node_ipvs_backend_connections_active{local_address="192.168.0.57",local_port="3306"} 2997
7+
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
8+
# TYPE node_ipvs_backend_connections_inactive gauge
9+
node_ipvs_backend_connections_inactive{local_address="",local_port="0"} 6
10+
node_ipvs_backend_connections_inactive{local_address="192.168.0.22",local_port="3306"} 5
11+
node_ipvs_backend_connections_inactive{local_address="192.168.0.55",local_port="3306"} 0
12+
node_ipvs_backend_connections_inactive{local_address="192.168.0.57",local_port="3306"} 0
13+
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
14+
# TYPE node_ipvs_backend_weight gauge
15+
node_ipvs_backend_weight{local_address="",local_port="0"} 120
16+
node_ipvs_backend_weight{local_address="192.168.0.22",local_port="3306"} 300
17+
node_ipvs_backend_weight{local_address="192.168.0.55",local_port="3306"} 100
18+
node_ipvs_backend_weight{local_address="192.168.0.57",local_port="3306"} 200
19+
# HELP node_ipvs_connections_total The total number of connections made.
20+
# TYPE node_ipvs_connections_total counter
21+
node_ipvs_connections_total 2.3765872e+07
22+
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
23+
# TYPE node_ipvs_incoming_bytes_total counter
24+
node_ipvs_incoming_bytes_total 8.9991519156915e+13
25+
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
26+
# TYPE node_ipvs_incoming_packets_total counter
27+
node_ipvs_incoming_packets_total 3.811989221e+09
28+
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
29+
# TYPE node_ipvs_outgoing_bytes_total counter
30+
node_ipvs_outgoing_bytes_total 0
31+
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
32+
# TYPE node_ipvs_outgoing_packets_total counter
33+
node_ipvs_outgoing_packets_total 0

‎collector/fixtures/ip_vs_result_lbs_local_port.txt

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
2+
# TYPE node_ipvs_backend_connections_active gauge
3+
node_ipvs_backend_connections_active{local_port="0"} 385
4+
node_ipvs_backend_connections_active{local_port="3306"} 3741
5+
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
6+
# TYPE node_ipvs_backend_connections_inactive gauge
7+
node_ipvs_backend_connections_inactive{local_port="0"} 6
8+
node_ipvs_backend_connections_inactive{local_port="3306"} 5
9+
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
10+
# TYPE node_ipvs_backend_weight gauge
11+
node_ipvs_backend_weight{local_port="0"} 120
12+
node_ipvs_backend_weight{local_port="3306"} 600
13+
# HELP node_ipvs_connections_total The total number of connections made.
14+
# TYPE node_ipvs_connections_total counter
15+
node_ipvs_connections_total 2.3765872e+07
16+
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
17+
# TYPE node_ipvs_incoming_bytes_total counter
18+
node_ipvs_incoming_bytes_total 8.9991519156915e+13
19+
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
20+
# TYPE node_ipvs_incoming_packets_total counter
21+
node_ipvs_incoming_packets_total 3.811989221e+09
22+
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
23+
# TYPE node_ipvs_outgoing_bytes_total counter
24+
node_ipvs_outgoing_bytes_total 0
25+
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
26+
# TYPE node_ipvs_outgoing_packets_total counter
27+
node_ipvs_outgoing_packets_total 0

‎collector/fixtures/ip_vs_result_lbs_none.txt

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# HELP node_ipvs_backend_connections_active The current active connections by local and remote address.
2+
# TYPE node_ipvs_backend_connections_active gauge
3+
node_ipvs_backend_connections_active 4126
4+
# HELP node_ipvs_backend_connections_inactive The current inactive connections by local and remote address.
5+
# TYPE node_ipvs_backend_connections_inactive gauge
6+
node_ipvs_backend_connections_inactive 11
7+
# HELP node_ipvs_backend_weight The current backend weight by local and remote address.
8+
# TYPE node_ipvs_backend_weight gauge
9+
node_ipvs_backend_weight 720
10+
# HELP node_ipvs_connections_total The total number of connections made.
11+
# TYPE node_ipvs_connections_total counter
12+
node_ipvs_connections_total 2.3765872e+07
13+
# HELP node_ipvs_incoming_bytes_total The total amount of incoming data.
14+
# TYPE node_ipvs_incoming_bytes_total counter
15+
node_ipvs_incoming_bytes_total 8.9991519156915e+13
16+
# HELP node_ipvs_incoming_packets_total The total number of incoming packets.
17+
# TYPE node_ipvs_incoming_packets_total counter
18+
node_ipvs_incoming_packets_total 3.811989221e+09
19+
# HELP node_ipvs_outgoing_bytes_total The total amount of outgoing data.
20+
# TYPE node_ipvs_outgoing_bytes_total counter
21+
node_ipvs_outgoing_bytes_total 0
22+
# HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
23+
# TYPE node_ipvs_outgoing_packets_total counter
24+
node_ipvs_outgoing_packets_total 0

‎collector/ipvs_linux.go

Lines changed: 100 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,53 @@ package collector
1818
import (
1919
"fmt"
2020
"os"
21+
"sort"
2122
"strconv"
23+
"strings"
2224

2325
"github.com/go-kit/kit/log"
2426
"github.com/go-kit/kit/log/level"
2527
"github.com/prometheus/client_golang/prometheus"
2628
"github.com/prometheus/procfs"
29+
kingpin "gopkg.in/alecthomas/kingpin.v2"
2730
)
2831

2932
type ipvsCollector struct {
3033
Collector
3134
fs procfs.FS
35+
backendLabels []string
3236
backendConnectionsActive, backendConnectionsInact, backendWeight typedDesc
3337
connections, incomingPackets, outgoingPackets, incomingBytes, outgoingBytes typedDesc
3438
logger log.Logger
3539
}
3640

41+
type ipvsBackendStatus struct {
42+
ActiveConn uint64
43+
InactConn uint64
44+
Weight uint64
45+
}
46+
47+
const (
48+
ipvsLabelLocalAddress = "local_address"
49+
ipvsLabelLocalPort = "local_port"
50+
ipvsLabelRemoteAddress = "remote_address"
51+
ipvsLabelRemotePort = "remote_port"
52+
ipvsLabelProto = "proto"
53+
ipvsLabelLocalMark = "local_mark"
54+
)
55+
56+
var (
57+
fullIpvsBackendLabels = []string{
58+
ipvsLabelLocalAddress,
59+
ipvsLabelLocalPort,
60+
ipvsLabelRemoteAddress,
61+
ipvsLabelRemotePort,
62+
ipvsLabelProto,
63+
ipvsLabelLocalMark,
64+
}
65+
ipvsLabels = kingpin.Flag("collector.ipvs.backend-labels", "Comma separated list for IPVS backend stats labels.").Default(strings.Join(fullIpvsBackendLabels, ",")).String()
66+
)
67+
3768
func init() {
3869
registerCollector("ipvs", defaultEnabled, NewIPVSCollector)
3970
}
@@ -46,19 +77,15 @@ func NewIPVSCollector(logger log.Logger) (Collector, error) {
4677

4778
func newIPVSCollector(logger log.Logger) (*ipvsCollector, error) {
4879
var (
49-
ipvsBackendLabelNames = []string{
50-
"local_address",
51-
"local_port",
52-
"remote_address",
53-
"remote_port",
54-
"proto",
55-
"local_mark",
56-
}
5780
c ipvsCollector
5881
err error
5982
subsystem = "ipvs"
6083
)
6184

85+
if c.backendLabels, err = c.parseIpvsLabels(*ipvsLabels); err != nil {
86+
return nil, err
87+
}
88+
6289
c.logger = logger
6390
c.fs, err = procfs.NewFS(*procPath)
6491
if err != nil {
@@ -93,17 +120,17 @@ func newIPVSCollector(logger log.Logger) (*ipvsCollector, error) {
93120
c.backendConnectionsActive = typedDesc{prometheus.NewDesc(
94121
prometheus.BuildFQName(namespace, subsystem, "backend_connections_active"),
95122
"The current active connections by local and remote address.",
96-
ipvsBackendLabelNames, nil,
123+
c.backendLabels, nil,
97124
), prometheus.GaugeValue}
98125
c.backendConnectionsInact = typedDesc{prometheus.NewDesc(
99126
prometheus.BuildFQName(namespace, subsystem, "backend_connections_inactive"),
100127
"The current inactive connections by local and remote address.",
101-
ipvsBackendLabelNames, nil,
128+
c.backendLabels, nil,
102129
), prometheus.GaugeValue}
103130
c.backendWeight = typedDesc{prometheus.NewDesc(
104131
prometheus.BuildFQName(namespace, subsystem, "backend_weight"),
105132
"The current backend weight by local and remote address.",
106-
ipvsBackendLabelNames, nil,
133+
c.backendLabels, nil,
107134
), prometheus.GaugeValue}
108135

109136
return &c, nil
@@ -130,22 +157,74 @@ func (c *ipvsCollector) Update(ch chan<- prometheus.Metric) error {
130157
return fmt.Errorf("could not get backend status: %s", err)
131158
}
132159

160+
sums := map[string]ipvsBackendStatus{}
161+
labelValues := map[string][]string{}
133162
for _, backend := range backendStats {
134163
localAddress := ""
135164
if backend.LocalAddress.String() != "<nil>" {
136165
localAddress = backend.LocalAddress.String()
137166
}
138-
labelValues := []string{
139-
localAddress,
140-
strconv.FormatUint(uint64(backend.LocalPort), 10),
141-
backend.RemoteAddress.String(),
142-
strconv.FormatUint(uint64(backend.RemotePort), 10),
143-
backend.Proto,
144-
backend.LocalMark,
167+
kv := make([]string, len(c.backendLabels))
168+
for i, label := range c.backendLabels {
169+
var labelValue string
170+
switch label {
171+
case ipvsLabelLocalAddress:
172+
labelValue = localAddress
173+
case ipvsLabelLocalPort:
174+
labelValue = strconv.FormatUint(uint64(backend.LocalPort), 10)
175+
case ipvsLabelRemoteAddress:
176+
labelValue = backend.RemoteAddress.String()
177+
case ipvsLabelRemotePort:
178+
labelValue = strconv.FormatUint(uint64(backend.RemotePort), 10)
179+
case ipvsLabelProto:
180+
labelValue = backend.Proto
181+
case ipvsLabelLocalMark:
182+
labelValue = backend.LocalMark
183+
}
184+
kv[i] = labelValue
145185
}
146-
ch <- c.backendConnectionsActive.mustNewConstMetric(float64(backend.ActiveConn), labelValues...)
147-
ch <- c.backendConnectionsInact.mustNewConstMetric(float64(backend.InactConn), labelValues...)
148-
ch <- c.backendWeight.mustNewConstMetric(float64(backend.Weight), labelValues...)
186+
key := strings.Join(kv, "-")
187+
status := sums[key]
188+
status.ActiveConn += backend.ActiveConn
189+
status.InactConn += backend.InactConn
190+
status.Weight += backend.Weight
191+
sums[key] = status
192+
labelValues[key] = kv
193+
}
194+
for key, status := range sums {
195+
kv := labelValues[key]
196+
ch <- c.backendConnectionsActive.mustNewConstMetric(float64(status.ActiveConn), kv...)
197+
ch <- c.backendConnectionsInact.mustNewConstMetric(float64(status.InactConn), kv...)
198+
ch <- c.backendWeight.mustNewConstMetric(float64(status.Weight), kv...)
149199
}
150200
return nil
151201
}
202+
203+
func (c *ipvsCollector) parseIpvsLabels(labelString string) ([]string, error) {
204+
labels := strings.Split(labelString, ",")
205+
labelSet := make(map[string]bool, len(labels))
206+
results := make([]string, 0, len(labels))
207+
for _, label := range labels {
208+
if label != "" {
209+
labelSet[label] = true
210+
}
211+
}
212+
213+
for _, label := range fullIpvsBackendLabels {
214+
if labelSet[label] {
215+
results = append(results, label)
216+
}
217+
delete(labelSet, label)
218+
}
219+
220+
if len(labelSet) > 0 {
221+
keys := make([]string, 0, len(labelSet))
222+
for label := range labelSet {
223+
keys = append(keys, label)
224+
}
225+
sort.Strings(keys)
226+
return nil, fmt.Errorf("unknown IPVS backend labels: %q", strings.Join(keys, ", "))
227+
}
228+
229+
return results, nil
230+
}

0 commit comments

Comments
 (0)