From c823879097f10ce62b63fde53af88a0367c30f9e Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 20 Mar 2023 18:06:07 +0100 Subject: [PATCH] Add prometheus metric requests_total with headers Co-authored-by: Julien Salleyron --- .../observability/metrics/prometheus.md | 63 +++++++++++++++++++ .../reference/static-configuration/cli-ref.md | 3 + .../reference/static-configuration/env-ref.md | 3 + .../reference/static-configuration/file.toml | 3 + .../reference/static-configuration/file.yaml | 3 + pkg/metrics/datadog.go | 6 +- pkg/metrics/datadog_test.go | 10 +-- pkg/metrics/headers.go | 57 +++++++++++++++++ pkg/metrics/influxdb.go | 6 +- pkg/metrics/influxdb2.go | 6 +- pkg/metrics/influxdb2_test.go | 10 +-- pkg/metrics/influxdb_test.go | 20 +++--- pkg/metrics/metrics.go | 30 ++++----- pkg/metrics/metrics_test.go | 21 ++++++- pkg/metrics/prometheus.go | 61 ++++++++++++++++-- pkg/metrics/prometheus_test.go | 50 ++++++++------- pkg/metrics/statsd.go | 6 +- pkg/metrics/statsd_test.go | 10 +-- pkg/middlewares/metrics/metrics.go | 4 +- pkg/types/metrics.go | 13 ++-- 20 files changed, 295 insertions(+), 90 deletions(-) create mode 100644 pkg/metrics/headers.go diff --git a/docs/content/observability/metrics/prometheus.md b/docs/content/observability/metrics/prometheus.md index da7f73773..be3e29cb6 100644 --- a/docs/content/observability/metrics/prometheus.md +++ b/docs/content/observability/metrics/prometheus.md @@ -165,3 +165,66 @@ metrics: ```bash tab="CLI" --metrics.prometheus.manualrouting=true ``` + +#### `headerLabels` + +_Optional_ + +Defines the extra labels for the `requests_total` metrics, and for each of them, the request header containing the value for this label. +Please note that if the header is not present in the request it will be added nonetheless with an empty value. +In addition, the label should be a valid label name for Prometheus metrics, +otherwise, the Prometheus metrics provider will fail to serve any Traefik-related metric. + +```yaml tab="File (YAML)" +metrics: + prometheus: + headerLabels: + label: headerKey +``` + +```toml tab="File (TOML)" +[metrics] + [metrics.prometheus] + [metrics.prometheus.headerLabels] + label = "headerKey" +``` + +```bash tab="CLI" +--metrics.prometheus.headerlabels.label=headerKey +``` + +##### Example + +Here is an example of the entryPoint `requests_total` metric with an additional "useragent" label. + +When configuring the label in Static Configuration: + +```yaml tab="File (YAML)" +metrics: + prometheus: + headerLabels: + useragent: User-Agent +``` + +```toml tab="File (TOML)" +[metrics] + [metrics.prometheus] + [metrics.prometheus.headerLabels] + useragent = "User-Agent" +``` + +```bash tab="CLI" +--metrics.prometheus.headerlabels.useragent=User-Agent +``` + +And performing a request with a custom User-Agent: + +```bash +curl -H "User-Agent: foobar" http://localhost +``` + +The following metric is produced : + +```bash +traefik_entrypoint_requests_total{code="200",entrypoint="web",method="GET",protocol="http",useragent="foobar"} 1 +``` \ No newline at end of file diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index 80c9436aa..a965ff977 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -354,6 +354,9 @@ Buckets for latency metrics. (Default: ```0.100000, 0.300000, 1.200000, 5.000000 `--metrics.prometheus.entrypoint`: EntryPoint (Default: ```traefik```) +`--metrics.prometheus.headerlabels.`: +Defines the extra labels for the requests_total metrics, and for each of them, the request header containing the value for this label. + `--metrics.prometheus.manualrouting`: Manual routing (Default: ```false```) diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index 4edfeb411..e38608aa7 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -354,6 +354,9 @@ Buckets for latency metrics. (Default: ```0.100000, 0.300000, 1.200000, 5.000000 `TRAEFIK_METRICS_PROMETHEUS_ENTRYPOINT`: EntryPoint (Default: ```traefik```) +`TRAEFIK_METRICS_PROMETHEUS_HEADERLABELS_`: +Defines the extra labels for the requests_total metrics, and for each of them, the request header containing the value for this label. + `TRAEFIK_METRICS_PROMETHEUS_MANUALROUTING`: Manual routing (Default: ```false```) diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index bf3d13125..57970c0a7 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -272,6 +272,9 @@ addServicesLabels = true entryPoint = "foobar" manualRouting = true + [metrics.prometheus.headerLabels] + label1 = "foobar" + label2 = "foobar" [metrics.datadog] address = "foobar" pushInterval = "42s" diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index b323f8b72..c0b4cbead 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -298,6 +298,9 @@ metrics: addServicesLabels: true entryPoint: foobar manualRouting: true + headerLabels: + label1: foobar + label2: foobar datadog: address: foobar pushInterval: 42s diff --git a/pkg/metrics/datadog.go b/pkg/metrics/datadog.go index 596bc700d..6ed76bbfd 100644 --- a/pkg/metrics/datadog.go +++ b/pkg/metrics/datadog.go @@ -75,7 +75,7 @@ func RegisterDatadog(ctx context.Context, config *types.Datadog) Registry { if config.AddEntryPointsLabels { registry.epEnabled = config.AddEntryPointsLabels - registry.entryPointReqsCounter = datadogClient.NewCounter(ddEntryPointReqsName, 1.0) + registry.entryPointReqsCounter = NewCounterWithNoopHeaders(datadogClient.NewCounter(ddEntryPointReqsName, 1.0)) registry.entryPointReqsTLSCounter = datadogClient.NewCounter(ddEntryPointReqsTLSName, 1.0) registry.entryPointReqDurationHistogram, _ = NewHistogramWithScale(datadogClient.NewHistogram(ddEntryPointReqDurationName, 1.0), time.Second) registry.entryPointOpenConnsGauge = datadogClient.NewGauge(ddEntryPointOpenConnsName) @@ -85,7 +85,7 @@ func RegisterDatadog(ctx context.Context, config *types.Datadog) Registry { if config.AddRoutersLabels { registry.routerEnabled = config.AddRoutersLabels - registry.routerReqsCounter = datadogClient.NewCounter(ddRouterReqsName, 1.0) + registry.routerReqsCounter = NewCounterWithNoopHeaders(datadogClient.NewCounter(ddRouterReqsName, 1.0)) registry.routerReqsTLSCounter = datadogClient.NewCounter(ddRouterReqsTLSName, 1.0) registry.routerReqDurationHistogram, _ = NewHistogramWithScale(datadogClient.NewHistogram(ddRouterReqsDurationName, 1.0), time.Second) registry.routerOpenConnsGauge = datadogClient.NewGauge(ddRouterOpenConnsName) @@ -95,7 +95,7 @@ func RegisterDatadog(ctx context.Context, config *types.Datadog) Registry { if config.AddServicesLabels { registry.svcEnabled = config.AddServicesLabels - registry.serviceReqsCounter = datadogClient.NewCounter(ddServiceReqsName, 1.0) + registry.serviceReqsCounter = NewCounterWithNoopHeaders(datadogClient.NewCounter(ddServiceReqsName, 1.0)) registry.serviceReqsTLSCounter = datadogClient.NewCounter(ddServiceReqsTLSName, 1.0) registry.serviceReqDurationHistogram, _ = NewHistogramWithScale(datadogClient.NewHistogram(ddServiceReqsDurationName, 1.0), time.Second) registry.serviceRetriesCounter = datadogClient.NewCounter(ddServiceRetriesName, 1.0) diff --git a/pkg/metrics/datadog_test.go b/pkg/metrics/datadog_test.go index 7a36c1d77..e4b13cb15 100644 --- a/pkg/metrics/datadog_test.go +++ b/pkg/metrics/datadog_test.go @@ -86,23 +86,23 @@ func testDatadogRegistry(t *testing.T, metricsPrefix string, datadogRegistry Reg datadogRegistry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1) - datadogRegistry.EntryPointReqsCounter().With("entrypoint", "test").Add(1) + datadogRegistry.EntryPointReqsCounter().With(nil, "entrypoint", "test").Add(1) datadogRegistry.EntryPointReqsTLSCounter().With("entrypoint", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) datadogRegistry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000) datadogRegistry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1) datadogRegistry.EntryPointReqsBytesCounter().With("entrypoint", "test").Add(1) datadogRegistry.EntryPointRespsBytesCounter().With("entrypoint", "test").Add(1) - datadogRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - datadogRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + datadogRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + datadogRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) datadogRegistry.RouterReqsTLSCounter().With("router", "demo", "service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) datadogRegistry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) datadogRegistry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) datadogRegistry.RouterReqsBytesCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) datadogRegistry.RouterRespsBytesCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - datadogRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - datadogRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + datadogRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + datadogRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) datadogRegistry.ServiceReqsTLSCounter().With("service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) datadogRegistry.ServiceReqDurationHistogram().With("service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) datadogRegistry.ServiceOpenConnsGauge().With("service", "test").Set(1) diff --git a/pkg/metrics/headers.go b/pkg/metrics/headers.go new file mode 100644 index 000000000..65443cd29 --- /dev/null +++ b/pkg/metrics/headers.go @@ -0,0 +1,57 @@ +package metrics + +import ( + "net/http" + + "github.com/go-kit/kit/metrics" +) + +// CounterWithHeaders represents a counter that can use http.Header values as label values. +type CounterWithHeaders interface { + Add(delta float64) + With(headers http.Header, labelValues ...string) CounterWithHeaders +} + +// MultiCounterWithHeaders collects multiple individual CounterWithHeaders and treats them as a unit. +type MultiCounterWithHeaders []CounterWithHeaders + +// NewMultiCounterWithHeaders returns a multi-counter, wrapping the passed CounterWithHeaders. +func NewMultiCounterWithHeaders(c ...CounterWithHeaders) MultiCounterWithHeaders { + return c +} + +// Add adds the given delta value to the counter value. +func (c MultiCounterWithHeaders) Add(delta float64) { + for _, counter := range c { + counter.Add(delta) + } +} + +// With creates a new counter by appending the given label values and http.Header as labels and returns it. +func (c MultiCounterWithHeaders) With(headers http.Header, labelValues ...string) CounterWithHeaders { + next := make(MultiCounterWithHeaders, len(c)) + for i := range c { + next[i] = c[i].With(headers, labelValues...) + } + return next +} + +// NewCounterWithNoopHeaders returns a CounterWithNoopHeaders. +func NewCounterWithNoopHeaders(counter metrics.Counter) CounterWithNoopHeaders { + return CounterWithNoopHeaders{counter: counter} +} + +// CounterWithNoopHeaders is a counter that satisfies CounterWithHeaders but ignores the given http.Header. +type CounterWithNoopHeaders struct { + counter metrics.Counter +} + +// Add adds the given delta value to the counter value. +func (c CounterWithNoopHeaders) Add(delta float64) { + c.counter.Add(delta) +} + +// With creates a new counter by appending the given label values and returns it. +func (c CounterWithNoopHeaders) With(_ http.Header, labelValues ...string) CounterWithHeaders { + return NewCounterWithNoopHeaders(c.counter.With(labelValues...)) +} diff --git a/pkg/metrics/influxdb.go b/pkg/metrics/influxdb.go index f9f5485b7..74a83a23b 100644 --- a/pkg/metrics/influxdb.go +++ b/pkg/metrics/influxdb.go @@ -77,7 +77,7 @@ func RegisterInfluxDB(ctx context.Context, config *types.InfluxDB) Registry { if config.AddEntryPointsLabels { registry.epEnabled = config.AddEntryPointsLabels - registry.entryPointReqsCounter = influxDBClient.NewCounter(influxDBEntryPointReqsName) + registry.entryPointReqsCounter = NewCounterWithNoopHeaders(influxDBClient.NewCounter(influxDBEntryPointReqsName)) registry.entryPointReqsTLSCounter = influxDBClient.NewCounter(influxDBEntryPointReqsTLSName) registry.entryPointReqDurationHistogram, _ = NewHistogramWithScale(influxDBClient.NewHistogram(influxDBEntryPointReqDurationName), time.Second) registry.entryPointOpenConnsGauge = influxDBClient.NewGauge(influxDBEntryPointOpenConnsName) @@ -87,7 +87,7 @@ func RegisterInfluxDB(ctx context.Context, config *types.InfluxDB) Registry { if config.AddRoutersLabels { registry.routerEnabled = config.AddRoutersLabels - registry.routerReqsCounter = influxDBClient.NewCounter(influxDBRouterReqsName) + registry.routerReqsCounter = NewCounterWithNoopHeaders(influxDBClient.NewCounter(influxDBRouterReqsName)) registry.routerReqsTLSCounter = influxDBClient.NewCounter(influxDBRouterReqsTLSName) registry.routerReqDurationHistogram, _ = NewHistogramWithScale(influxDBClient.NewHistogram(influxDBRouterReqsDurationName), time.Second) registry.routerOpenConnsGauge = influxDBClient.NewGauge(influxDBORouterOpenConnsName) @@ -97,7 +97,7 @@ func RegisterInfluxDB(ctx context.Context, config *types.InfluxDB) Registry { if config.AddServicesLabels { registry.svcEnabled = config.AddServicesLabels - registry.serviceReqsCounter = influxDBClient.NewCounter(influxDBServiceReqsName) + registry.serviceReqsCounter = NewCounterWithNoopHeaders(influxDBClient.NewCounter(influxDBServiceReqsName)) registry.serviceReqsTLSCounter = influxDBClient.NewCounter(influxDBServiceReqsTLSName) registry.serviceReqDurationHistogram, _ = NewHistogramWithScale(influxDBClient.NewHistogram(influxDBServiceReqsDurationName), time.Second) registry.serviceRetriesCounter = influxDBClient.NewCounter(influxDBServiceRetriesTotalName) diff --git a/pkg/metrics/influxdb2.go b/pkg/metrics/influxdb2.go index 27be18f2a..fd41590e4 100644 --- a/pkg/metrics/influxdb2.go +++ b/pkg/metrics/influxdb2.go @@ -61,7 +61,7 @@ func RegisterInfluxDB2(ctx context.Context, config *types.InfluxDB2) Registry { if config.AddEntryPointsLabels { registry.epEnabled = config.AddEntryPointsLabels - registry.entryPointReqsCounter = influxDB2Store.NewCounter(influxDBEntryPointReqsName) + registry.entryPointReqsCounter = NewCounterWithNoopHeaders(influxDB2Store.NewCounter(influxDBEntryPointReqsName)) registry.entryPointReqsTLSCounter = influxDB2Store.NewCounter(influxDBEntryPointReqsTLSName) registry.entryPointReqDurationHistogram, _ = NewHistogramWithScale(influxDB2Store.NewHistogram(influxDBEntryPointReqDurationName), time.Second) registry.entryPointOpenConnsGauge = influxDB2Store.NewGauge(influxDBEntryPointOpenConnsName) @@ -71,7 +71,7 @@ func RegisterInfluxDB2(ctx context.Context, config *types.InfluxDB2) Registry { if config.AddRoutersLabels { registry.routerEnabled = config.AddRoutersLabels - registry.routerReqsCounter = influxDB2Store.NewCounter(influxDBRouterReqsName) + registry.routerReqsCounter = NewCounterWithNoopHeaders(influxDB2Store.NewCounter(influxDBRouterReqsName)) registry.routerReqsTLSCounter = influxDB2Store.NewCounter(influxDBRouterReqsTLSName) registry.routerReqDurationHistogram, _ = NewHistogramWithScale(influxDB2Store.NewHistogram(influxDBRouterReqsDurationName), time.Second) registry.routerOpenConnsGauge = influxDB2Store.NewGauge(influxDBORouterOpenConnsName) @@ -81,7 +81,7 @@ func RegisterInfluxDB2(ctx context.Context, config *types.InfluxDB2) Registry { if config.AddServicesLabels { registry.svcEnabled = config.AddServicesLabels - registry.serviceReqsCounter = influxDB2Store.NewCounter(influxDBServiceReqsName) + registry.serviceReqsCounter = NewCounterWithNoopHeaders(influxDB2Store.NewCounter(influxDBServiceReqsName)) registry.serviceReqsTLSCounter = influxDB2Store.NewCounter(influxDBServiceReqsTLSName) registry.serviceReqDurationHistogram, _ = NewHistogramWithScale(influxDB2Store.NewHistogram(influxDBServiceReqsDurationName), time.Second) registry.serviceRetriesCounter = influxDB2Store.NewCounter(influxDBServiceRetriesTotalName) diff --git a/pkg/metrics/influxdb2_test.go b/pkg/metrics/influxdb2_test.go index 3fc628d67..de20bb4b3 100644 --- a/pkg/metrics/influxdb2_test.go +++ b/pkg/metrics/influxdb2_test.go @@ -77,7 +77,7 @@ func TestInfluxDB2(t *testing.T) { `(traefik\.entrypoint\.responses\.bytes\.total,code=200,entrypoint=test,method=GET count=1) [\d]{19}`, } - influxDB2Registry.EntryPointReqsCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDB2Registry.EntryPointReqsCounter().With(nil, "entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDB2Registry.EntryPointReqsTLSCounter().With("entrypoint", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDB2Registry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000) influxDB2Registry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1) @@ -97,8 +97,8 @@ func TestInfluxDB2(t *testing.T) { `(traefik\.router\.responses\.bytes\.total,code=200,method=GET,router=demo,service=test count=1) [\d]{19}`, } - influxDB2Registry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) - influxDB2Registry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDB2Registry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDB2Registry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDB2Registry.RouterReqsTLSCounter().With("router", "demo", "service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDB2Registry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDB2Registry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) @@ -118,8 +118,8 @@ func TestInfluxDB2(t *testing.T) { `(traefik\.service\.responses\.bytes\.total,code=200,method=GET,service=test count=1) [\d]{19}`, } - influxDB2Registry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - influxDB2Registry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDB2Registry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDB2Registry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) influxDB2Registry.ServiceReqsTLSCounter().With("service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDB2Registry.ServiceReqDurationHistogram().With("service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDB2Registry.ServiceServerUpGauge().With("service", "test", "url", "http://127.0.0.1").Set(1) diff --git a/pkg/metrics/influxdb_test.go b/pkg/metrics/influxdb_test.go index 38a146697..bcec37b3f 100644 --- a/pkg/metrics/influxdb_test.go +++ b/pkg/metrics/influxdb_test.go @@ -74,7 +74,7 @@ func TestInfluxDB(t *testing.T) { } msgEntrypoint := udp.ReceiveString(t, func() { - influxDBRegistry.EntryPointReqsCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.EntryPointReqsCounter().With(nil, "entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDBRegistry.EntryPointReqsTLSCounter().With("entrypoint", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000) influxDBRegistry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1) @@ -95,8 +95,8 @@ func TestInfluxDB(t *testing.T) { } msgRouter := udp.ReceiveString(t, func() { - influxDBRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) - influxDBRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDBRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDBRegistry.RouterReqsTLSCounter().With("router", "demo", "service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDBRegistry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) @@ -119,8 +119,8 @@ func TestInfluxDB(t *testing.T) { } msgService := udp.ReceiveString(t, func() { - influxDBRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - influxDBRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDBRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) influxDBRegistry.ServiceReqsTLSCounter().With("service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.ServiceReqDurationHistogram().With("service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDBRegistry.ServiceOpenConnsGauge().With("service", "test").Set(1) @@ -197,7 +197,7 @@ func TestInfluxDBHTTP(t *testing.T) { `(traefik\.entrypoint\.responses\.bytes\.total,code=200,entrypoint=test,method=GET count=1) [\d]{19}`, } - influxDBRegistry.EntryPointReqsCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.EntryPointReqsCounter().With(nil, "entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDBRegistry.EntryPointReqsTLSCounter().With("entrypoint", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000) influxDBRegistry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1) @@ -217,8 +217,8 @@ func TestInfluxDBHTTP(t *testing.T) { `(traefik\.router\.responses\.bytes\.total,code=200,method=GET,router=demo,service=test count=1) [\d]{19}`, } - influxDBRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) - influxDBRegistry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDBRegistry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) influxDBRegistry.RouterReqsTLSCounter().With("router", "demo", "service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDBRegistry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) @@ -240,8 +240,8 @@ func TestInfluxDBHTTP(t *testing.T) { `(traefik\.service\.responses\.bytes\.total,code=200,method=GET,service=test count=1) [\d]{19}`, } - influxDBRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - influxDBRegistry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + influxDBRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + influxDBRegistry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) influxDBRegistry.ServiceReqsTLSCounter().With("service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) influxDBRegistry.ServiceReqDurationHistogram().With("service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) influxDBRegistry.ServiceOpenConnsGauge().With("service", "test").Set(1) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ac49d389a..86dfd3f52 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -32,7 +32,7 @@ type Registry interface { // entry point metrics - EntryPointReqsCounter() metrics.Counter + EntryPointReqsCounter() CounterWithHeaders EntryPointReqsTLSCounter() metrics.Counter EntryPointReqDurationHistogram() ScalableHistogram EntryPointOpenConnsGauge() metrics.Gauge @@ -41,7 +41,7 @@ type Registry interface { // router metrics - RouterReqsCounter() metrics.Counter + RouterReqsCounter() CounterWithHeaders RouterReqsTLSCounter() metrics.Counter RouterReqDurationHistogram() ScalableHistogram RouterOpenConnsGauge() metrics.Gauge @@ -50,7 +50,7 @@ type Registry interface { // service metrics - ServiceReqsCounter() metrics.Counter + ServiceReqsCounter() CounterWithHeaders ServiceReqsTLSCounter() metrics.Counter ServiceReqDurationHistogram() ScalableHistogram ServiceOpenConnsGauge() metrics.Gauge @@ -75,19 +75,19 @@ func NewMultiRegistry(registries []Registry) Registry { var lastConfigReloadSuccessGauge []metrics.Gauge var lastConfigReloadFailureGauge []metrics.Gauge var tlsCertsNotAfterTimestampGauge []metrics.Gauge - var entryPointReqsCounter []metrics.Counter + var entryPointReqsCounter []CounterWithHeaders var entryPointReqsTLSCounter []metrics.Counter var entryPointReqDurationHistogram []ScalableHistogram var entryPointOpenConnsGauge []metrics.Gauge var entryPointReqsBytesCounter []metrics.Counter var entryPointRespsBytesCounter []metrics.Counter - var routerReqsCounter []metrics.Counter + var routerReqsCounter []CounterWithHeaders var routerReqsTLSCounter []metrics.Counter var routerReqDurationHistogram []ScalableHistogram var routerOpenConnsGauge []metrics.Gauge var routerReqsBytesCounter []metrics.Counter var routerRespsBytesCounter []metrics.Counter - var serviceReqsCounter []metrics.Counter + var serviceReqsCounter []CounterWithHeaders var serviceReqsTLSCounter []metrics.Counter var serviceReqDurationHistogram []ScalableHistogram var serviceOpenConnsGauge []metrics.Gauge @@ -183,19 +183,19 @@ func NewMultiRegistry(registries []Registry) Registry { lastConfigReloadSuccessGauge: multi.NewGauge(lastConfigReloadSuccessGauge...), lastConfigReloadFailureGauge: multi.NewGauge(lastConfigReloadFailureGauge...), tlsCertsNotAfterTimestampGauge: multi.NewGauge(tlsCertsNotAfterTimestampGauge...), - entryPointReqsCounter: multi.NewCounter(entryPointReqsCounter...), + entryPointReqsCounter: NewMultiCounterWithHeaders(entryPointReqsCounter...), entryPointReqsTLSCounter: multi.NewCounter(entryPointReqsTLSCounter...), entryPointReqDurationHistogram: MultiHistogram(entryPointReqDurationHistogram), entryPointOpenConnsGauge: multi.NewGauge(entryPointOpenConnsGauge...), entryPointReqsBytesCounter: multi.NewCounter(entryPointReqsBytesCounter...), entryPointRespsBytesCounter: multi.NewCounter(entryPointRespsBytesCounter...), - routerReqsCounter: multi.NewCounter(routerReqsCounter...), + routerReqsCounter: NewMultiCounterWithHeaders(routerReqsCounter...), routerReqsTLSCounter: multi.NewCounter(routerReqsTLSCounter...), routerReqDurationHistogram: MultiHistogram(routerReqDurationHistogram), routerOpenConnsGauge: multi.NewGauge(routerOpenConnsGauge...), routerReqsBytesCounter: multi.NewCounter(routerReqsBytesCounter...), routerRespsBytesCounter: multi.NewCounter(routerRespsBytesCounter...), - serviceReqsCounter: multi.NewCounter(serviceReqsCounter...), + serviceReqsCounter: NewMultiCounterWithHeaders(serviceReqsCounter...), serviceReqsTLSCounter: multi.NewCounter(serviceReqsTLSCounter...), serviceReqDurationHistogram: MultiHistogram(serviceReqDurationHistogram), serviceOpenConnsGauge: multi.NewGauge(serviceOpenConnsGauge...), @@ -215,19 +215,19 @@ type standardRegistry struct { lastConfigReloadSuccessGauge metrics.Gauge lastConfigReloadFailureGauge metrics.Gauge tlsCertsNotAfterTimestampGauge metrics.Gauge - entryPointReqsCounter metrics.Counter + entryPointReqsCounter CounterWithHeaders entryPointReqsTLSCounter metrics.Counter entryPointReqDurationHistogram ScalableHistogram entryPointOpenConnsGauge metrics.Gauge entryPointReqsBytesCounter metrics.Counter entryPointRespsBytesCounter metrics.Counter - routerReqsCounter metrics.Counter + routerReqsCounter CounterWithHeaders routerReqsTLSCounter metrics.Counter routerReqDurationHistogram ScalableHistogram routerOpenConnsGauge metrics.Gauge routerReqsBytesCounter metrics.Counter routerRespsBytesCounter metrics.Counter - serviceReqsCounter metrics.Counter + serviceReqsCounter CounterWithHeaders serviceReqsTLSCounter metrics.Counter serviceReqDurationHistogram ScalableHistogram serviceOpenConnsGauge metrics.Gauge @@ -269,7 +269,7 @@ func (r *standardRegistry) TLSCertsNotAfterTimestampGauge() metrics.Gauge { return r.tlsCertsNotAfterTimestampGauge } -func (r *standardRegistry) EntryPointReqsCounter() metrics.Counter { +func (r *standardRegistry) EntryPointReqsCounter() CounterWithHeaders { return r.entryPointReqsCounter } @@ -293,7 +293,7 @@ func (r *standardRegistry) EntryPointRespsBytesCounter() metrics.Counter { return r.entryPointRespsBytesCounter } -func (r *standardRegistry) RouterReqsCounter() metrics.Counter { +func (r *standardRegistry) RouterReqsCounter() CounterWithHeaders { return r.routerReqsCounter } @@ -317,7 +317,7 @@ func (r *standardRegistry) RouterRespsBytesCounter() metrics.Counter { return r.routerRespsBytesCounter } -func (r *standardRegistry) ServiceReqsCounter() metrics.Counter { +func (r *standardRegistry) ServiceReqsCounter() CounterWithHeaders { return r.serviceReqsCounter } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 7036335ea..6871aa0a5 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -2,6 +2,7 @@ package metrics import ( "bytes" + "net/http" "strings" "testing" "time" @@ -37,12 +38,12 @@ func TestNewMultiRegistry(t *testing.T) { registries := []Registry{newCollectingRetryMetrics(), newCollectingRetryMetrics()} registry := NewMultiRegistry(registries) - registry.ServiceReqsCounter().With("key", "requests").Add(1) + registry.ServiceReqsCounter().With(nil, "key", "requests").Add(1) registry.ServiceReqDurationHistogram().With("key", "durations").Observe(float64(2)) registry.ServiceRetriesCounter().With("key", "retries").Add(3) for _, collectingRegistry := range registries { - cReqsCounter := collectingRegistry.ServiceReqsCounter().(*counterMock) + cReqsCounter := collectingRegistry.ServiceReqsCounter().(*counterWithHeadersMock) cReqDurationHistogram := collectingRegistry.ServiceReqDurationHistogram().(*histogramMock) cRetriesCounter := collectingRegistry.ServiceRetriesCounter().(*counterMock) @@ -67,7 +68,7 @@ func TestNewMultiRegistry(t *testing.T) { func newCollectingRetryMetrics() Registry { return &standardRegistry{ - serviceReqsCounter: &counterMock{}, + serviceReqsCounter: &counterWithHeadersMock{}, serviceReqDurationHistogram: &histogramMock{}, serviceRetriesCounter: &counterMock{}, } @@ -87,6 +88,20 @@ func (c *counterMock) Add(delta float64) { c.counterValue += delta } +type counterWithHeadersMock struct { + counterValue float64 + lastLabelValues []string +} + +func (c *counterWithHeadersMock) With(_ http.Header, labelValues ...string) CounterWithHeaders { + c.lastLabelValues = labelValues + return c +} + +func (c *counterWithHeadersMock) Add(delta float64) { + c.counterValue += delta +} + type histogramMock struct { lastHistogramValue float64 lastLabelValues []string diff --git a/pkg/metrics/prometheus.go b/pkg/metrics/prometheus.go index 324864087..3f35fe72b 100644 --- a/pkg/metrics/prometheus.go +++ b/pkg/metrics/prometheus.go @@ -155,10 +155,10 @@ func initStandardRegistry(config *types.Prometheus) Registry { } if config.AddEntryPointsLabels { - entryPointReqs := newCounterFrom(stdprometheus.CounterOpts{ + entryPointReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{ Name: entryPointReqsTotalName, Help: "How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.", - }, []string{"code", "method", "protocol", "entrypoint"}) + }, config.HeaderLabels, []string{"code", "method", "protocol", "entrypoint"}) entryPointReqsTLS := newCounterFrom(stdprometheus.CounterOpts{ Name: entryPointReqsTLSTotalName, Help: "How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.", @@ -199,10 +199,10 @@ func initStandardRegistry(config *types.Prometheus) Registry { } if config.AddRoutersLabels { - routerReqs := newCounterFrom(stdprometheus.CounterOpts{ + routerReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{ Name: routerReqsTotalName, Help: "How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.", - }, []string{"code", "method", "protocol", "router", "service"}) + }, config.HeaderLabels, []string{"code", "method", "protocol", "router", "service"}) routerReqsTLS := newCounterFrom(stdprometheus.CounterOpts{ Name: routerReqsTLSTotalName, Help: "How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.", @@ -242,10 +242,10 @@ func initStandardRegistry(config *types.Prometheus) Registry { } if config.AddServicesLabels { - serviceReqs := newCounterFrom(stdprometheus.CounterOpts{ + serviceReqs := newCounterWithHeadersFrom(stdprometheus.CounterOpts{ Name: serviceReqsTotalName, Help: "How many HTTP requests processed on a service, partitioned by status code, protocol, and method.", - }, []string{"code", "method", "protocol", "service"}) + }, config.HeaderLabels, []string{"code", "method", "protocol", "service"}) serviceReqsTLS := newCounterFrom(stdprometheus.CounterOpts{ Name: serviceReqsTLSTotalName, Help: "How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.", @@ -508,6 +508,55 @@ func (d *dynamicConfig) hasServerURL(serviceName, serverURL string) bool { return false } +func newCounterWithHeadersFrom(opts stdprometheus.CounterOpts, headers map[string]string, labelNames []string) *counterWithHeaders { + var headerLabels []string + for k := range headers { + headerLabels = append(headerLabels, k) + } + + cv := stdprometheus.NewCounterVec(opts, append(labelNames, headerLabels...)) + c := &counterWithHeaders{ + name: opts.Name, + headers: headers, + cv: cv, + } + if len(labelNames) == 0 && len(headerLabels) == 0 { + c.collector = cv.WithLabelValues() + c.Add(0) + } + return c +} + +type counterWithHeaders struct { + name string + cv *stdprometheus.CounterVec + labelNamesValues labelNamesValues + headers map[string]string + collector stdprometheus.Counter +} + +func (c *counterWithHeaders) With(headers http.Header, labelValues ...string) CounterWithHeaders { + for headerLabel, headerKey := range c.headers { + labelValues = append(labelValues, headerLabel, headers.Get(headerKey)) + } + lnv := c.labelNamesValues.With(labelValues...) + return &counterWithHeaders{ + name: c.name, + headers: c.headers, + cv: c.cv, + labelNamesValues: lnv, + collector: c.cv.With(lnv.ToLabels()), + } +} + +func (c *counterWithHeaders) Add(delta float64) { + c.collector.Add(delta) +} + +func (c *counterWithHeaders) Describe(ch chan<- *stdprometheus.Desc) { + c.cv.Describe(ch) +} + func newCounterFrom(opts stdprometheus.CounterOpts, labelNames []string) *counter { cv := stdprometheus.NewCounterVec(opts, labelNames) c := &counter{ diff --git a/pkg/metrics/prometheus_test.go b/pkg/metrics/prometheus_test.go index 744a7615d..e347e1e5c 100644 --- a/pkg/metrics/prometheus_test.go +++ b/pkg/metrics/prometheus_test.go @@ -92,7 +92,12 @@ func TestPrometheus(t *testing.T) { promRegistry = prometheus.NewRegistry() t.Cleanup(promState.reset) - prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddRoutersLabels: true, AddServicesLabels: true}) + prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{ + AddEntryPointsLabels: true, + AddRoutersLabels: true, + AddServicesLabels: true, + HeaderLabels: map[string]string{"useragent": "User-Agent"}, + }) defer promRegistry.Unregister(promState) if !prometheusRegistry.IsEpEnabled() || !prometheusRegistry.IsRouterEnabled() || !prometheusRegistry.IsSvcEnabled() { @@ -111,7 +116,7 @@ func TestPrometheus(t *testing.T) { prometheusRegistry. EntryPointReqsCounter(). - With("code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http", "entrypoint", "http"). + With(map[string][]string{"User-Agent": {"foobar"}}, "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http", "entrypoint", "http"). Add(1) prometheusRegistry. EntryPointReqDurationHistogram(). @@ -132,7 +137,7 @@ func TestPrometheus(t *testing.T) { prometheusRegistry. RouterReqsCounter(). - With("router", "demo", "service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "router", "demo", "service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. RouterReqsTLSCounter(). @@ -157,7 +162,7 @@ func TestPrometheus(t *testing.T) { prometheusRegistry. ServiceReqsCounter(). - With("service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(map[string][]string{"User-Agent": {"foobar"}}, "service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. ServiceReqsTLSCounter(). @@ -229,6 +234,7 @@ func TestPrometheus(t *testing.T) { "method": http.MethodGet, "protocol": "http", "entrypoint": "http", + "useragent": "foobar", }, assert: buildCounterAssert(t, entryPointReqsTotalName, 1), }, @@ -274,11 +280,12 @@ func TestPrometheus(t *testing.T) { { name: routerReqsTotalName, labels: map[string]string{ - "code": "200", - "method": http.MethodGet, - "protocol": "http", - "service": "service1", - "router": "demo", + "code": "200", + "method": http.MethodGet, + "protocol": "http", + "service": "service1", + "router": "demo", + "useragent": "", }, assert: buildCounterAssert(t, routerReqsTotalName, 1), }, @@ -338,10 +345,11 @@ func TestPrometheus(t *testing.T) { { name: serviceReqsTotalName, labels: map[string]string{ - "code": "200", - "method": http.MethodGet, - "protocol": "http", - "service": "service1", + "code": "200", + "method": http.MethodGet, + "protocol": "http", + "service": "service1", + "useragent": "foobar", }, assert: buildCounterAssert(t, serviceReqsTotalName, 1), }, @@ -476,15 +484,15 @@ func TestPrometheusMetricRemoval(t *testing.T) { // should be removed after that scrape. prometheusRegistry. EntryPointReqsCounter(). - With("entrypoint", "entrypoint2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "entrypoint", "entrypoint2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. RouterReqsCounter(). - With("router", "router2", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "router", "router2", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. ServiceReqsCounter(). - With("service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. ServiceServerUpGauge(). @@ -502,15 +510,15 @@ func TestPrometheusMetricRemoval(t *testing.T) { // here the counter examples. prometheusRegistry. EntryPointReqsCounter(). - With("entrypoint", "entrypoint1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "entrypoint", "entrypoint1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. RouterReqsCounter(). - With("router", "foo@providerName", "service", "bar", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "router", "foo@providerName", "service", "bar", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. ServiceReqsCounter(). - With("service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). + With(nil, "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http"). Add(1) prometheusRegistry. ServiceServerUpGauge(). @@ -588,7 +596,7 @@ func TestPrometheusRemovedMetricsReset(t *testing.T) { } prometheusRegistry. ServiceReqsCounter(). - With(labelNamesValues...). + With(nil, labelNamesValues...). Add(3) delayForTrackingCompletion() @@ -602,7 +610,7 @@ func TestPrometheusRemovedMetricsReset(t *testing.T) { prometheusRegistry. ServiceReqsCounter(). - With(labelNamesValues...). + With(nil, labelNamesValues...). Add(1) delayForTrackingCompletion() diff --git a/pkg/metrics/statsd.go b/pkg/metrics/statsd.go index c84a7cf5a..f3bfaadfe 100644 --- a/pkg/metrics/statsd.go +++ b/pkg/metrics/statsd.go @@ -74,7 +74,7 @@ func RegisterStatsd(ctx context.Context, config *types.Statsd) Registry { if config.AddEntryPointsLabels { registry.epEnabled = config.AddEntryPointsLabels - registry.entryPointReqsCounter = statsdClient.NewCounter(statsdEntryPointReqsName, 1.0) + registry.entryPointReqsCounter = NewCounterWithNoopHeaders(statsdClient.NewCounter(statsdEntryPointReqsName, 1.0)) registry.entryPointReqsTLSCounter = statsdClient.NewCounter(statsdEntryPointReqsTLSName, 1.0) registry.entryPointReqDurationHistogram, _ = NewHistogramWithScale(statsdClient.NewTiming(statsdEntryPointReqDurationName, 1.0), time.Millisecond) registry.entryPointOpenConnsGauge = statsdClient.NewGauge(statsdEntryPointOpenConnsName) @@ -84,7 +84,7 @@ func RegisterStatsd(ctx context.Context, config *types.Statsd) Registry { if config.AddRoutersLabels { registry.routerEnabled = config.AddRoutersLabels - registry.routerReqsCounter = statsdClient.NewCounter(statsdRouterReqsName, 1.0) + registry.routerReqsCounter = NewCounterWithNoopHeaders(statsdClient.NewCounter(statsdRouterReqsName, 1.0)) registry.routerReqsTLSCounter = statsdClient.NewCounter(statsdRouterReqsTLSName, 1.0) registry.routerReqDurationHistogram, _ = NewHistogramWithScale(statsdClient.NewTiming(statsdRouterReqsDurationName, 1.0), time.Millisecond) registry.routerOpenConnsGauge = statsdClient.NewGauge(statsdRouterOpenConnsName) @@ -94,7 +94,7 @@ func RegisterStatsd(ctx context.Context, config *types.Statsd) Registry { if config.AddServicesLabels { registry.svcEnabled = config.AddServicesLabels - registry.serviceReqsCounter = statsdClient.NewCounter(statsdServiceReqsName, 1.0) + registry.serviceReqsCounter = NewCounterWithNoopHeaders(statsdClient.NewCounter(statsdServiceReqsName, 1.0)) registry.serviceReqsTLSCounter = statsdClient.NewCounter(statsdServiceReqsTLSName, 1.0) registry.serviceReqDurationHistogram, _ = NewHistogramWithScale(statsdClient.NewTiming(statsdServiceReqsDurationName, 1.0), time.Millisecond) registry.serviceRetriesCounter = statsdClient.NewCounter(statsdServiceRetriesTotalName, 1.0) diff --git a/pkg/metrics/statsd_test.go b/pkg/metrics/statsd_test.go index 3eb8e4705..1031d5929 100644 --- a/pkg/metrics/statsd_test.go +++ b/pkg/metrics/statsd_test.go @@ -87,23 +87,23 @@ func testRegistry(t *testing.T, metricsPrefix string, registry Registry) { registry.TLSCertsNotAfterTimestampGauge().With("key", "value").Set(1) - registry.EntryPointReqsCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + registry.EntryPointReqsCounter().With(nil, "entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.EntryPointReqsTLSCounter().With("entrypoint", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) registry.EntryPointReqDurationHistogram().With("entrypoint", "test").Observe(10000) registry.EntryPointOpenConnsGauge().With("entrypoint", "test").Set(1) registry.EntryPointReqsBytesCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.EntryPointRespsBytesCounter().With("entrypoint", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - registry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) - registry.RouterReqsCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + registry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + registry.RouterReqsCounter().With(nil, "router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.RouterReqsTLSCounter().With("router", "demo", "service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) registry.RouterReqDurationHistogram().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) registry.RouterOpenConnsGauge().With("router", "demo", "service", "test").Set(1) registry.RouterReqsBytesCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) registry.RouterRespsBytesCounter().With("router", "demo", "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - registry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) - registry.ServiceReqsCounter().With("service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) + registry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet).Add(1) + registry.ServiceReqsCounter().With(nil, "service", "test", "code", strconv.Itoa(http.StatusNotFound), "method", http.MethodGet).Add(1) registry.ServiceReqsTLSCounter().With("service", "test", "tls_version", "foo", "tls_cipher", "bar").Add(1) registry.ServiceReqDurationHistogram().With("service", "test", "code", strconv.Itoa(http.StatusOK)).Observe(10000) registry.ServiceOpenConnsGauge().With("service", "test").Set(1) diff --git a/pkg/middlewares/metrics/metrics.go b/pkg/middlewares/metrics/metrics.go index 48ff699c7..c4fc78801 100644 --- a/pkg/middlewares/metrics/metrics.go +++ b/pkg/middlewares/metrics/metrics.go @@ -30,7 +30,7 @@ const ( type metricsMiddleware struct { next http.Handler - reqsCounter gokitmetrics.Counter + reqsCounter metrics.CounterWithHeaders reqsTLSCounter gokitmetrics.Counter reqDurationHistogram metrics.ScalableHistogram openConnsGauge gokitmetrics.Gauge @@ -147,7 +147,7 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request) labels = append(labels, "code", strconv.Itoa(capt.StatusCode())) m.reqDurationHistogram.With(labels...).ObserveFromStart(start) - m.reqsCounter.With(labels...).Add(1) + m.reqsCounter.With(req.Header, labels...).Add(1) m.respsBytesCounter.With(labels...).Add(float64(capt.ResponseSize())) m.reqsBytesCounter.With(labels...).Add(float64(capt.RequestSize())) } diff --git a/pkg/types/metrics.go b/pkg/types/metrics.go index de3b85a06..26eea2f1e 100644 --- a/pkg/types/metrics.go +++ b/pkg/types/metrics.go @@ -19,12 +19,13 @@ type Metrics struct { // Prometheus can contain specific configuration used by the Prometheus Metrics exporter. type Prometheus struct { - Buckets []float64 `description:"Buckets for latency metrics." json:"buckets,omitempty" toml:"buckets,omitempty" yaml:"buckets,omitempty" export:"true"` - AddEntryPointsLabels bool `description:"Enable metrics on entry points." json:"addEntryPointsLabels,omitempty" toml:"addEntryPointsLabels,omitempty" yaml:"addEntryPointsLabels,omitempty" export:"true"` - AddRoutersLabels bool `description:"Enable metrics on routers." json:"addRoutersLabels,omitempty" toml:"addRoutersLabels,omitempty" yaml:"addRoutersLabels,omitempty" export:"true"` - AddServicesLabels bool `description:"Enable metrics on services." json:"addServicesLabels,omitempty" toml:"addServicesLabels,omitempty" yaml:"addServicesLabels,omitempty" export:"true"` - EntryPoint string `description:"EntryPoint" json:"entryPoint,omitempty" toml:"entryPoint,omitempty" yaml:"entryPoint,omitempty" export:"true"` - ManualRouting bool `description:"Manual routing" json:"manualRouting,omitempty" toml:"manualRouting,omitempty" yaml:"manualRouting,omitempty" export:"true"` + Buckets []float64 `description:"Buckets for latency metrics." json:"buckets,omitempty" toml:"buckets,omitempty" yaml:"buckets,omitempty" export:"true"` + AddEntryPointsLabels bool `description:"Enable metrics on entry points." json:"addEntryPointsLabels,omitempty" toml:"addEntryPointsLabels,omitempty" yaml:"addEntryPointsLabels,omitempty" export:"true"` + AddRoutersLabels bool `description:"Enable metrics on routers." json:"addRoutersLabels,omitempty" toml:"addRoutersLabels,omitempty" yaml:"addRoutersLabels,omitempty" export:"true"` + AddServicesLabels bool `description:"Enable metrics on services." json:"addServicesLabels,omitempty" toml:"addServicesLabels,omitempty" yaml:"addServicesLabels,omitempty" export:"true"` + EntryPoint string `description:"EntryPoint" json:"entryPoint,omitempty" toml:"entryPoint,omitempty" yaml:"entryPoint,omitempty" export:"true"` + ManualRouting bool `description:"Manual routing" json:"manualRouting,omitempty" toml:"manualRouting,omitempty" yaml:"manualRouting,omitempty" export:"true"` + HeaderLabels map[string]string `description:"Defines the extra labels for the requests_total metrics, and for each of them, the request header containing the value for this label." json:"headerLabels,omitempty" toml:"headerLabels,omitempty" yaml:"headerLabels,omitempty" export:"true"` } // SetDefaults sets the default values.