baalajimaestro/traefik
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Semconv OTLP stable HTTP metrics

Browse source
This commit is contained in:
Michael 2024-03-12 09:48:04 +01:00 committed by GitHub
parent 709ff6fb09
commit 6c9687f410
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 803 additions and 432 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
cmd/traefik/traefik.go
View file

@ -195,10 +195,17 @@ func setupServer(staticConfiguration *static.Configuration) (*server.Server, err
// Observability
metricRegistries := registerMetricClients(staticConfiguration.Metrics)
var semConvMetricRegistry *metrics.SemConvMetricsRegistry
if staticConfiguration.Metrics != nil && staticConfiguration.Metrics.OTLP != nil {
semConvMetricRegistry, err = metrics.NewSemConvMetricRegistry(ctx, staticConfiguration.Metrics.OTLP)
if err != nil {
return nil, fmt.Errorf("unable to create SemConv metric registry: %w", err)
}
}
metricsRegistry := metrics.NewMultiRegistry(metricRegistries)
accessLog := setupAccessLog(staticConfiguration.AccessLog)
tracer, tracerCloser := setupTracing(staticConfiguration.Tracing)
observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, accessLog, tracer, tracerCloser)
observabilityMgr := middleware.NewObservabilityMgr(*staticConfiguration, metricsRegistry, semConvMetricRegistry, accessLog, tracer, tracerCloser)
// Entrypoints

59
docs/content/observability/metrics/overview.md
View file

@ -12,7 +12,8 @@ Traefik supports these metrics backends:
- [Prometheus](./prometheus.md)
- [StatsD](./statsd.md)
Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346) and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments.
Traefik Proxy hosts an official Grafana dashboard for both [on-premises](https://grafana.com/grafana/dashboards/17346)
and [Kubernetes](https://grafana.com/grafana/dashboards/17347) deployments.
## Common Options
@ -29,7 +30,7 @@ metrics:
```toml tab="File (TOML)"
[metrics]
addInternals = true
addInternals = true
```
```bash tab="CLI"
@ -85,10 +86,10 @@ traefik_tls_certs_not_after
Here is a comprehensive list of labels that are provided by the global metrics:
| Label | Description | example |
|---------------|----------------------------------------|----------------------|
| `entrypoint` | Entrypoint that handled the connection | "example_entrypoint" |
| `protocol` | Connection protocol | "TCP" |
| Label | Description | example |
|--------------|----------------------------------------|----------------------|
| `entrypoint` | Entrypoint that handled the connection | "example_entrypoint" |
| `protocol` | Connection protocol | "TCP" |
## HTTP Metrics
@ -281,3 +282,49 @@ Here is a comprehensive list of labels that are provided by the metrics:
If the HTTP method verb on a request is not one defined in the set of common methods for [`HTTP/1.1`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods)
or the [`PRI`](https://datatracker.ietf.org/doc/html/rfc7540#section-11.6) verb (for `HTTP/2`),
then the value for the method label becomes `EXTENSION_METHOD`.
## Semantic Conventions for HTTP Metrics
Traefik Proxy follows [official OTLP semantic conventions v1.23.1](https://github.com/open-telemetry/semantic-conventions/blob/v1.23.1/docs/http/http-metrics.md).
### HTTP Server
| Metric | Type | [Labels](#labels) | Description |
|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|
| http.server.request.duration | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP server requests |
#### Labels
Here is a comprehensive list of labels that are provided by the metrics:
| Label | Description | example |
|-----------------------------|--------------------------------------------------------------|---------------|
| `error.type` | Describes a class of error the operation ended with | "500" |
| `http.request.method` | HTTP request method | "GET" |
| `http.response.status_code` | HTTP response status code | "200" |
| `network.protocol.name` | OSI application layer or non-OSI equivalent | "http/1.1" |
| `network.protocol.version` | Version of the protocol specified in `network.protocol.name` | "1.1" |
| `server.address` | Name of the local HTTP server that received the request | "example.com" |
| `server.port` | Port of the local HTTP server that received the request | "80" |
| `url.scheme` | The URI scheme component identifying the used protocol | "http" |
### HTTP Client
| Metric | Type | [Labels](#labels) | Description |
|-------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|
| http.client.request.duration | Histogram | `error.type`, `http.request.method`, `http.response.status_code`, `network.protocol.name`, `server.address`, `server.port`, `url.scheme` | Duration of HTTP client requests |
#### Labels
Here is a comprehensive list of labels that are provided by the metrics:
| Label | Description | example |
|-----------------------------|--------------------------------------------------------------|---------------|
| `error.type` | Describes a class of error the operation ended with | "500" |
| `http.request.method` | HTTP request method | "GET" |
| `http.response.status_code` | HTTP response status code | "200" |
| `network.protocol.name` | OSI application layer or non-OSI equivalent | "http/1.1" |
| `network.protocol.version` | Version of the protocol specified in `network.protocol.name` | "1.1" |
| `server.address` | Name of the local HTTP server that received the request | "example.com" |
| `server.port` | Port of the local HTTP server that received the request | "80" |
| `url.scheme` | The URI scheme component identifying the used protocol | "http" |

2
docs/content/reference/static-configuration/cli-ref.md
View file

@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```)
Enable metrics on services. (Default: ```true```)
`--metrics.otlp.explicitboundaries`:
Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```)
Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```)
`--metrics.otlp.grpc.endpoint`:
Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```)

2
docs/content/reference/static-configuration/env-ref.md
View file

@ -337,7 +337,7 @@ Enable metrics on routers. (Default: ```false```)
Enable metrics on services. (Default: ```true```)
`TRAEFIK_METRICS_OTLP_EXPLICITBOUNDARIES`:
Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.100000, 0.250000, 0.500000, 1.000000, 2.500000, 5.000000, 10.000000```)
Boundaries for latency metrics. (Default: ```0.005000, 0.010000, 0.025000, 0.050000, 0.075000, 0.100000, 0.250000, 0.500000, 0.750000, 1.000000, 2.500000, 5.000000, 7.500000, 10.000000```)
`TRAEFIK_METRICS_OTLP_GRPC_ENDPOINT`:
Sets the gRPC endpoint (host:port) of the collector. (Default: ```localhost:4317```)

4
integration/conf_throttling_test.go
View file

@ -34,11 +34,11 @@ func (s *ThrottlingSuite) TestThrottleConfReload() {
s.traefikCmd(withConfigFile("fixtures/throttling/simple.toml"))
// wait for Traefik
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 1000*time.Millisecond, try.BodyContains("rest@internal"))
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 5*time.Second, try.BodyContains("rest@internal"))
require.NoError(s.T(), err)
// Expected a 404 as we did not configure anything.
err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
err = try.GetRequest("http://127.0.0.1:8000/", 2*time.Second, try.StatusCodeIs(http.StatusNotFound))
require.NoError(s.T(), err)
config := &dynamic.Configuration{

11
integration/fixtures/tracing/simple-opentelemetry.toml
View file

@ -15,6 +15,17 @@
[entryPoints.web]
address = ":8000"
# Adding metrics to confirm that there is no wrong interaction with tracing.
[metrics]
{{if .IsHTTP}}
[metrics.otlp.http]
endpoint = "http://{{.IP}}:4318"
{{else}}
[metrics.otlp.grpc]
endpoint = "{{.IP}}:4317"
insecure = true
{{end}}
[tracing]
servicename = "tracing"
sampleRate = 1.0

4
integration/tracing_test.go
View file

@ -302,7 +302,7 @@ func (s *TracingSuite) TestOpentelemetryRetry() {
s.traefikCmd(withConfigFile(file))
// wait for traefik
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth"))
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth"))
require.NoError(s.T(), err)
err = try.GetRequest("http://127.0.0.1:8000/retry", 500*time.Millisecond, try.StatusCodeIs(http.StatusBadGateway))
@ -425,7 +425,7 @@ func (s *TracingSuite) TestNoInternals() {
s.traefikCmd(withConfigFile(file))
// wait for traefik
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", time.Second, try.BodyContains("basic-auth"))
err := try.GetRequest("http://127.0.0.1:8080/api/rawdata", 2*time.Second, try.BodyContains("basic-auth"))
require.NoError(s.T(), err)
err = try.GetRequest("http://127.0.0.1:8000/ratelimit", 500*time.Millisecond, try.StatusCodeIs(http.StatusOK))

68
pkg/metrics/opentelemetry.go
View file

@ -30,6 +30,74 @@ var (
openTelemetryGaugeCollector *gaugeCollector
)
// SetMeterProvider sets the meter provider for the tests.
func SetMeterProvider(meterProvider *sdkmetric.MeterProvider) {
openTelemetryMeterProvider = meterProvider
otel.SetMeterProvider(meterProvider)
}
// SemConvMetricsRegistry holds stables semantic conventions metric instruments.
type SemConvMetricsRegistry struct {
// server metrics
httpServerRequestDuration metric.Float64Histogram
// client metrics
httpClientRequestDuration metric.Float64Histogram
}
// NewSemConvMetricRegistry registers all stables semantic conventions metrics.
func NewSemConvMetricRegistry(ctx context.Context, config *types.OTLP) (*SemConvMetricsRegistry, error) {
if openTelemetryMeterProvider == nil {
var err error
if openTelemetryMeterProvider, err = newOpenTelemetryMeterProvider(ctx, config); err != nil {
log.Ctx(ctx).Err(err).Msg("Unable to create OpenTelemetry meter provider")
return nil, nil
}
}
meter := otel.Meter("github.com/traefik/traefik",
metric.WithInstrumentationVersion(version.Version))
httpServerRequestDuration, err := meter.Float64Histogram("http.server.request.duration",
metric.WithDescription("Duration of HTTP server requests."),
metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...))
if err != nil {
return nil, fmt.Errorf("can't build httpServerRequestDuration histogram: %w", err)
}
httpClientRequestDuration, err := meter.Float64Histogram("http.client.request.duration",
metric.WithDescription("Duration of HTTP client requests."),
metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(config.ExplicitBoundaries...))
if err != nil {
return nil, fmt.Errorf("can't build httpClientRequestDuration histogram: %w", err)
}
return &SemConvMetricsRegistry{
httpServerRequestDuration: httpServerRequestDuration,
httpClientRequestDuration: httpClientRequestDuration,
}, nil
}
// HTTPServerRequestDuration returns the HTTP server request duration histogram.
func (s *SemConvMetricsRegistry) HTTPServerRequestDuration() metric.Float64Histogram {
if s == nil {
return nil
}
return s.httpServerRequestDuration
}
// HTTPClientRequestDuration returns the HTTP client request duration histogram.
func (s *SemConvMetricsRegistry) HTTPClientRequestDuration() metric.Float64Histogram {
if s == nil {
return nil
}
return s.httpClientRequestDuration
}
// RegisterOpenTelemetry registers all OpenTelemetry metrics.
func RegisterOpenTelemetry(ctx context.Context, config *types.OTLP) Registry {
if openTelemetryMeterProvider == nil {

8
pkg/metrics/opentelemetry_test.go
View file

@ -361,7 +361,7 @@ func TestOpenTelemetry(t *testing.T) {
expectedEntryPoints := []string{
`({"name":"traefik_entrypoint_requests_total","description":"How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_entrypoint_requests_tls_total","description":"How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test2"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_entrypoint_request_duration_seconds","description":"How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"entrypoint","value":{"stringValue":"test3"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_entrypoint_requests_bytes_total","description":"The total size of requests in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_entrypoint_responses_bytes_total","description":"The total size of responses in bytes handled by an entrypoint, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"entrypoint","value":{"stringValue":"test1"}},{"key":"method","value":{"stringValue":"GET"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
}
@ -377,7 +377,7 @@ func TestOpenTelemetry(t *testing.T) {
expectedRouters := []string{
`({"name":"traefik_router_requests_total","description":"How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_router_requests_tls_total","description":"How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_router_request_duration_seconds","description":"How long it took to process the request on a router, partitioned by service, status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"router","value":{"stringValue":"demo"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_router_requests_bytes_total","description":"The total size of requests in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_router_responses_bytes_total","description":"The total size of responses in bytes handled by a router, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"router","value":{"stringValue":"RouterReqsCounter"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
}
@ -394,7 +394,7 @@ func TestOpenTelemetry(t *testing.T) {
expectedServices := []string{
`({"name":"traefik_service_requests_total","description":"How many HTTP requests processed on a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1},{"attributes":\[{"key":"code","value":{"stringValue":"(?:200|404)"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_service_requests_tls_total","description":"How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"tls_cipher","value":{"stringValue":"bar"}},{"key":"tls_version","value":{"stringValue":"foo"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_service_request_duration_seconds","description":"How long it took to process the request on a service, partitioned by status code, protocol, and method.","unit":"ms","histogram":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"200"}},{"key":"service","value":{"stringValue":"test"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"1","sum":10000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","1"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":10000}\],"aggregationTemporality":2}})`,
`({"name":"traefik_service_server_up","description":"service server is up, described by gauge value of 0 or 1.","unit":"1","gauge":{"dataPoints":\[{"attributes":\[{"key":"service","value":{"stringValue":"test"}},{"key":"url","value":{"stringValue":"http://127.0.0.1"}}\],"timeUnixNano":"[\d]{19}","asDouble":1}\]}})`,
`({"name":"traefik_service_requests_bytes_total","description":"The total size of requests in bytes received by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
`({"name":"traefik_service_responses_bytes_total","description":"The total size of responses in bytes returned by a service, partitioned by status code, protocol, and method.","unit":"1","sum":{"dataPoints":\[{"attributes":\[{"key":"code","value":{"stringValue":"404"}},{"key":"method","value":{"stringValue":"GET"}},{"key":"service","value":{"stringValue":"ServiceReqsCounter"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","asDouble":1}\],"aggregationTemporality":2,"isMonotonic":true}})`,
@ -426,7 +426,7 @@ func TestOpenTelemetry(t *testing.T) {
// and as soon as the EntryPointReqDurationHistogram.Observe is called,
// it adds a new dataPoint to the histogram.
expectedEntryPointReqDuration := []string{
`({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10\],"min":10000,"max":20000})`,
`({"attributes":\[{"key":"entrypoint","value":{"stringValue":"myEntrypoint"}}\],"startTimeUnixNano":"[\d]{19}","timeUnixNano":"[\d]{19}","count":"2","sum":30000,"bucketCounts":\["0","0","0","0","0","0","0","0","0","0","0","0","0","0","2"\],"explicitBounds":\[0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10\],"min":10000,"max":20000})`,
}
registry.EntryPointReqDurationHistogram().With("entrypoint", "myEntrypoint").Observe(10000)

4
pkg/middlewares/auth/basic_auth.go
View file

@ -11,7 +11,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"go.opentelemetry.io/otel/trace"
)
@ -77,7 +77,7 @@ func (b *basicAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if !ok {
logger.Debug().Msg("Authentication failed")
tracing.SetStatusErrorf(req.Context(), "Authentication failed")
observability.SetStatusErrorf(req.Context(), "Authentication failed")
b.auth.RequireAuth(rw, req)
return

6
pkg/middlewares/auth/digest_auth.go
View file

@ -11,7 +11,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"go.opentelemetry.io/otel/trace"
)
@ -78,13 +78,13 @@ func (d *digestAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if authinfo != nil && *authinfo == "stale" {
logger.Debug().Msg("Digest authentication failed, possibly because out of order requests")
tracing.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests")
observability.SetStatusErrorf(req.Context(), "Digest authentication failed, possibly because out of order requests")
d.auth.RequireAuthStale(rw, req)
return
}
logger.Debug().Msg("Digest authentication failed")
tracing.SetStatusErrorf(req.Context(), "Digest authentication failed")
observability.SetStatusErrorf(req.Context(), "Digest authentication failed")
d.auth.RequireAuth(rw, req)
return
}

9
pkg/middlewares/auth/forward.go
View file

@ -14,6 +14,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/middlewares/connectionheader"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/types"
"github.com/vulcand/oxy/v2/forward"
@ -126,7 +127,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if err != nil {
logMessage := fmt.Sprintf("Error calling %s. Cause %s", fa.address, err)
logger.Debug().Msg(logMessage)
tracing.SetStatusErrorf(req.Context(), logMessage)
observability.SetStatusErrorf(req.Context(), logMessage)
rw.WriteHeader(http.StatusInternalServerError)
return
}
@ -150,7 +151,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if forwardErr != nil {
logMessage := fmt.Sprintf("Error calling %s. Cause: %s", fa.address, forwardErr)
logger.Debug().Msg(logMessage)
tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
observability.SetStatusErrorf(forwardReq.Context(), logMessage)
rw.WriteHeader(http.StatusInternalServerError)
return
@ -161,7 +162,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if readError != nil {
logMessage := fmt.Sprintf("Error reading body %s. Cause: %s", fa.address, readError)
logger.Debug().Msg(logMessage)
tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
observability.SetStatusErrorf(forwardReq.Context(), logMessage)
rw.WriteHeader(http.StatusInternalServerError)
return
@ -188,7 +189,7 @@ func (fa *forwardAuth) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if !errors.Is(err, http.ErrNoLocation) {
logMessage := fmt.Sprintf("Error reading response location header %s. Cause: %s", fa.address, err)
logger.Debug().Msg(logMessage)
tracing.SetStatusErrorf(forwardReq.Context(), logMessage)
observability.SetStatusErrorf(forwardReq.Context(), logMessage)
rw.WriteHeader(http.StatusInternalServerError)
return

4
pkg/middlewares/circuitbreaker/circuit_breaker.go
View file

@ -10,7 +10,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/vulcand/oxy/v2/cbreaker"
"go.opentelemetry.io/otel/trace"
)
@ -34,7 +34,7 @@ func New(ctx context.Context, next http.Handler, confCircuitBreaker dynamic.Circ
cbOpts := []cbreaker.Option{
cbreaker.Fallback(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
tracing.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression)
observability.SetStatusErrorf(req.Context(), "blocked by circuit-breaker (%q)", expression)
rw.WriteHeader(responseCode)
if _, err := rw.Write([]byte(http.StatusText(responseCode))); err != nil {

6
pkg/middlewares/customerrors/custom_errors.go
View file

@ -12,7 +12,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/types"
"github.com/vulcand/oxy/v2/utils"
"go.opentelemetry.io/otel/trace"
@ -71,7 +71,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if c.backendHandler == nil {
logger.Error().Msg("Error pages: no backend handler.")
tracing.SetStatusErrorf(req.Context(), "Error pages: no backend handler.")
observability.SetStatusErrorf(req.Context(), "Error pages: no backend handler.")
c.next.ServeHTTP(rw, req)
return
}
@ -96,7 +96,7 @@ func (c *customErrors) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
pageReq, err := newRequest("http://" + req.Host + query)
if err != nil {
logger.Error().Err(err).Send()
tracing.SetStatusErrorf(req.Context(), err.Error())
observability.SetStatusErrorf(req.Context(), err.Error())
http.Error(rw, http.StatusText(code), code)
return
}

4
pkg/middlewares/ipallowlist/ip_allowlist.go
View file

@ -10,7 +10,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/ip"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"go.opentelemetry.io/otel/trace"
)
@ -78,7 +78,7 @@ func (al *ipAllowLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if err != nil {
msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err)
logger.Debug().Msg(msg)
tracing.SetStatusErrorf(req.Context(), msg)
observability.SetStatusErrorf(req.Context(), msg)
reject(ctx, al.rejectStatusCode, rw)
return
}

4
pkg/middlewares/ipwhitelist/ip_whitelist.go
View file

@ -10,7 +10,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/ip"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"go.opentelemetry.io/otel/trace"
)
@ -68,7 +68,7 @@ func (wl *ipWhiteLister) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if err != nil {
msg := fmt.Sprintf("Rejecting IP %s: %v", clientIP, err)
logger.Debug().Msg(msg)
tracing.SetStatusErrorf(req.Context(), msg)
observability.SetStatusErrorf(req.Context(), msg)
reject(ctx, rw)
return
}

4
pkg/middlewares/metrics/metrics.go
View file

@ -14,9 +14,9 @@ import (
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/middlewares/capture"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/middlewares/retry"
traefiktls "github.com/traefik/traefik/v3/pkg/tls"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc/codes"
)
@ -144,7 +144,7 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request)
}
logger := with.Logger()
logger.Error().Err(err).Msg("Could not get Capture")
tracing.SetStatusErrorf(req.Context(), "Could not get Capture")
observability.SetStatusErrorf(req.Context(), "Could not get Capture")
return
}

98
pkg/middlewares/observability/entrypoint.go Normal file
View file

@ -0,0 +1,98 @@
package observability
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/containous/alice"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
const (
entryPointTypeName = "TracingEntryPoint"
)
type entryPointTracing struct {
tracer *tracing.Tracer
entryPoint string
next http.Handler
semConvMetricRegistry *metrics.SemConvMetricsRegistry
}
// WrapEntryPointHandler Wraps tracing to alice.Constructor.
func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
if tracer == nil {
tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
}
return newEntryPoint(ctx, tracer, semConvMetricRegistry, entryPointName, next), nil
}
}
// newEntryPoint creates a new tracing middleware for incoming requests.
func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
if tracer == nil {
tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
}
return &entryPointTracing{
entryPoint: entryPointName,
tracer: tracer,
semConvMetricRegistry: semConvMetricRegistry,
next: next,
}
}
func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header)
start := time.Now()
tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer), trace.WithTimestamp(start))
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("entry_point", e.entryPoint))
e.tracer.CaptureServerRequest(span, req)
recorder := newStatusCodeRecorder(rw, http.StatusOK)
e.next.ServeHTTP(recorder, req)
e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
end := time.Now()
span.End(trace.WithTimestamp(end))
if e.semConvMetricRegistry != nil && e.semConvMetricRegistry.HTTPServerRequestDuration() != nil {
var attrs []attribute.KeyValue
if recorder.Status() < 100 || recorder.Status() >= 600 {
attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code ; %d", recorder.Status())))
} else if recorder.Status() >= 400 {
attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(recorder.Status())))
}
attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method))
attrs = append(attrs, semconv.HTTPResponseStatusCode(recorder.Status()))
attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto)))
attrs = append(attrs, semconv.NetworkProtocolVersion(Proto(req.Proto)))
attrs = append(attrs, semconv.ServerAddress(req.Host))
attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto")))
e.semConvMetricRegistry.HTTPServerRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...))
}
}

185
pkg/middlewares/observability/entrypoint_test.go Normal file
View file

@ -0,0 +1,185 @@
package observability
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ptypes "github.com/traefik/paerser/types"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/types"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
)
func TestEntryPointMiddleware_tracing(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
testCases := []struct {
desc string
entryPoint string
expected expected
}{
{
desc: "basic test",
entryPoint: "test",
expected: expected{
name: "EntryPoint",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "server"),
attribute.String("entry_point", "test"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.Int64("http.request.body.size", int64(0)),
attribute.String("url.path", "/search"),
attribute.String("url.query", "q=Opentelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", "entrypoint-test"),
attribute.String("server.address", "www.test.com"),
attribute.String("network.peer.address", "10.0.0.1"),
attribute.String("network.peer.port", "1234"),
attribute.String("client.address", "10.0.0.1"),
attribute.Int64("client.port", int64(1234)),
attribute.String("client.socket.address", ""),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
rw := httptest.NewRecorder()
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.Header().Set("X-Bar", "foo")
rw.Header().Add("X-Bar", "bar")
rw.WriteHeader(http.StatusNotFound)
})
tracer := &mockTracer{}
handler := newEntryPoint(context.Background(), tracing.NewTracer(tracer, []string{"X-Foo"}, []string{"X-Bar"}), nil, test.entryPoint, next)
handler.ServeHTTP(rw, req)
for _, span := range tracer.spans {
assert.Equal(t, test.expected.name, span.name)
assert.Equal(t, test.expected.attributes, span.attributes)
}
})
}
}
func TestEntryPointMiddleware_metrics(t *testing.T) {
tests := []struct {
desc string
statusCode int
wantAttributes attribute.Set
}{
{
desc: "not found status",
statusCode: http.StatusNotFound,
wantAttributes: attribute.NewSet(
attribute.Key("error.type").String("404"),
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(404),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("url.scheme").String("http"),
),
},
{
desc: "created status",
statusCode: http.StatusCreated,
wantAttributes: attribute.NewSet(
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(201),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("url.scheme").String("http"),
),
},
}
for _, test := range tests {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
var cfg types.OTLP
(&cfg).SetDefaults()
cfg.AddRoutersLabels = true
cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
rdr := sdkmetric.NewManualReader()
meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr))
// force the meter provider with manual reader to collect metrics for the test.
metrics.SetMeterProvider(meterProvider)
semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg)
require.NoError(t, err)
require.NotNil(t, semConvMetricRegistry)
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
rw := httptest.NewRecorder()
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.WriteHeader(test.statusCode)
})
handler := newEntryPoint(context.Background(), nil, semConvMetricRegistry, "test", next)
handler.ServeHTTP(rw, req)
got := metricdata.ResourceMetrics{}
err = rdr.Collect(context.Background(), &got)
require.NoError(t, err)
require.Len(t, got.ScopeMetrics, 1)
expected := metricdata.Metrics{
Name: "http.server.request.duration",
Description: "Duration of HTTP server requests.",
Unit: "s",
Data: metricdata.Histogram[float64]{
DataPoints: []metricdata.HistogramDataPoint[float64]{
{
Attributes: test.wantAttributes,
Count: 1,
Bounds: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10},
BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
Min: metricdata.NewExtrema[float64](1),
Max: metricdata.NewExtrema[float64](1),
Sum: 1,
},
},
Temporality: metricdata.CumulativeTemporality,
},
}
metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue())
})
}
}

2
pkg/middlewares/tracing/middleware.go → pkg/middlewares/observability/middleware.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"

10
pkg/middlewares/tracing/mock_tracing_test.go → pkg/middlewares/observability/mock_tracing_test.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"
@ -47,7 +47,9 @@ type mockSpan struct {
var _ trace.Span = &mockSpan{}
func (*mockSpan) SpanContext() trace.SpanContext { return trace.SpanContext{} }
func (*mockSpan) SpanContext() trace.SpanContext {
return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
}
func (*mockSpan) IsRecording() bool { return false }
func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
@ -59,4 +61,6 @@ func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {}
func (s *mockSpan) SetName(name string) { s.name = name }
func (*mockSpan) TracerProvider() trace.TracerProvider { return mockTracerProvider{} }
func (s *mockSpan) TracerProvider() trace.TracerProvider {
return nil
}

31
pkg/middlewares/observability/observability.go Normal file
View file

@ -0,0 +1,31 @@
package observability
import (
"context"
"fmt"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// SetStatusErrorf flags the span as in error and log an event.
func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) {
if span := trace.SpanFromContext(ctx); span != nil {
span.SetStatus(codes.Error, fmt.Sprintf(format, args...))
}
}
func Proto(proto string) string {
switch proto {
case "HTTP/1.0":
return "1.0"
case "HTTP/1.1":
return "1.1"
case "HTTP/2":
return "2"
case "HTTP/3":
return "3"
default:
return proto
}
}

2
pkg/middlewares/tracing/router.go → pkg/middlewares/observability/router.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"

2
pkg/middlewares/tracing/router_test.go → pkg/middlewares/observability/router_test.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"

2
pkg/middlewares/tracing/service.go → pkg/middlewares/observability/service.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"

2
pkg/middlewares/tracing/service_test.go → pkg/middlewares/observability/service_test.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"context"

2
pkg/middlewares/tracing/status_code.go → pkg/middlewares/observability/status_code.go
View file

@ -1,4 +1,4 @@
package tracing
package observability
import (
"bufio"

6
pkg/middlewares/ratelimiter/rate_limiter.go
View file

@ -12,7 +12,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/vulcand/oxy/v2/utils"
"go.opentelemetry.io/otel/trace"
"golang.org/x/time/rate"
@ -153,14 +153,14 @@ func (rl *rateLimiter) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
// as the expiryTime is supposed to reflect the activity (or lack thereof) on that source.
if err := rl.buckets.Set(source, bucket, rl.ttl); err != nil {
logger.Error().Err(err).Msg("Could not insert/update bucket")
tracing.SetStatusErrorf(req.Context(), "Could not insert/update bucket")
observability.SetStatusErrorf(req.Context(), "Could not insert/update bucket")
http.Error(rw, "could not insert/update bucket", http.StatusInternalServerError)
return
}
res := bucket.Reserve()
if !res.OK() {
tracing.SetStatusErrorf(req.Context(), "No bursty traffic allowed")
observability.SetStatusErrorf(req.Context(), "No bursty traffic allowed")
http.Error(rw, "No bursty traffic allowed", http.StatusTooManyRequests)
return
}

4
pkg/middlewares/replacepath/replace_path.go
View file

@ -7,7 +7,7 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"go.opentelemetry.io/otel/trace"
)
@ -52,7 +52,7 @@ func (r *replacePath) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
req.URL.Path, err = url.PathUnescape(req.URL.RawPath)
if err != nil {
middlewares.GetLogger(context.Background(), r.name, typeName).Error().Err(err).Send()
tracing.SetStatusErrorf(req.Context(), err.Error())
observability.SetStatusErrorf(req.Context(), err.Error())
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}

4
pkg/middlewares/replacepathregex/replace_path_regex.go
View file

@ -10,8 +10,8 @@ import (
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/middlewares/replacepath"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/trace"
)
@ -63,7 +63,7 @@ func (rp *replacePathRegex) ServeHTTP(rw http.ResponseWriter, req *http.Request)
req.URL.Path, err = url.PathUnescape(req.URL.RawPath)
if err != nil {
middlewares.GetLogger(context.Background(), rp.name, typeName).Error().Err(err).Send()
tracing.SetStatusErrorf(req.Context(), err.Error())
observability.SetStatusErrorf(req.Context(), err.Error())
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}

62
pkg/middlewares/tracing/entrypoint.go
View file

@ -1,62 +0,0 @@
package tracing
import (
"context"
"errors"
"net/http"
"github.com/containous/alice"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const (
entryPointTypeName = "TracingEntryPoint"
)
type entryPointTracing struct {
tracer *tracing.Tracer
entryPoint string
next http.Handler
}
// WrapEntryPointHandler Wraps tracing to alice.Constructor.
func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, entryPointName string) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
if tracer == nil {
return nil, errors.New("unexpected nil tracer")
}
return newEntryPoint(ctx, tracer, entryPointName, next), nil
}
}
// newEntryPoint creates a new tracing middleware for incoming requests.
func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, entryPointName string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
return &entryPointTracing{
entryPoint: entryPointName,
tracer: tracer,
next: next,
}
}
func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header)
tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer))
defer span.End()
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("entry_point", e.entryPoint))
e.tracer.CaptureServerRequest(span, req)
recorder := newStatusCodeRecorder(rw, http.StatusOK)
e.next.ServeHTTP(recorder, req)
e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
}

80
pkg/middlewares/tracing/entrypoint_test.go
View file

@ -1,80 +0,0 @@
package tracing
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
)
func TestEntryPointMiddleware(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
testCases := []struct {
desc string
entryPoint string
expected expected
}{
{
desc: "basic test",
entryPoint: "test",
expected: expected{
name: "EntryPoint",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "server"),
attribute.String("entry_point", "test"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.Int64("http.request.body.size", int64(0)),
attribute.String("url.path", "/search"),
attribute.String("url.query", "q=Opentelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", "entrypoint-test"),
attribute.String("server.address", "www.test.com"),
attribute.String("network.peer.address", "10.0.0.1"),
attribute.String("network.peer.port", "1234"),
attribute.String("client.address", "10.0.0.1"),
attribute.Int64("client.port", int64(1234)),
attribute.String("client.socket.address", ""),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
rw := httptest.NewRecorder()
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.Header().Set("X-Bar", "foo")
rw.Header().Add("X-Bar", "bar")
rw.WriteHeader(http.StatusNotFound)
})
mockTracer := &mockTracer{}
handler := newEntryPoint(context.Background(), tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"}), test.entryPoint, next)
handler.ServeHTTP(rw, req)
for _, span := range mockTracer.spans {
assert.Equal(t, test.expected.name, span.name)
assert.Equal(t, test.expected.attributes, span.attributes)
}
})
}
}

4
pkg/server/middleware/middlewares.go
View file

@ -24,6 +24,7 @@ import (
"github.com/traefik/traefik/v3/pkg/middlewares/inflightreq"
"github.com/traefik/traefik/v3/pkg/middlewares/ipallowlist"
"github.com/traefik/traefik/v3/pkg/middlewares/ipwhitelist"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/middlewares/passtlsclientcert"
"github.com/traefik/traefik/v3/pkg/middlewares/ratelimiter"
"github.com/traefik/traefik/v3/pkg/middlewares/redirect"
@ -32,7 +33,6 @@ import (
"github.com/traefik/traefik/v3/pkg/middlewares/retry"
"github.com/traefik/traefik/v3/pkg/middlewares/stripprefix"
"github.com/traefik/traefik/v3/pkg/middlewares/stripprefixregex"
"github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/server/provider"
)
@ -390,7 +390,7 @@ func (b *Builder) buildConstructor(ctx context.Context, middlewareName string) (
// The tracing middleware is a NOOP if tracing is not setup on the middleware chain.
// Hence, regarding internal resources' observability deactivation,
// this would not enable tracing.
return tracing.WrapMiddleware(ctx, middleware), nil
return observability.WrapMiddleware(ctx, middleware), nil
}
func inSlice(element string, stack []string) bool {

81
pkg/server/middleware/observability.go
View file

@ -14,7 +14,7 @@ import (
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
"github.com/traefik/traefik/v3/pkg/middlewares/capture"
metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/tracing"
)
@ -23,15 +23,17 @@ type ObservabilityMgr struct {
config static.Configuration
accessLoggerMiddleware *accesslog.Handler
metricsRegistry metrics.Registry
semConvMetricRegistry *metrics.SemConvMetricsRegistry
tracer *tracing.Tracer
tracerCloser io.Closer
}
// NewObservabilityMgr creates a new ObservabilityMgr.
func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Registry, semConvMetricRegistry *metrics.SemConvMetricsRegistry, accessLoggerMiddleware *accesslog.Handler, tracer *tracing.Tracer, tracerCloser io.Closer) *ObservabilityMgr {
return &ObservabilityMgr{
config: config,
metricsRegistry: metricsRegistry,
semConvMetricRegistry: semConvMetricRegistry,
accessLoggerMiddleware: accessLoggerMiddleware,
tracer: tracer,
tracerCloser: tracerCloser,
@ -39,35 +41,35 @@ func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Re
}
// BuildEPChain an observability middleware chain by entry point.
func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain {
func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain {
chain := alice.New()
if c == nil {
if o == nil {
return chain
}
if c.accessLoggerMiddleware != nil || c.metricsRegistry != nil && (c.metricsRegistry.IsEpEnabled() || c.metricsRegistry.IsRouterEnabled() || c.metricsRegistry.IsSvcEnabled()) {
if c.ShouldAddAccessLogs(resourceName) || c.ShouldAddMetrics(resourceName) {
if o.accessLoggerMiddleware != nil || o.metricsRegistry != nil && (o.metricsRegistry.IsEpEnabled() || o.metricsRegistry.IsRouterEnabled() || o.metricsRegistry.IsSvcEnabled()) {
if o.ShouldAddAccessLogs(resourceName) || o.ShouldAddMetrics(resourceName) {
chain = chain.Append(capture.Wrap)
}
}
if c.accessLoggerMiddleware != nil && c.ShouldAddAccessLogs(resourceName) {
chain = chain.Append(accesslog.WrapHandler(c.accessLoggerMiddleware))
if o.accessLoggerMiddleware != nil && o.ShouldAddAccessLogs(resourceName) {
chain = chain.Append(accesslog.WrapHandler(o.accessLoggerMiddleware))
chain = chain.Append(func(next http.Handler) (http.Handler, error) {
return accesslog.NewFieldHandler(next, logs.EntryPointName, entryPointName, accesslog.InitServiceFields), nil
})
}
if c.tracer != nil && c.ShouldAddTracing(resourceName) {
chain = chain.Append(tracingMiddle.WrapEntryPointHandler(ctx, c.tracer, entryPointName))
if (o.tracer != nil && o.ShouldAddTracing(resourceName)) || (o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName)) {
chain = chain.Append(observability.WrapEntryPointHandler(ctx, o.tracer, o.semConvMetricRegistry, entryPointName))
}
if c.metricsRegistry != nil && c.metricsRegistry.IsEpEnabled() && c.ShouldAddMetrics(resourceName) {
metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, c.metricsRegistry, entryPointName)
if o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName) {
metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, o.metricsRegistry, entryPointName)
if c.tracer != nil && c.ShouldAddTracing(resourceName) {
chain = chain.Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler))
if o.tracer != nil && o.ShouldAddTracing(resourceName) {
chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler))
} else {
chain = chain.Append(metricsHandler)
}
@ -77,64 +79,73 @@ func (c *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName stri
}
// ShouldAddAccessLogs returns whether the access logs should be enabled for the given resource.
func (c *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool {
if c == nil {
func (o *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool {
if o == nil {
return false
}
return c.config.AccessLog != nil && (c.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
return o.config.AccessLog != nil && (o.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
}
// ShouldAddMetrics returns whether the metrics should be enabled for the given resource.
func (c *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool {
if c == nil {
func (o *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool {
if o == nil {
return false
}
return c.config.Metrics != nil && (c.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
return o.config.Metrics != nil && (o.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
}
// ShouldAddTracing returns whether the tracing should be enabled for the given resource.
func (c *ObservabilityMgr) ShouldAddTracing(resourceName string) bool {
if c == nil {
func (o *ObservabilityMgr) ShouldAddTracing(resourceName string) bool {
if o == nil {
return false
}
return c.config.Tracing != nil && (c.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
return o.config.Tracing != nil && (o.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
}
// MetricsRegistry is an accessor to the metrics registry.
func (c *ObservabilityMgr) MetricsRegistry() metrics.Registry {
if c == nil {
func (o *ObservabilityMgr) MetricsRegistry() metrics.Registry {
if o == nil {
return nil
}
return c.metricsRegistry
return o.metricsRegistry
}
// SemConvMetricsRegistry is an accessor to the semantic conventions metrics registry.
func (o *ObservabilityMgr) SemConvMetricsRegistry() *metrics.SemConvMetricsRegistry {
if o == nil {
return nil
}
return o.semConvMetricRegistry
}
// Close closes the accessLogger and tracer.
func (c *ObservabilityMgr) Close() {
if c == nil {
func (o *ObservabilityMgr) Close() {
if o == nil {
return
}
if c.accessLoggerMiddleware != nil {
if err := c.accessLoggerMiddleware.Close(); err != nil {
if o.accessLoggerMiddleware != nil {
if err := o.accessLoggerMiddleware.Close(); err != nil {
log.Error().Err(err).Msg("Could not close the access log file")
}
}
if c.tracerCloser != nil {
if err := c.tracerCloser.Close(); err != nil {
if o.tracerCloser != nil {
if err := o.tracerCloser.Close(); err != nil {
log.Error().Err(err).Msg("Could not close the tracer")
}
}
}
func (c *ObservabilityMgr) RotateAccessLogs() error {
if c.accessLoggerMiddleware == nil {
func (o *ObservabilityMgr) RotateAccessLogs() error {
if o.accessLoggerMiddleware == nil {
return nil
}
return c.accessLoggerMiddleware.Rotate()
return o.accessLoggerMiddleware.Rotate()
}

6
pkg/server/router/router.go
View file

@ -13,8 +13,8 @@ import (
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
"github.com/traefik/traefik/v3/pkg/middlewares/denyrouterrecursion"
metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/middlewares/recovery"
"github.com/traefik/traefik/v3/pkg/middlewares/tracing"
httpmuxer "github.com/traefik/traefik/v3/pkg/muxer/http"
"github.com/traefik/traefik/v3/pkg/server/middleware"
"github.com/traefik/traefik/v3/pkg/server/provider"
@ -221,11 +221,11 @@ func (m *Manager) buildHTTPHandler(ctx context.Context, router *runtime.RouterIn
return chain.Extend(*mHandler).Then(sHandler)
}
chain = chain.Append(tracing.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service)))
chain = chain.Append(observability.WrapRouterHandler(ctx, routerName, router.Rule, provider.GetQualifiedName(ctx, router.Service)))
if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsRouterEnabled() {
metricsHandler := metricsMiddle.WrapRouterHandler(ctx, m.observabilityMgr.MetricsRegistry(), routerName, provider.GetQualifiedName(ctx, router.Service))
chain = chain.Append(tracing.WrapMiddleware(ctx, metricsHandler))
chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler))
}
if router.DefaultRule {

2
pkg/server/routerfactory_test.go
View file

@ -193,7 +193,7 @@ func TestServerResponseEmptyBackend(t *testing.T) {
dialerManager := tcp.NewDialerManager(nil)
dialerManager.Update(map[string]*dynamic.TCPServersTransport{"default@internal": {}})
observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil)
observabiltyMgr := middleware.NewObservabilityMgr(staticConfig, nil, nil, nil, nil, nil)
factory := NewRouterFactory(staticConfig, managerFactory, tlsManager, observabiltyMgr, nil, dialerManager)
entryPointsHandlers, _ := factory.CreateRouters(runtime.NewConfig(dynamic.Configuration{HTTP: test.config(testServer.URL)}))

105
pkg/server/service/observability_roundtripper.go Normal file
View file

@ -0,0 +1,105 @@
package service
import (
"context"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
)
type wrapper struct {
semConvMetricRegistry *metrics.SemConvMetricsRegistry
rt http.RoundTripper
}
func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
start := time.Now()
var span trace.Span
var tracingCtx context.Context
var tracer *tracing.Tracer
if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
req = req.WithContext(tracingCtx)
tracer.CaptureClientRequest(span, req)
tracing.InjectContextIntoCarrier(req)
}
var statusCode int
var headers http.Header
response, err := t.rt.RoundTrip(req)
if err != nil {
statusCode = computeStatusCode(err)
}
if response != nil {
statusCode = response.StatusCode
headers = response.Header
}
if tracer != nil {
tracer.CaptureResponse(span, headers, statusCode, trace.SpanKindClient)
}
end := time.Now()
// Ending the span as soon as the response is handled because we want to use the same end time for the trace and the metric.
// If any errors happen earlier, this span will be close by the defer instruction.
if span != nil {
span.End(trace.WithTimestamp(end))
}
if t.semConvMetricRegistry != nil && t.semConvMetricRegistry.HTTPClientRequestDuration() != nil {
var attrs []attribute.KeyValue
if statusCode < 100 || statusCode >= 600 {
attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code %d", statusCode)))
} else if statusCode >= 400 {
attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(statusCode)))
}
attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method))
attrs = append(attrs, semconv.HTTPResponseStatusCode(statusCode))
attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto)))
attrs = append(attrs, semconv.NetworkProtocolVersion(observability.Proto(req.Proto)))
attrs = append(attrs, semconv.ServerAddress(req.URL.Host))
_, port, err := net.SplitHostPort(req.URL.Host)
if err != nil {
switch req.URL.Scheme {
case "http":
attrs = append(attrs, semconv.ServerPort(80))
case "https":
attrs = append(attrs, semconv.ServerPort(443))
}
} else {
intPort, _ := strconv.Atoi(port)
attrs = append(attrs, semconv.ServerPort(intPort))
}
attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto")))
t.semConvMetricRegistry.HTTPClientRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...))
}
return response, err
}
func newObservabilityRoundTripper(semConvMetricRegistry *metrics.SemConvMetricsRegistry, rt http.RoundTripper) http.RoundTripper {
return &wrapper{
semConvMetricRegistry: semConvMetricRegistry,
rt: rt,
}
}

123
pkg/server/service/observability_roundtripper_test.go Normal file
View file

@ -0,0 +1,123 @@
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
ptypes "github.com/traefik/paerser/types"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/types"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
)
func TestObservabilityRoundTripper_metrics(t *testing.T) {
tests := []struct {
desc string
serverURL string
statusCode int
wantAttributes attribute.Set
}{
{
desc: "not found status",
serverURL: "http://www.test.com",
statusCode: http.StatusNotFound,
wantAttributes: attribute.NewSet(
attribute.Key("error.type").String("404"),
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(404),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("server.port").Int(80),
attribute.Key("url.scheme").String("http"),
),
},
{
desc: "created status",
serverURL: "https://www.test.com",
statusCode: http.StatusCreated,
wantAttributes: attribute.NewSet(
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(201),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("server.port").Int(443),
attribute.Key("url.scheme").String("http"),
),
},
}
for _, test := range tests {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
var cfg types.OTLP
(&cfg).SetDefaults()
cfg.AddRoutersLabels = true
cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
rdr := sdkmetric.NewManualReader()
meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr))
// force the meter provider with manual reader to collect metrics for the test.
metrics.SetMeterProvider(meterProvider)
semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg)
require.NoError(t, err)
require.NotNil(t, semConvMetricRegistry)
req := httptest.NewRequest(http.MethodGet, test.serverURL+"/search?q=Opentelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "rt-test")
req.Header.Set("X-Forwarded-Proto", "http")
ort := newObservabilityRoundTripper(semConvMetricRegistry, mockRoundTripper{statusCode: test.statusCode})
_, err = ort.RoundTrip(req)
require.NoError(t, err)
got := metricdata.ResourceMetrics{}
err = rdr.Collect(context.Background(), &got)
require.NoError(t, err)
require.Len(t, got.ScopeMetrics, 1)
expected := metricdata.Metrics{
Name: "http.client.request.duration",
Description: "Duration of HTTP client requests.",
Unit: "s",
Data: metricdata.Histogram[float64]{
DataPoints: []metricdata.HistogramDataPoint[float64]{
{
Attributes: test.wantAttributes,
Count: 1,
Bounds: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10},
BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
Min: metricdata.NewExtrema[float64](1),
Max: metricdata.NewExtrema[float64](1),
Sum: 1,
},
},
Temporality: metricdata.CumulativeTemporality,
},
}
metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue())
})
}
}
type mockRoundTripper struct {
statusCode int
}
func (m mockRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: m.statusCode}, nil
}

6
pkg/server/service/proxy.go
View file

@ -22,13 +22,9 @@ const StatusClientClosedRequest = 499
const StatusClientClosedRequestText = "Client Closed Request"
func buildSingleHostProxy(target *url.URL, passHostHeader bool, flushInterval time.Duration, roundTripper http.RoundTripper, bufferPool httputil.BufferPool) http.Handler {
// Wrapping the roundTripper with the Tracing roundTripper,
// to handle the reverseProxy client span creation.
tracingRoundTripper := newTracingRoundTripper(roundTripper)
return &httputil.ReverseProxy{
Director: directorBuilder(target, passHostHeader),
Transport: tracingRoundTripper,
Transport: roundTripper,
FlushInterval: flushInterval,
BufferPool: bufferPool,
ErrorHandler: errorHandler,

20
pkg/server/service/service.go
View file

@ -22,7 +22,7 @@ import (
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/safe"
"github.com/traefik/traefik/v3/pkg/server/cookie"
"github.com/traefik/traefik/v3/pkg/server/middleware"
@ -300,30 +300,38 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName
logger.Debug().Str(logs.ServerName, proxyName).Stringer("target", target).
Msg("Creating server")
qualifiedSvcName := provider.GetQualifiedName(ctx, serviceName)
if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
// Wrapping the roundTripper with the Tracing roundTripper,
// to handle the reverseProxy client span creation.
roundTripper = newObservabilityRoundTripper(m.observabilityMgr.SemConvMetricsRegistry(), roundTripper)
}
proxy := buildSingleHostProxy(target, passHostHeader, time.Duration(flushInterval), roundTripper, m.bufferPool)
// Prevents from enabling observability for internal resources.
if m.observabilityMgr.ShouldAddAccessLogs(provider.GetQualifiedName(ctx, serviceName)) {
if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName) {
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceURL, target.String(), nil)
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceAddr, target.Host, nil)
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceName, serviceName, accesslog.AddServiceFields)
}
if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsSvcEnabled() &&
m.observabilityMgr.ShouldAddMetrics(provider.GetQualifiedName(ctx, serviceName)) {
m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
metricsHandler := metricsMiddle.WrapServiceHandler(ctx, m.observabilityMgr.MetricsRegistry(), serviceName)
proxy, err = alice.New().
Append(tracingMiddle.WrapMiddleware(ctx, metricsHandler)).
Append(observability.WrapMiddleware(ctx, metricsHandler)).
Then(proxy)
if err != nil {
return nil, fmt.Errorf("error wrapping metrics handler: %w", err)
}
}
if m.observabilityMgr.ShouldAddTracing(provider.GetQualifiedName(ctx, serviceName)) {
proxy = tracingMiddle.NewService(ctx, serviceName, proxy)
if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) {
proxy = observability.NewService(ctx, serviceName, proxy)
}
lb.Add(proxyName, proxy, server.Weight)

44
pkg/server/service/tracing_roundtripper.go
View file

@ -1,44 +0,0 @@
package service
import (
"context"
"net/http"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/trace"
)
type wrapper struct {
rt http.RoundTripper
}
func (t *wrapper) RoundTrip(req *http.Request) (*http.Response, error) {
var span trace.Span
var tracer *tracing.Tracer
if tracer = tracing.TracerFromContext(req.Context()); tracer != nil {
var tracingCtx context.Context
tracingCtx, span = tracer.Start(req.Context(), "ReverseProxy", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
req = req.WithContext(tracingCtx)
tracer.CaptureClientRequest(span, req)
tracing.InjectContextIntoCarrier(req)
}
response, err := t.rt.RoundTrip(req)
if err != nil {
statusCode := computeStatusCode(err)
tracer.CaptureResponse(span, nil, statusCode, trace.SpanKindClient)
return response, err
}
tracer.CaptureResponse(span, response.Header, response.StatusCode, trace.SpanKindClient)
return response, nil
}
func newTracingRoundTripper(rt http.RoundTripper) http.RoundTripper {
return &wrapper{rt: rt}
}

138
pkg/server/service/tracing_roundtripper_test.go
View file

@ -1,138 +0,0 @@
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/embedded"
)
func TestTracingRoundTripper(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
testCases := []struct {
desc string
expected []expected
}{
{
desc: "basic test",
expected: []expected{
{
name: "initial",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "unspecified"),
},
},
{
name: "ReverseProxy",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "client"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.String("url.full", "http://www.test.com/search?q=Opentelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", "reverse-test"),
attribute.String("network.peer.address", ""),
attribute.String("server.address", "www.test.com"),
attribute.String("network.peer.port", "80"),
attribute.Int64("server.port", int64(80)),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "reverse-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
mockTracer := &mockTracer{}
tracer := tracing.NewTracer(mockTracer, []string{"X-Foo"}, []string{"X-Bar"})
initialCtx, initialSpan := tracer.Start(req.Context(), "initial")
defer initialSpan.End()
req = req.WithContext(initialCtx)
tracingRoundTripper := newTracingRoundTripper(roundTripperFn(func(req *http.Request) (*http.Response, error) {
return &http.Response{
Header: map[string][]string{
"X-Bar": {"foo", "bar"},
},
StatusCode: http.StatusNotFound,
}, nil
}))
_, err := tracingRoundTripper.RoundTrip(req)
require.NoError(t, err)
for i, span := range mockTracer.spans {
assert.Equal(t, test.expected[i].name, span.name)
assert.Equal(t, test.expected[i].attributes, span.attributes)
}
})
}
}
type mockTracer struct {
embedded.Tracer
spans []*mockSpan
}
var _ trace.Tracer = &mockTracer{}
func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
config := trace.NewSpanStartConfig(opts...)
span := &mockSpan{}
span.SetName(name)
span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
span.SetAttributes(config.Attributes()...)
t.spans = append(t.spans, span)
return trace.ContextWithSpan(ctx, span), span
}
// mockSpan is an implementation of Span that preforms no operations.
type mockSpan struct {
embedded.Span
name string
attributes []attribute.KeyValue
}
var _ trace.Span = &mockSpan{}
func (*mockSpan) SpanContext() trace.SpanContext {
return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
}
func (*mockSpan) IsRecording() bool { return false }
func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
s.attributes = append(s.attributes, kv...)
}
func (s *mockSpan) End(...trace.SpanEndOption) {}
func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {}
func (s *mockSpan) SetName(name string) { s.name = name }
func (s *mockSpan) TracerProvider() trace.TracerProvider {
return nil
}

4
pkg/tracing/opentelemetry/opentelemetry_test.go
View file

@ -14,7 +14,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/traefik/traefik/v3/pkg/config/static"
tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
"github.com/traefik/traefik/v3/pkg/types"
@ -296,7 +296,7 @@ func TestTracing(t *testing.T) {
_ = closer.Close()
})
chain := alice.New(tracingMiddle.WrapEntryPointHandler(context.Background(), newTracing, "test"))
chain := alice.New(observability.WrapEntryPointHandler(context.Background(), newTracing, nil, "test"))
epHandler, err := chain.Then(service)
require.NoError(t, err)

2
pkg/types/metrics.go
View file

@ -125,7 +125,7 @@ func (o *OTLP) SetDefaults() {
o.AddEntryPointsLabels = true
o.AddServicesLabels = true
o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 7.5, 10}
o.PushInterval = types.Duration(10 * time.Second)
}