diff --git a/cmd/configuration.go b/cmd/configuration.go index c2fd0675a..acd2083b5 100644 --- a/cmd/configuration.go +++ b/cmd/configuration.go @@ -9,6 +9,7 @@ import ( "github.com/containous/traefik/configuration" "github.com/containous/traefik/middlewares/accesslog" "github.com/containous/traefik/middlewares/tracing" + "github.com/containous/traefik/middlewares/tracing/datadog" "github.com/containous/traefik/middlewares/tracing/jaeger" "github.com/containous/traefik/middlewares/tracing/zipkin" "github.com/containous/traefik/ping" @@ -218,8 +219,9 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration { // default Tracing defaultTracing := tracing.Tracing{ - Backend: "jaeger", - ServiceName: "traefik", + Backend: "jaeger", + ServiceName: "traefik", + SpanNameLimit: 0, Jaeger: &jaeger.Config{ SamplingServerURL: "http://localhost:5778/sampling", SamplingType: "const", @@ -232,6 +234,11 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration { ID128Bit: true, Debug: false, }, + DataDog: &datadog.Config{ + LocalAgentHostPort: "localhost:8126", + GlobalTag: "", + Debug: false, + }, } // default LifeCycle diff --git a/docs/configuration/tracing.md b/docs/configuration/tracing.md index f4ce62a4f..abfdadd99 100644 --- a/docs/configuration/tracing.md +++ b/docs/configuration/tracing.md @@ -22,6 +22,13 @@ Træfik supports two backends: Jaeger and Zipkin. # Default: "traefik" # serviceName = "traefik" + + # Span name limit allows for name truncation in case of very long Frontend/Backend names + # This can prevent certain tracing providers to drop traces that exceed their length limits + # + # Default: 0 - no truncation will occur + # + spanNameLimit = 0 [tracing.jaeger] # Sampling Server URL is the address of jaeger-agent's HTTP sampling server @@ -72,6 +79,13 @@ Træfik supports two backends: Jaeger and Zipkin. # Default: "traefik" # serviceName = "traefik" + + # Span name limit allows for name truncation in case of very long Frontend/Backend names + # This can prevent certain tracing providers to drop traces that exceed their length limits + # + # Default: 0 - no truncation will occur + # + spanNameLimit = 150 [tracing.zipkin] # Zipking HTTP endpoint used to send data @@ -115,6 +129,13 @@ Træfik supports two backends: Jaeger and Zipkin. # Default: "traefik" # serviceName = "traefik" + + # Span name limit allows for name truncation in case of very long Frontend/Backend names + # This can prevent certain tracing providers to drop traces that exceed their length limits + # + # Default: 0 - no truncation will occur + # + spanNameLimit = 100 [tracing.datadog] # Local Agent Host Port instructs reporter to send spans to datadog-tracing-agent at this address diff --git a/middlewares/tracing/entrypoint.go b/middlewares/tracing/entrypoint.go index eb2f6aca6..a35bcea92 100644 --- a/middlewares/tracing/entrypoint.go +++ b/middlewares/tracing/entrypoint.go @@ -22,12 +22,10 @@ func (t *Tracing) NewEntryPoint(name string) negroni.Handler { } func (e *entryPointMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - opNameFunc := func(r *http.Request) string { - return fmt.Sprintf("Entrypoint %s %s", e.entryPoint, r.Host) - } + opNameFunc := generateEntryPointSpanName ctx, _ := e.Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(r.Header)) - span := e.StartSpan(opNameFunc(r), ext.RPCServerOption(ctx)) + span := e.StartSpan(opNameFunc(r, e.entryPoint, e.SpanNameLimit), ext.RPCServerOption(ctx)) ext.Component.Set(span, e.ServiceName) LogRequest(span, r) ext.SpanKindRPCServer.Set(span) @@ -40,3 +38,20 @@ func (e *entryPointMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, LogResponseCode(span, recorder.Status()) span.Finish() } + +// generateEntryPointSpanName will return a Span name of an appropriate lenth based on the 'spanLimit' argument. If needed, it will be truncated, but will not be less than 24 characters. +func generateEntryPointSpanName(r *http.Request, entryPoint string, spanLimit int) string { + name := fmt.Sprintf("Entrypoint %s %s", entryPoint, r.Host) + + if spanLimit > 0 && len(name) > spanLimit { + if spanLimit < EntryPointMaxLengthNumber { + log.Warnf("SpanNameLimit is set to be less than required static number of characters, defaulting to %d + 3", EntryPointMaxLengthNumber) + spanLimit = EntryPointMaxLengthNumber + 3 + } + hash := computeHash(name) + limit := (spanLimit - EntryPointMaxLengthNumber) / 2 + name = fmt.Sprintf("Entrypoint %s %s %s", truncateString(entryPoint, limit), truncateString(r.Host, limit), hash) + } + + return name +} diff --git a/middlewares/tracing/entrypoint_test.go b/middlewares/tracing/entrypoint_test.go new file mode 100644 index 000000000..f00b74ce1 --- /dev/null +++ b/middlewares/tracing/entrypoint_test.go @@ -0,0 +1,69 @@ +package tracing + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/opentracing/opentracing-go/ext" + "github.com/stretchr/testify/assert" +) + +func TestEntryPointMiddlewareServeHTTP(t *testing.T) { + expectedTags := map[string]interface{}{ + "span.kind": ext.SpanKindRPCServerEnum, + "http.method": "GET", + "component": "", + "http.url": "http://www.test.com", + "http.host": "www.test.com", + } + testCases := []struct { + desc string + entryPoint string + tracing *Tracing + expectedTags map[string]interface{} + expectedName string + }{ + { + desc: "no truncation test", + entryPoint: "test", + tracing: &Tracing{ + SpanNameLimit: 0, + tracer: &MockTracer{Span: &MockSpan{Tags: make(map[string]interface{})}}, + }, + expectedTags: expectedTags, + expectedName: "Entrypoint test www.test.com", + }, { + desc: "basic test", + entryPoint: "test", + tracing: &Tracing{ + SpanNameLimit: 25, + tracer: &MockTracer{Span: &MockSpan{Tags: make(map[string]interface{})}}, + }, + expectedTags: expectedTags, + expectedName: "Entrypoint te... ww... 39b97e58", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + e := &entryPointMiddleware{ + entryPoint: test.entryPoint, + Tracing: test.tracing, + } + + next := func(http.ResponseWriter, *http.Request) { + span := test.tracing.tracer.(*MockTracer).Span + + actual := span.Tags + assert.Equal(t, test.expectedTags, actual) + assert.Equal(t, test.expectedName, span.OpName) + } + + e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "http://www.test.com", nil), next) + }) + } +} diff --git a/middlewares/tracing/forwarder.go b/middlewares/tracing/forwarder.go index aa80486c5..fd4f243bf 100644 --- a/middlewares/tracing/forwarder.go +++ b/middlewares/tracing/forwarder.go @@ -23,7 +23,7 @@ func (t *Tracing) NewForwarderMiddleware(frontend, backend string) negroni.Handl Tracing: t, frontend: frontend, backend: backend, - opName: fmt.Sprintf("forward %s/%s", frontend, backend), + opName: generateForwardSpanName(frontend, backend, t.SpanNameLimit), } } @@ -44,3 +44,20 @@ func (f *forwarderMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, LogResponseCode(span, recorder.Status()) } + +// generateForwardSpanName will return a Span name of an appropriate lenth based on the 'spanLimit' argument. If needed, it will be truncated, but will not be less than 21 characters +func generateForwardSpanName(frontend, backend string, spanLimit int) string { + name := fmt.Sprintf("forward %s/%s", frontend, backend) + + if spanLimit > 0 && len(name) > spanLimit { + if spanLimit < ForwardMaxLengthNumber { + log.Warnf("SpanNameLimit is set to be less than required static number of characters, defaulting to %d + 3", ForwardMaxLengthNumber) + spanLimit = ForwardMaxLengthNumber + 3 + } + hash := computeHash(name) + limit := (spanLimit - ForwardMaxLengthNumber) / 2 + name = fmt.Sprintf("forward %s/%s/%s", truncateString(frontend, limit), truncateString(backend, limit), hash) + } + + return name +} diff --git a/middlewares/tracing/forwarder_test.go b/middlewares/tracing/forwarder_test.go new file mode 100644 index 000000000..00c90c293 --- /dev/null +++ b/middlewares/tracing/forwarder_test.go @@ -0,0 +1,93 @@ +package tracing + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTracingNewForwarderMiddleware(t *testing.T) { + testCases := []struct { + desc string + tracer *Tracing + frontend string + backend string + expected *forwarderMiddleware + }{ + { + desc: "Simple Forward Tracer without truncation and hashing", + tracer: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service.domain.tld", + backend: "some-service.domain.tld", + expected: &forwarderMiddleware{ + Tracing: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service.domain.tld", + backend: "some-service.domain.tld", + opName: "forward some-service.domain.tld/some-service.domain.tld", + }, + }, { + desc: "Simple Forward Tracer with truncation and hashing", + tracer: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service-100.slug.namespace.environment.domain.tld", + backend: "some-service-100.slug.namespace.environment.domain.tld", + expected: &forwarderMiddleware{ + Tracing: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service-100.slug.namespace.environment.domain.tld", + backend: "some-service-100.slug.namespace.environment.domain.tld", + opName: "forward some-service-100.slug.namespace.enviro.../some-service-100.slug.namespace.enviro.../bc4a0d48", + }, + }, + { + desc: "Exactly 101 chars", + tracer: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service1.namespace.environment.domain.tld", + backend: "some-service1.namespace.environment.domain.tld", + expected: &forwarderMiddleware{ + Tracing: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service1.namespace.environment.domain.tld", + backend: "some-service1.namespace.environment.domain.tld", + opName: "forward some-service1.namespace.environment.domain.tld/some-service1.namespace.environment.domain.tld", + }, + }, + { + desc: "More than 101 chars", + tracer: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service1.frontend.namespace.environment.domain.tld", + backend: "some-service1.backend.namespace.environment.domain.tld", + expected: &forwarderMiddleware{ + Tracing: &Tracing{ + SpanNameLimit: 101, + }, + frontend: "some-service1.frontend.namespace.environment.domain.tld", + backend: "some-service1.backend.namespace.environment.domain.tld", + opName: "forward some-service1.frontend.namespace.envir.../some-service1.backend.namespace.enviro.../fa49dd23", + }, + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := test.tracer.NewForwarderMiddleware(test.frontend, test.backend) + + assert.Equal(t, test.expected, actual) + assert.True(t, len(test.expected.opName) <= test.tracer.SpanNameLimit) + }) + } +} diff --git a/middlewares/tracing/tracing.go b/middlewares/tracing/tracing.go index ac7f5b7a9..a18c81806 100644 --- a/middlewares/tracing/tracing.go +++ b/middlewares/tracing/tracing.go @@ -1,6 +1,7 @@ package tracing import ( + "crypto/sha256" "fmt" "io" "net/http" @@ -13,13 +14,23 @@ import ( "github.com/opentracing/opentracing-go/ext" ) +// ForwardMaxLengthNumber defines the number of static characters in the Forwarding Span Trace name : 8 chars for 'forward ' + 8 chars for hash + 2 chars for '_'. +const ForwardMaxLengthNumber = 18 + +// EntryPointMaxLengthNumber defines the number of static characters in the Entrypoint Span Trace name : 11 chars for 'Entrypoint ' + 8 chars for hash + 2 chars for '_'. +const EntryPointMaxLengthNumber = 21 + +// TraceNameHashLength defines the number of characters to use from the head of the generated hash. +const TraceNameHashLength = 8 + // Tracing middleware type Tracing struct { - Backend string `description:"Selects the tracking backend ('jaeger','zipkin', 'datadog')." export:"true"` - ServiceName string `description:"Set the name for this service" export:"true"` - Jaeger *jaeger.Config `description:"Settings for jaeger"` - Zipkin *zipkin.Config `description:"Settings for zipkin"` - DataDog *datadog.Config `description:"Settings for DataDog"` + Backend string `description:"Selects the tracking backend ('jaeger','zipkin', 'datadog')." export:"true"` + ServiceName string `description:"Set the name for this service" export:"true"` + SpanNameLimit int `description:"Set the maximum character limit for Span names (default 0 = no limit)" export:"true"` + Jaeger *jaeger.Config `description:"Settings for jaeger"` + Zipkin *zipkin.Config `description:"Settings for zipkin"` + DataDog *datadog.Config `description:"Settings for DataDog"` tracer opentracing.Tracer closer io.Closer @@ -147,16 +158,40 @@ func SetError(r *http.Request) { } } -// SetErrorAndDebugLog flags the span associated with this request as in error and create a debug log +// SetErrorAndDebugLog flags the span associated with this request as in error and create a debug log. func SetErrorAndDebugLog(r *http.Request, format string, args ...interface{}) { SetError(r) log.Debugf(format, args...) LogEventf(r, format, args...) } -// SetErrorAndWarnLog flags the span associated with this request as in error and create a debug log +// SetErrorAndWarnLog flags the span associated with this request as in error and create a debug log. func SetErrorAndWarnLog(r *http.Request, format string, args ...interface{}) { SetError(r) log.Warnf(format, args...) LogEventf(r, format, args...) } + +// truncateString reduces the length of the 'str' argument to 'num' - 3 and adds a '...' suffix to the tail. +func truncateString(str string, num int) string { + text := str + if len(str) > num { + if num > 3 { + num -= 3 + } + text = str[0:num] + "..." + } + return text +} + +// computeHash returns the first TraceNameHashLength character of the sha256 hash for 'name' argument. +func computeHash(name string) string { + data := []byte(name) + hash := sha256.New() + if _, err := hash.Write(data); err != nil { + // Impossible case + log.Errorf("Fail to create Span name hash for %s: %v", name, err) + } + + return fmt.Sprintf("%x", hash.Sum(nil))[:TraceNameHashLength] +} diff --git a/middlewares/tracing/tracing_test.go b/middlewares/tracing/tracing_test.go new file mode 100644 index 000000000..d4a631312 --- /dev/null +++ b/middlewares/tracing/tracing_test.go @@ -0,0 +1,133 @@ +package tracing + +import ( + "testing" + + "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/log" + "github.com/stretchr/testify/assert" +) + +type MockTracer struct { + Span *MockSpan +} + +type MockSpan struct { + OpName string + Tags map[string]interface{} +} + +type MockSpanContext struct { +} + +// MockSpanContext: +func (n MockSpanContext) ForeachBaggageItem(handler func(k, v string) bool) {} + +// MockSpan: +func (n MockSpan) Context() opentracing.SpanContext { return MockSpanContext{} } +func (n MockSpan) SetBaggageItem(key, val string) opentracing.Span { + return MockSpan{Tags: make(map[string]interface{})} +} +func (n MockSpan) BaggageItem(key string) string { return "" } +func (n MockSpan) SetTag(key string, value interface{}) opentracing.Span { + n.Tags[key] = value + return n +} +func (n MockSpan) LogFields(fields ...log.Field) {} +func (n MockSpan) LogKV(keyVals ...interface{}) {} +func (n MockSpan) Finish() {} +func (n MockSpan) FinishWithOptions(opts opentracing.FinishOptions) {} +func (n MockSpan) SetOperationName(operationName string) opentracing.Span { return n } +func (n MockSpan) Tracer() opentracing.Tracer { return MockTracer{} } +func (n MockSpan) LogEvent(event string) {} +func (n MockSpan) LogEventWithPayload(event string, payload interface{}) {} +func (n MockSpan) Log(data opentracing.LogData) {} +func (n MockSpan) Reset() { + n.Tags = make(map[string]interface{}) +} + +// StartSpan belongs to the Tracer interface. +func (n MockTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span { + n.Span.OpName = operationName + return n.Span +} + +// Inject belongs to the Tracer interface. +func (n MockTracer) Inject(sp opentracing.SpanContext, format interface{}, carrier interface{}) error { + return nil +} + +// Extract belongs to the Tracer interface. +func (n MockTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) { + return nil, opentracing.ErrSpanContextNotFound +} + +func TestTruncateString(t *testing.T) { + testCases := []struct { + desc string + text string + limit int + expected string + }{ + { + desc: "short text less than limit 10", + text: "short", + limit: 10, + expected: "short", + }, + { + desc: "basic truncate with limit 10", + text: "some very long pice of text", + limit: 10, + expected: "some ve...", + }, + { + desc: "truncate long FQDN to 39 chars", + text: "some-service-100.slug.namespace.environment.domain.tld", + limit: 39, + expected: "some-service-100.slug.namespace.envi...", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := truncateString(test.text, test.limit) + + assert.Equal(t, test.expected, actual) + assert.True(t, len(actual) <= test.limit) + }) + } +} + +func TestComputeHash(t *testing.T) { + testCases := []struct { + desc string + text string + expected string + }{ + { + desc: "hashing", + text: "some very long pice of text", + expected: "0258ea1c", + }, + { + desc: "short text less than limit 10", + text: "short", + expected: "f9b0078b", + }, + } + + for _, test := range testCases { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + actual := computeHash(test.text) + + assert.Equal(t, test.expected, actual) + }) + } +}