From 0ca65f955da2d4f32dfb882b17fc0bce7a421f1c Mon Sep 17 00:00:00 2001 From: Ludovic Fernandez Date: Sat, 25 Nov 2017 13:36:03 +0100 Subject: [PATCH] Stats collection. --- cmd/traefik/traefik.go | 41 +- collector/collector.go | 79 ++++ configuration/configuration.go | 1 + docs/basics.md | 119 ++++++ glide.lock | 6 +- glide.yaml | 1 + .../mitchellh/hashstructure/LICENSE | 21 + .../mitchellh/hashstructure/hashstructure.go | 358 ++++++++++++++++++ .../mitchellh/hashstructure/include.go | 15 + 9 files changed, 632 insertions(+), 9 deletions(-) create mode 100644 collector/collector.go create mode 100644 vendor/github.com/mitchellh/hashstructure/LICENSE create mode 100644 vendor/github.com/mitchellh/hashstructure/hashstructure.go create mode 100644 vendor/github.com/mitchellh/hashstructure/include.go diff --git a/cmd/traefik/traefik.go b/cmd/traefik/traefik.go index cad69e1fb..cc8179bcb 100644 --- a/cmd/traefik/traefik.go +++ b/cmd/traefik/traefik.go @@ -15,6 +15,7 @@ import ( "github.com/containous/flaeg" "github.com/containous/staert" "github.com/containous/traefik/acme" + "github.com/containous/traefik/collector" "github.com/containous/traefik/configuration" "github.com/containous/traefik/job" "github.com/containous/traefik/log" @@ -149,6 +150,8 @@ func run(globalConfiguration *configuration.GlobalConfiguration, configFile stri checkNewVersion() } + stats(globalConfiguration) + log.Debugf("Global configuration loaded %s", string(jsonConf)) svr := server.NewServer(*globalConfiguration) svr.Start() @@ -244,14 +247,38 @@ func configureLogging(globalConfiguration *configuration.GlobalConfiguration) { } func checkNewVersion() { - ticker := time.NewTicker(24 * time.Hour) + ticker := time.Tick(24 * time.Hour) safe.Go(func() { - time.Sleep(10 * time.Minute) - version.CheckNewVersion() - for { - select { - case <-ticker.C: - version.CheckNewVersion() + for time.Sleep(10 * time.Minute); ; <-ticker { + version.CheckNewVersion() + } + }) +} + +func stats(globalConfiguration *configuration.GlobalConfiguration) { + if globalConfiguration.SendAnonymousUsage { + log.Info(` +Stats collection is enabled. +Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration. +Help us improve Traefik by leaving this feature on :) +More details on: https://docs.traefik.io/basic/#collected-data +`) + collect(globalConfiguration) + } else { + log.Info(` +Stats collection is disabled. +Help us improve Traefik by turning this feature on :) +More details on: https://docs.traefik.io/basic/#collected-data +`) + } +} + +func collect(globalConfiguration *configuration.GlobalConfiguration) { + ticker := time.Tick(24 * time.Hour) + safe.Go(func() { + for time.Sleep(10 * time.Minute); ; <-ticker { + if err := collector.Collect(globalConfiguration); err != nil { + log.Debug(err) } } }) diff --git a/collector/collector.go b/collector/collector.go new file mode 100644 index 000000000..63d1fb81e --- /dev/null +++ b/collector/collector.go @@ -0,0 +1,79 @@ +package collector + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "net" + "net/http" + "strconv" + "time" + + "github.com/containous/traefik/cmd/traefik/anonymize" + "github.com/containous/traefik/configuration" + "github.com/containous/traefik/log" + "github.com/containous/traefik/version" + "github.com/mitchellh/hashstructure" +) + +// collectorURL URL where the stats are send +const collectorURL = "https://collect.traefik.io/619df80498b60f985d766ce62f912b7c" + +// Collected data +type data struct { + Version string + Codename string + BuildDate string + Configuration string + Hash string +} + +// Collect anonymous data. +func Collect(globalConfiguration *configuration.GlobalConfiguration) error { + anonConfig, err := anonymize.Do(globalConfiguration, false) + if err != nil { + return err + } + + log.Infof("Anonymous stats sent to %s: %s", collectorURL, anonConfig) + + hashConf, err := hashstructure.Hash(globalConfiguration, nil) + if err != nil { + return err + } + + data := &data{ + Version: version.Version, + Codename: version.Codename, + BuildDate: version.BuildDate, + Hash: strconv.FormatUint(hashConf, 10), + Configuration: base64.StdEncoding.EncodeToString([]byte(anonConfig)), + } + + buf := new(bytes.Buffer) + err = json.NewEncoder(buf).Encode(data) + if err != nil { + return err + } + + _, err = makeHTTPClient().Post(collectorURL, "application/json; charset=utf-8", buf) + return err +} + +func makeHTTPClient() *http.Client { + dialer := &net.Dialer{ + Timeout: configuration.DefaultDialTimeout, + KeepAlive: 30 * time.Second, + DualStack: true, + } + + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: dialer.DialContext, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + return &http.Client{Transport: transport} +} diff --git a/configuration/configuration.go b/configuration/configuration.go index 0eac04901..0eba373f8 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -53,6 +53,7 @@ type GlobalConfiguration struct { GraceTimeOut flaeg.Duration `short:"g" description:"(Deprecated) Duration to give active requests a chance to finish before Traefik stops" export:"true"` // Deprecated Debug bool `short:"d" description:"Enable debug mode" export:"true"` CheckNewVersion bool `description:"Periodically check if a new version has been released" export:"true"` + SendAnonymousUsage bool `description:"send periodically anonymous usage statistics" export:"true"` AccessLogsFile string `description:"(Deprecated) Access logs file" export:"true"` // Deprecated AccessLog *types.AccessLog `description:"Access log settings" export:"true"` TraefikLogsFile string `description:"(Deprecated) Traefik logs file. Stdout is used when omitted or empty" export:"true"` // Deprecated diff --git a/docs/basics.md b/docs/basics.md index cddabeb78..ffe71ec84 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -628,3 +628,122 @@ traefik healthcheck ```bash OK: http://:8082/ping ``` + + +## Collected Data + +**This feature is disabled by default.** + +You can read the public proposal on this topic [here](https://github.com/containous/traefik/issues/2369). + +### Why ? + +In order to help us learn more about how Træfik is being used and improve it, we collect anonymous usage statistics from running instances. +Those data help us prioritize our developments and focus on what's more important (for example, which configuration backend is used and which is not used). + +### What ? + +Once a day (the first call begins 10 minutes after the start of Træfik), we collect: +- the Træfik version +- a hash of the configuration +- an **anonymous version** of the static configuration: + - token, user name, password, URL, IP, domain, email, etc, are removed + +!!! note + We do not collect the dynamic configuration (frontends & backends). + +!!! note + We do not collect data behind the scenes to run advertising programs or to sell such data to third-party. + +#### Here is an example + +- Source configuration: + +```toml +[entryPoints] + [entryPoints.http] + address = ":80" + +[web] + address = ":8080" + +[Docker] + endpoint = "tcp://10.10.10.10:2375" + domain = "foo.bir" + exposedByDefault = true + swarmMode = true + + [Docker.TLS] + CA = "dockerCA" + Cert = "dockerCert" + Key = "dockerKey" + InsecureSkipVerify = true + +[ECS] + Domain = "foo.bar" + ExposedByDefault = true + Clusters = ["foo-bar"] + Region = "us-west-2" + AccessKeyID = "AccessKeyID" + SecretAccessKey = "SecretAccessKey" +``` + +- Obfuscated and anonymous configuration: + +```toml +[entryPoints] + [entryPoints.http] + address = ":80" + +[web] + address = ":8080" + +[Docker] + Endpoint = "xxxx" + Domain = "xxxx" + ExposedByDefault = true + SwarmMode = true + + [Docker.TLS] + CA = "xxxx" + Cert = "xxxx" + Key = "xxxx" + InsecureSkipVerify = false + +[ECS] + Domain = "xxxx" + ExposedByDefault = true + Clusters = [] + Region = "us-west-2" + AccessKeyID = "xxxx" + SecretAccessKey = "xxxx" +``` + +### Show me the code ! + +If you want to dig into more details, here is the source code of the collecting system: [collector.go](https://github.com/containous/traefik/blob/master/collector/collector.go) + +By default we anonymize all configuration fields, except fields tagged with `export=true`. + +You can check all fields in the [godoc](https://godoc.org/github.com/containous/traefik/configuration#GlobalConfiguration). + +### How to enable this ? + +You can enable the collecting system by: + +- adding this line in the configuration TOML file: + +```toml +# Send anonymous usage data +# +# Optional +# Default: false +# +sendAnonymousUsage = true +``` + +- adding this flag in the CLI: + +```bash +./traefik --sendAnonymousUsage=true +``` diff --git a/glide.lock b/glide.lock index aff28ba38..0954ce58e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 6deb9adeca5f1724f9ef2b31b122f85a00cf47cf4308527d6d3ff68a6ac0e705 -updated: 2017-11-17T14:21:55.148450413+01:00 +hash: 4ac4017b19d4a7355894a09cd6d1f2b92729f252578b6a044a3ff2dea22133c4 +updated: 2017-11-21T21:24:24.601164724+01:00 imports: - name: cloud.google.com/go version: 2e6a95edb1071d750f6d7db777bf66cd2997af6c @@ -409,6 +409,8 @@ imports: version: 8060d9f51305bbe024b99679454e62f552cd0b0b - name: github.com/mitchellh/copystructure version: d23ffcb85de31694d6ccaa23ccb4a03e55c1303f +- name: github.com/mitchellh/hashstructure + version: 2bca23e0e452137f789efbc8610126fd8b94f73b - name: github.com/mitchellh/mapstructure version: d0303fe809921458f417bcf828397a65db30a7e4 - name: github.com/mitchellh/reflectwalk diff --git a/glide.yaml b/glide.yaml index 6da32ec0f..43683c085 100644 --- a/glide.yaml +++ b/glide.yaml @@ -215,6 +215,7 @@ import: - package: github.com/armon/go-proxyproto version: 48572f11356f1843b694f21a290d4f1006bc5e47 - package: github.com/mitchellh/copystructure +- package: github.com/mitchellh/hashstructure testImport: - package: github.com/stvp/go-udp-testing - package: github.com/docker/libcompose diff --git a/vendor/github.com/mitchellh/hashstructure/LICENSE b/vendor/github.com/mitchellh/hashstructure/LICENSE new file mode 100644 index 000000000..a3866a291 --- /dev/null +++ b/vendor/github.com/mitchellh/hashstructure/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/hashstructure/hashstructure.go b/vendor/github.com/mitchellh/hashstructure/hashstructure.go new file mode 100644 index 000000000..ea13a1583 --- /dev/null +++ b/vendor/github.com/mitchellh/hashstructure/hashstructure.go @@ -0,0 +1,358 @@ +package hashstructure + +import ( + "encoding/binary" + "fmt" + "hash" + "hash/fnv" + "reflect" +) + +// ErrNotStringer is returned when there's an error with hash:"string" +type ErrNotStringer struct { + Field string +} + +// Error implements error for ErrNotStringer +func (ens *ErrNotStringer) Error() string { + return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field) +} + +// HashOptions are options that are available for hashing. +type HashOptions struct { + // Hasher is the hash function to use. If this isn't set, it will + // default to FNV. + Hasher hash.Hash64 + + // TagName is the struct tag to look at when hashing the structure. + // By default this is "hash". + TagName string + + // ZeroNil is flag determining if nil pointer should be treated equal + // to a zero value of pointed type. By default this is false. + ZeroNil bool +} + +// Hash returns the hash value of an arbitrary value. +// +// If opts is nil, then default options will be used. See HashOptions +// for the default values. The same *HashOptions value cannot be used +// concurrently. None of the values within a *HashOptions struct are +// safe to read/write while hashing is being done. +// +// Notes on the value: +// +// * Unexported fields on structs are ignored and do not affect the +// hash value. +// +// * Adding an exported field to a struct with the zero value will change +// the hash value. +// +// For structs, the hashing can be controlled using tags. For example: +// +// struct { +// Name string +// UUID string `hash:"ignore"` +// } +// +// The available tag values are: +// +// * "ignore" or "-" - The field will be ignored and not affect the hash code. +// +// * "set" - The field will be treated as a set, where ordering doesn't +// affect the hash code. This only works for slices. +// +// * "string" - The field will be hashed as a string, only works when the +// field implements fmt.Stringer +// +func Hash(v interface{}, opts *HashOptions) (uint64, error) { + // Create default options + if opts == nil { + opts = &HashOptions{} + } + if opts.Hasher == nil { + opts.Hasher = fnv.New64() + } + if opts.TagName == "" { + opts.TagName = "hash" + } + + // Reset the hash + opts.Hasher.Reset() + + // Create our walker and walk the structure + w := &walker{ + h: opts.Hasher, + tag: opts.TagName, + zeronil: opts.ZeroNil, + } + return w.visit(reflect.ValueOf(v), nil) +} + +type walker struct { + h hash.Hash64 + tag string + zeronil bool +} + +type visitOpts struct { + // Flags are a bitmask of flags to affect behavior of this visit + Flags visitFlag + + // Information about the struct containing this field + Struct interface{} + StructField string +} + +func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) { + t := reflect.TypeOf(0) + + // Loop since these can be wrapped in multiple layers of pointers + // and interfaces. + for { + // If we have an interface, dereference it. We have to do this up + // here because it might be a nil in there and the check below must + // catch that. + if v.Kind() == reflect.Interface { + v = v.Elem() + continue + } + + if v.Kind() == reflect.Ptr { + if w.zeronil { + t = v.Type().Elem() + } + v = reflect.Indirect(v) + continue + } + + break + } + + // If it is nil, treat it like a zero. + if !v.IsValid() { + v = reflect.Zero(t) + } + + // Binary writing can use raw ints, we have to convert to + // a sized-int, we'll choose the largest... + switch v.Kind() { + case reflect.Int: + v = reflect.ValueOf(int64(v.Int())) + case reflect.Uint: + v = reflect.ValueOf(uint64(v.Uint())) + case reflect.Bool: + var tmp int8 + if v.Bool() { + tmp = 1 + } + v = reflect.ValueOf(tmp) + } + + k := v.Kind() + + // We can shortcut numeric values by directly binary writing them + if k >= reflect.Int && k <= reflect.Complex64 { + // A direct hash calculation + w.h.Reset() + err := binary.Write(w.h, binary.LittleEndian, v.Interface()) + return w.h.Sum64(), err + } + + switch k { + case reflect.Array: + var h uint64 + l := v.Len() + for i := 0; i < l; i++ { + current, err := w.visit(v.Index(i), nil) + if err != nil { + return 0, err + } + + h = hashUpdateOrdered(w.h, h, current) + } + + return h, nil + + case reflect.Map: + var includeMap IncludableMap + if opts != nil && opts.Struct != nil { + if v, ok := opts.Struct.(IncludableMap); ok { + includeMap = v + } + } + + // Build the hash for the map. We do this by XOR-ing all the key + // and value hashes. This makes it deterministic despite ordering. + var h uint64 + for _, k := range v.MapKeys() { + v := v.MapIndex(k) + if includeMap != nil { + incl, err := includeMap.HashIncludeMap( + opts.StructField, k.Interface(), v.Interface()) + if err != nil { + return 0, err + } + if !incl { + continue + } + } + + kh, err := w.visit(k, nil) + if err != nil { + return 0, err + } + vh, err := w.visit(v, nil) + if err != nil { + return 0, err + } + + fieldHash := hashUpdateOrdered(w.h, kh, vh) + h = hashUpdateUnordered(h, fieldHash) + } + + return h, nil + + case reflect.Struct: + parent := v.Interface() + var include Includable + if impl, ok := parent.(Includable); ok { + include = impl + } + + t := v.Type() + h, err := w.visit(reflect.ValueOf(t.Name()), nil) + if err != nil { + return 0, err + } + + l := v.NumField() + for i := 0; i < l; i++ { + if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" { + var f visitFlag + fieldType := t.Field(i) + if fieldType.PkgPath != "" { + // Unexported + continue + } + + tag := fieldType.Tag.Get(w.tag) + if tag == "ignore" || tag == "-" { + // Ignore this field + continue + } + + // if string is set, use the string value + if tag == "string" { + if impl, ok := innerV.Interface().(fmt.Stringer); ok { + innerV = reflect.ValueOf(impl.String()) + } else { + return 0, &ErrNotStringer{ + Field: v.Type().Field(i).Name, + } + } + } + + // Check if we implement includable and check it + if include != nil { + incl, err := include.HashInclude(fieldType.Name, innerV) + if err != nil { + return 0, err + } + if !incl { + continue + } + } + + switch tag { + case "set": + f |= visitFlagSet + } + + kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil) + if err != nil { + return 0, err + } + + vh, err := w.visit(innerV, &visitOpts{ + Flags: f, + Struct: parent, + StructField: fieldType.Name, + }) + if err != nil { + return 0, err + } + + fieldHash := hashUpdateOrdered(w.h, kh, vh) + h = hashUpdateUnordered(h, fieldHash) + } + } + + return h, nil + + case reflect.Slice: + // We have two behaviors here. If it isn't a set, then we just + // visit all the elements. If it is a set, then we do a deterministic + // hash code. + var h uint64 + var set bool + if opts != nil { + set = (opts.Flags & visitFlagSet) != 0 + } + l := v.Len() + for i := 0; i < l; i++ { + current, err := w.visit(v.Index(i), nil) + if err != nil { + return 0, err + } + + if set { + h = hashUpdateUnordered(h, current) + } else { + h = hashUpdateOrdered(w.h, h, current) + } + } + + return h, nil + + case reflect.String: + // Directly hash + w.h.Reset() + _, err := w.h.Write([]byte(v.String())) + return w.h.Sum64(), err + + default: + return 0, fmt.Errorf("unknown kind to hash: %s", k) + } + +} + +func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 { + // For ordered updates, use a real hash function + h.Reset() + + // We just panic if the binary writes fail because we are writing + // an int64 which should never be fail-able. + e1 := binary.Write(h, binary.LittleEndian, a) + e2 := binary.Write(h, binary.LittleEndian, b) + if e1 != nil { + panic(e1) + } + if e2 != nil { + panic(e2) + } + + return h.Sum64() +} + +func hashUpdateUnordered(a, b uint64) uint64 { + return a ^ b +} + +// visitFlag is used as a bitmask for affecting visit behavior +type visitFlag uint + +const ( + visitFlagInvalid visitFlag = iota + visitFlagSet = iota << 1 +) diff --git a/vendor/github.com/mitchellh/hashstructure/include.go b/vendor/github.com/mitchellh/hashstructure/include.go new file mode 100644 index 000000000..b6289c0be --- /dev/null +++ b/vendor/github.com/mitchellh/hashstructure/include.go @@ -0,0 +1,15 @@ +package hashstructure + +// Includable is an interface that can optionally be implemented by +// a struct. It will be called for each field in the struct to check whether +// it should be included in the hash. +type Includable interface { + HashInclude(field string, v interface{}) (bool, error) +} + +// IncludableMap is an interface that can optionally be implemented by +// a struct. It will be called when a map-type field is found to ask the +// struct if the map item should be included in the hash. +type IncludableMap interface { + HashIncludeMap(field string, k, v interface{}) (bool, error) +}