From 9db12374eac41bd2b32c769f19f853acca04ed0d Mon Sep 17 00:00:00 2001 From: "Philippe M. Chiasson" Date: Fri, 22 Sep 2017 10:14:03 -0400 Subject: [PATCH 01/14] Be certain to clear our marshalled representation before reloading it --- cluster/datastore.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster/datastore.go b/cluster/datastore.go index ee8f56d9d..47bfd89ba 100644 --- a/cluster/datastore.go +++ b/cluster/datastore.go @@ -199,6 +199,10 @@ func (d *Datastore) get() *Metadata { func (d *Datastore) Load() (Object, error) { d.localLock.Lock() defer d.localLock.Unlock() + + // clear Object first, as mapstructure's decoder doesn't have ZeroFields set to true for merging purposes + d.meta.Object = d.meta.Object[:0] + err := d.kv.LoadConfig(d.meta) if err != nil { return nil, err From 058fa1367b49587133c48aa8f1e06d6e615a1884 Mon Sep 17 00:00:00 2001 From: Ludovic Fernandez Date: Fri, 22 Sep 2017 16:46:03 +0200 Subject: [PATCH 02/14] CI: speed up pull images. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5799a0822..2eede96cd 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ fmt: gofmt -s -l -w $(SRCS) pull-images: - cat ./integration/resources/compose/*.yml | grep -E '^\s+image:' | awk '{print $$2}' | sort | uniq | xargs -n 1 docker pull + cat ./integration/resources/compose/*.yml | grep -E '^\s+image:' | awk '{print $$2}' | sort | uniq | xargs -P 6 -n 1 docker pull help: ## this help @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) From dd23ceeeadadb4c2e5c2cf0f42944e4f613b9207 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Fri, 22 Sep 2017 16:22:03 +0100 Subject: [PATCH 03/14] Updating Docker output and curl for sticky sessions --- docs/user-guide/swarm-mode.md | 207 ++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 99 deletions(-) diff --git a/docs/user-guide/swarm-mode.md b/docs/user-guide/swarm-mode.md index 5f419d0a6..dc5a84ccb 100644 --- a/docs/user-guide/swarm-mode.md +++ b/docs/user-guide/swarm-mode.md @@ -7,14 +7,15 @@ The cluster consists of: - 3 servers - 1 manager - 2 workers -- 1 [overlay](https://docs.docker.com/engine/userguide/networking/dockernetworks/#an-overlay-network) network -(multi-host networking) +- 1 [overlay](https://docs.docker.com/engine/userguide/networking/dockernetworks/#an-overlay-network) network (multi-host networking) + ## Prerequisites 1. You will need to install [docker-machine](https://docs.docker.com/machine/) 2. You will need the latest [VirtualBox](https://www.virtualbox.org/wiki/Downloads) + ## Cluster provisioning First, let's create all the required nodes. @@ -26,7 +27,7 @@ docker-machine create -d virtualbox worker1 docker-machine create -d virtualbox worker2 ``` -Then, let's setup the cluster, in order : +Then, let's setup the cluster, in order: 1. initialize the cluster 1. get the token for other host to join @@ -60,9 +61,9 @@ docker-machine ssh manager docker node ls ``` ``` ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS -2a770ov9vixeadep674265u1n worker1 Ready Active -dbi3or4q8ii8elbws70g4hkdh * manager Ready Active Leader -esbhhy6vnqv90xomjaomdgy46 worker2 Ready Active +013v16l1sbuwjqcn7ucbu4jwt worker1 Ready Active +8buzkquycd17jqjber0mo2gn8 worker2 Ready Active +fnpj8ozfc85zvahx2r540xfcf * manager Ready Active Leader ``` Finally, let's create a network for Træfik to use. @@ -71,11 +72,11 @@ Finally, let's create a network for Træfik to use. docker-machine ssh manager "docker network create --driver=overlay traefik-net" ``` + ## Deploy Træfik Let's deploy Træfik as a docker service in our cluster. -The only requirement for Træfik to work with swarm mode is that it needs to run on a manager node — we are going to use a -[constraint](https://docs.docker.com/engine/reference/commandline/service_create/#/specify-service-constraints-constraint) for that. +The only requirement for Træfik to work with swarm mode is that it needs to run on a manager node - we are going to use a [constraint](https://docs.docker.com/engine/reference/commandline/service_create/#/specify-service-constraints-constraint) for that. ```shell docker-machine ssh manager "docker service create \ @@ -103,6 +104,7 @@ Let's explain this command: | `--docker` | enable docker backend, and `--docker.swarmmode` to enable the swarm mode on Træfik. | | `--web` | activate the webUI on port 8080 | + ## Deploy your apps We can now deploy our app on the cluster, here [whoami](https://github.com/emilevauge/whoami), a simple web server in Go. @@ -124,7 +126,7 @@ docker-machine ssh manager "docker service create \ ``` !!! note - We set whoami1 to use sticky sessions (`--label traefik.backend.loadbalancer.sticky=true`). + We set `whoami1` to use sticky sessions (`--label traefik.backend.loadbalancer.sticky=true`). We'll demonstrate that later. !!! note @@ -136,55 +138,52 @@ Check that everything is scheduled and started: docker-machine ssh manager "docker service ls" ``` ``` -ID NAME REPLICAS IMAGE COMMAND -ab046gpaqtln whoami0 1/1 emilevauge/whoami -cgfg5ifzrpgm whoami1 1/1 emilevauge/whoami -dtpl249tfghc traefik 1/1 traefik --docker --docker.swarmmode --docker.domain=traefik --docker.watch --web +ID NAME MODE REPLICAS IMAGE PORTS +moq3dq4xqv6t traefik replicated 1/1 traefik:latest *:80->80/tcp,*:8080->8080/tcp +ysil6oto1wim whoami0 replicated 1/1 emilevauge/whoami:latest +z9re2mnl34k4 whoami1 replicated 1/1 emilevauge/whoami:latest ``` + ## Access to your apps through Træfik ```shell curl -H Host:whoami0.traefik http://$(docker-machine ip manager) ``` ```yaml -Hostname: 8147a7746e7a +Hostname: 5b0b3d148359 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.3 -IP: fe80::42:aff:fe00:903 -IP: 172.18.0.3 -IP: fe80::42:acff:fe12:3 +IP: 10.0.0.8 +IP: 10.0.0.4 +IP: 172.18.0.5 GET / HTTP/1.1 -Host: 10.0.9.3:80 -User-Agent: curl/7.35.0 +Host: whoami0.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.3:80 +X-Forwarded-For: 10.255.0.2 +X-Forwarded-Host: whoami0.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` ```shell curl -H Host:whoami1.traefik http://$(docker-machine ip manager) ``` ```yaml -Hostname: ba2c21488299 +Hostname: 3633163970f6 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.4 -IP: fe80::42:aff:fe00:904 -IP: 172.18.0.2 -IP: fe80::42:acff:fe12:2 +IP: 10.0.0.14 +IP: 10.0.0.6 +IP: 172.18.0.5 GET / HTTP/1.1 -Host: 10.0.9.4:80 -User-Agent: curl/7.35.0 +Host: whoami1.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.4:80 +X-Forwarded-For: 10.255.0.2 +X-Forwarded-Host: whoami1.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` !!! note @@ -194,43 +193,39 @@ X-Forwarded-Server: 8fbc39271b4c curl -H Host:whoami0.traefik http://$(docker-machine ip worker1) ``` ```yaml -Hostname: 8147a7746e7a +Hostname: 5b0b3d148359 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.3 -IP: fe80::42:aff:fe00:903 -IP: 172.18.0.3 -IP: fe80::42:acff:fe12:3 +IP: 10.0.0.8 +IP: 10.0.0.4 +IP: 172.18.0.5 GET / HTTP/1.1 -Host: 10.0.9.3:80 -User-Agent: curl/7.35.0 +Host: whoami0.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.3:80 +X-Forwarded-For: 10.255.0.3 +X-Forwarded-Host: whoami0.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` ```shell curl -H Host:whoami1.traefik http://$(docker-machine ip worker2) ``` ```yaml -Hostname: ba2c21488299 +Hostname: 3633163970f6 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.4 -IP: fe80::42:aff:fe00:904 -IP: 172.18.0.2 -IP: fe80::42:acff:fe12:2 +IP: 10.0.0.14 +IP: 10.0.0.6 +IP: 172.18.0.5 GET / HTTP/1.1 -Host: 10.0.9.4:80 -User-Agent: curl/7.35.0 +Host: whoami1.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.4:80 +X-Forwarded-For: 10.255.0.4 +X-Forwarded-Host: whoami1.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` ## Scale both services @@ -246,79 +241,93 @@ Check that we now have 5 replicas of each `whoami` service: docker-machine ssh manager "docker service ls" ``` ``` -ID NAME REPLICAS IMAGE COMMAND -ab046gpaqtln whoami0 5/5 emilevauge/whoami -cgfg5ifzrpgm whoami1 5/5 emilevauge/whoami -dtpl249tfghc traefik 1/1 traefik --docker --docker.swarmmode --docker.domain=traefik --docker.watch --web +ID NAME MODE REPLICAS IMAGE PORTS +moq3dq4xqv6t traefik replicated 1/1 traefik:latest *:80->80/tcp,*:8080->8080/tcp +ysil6oto1wim whoami0 replicated 5/5 emilevauge/whoami:latest +z9re2mnl34k4 whoami1 replicated 5/5 emilevauge/whoami:latest ``` -## Access to your whoami0 through Træfik multiple times. + +## Access to your `whoami0` through Træfik multiple times. Repeat the following command multiple times and note that the Hostname changes each time as Traefik load balances each request against the 5 tasks: ```shell curl -H Host:whoami0.traefik http://$(docker-machine ip manager) ``` - ```yaml -Hostname: 8147a7746e7a +Hostname: f3138d15b567 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.3 -IP: fe80::42:aff:fe00:903 +IP: 10.0.0.5 +IP: 10.0.0.4 IP: 172.18.0.3 -IP: fe80::42:acff:fe12:3 GET / HTTP/1.1 -Host: 10.0.9.3:80 -User-Agent: curl/7.35.0 +Host: whoami0.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.3:80 +X-Forwarded-For: 10.255.0.2 +X-Forwarded-Host: whoami0.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` -Do the same against whoami1: +Do the same against `whoami1`: ```shell -curl -H Host:whoami1.traefik http://$(docker-machine ip manager) +curl -c cookies.txt -H Host:whoami1.traefik http://$(docker-machine ip manager) ``` - ```yaml -Hostname: ba2c21488299 +Hostname: 348e2f7bf432 IP: 127.0.0.1 -IP: ::1 -IP: 10.0.9.4 -IP: fe80::42:aff:fe00:904 -IP: 172.18.0.2 -IP: fe80::42:acff:fe12:2 +IP: 10.0.0.15 +IP: 10.0.0.6 +IP: 172.18.0.6 GET / HTTP/1.1 -Host: 10.0.9.4:80 -User-Agent: curl/7.35.0 +Host: whoami1.traefik +User-Agent: curl/7.55.1 Accept: */* Accept-Encoding: gzip -X-Forwarded-For: 192.168.99.1 -X-Forwarded-Host: 10.0.9.4:80 +X-Forwarded-For: 10.255.0.2 +X-Forwarded-Host: whoami1.traefik X-Forwarded-Proto: http -X-Forwarded-Server: 8fbc39271b4c +X-Forwarded-Server: 77fc29c69fe4 ``` -Wait, I thought we added the sticky flag to `whoami1`? -Traefik relies on a cookie to maintain stickyness so you'll need to test this with a browser. - -First you need to add `whoami1.traefik` to your hosts file: +Because the sticky sessions require cookies to work, we used the `-c cookies.txt` option to store the cookie into a file. +The cookie contains the IP of the container to which the session sticks: ```shell -if [ -n "$(grep whoami1.traefik /etc/hosts)" ]; -then - echo "whoami1.traefik already exists (make sure the ip is current)"; -else - sudo -- sh -c -e "echo '$(docker-machine ip manager)\twhoami1.traefik' >> /etc/hosts"; -fi +cat ./cookies.txt +``` +``` +# Netscape HTTP Cookie File +# https://curl.haxx.se/docs/http-cookies.html +# This file was generated by libcurl! Edit at your own risk. + +whoami1.traefik FALSE / FALSE 0 _TRAEFIK_BACKEND http://10.0.0.15:80 ``` -Now open your browser and go to http://whoami1.traefik/ +If you load the cookies file (`-b cookies.txt`) for the next request, you will see that stickyness is maintained: -You will now see that stickyness is maintained. +```shell +curl -b cookies.txt -H Host:whoami1.traefik http://$(docker-machine ip manager) +``` +```yaml +Hostname: 348e2f7bf432 +IP: 127.0.0.1 +IP: 10.0.0.15 +IP: 10.0.0.6 +IP: 172.18.0.6 +GET / HTTP/1.1 +Host: whoami1.traefik +User-Agent: curl/7.55.1 +Accept: */* +Accept-Encoding: gzip +Cookie: _TRAEFIK_BACKEND=http://10.0.0.15:80 +X-Forwarded-For: 10.255.0.2 +X-Forwarded-Host: whoami1.traefik +X-Forwarded-Proto: http +X-Forwarded-Server: 77fc29c69fe4 +``` ![](https://i.giphy.com/ujUdrdpX7Ok5W.gif) From 1ba7fd91ffe8ba8d02345a0348da85f986688e49 Mon Sep 17 00:00:00 2001 From: Timo Reimann Date: Tue, 26 Sep 2017 14:44:03 +0200 Subject: [PATCH 04/14] grep to-be-pulled-images directly to avoid newline issue. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2eede96cd..0ffb5e819 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ fmt: gofmt -s -l -w $(SRCS) pull-images: - cat ./integration/resources/compose/*.yml | grep -E '^\s+image:' | awk '{print $$2}' | sort | uniq | xargs -P 6 -n 1 docker pull + grep --no-filename -E '^\s+image:' ./integration/resources/compose/*.yml | awk '{print $$2}' | sort | uniq | xargs -n 1 docker pull help: ## this help @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) From 691a678b1912599686205adaa0ae653cfa6390ee Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Fri, 29 Sep 2017 09:34:03 +0100 Subject: [PATCH 05/14] Improve compression documentation --- docs/configuration/entrypoints.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration/entrypoints.md b/docs/configuration/entrypoints.md index ed8a08ede..9fe9799ff 100644 --- a/docs/configuration/entrypoints.md +++ b/docs/configuration/entrypoints.md @@ -171,6 +171,12 @@ To enable compression support using gzip format. compress = true ``` +Responses are compressed when: + +* The response body is larger than `512` bytes +* And the `Accept-Encoding` request header contains `gzip` +* And the response is not already compressed, i.e. the `Content-Encoding` response header is not already set. + ## Whitelisting To enable IP whitelisting at the entrypoint level. From 64c52a6921993a37c509e6211570070a80c7045b Mon Sep 17 00:00:00 2001 From: SALLEYRON Julien Date: Fri, 29 Sep 2017 16:30:03 +0200 Subject: [PATCH 06/14] Consul catalog remove service failed --- integration/consul_catalog_test.go | 6 +++++- provider/consul/consul_catalog.go | 8 ++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/integration/consul_catalog_test.go b/integration/consul_catalog_test.go index ca913983f..1ec14f6a1 100644 --- a/integration/consul_catalog_test.go +++ b/integration/consul_catalog_test.go @@ -138,7 +138,6 @@ func (s *ConsulCatalogSuite) TestSingleService(c *check.C) { err = s.registerService("test", nginx.NetworkSettings.IPAddress, 80, []string{}) c.Assert(err, checker.IsNil, check.Commentf("Error registering service")) - defer s.deregisterService("test", nginx.NetworkSettings.IPAddress) req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000/", nil) c.Assert(err, checker.IsNil) @@ -146,6 +145,11 @@ func (s *ConsulCatalogSuite) TestSingleService(c *check.C) { err = try.Request(req, 10*time.Second, try.StatusCodeIs(http.StatusOK), try.HasBody()) c.Assert(err, checker.IsNil) + + s.deregisterService("test", nginx.NetworkSettings.IPAddress) + err = try.Request(req, 10*time.Second, try.StatusCodeIs(http.StatusNotFound), try.HasBody()) + c.Assert(err, checker.IsNil) + } func (s *ConsulCatalogSuite) TestExposedByDefaultFalseSingleService(c *check.C) { diff --git a/provider/consul/consul_catalog.go b/provider/consul/consul_catalog.go index 2ae332792..e34a07a0d 100644 --- a/provider/consul/consul_catalog.go +++ b/provider/consul/consul_catalog.go @@ -190,7 +190,6 @@ func (p *CatalogProvider) watchCatalogServices(stopCh <-chan struct{}, watchCh c catalog := p.client.Catalog() safe.Go(func() { - current := make(map[string]Service) // variable to hold previous state var flashback map[string]Service @@ -216,7 +215,7 @@ func (p *CatalogProvider) watchCatalogServices(stopCh <-chan struct{}, watchCh c options.WaitIndex = meta.LastIndex if data != nil { - + current := make(map[string]Service) for key, value := range data { nodes, _, err := catalog.Service(key, "", &api.QueryOptions{}) if err != nil { @@ -246,10 +245,7 @@ func (p *CatalogProvider) watchCatalogServices(stopCh <-chan struct{}, watchCh c if len(removedServiceKeys) > 0 || len(removedServiceNodeKeys) > 0 || len(addedServiceKeys) > 0 || len(addedServiceNodeKeys) > 0 { log.WithField("MissingServices", removedServiceKeys).WithField("DiscoveredServices", addedServiceKeys).Debug("Catalog Services change detected.") watchCh <- data - flashback = make(map[string]Service, len(current)) - for key, value := range current { - flashback[key] = value - } + flashback = current } } } From d41e28fc36772cd8b55967bad98d97e4a0f7716a Mon Sep 17 00:00:00 2001 From: jeffreykoetsier <5791868+jeffreykoetsier@users.noreply.github.com> Date: Fri, 29 Sep 2017 16:56:03 +0200 Subject: [PATCH 07/14] Handle empty ECS Clusters properly --- provider/ecs/ecs.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/provider/ecs/ecs.go b/provider/ecs/ecs.go index e48ebe5bc..32e9af5c6 100644 --- a/provider/ecs/ecs.go +++ b/provider/ecs/ecs.go @@ -214,7 +214,6 @@ func (p *Provider) loadECSConfig(ctx context.Context, client *awsClient) (*types // Find all running Provider tasks in a cluster, also collect the task definitions (for docker labels) // and the EC2 instance data func (p *Provider) listInstances(ctx context.Context, client *awsClient) ([]ecsInstance, error) { - var taskArns []*string var instances []ecsInstance var clustersArn []*string var clusters Clusters @@ -255,6 +254,8 @@ func (p *Provider) listInstances(ctx context.Context, client *awsClient) ([]ecsI DesiredStatus: aws.String(ecs.DesiredStatusRunning), }) + var taskArns []*string + for ; req != nil; req = req.NextPage() { if err := wrapAws(ctx, req); err != nil { return nil, err @@ -263,12 +264,10 @@ func (p *Provider) listInstances(ctx context.Context, client *awsClient) ([]ecsI taskArns = append(taskArns, req.Data.(*ecs.ListTasksOutput).TaskArns...) } - // Early return: if we can't list tasks we have nothing to - // describe below - likely empty cluster/permissions are bad. This - // stops the AWS API from returning a 401 when you DescribeTasks - // with no input. + // Skip to the next cluster if there are no tasks found on + // this cluster. if len(taskArns) == 0 { - return []ecsInstance{}, nil + continue } chunkedTaskArns := p.chunkedTaskArns(taskArns) From b6752a2c0227dc198b6a8061dfc5d1459b491092 Mon Sep 17 00:00:00 2001 From: SALLEYRON Julien Date: Fri, 29 Sep 2017 21:04:03 +0200 Subject: [PATCH 08/14] Forward upgrade error from backend --- glide.lock | 4 +- glide.yaml | 2 +- integration/websocket_test.go | 95 ++++++++++++++++++++ vendor/github.com/vulcand/oxy/forward/fwd.go | 29 +++++- 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/glide.lock b/glide.lock index 092eb0694..1316462ff 100644 --- a/glide.lock +++ b/glide.lock @@ -1,4 +1,4 @@ -hash: 123a0f00c37d07cd6d0583437b70092176e8d5c2445e127afef40acdc4e5aa32 +hash: 17b4abe35874c0990cf9ac68f4cdfae0229f4395097bcaab3105b698d91bd272 updated: 2017-09-18T11:52:16.848940186+02:00 imports: - name: cloud.google.com/go @@ -477,7 +477,7 @@ imports: - name: github.com/urfave/negroni version: 490e6a555d47ca891a89a150d0c1ef3922dfffe9 - name: github.com/vulcand/oxy - version: 6c94d2888dba2b1a15a89b8a2ca515fc85e07477 + version: 648088ee0902cf8d8337826ae2a82444008720e2 repo: https://github.com/containous/oxy.git vcs: git subpackages: diff --git a/glide.yaml b/glide.yaml index 0a9c81934..6878be391 100644 --- a/glide.yaml +++ b/glide.yaml @@ -12,7 +12,7 @@ import: - package: github.com/cenk/backoff - package: github.com/containous/flaeg - package: github.com/vulcand/oxy - version: 6c94d2888dba2b1a15a89b8a2ca515fc85e07477 + version: 648088ee0902cf8d8337826ae2a82444008720e2 repo: https://github.com/containous/oxy.git vcs: git subpackages: diff --git a/integration/websocket_test.go b/integration/websocket_test.go index 220562eb8..2ad267fae 100644 --- a/integration/websocket_test.go +++ b/integration/websocket_test.go @@ -3,6 +3,7 @@ package integration import ( "crypto/tls" "crypto/x509" + "encoding/base64" "io/ioutil" "net" "net/http" @@ -295,3 +296,97 @@ func (s *WebsocketSuite) TestSSLTermination(c *check.C) { c.Assert(err, checker.IsNil) c.Assert(string(msg), checker.Equals, "OK") } + +func (s *WebsocketSuite) TestBasicAuth(c *check.C) { + var upgrader = gorillawebsocket.Upgrader{} // use default options + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + + if err != nil { + return + } + defer conn.Close() + + user, password, _ := r.BasicAuth() + c.Assert(user, check.Equals, "traefiker") + c.Assert(password, check.Equals, "secret") + + for { + mt, message, err := conn.ReadMessage() + if err != nil { + break + } + err = conn.WriteMessage(mt, message) + if err != nil { + break + } + } + })) + file := s.adaptFile(c, "fixtures/websocket/config.toml", struct { + WebsocketServer string + }{ + WebsocketServer: srv.URL, + }) + + defer os.Remove(file) + cmd, display := s.traefikCmd(withConfigFile(file), "--debug") + defer display(c) + + err := cmd.Start() + c.Assert(err, check.IsNil) + defer cmd.Process.Kill() + + // wait for traefik + err = try.GetRequest("http://127.0.0.1:8080/api/providers", 10*time.Second, try.BodyContains("127.0.0.1")) + c.Assert(err, checker.IsNil) + + config, err := websocket.NewConfig("ws://127.0.0.1:8000/ws", "ws://127.0.0.1:8000") + auth := "traefiker:secret" + config.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth))) + + c.Assert(err, check.IsNil) + + conn, err := net.DialTimeout("tcp", "127.0.0.1:8000", time.Second) + c.Assert(err, checker.IsNil) + client, err := websocket.NewClient(config, conn) + c.Assert(err, checker.IsNil) + + n, err := client.Write([]byte("OK")) + c.Assert(err, checker.IsNil) + c.Assert(n, checker.Equals, 2) + + msg := make([]byte, 2) + n, err = client.Read(msg) + c.Assert(err, checker.IsNil) + c.Assert(n, checker.Equals, 2) + c.Assert(string(msg), checker.Equals, "OK") +} + +func (s *WebsocketSuite) TestSpecificResponseFromBackend(c *check.C) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(401) + })) + file := s.adaptFile(c, "fixtures/websocket/config.toml", struct { + WebsocketServer string + }{ + WebsocketServer: srv.URL, + }) + + defer os.Remove(file) + cmd, display := s.traefikCmd(withConfigFile(file), "--debug") + defer display(c) + + err := cmd.Start() + c.Assert(err, check.IsNil) + defer cmd.Process.Kill() + + // wait for traefik + err = try.GetRequest("http://127.0.0.1:8080/api/providers", 10*time.Second, try.BodyContains("127.0.0.1")) + c.Assert(err, checker.IsNil) + + _, resp, err := gorillawebsocket.DefaultDialer.Dial("ws://127.0.0.1:8000/ws", nil) + c.Assert(err, checker.NotNil) + c.Assert(resp.StatusCode, check.Equals, 401) + +} diff --git a/vendor/github.com/vulcand/oxy/forward/fwd.go b/vendor/github.com/vulcand/oxy/forward/fwd.go index 04974ad4d..a30c76ba0 100644 --- a/vendor/github.com/vulcand/oxy/forward/fwd.go +++ b/vendor/github.com/vulcand/oxy/forward/fwd.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "reflect" "strconv" "strings" "time" @@ -261,8 +262,32 @@ func (f *websocketForwarder) serveHTTP(w http.ResponseWriter, req *http.Request, } targetConn, resp, err := dialer.Dial(outReq.URL.String(), outReq.Header) if err != nil { - ctx.log.Errorf("Error dialing `%v`: %v", outReq.Host, err) - ctx.errHandler.ServeHTTP(w, req, err) + if resp == nil { + ctx.errHandler.ServeHTTP(w, req, err) + } else { + ctx.log.Errorf("Error dialing %q: %v with resp: %d %s", outReq.Host, err, resp.StatusCode, resp.Status) + hijacker, ok := w.(http.Hijacker) + if !ok { + ctx.log.Errorf("%s can not be hijack", reflect.TypeOf(w)) + ctx.errHandler.ServeHTTP(w, req, err) + return + } + + conn, _, err := hijacker.Hijack() + if err != nil { + ctx.log.Errorf("Failed to hijack responseWriter") + ctx.errHandler.ServeHTTP(w, req, err) + return + } + defer conn.Close() + + err = resp.Write(conn) + if err != nil { + ctx.log.Errorf("Failed to forward response") + ctx.errHandler.ServeHTTP(w, req, err) + return + } + } return } From 5cc49e29315c46dca68e8b13cc75689f01139c56 Mon Sep 17 00:00:00 2001 From: Ludovic Fernandez Date: Mon, 2 Oct 2017 10:32:02 +0200 Subject: [PATCH 09/14] `bug` command. --- cmd/traefik/anonymize/anonymize.go | 136 ++++ .../anonymize/anonymize_config_test.go | 666 ++++++++++++++++++ .../anonymize/anonymize_doOnJSON_test.go | 238 +++++++ .../anonymize/anonymize_doOnStruct_test.go | 176 +++++ cmd/traefik/bug.go | 107 +-- cmd/traefik/bug_test.go | 226 ++---- cmd/traefik/configuration.go | 4 +- cmd/traefik/traefik.go | 30 +- configuration/configuration.go | 102 ++- docs/configuration/backends/web.md | 4 +- glide.lock | 8 +- glide.yaml | 1 + integration/fixtures/access_log_config.toml | 2 + integration/fixtures/log_rotation_config.toml | 2 + integration/fixtures/traefik_log_config.toml | 4 +- middlewares/auth/authenticator.go | 2 +- middlewares/metrics.go | 2 +- provider/boltdb/boltdb.go | 2 +- provider/consul/consul.go | 2 +- provider/consul/consul_catalog.go | 8 +- provider/docker/docker.go | 10 +- provider/dynamodb/dynamodb.go | 15 +- provider/ecs/ecs.go | 10 +- provider/etcd/etcd.go | 2 +- provider/eureka/eureka.go | 4 +- provider/file/file.go | 4 +- provider/kubernetes/kubernetes.go | 8 +- provider/kv/kv.go | 6 +- provider/marathon/marathon.go | 24 +- provider/mesos/mesos.go | 12 +- provider/provider.go | 10 +- provider/rancher/rancher.go | 16 +- provider/web/web.go | 16 +- provider/zk/zk.go | 2 +- server/uuid/uuid.go | 14 + types/types.go | 53 +- .../mitchellh/copystructure/LICENSE | 21 + .../mitchellh/copystructure/copier_time.go | 15 + .../mitchellh/copystructure/copystructure.go | 548 ++++++++++++++ .../github.com/mitchellh/reflectwalk/LICENSE | 21 + .../mitchellh/reflectwalk/location.go | 19 + .../mitchellh/reflectwalk/location_string.go | 16 + .../mitchellh/reflectwalk/reflectwalk.go | 401 +++++++++++ 43 files changed, 2560 insertions(+), 409 deletions(-) create mode 100644 cmd/traefik/anonymize/anonymize.go create mode 100644 cmd/traefik/anonymize/anonymize_config_test.go create mode 100644 cmd/traefik/anonymize/anonymize_doOnJSON_test.go create mode 100644 cmd/traefik/anonymize/anonymize_doOnStruct_test.go create mode 100644 server/uuid/uuid.go create mode 100644 vendor/github.com/mitchellh/copystructure/LICENSE create mode 100644 vendor/github.com/mitchellh/copystructure/copier_time.go create mode 100644 vendor/github.com/mitchellh/copystructure/copystructure.go create mode 100644 vendor/github.com/mitchellh/reflectwalk/LICENSE create mode 100644 vendor/github.com/mitchellh/reflectwalk/location.go create mode 100644 vendor/github.com/mitchellh/reflectwalk/location_string.go create mode 100644 vendor/github.com/mitchellh/reflectwalk/reflectwalk.go diff --git a/cmd/traefik/anonymize/anonymize.go b/cmd/traefik/anonymize/anonymize.go new file mode 100644 index 000000000..7be585ee5 --- /dev/null +++ b/cmd/traefik/anonymize/anonymize.go @@ -0,0 +1,136 @@ +package anonymize + +import ( + "encoding/json" + "fmt" + "reflect" + "regexp" + + "github.com/mitchellh/copystructure" + "github.com/mvdan/xurls" +) + +const ( + maskShort = "xxxx" + maskLarge = maskShort + maskShort + maskShort + maskShort + maskShort + maskShort + maskShort + maskShort +) + +// Do configuration. +func Do(baseConfig interface{}, indent bool) (string, error) { + anomConfig, err := copystructure.Copy(baseConfig) + if err != nil { + return "", err + } + + val := reflect.ValueOf(anomConfig) + + err = doOnStruct(val) + if err != nil { + return "", err + } + + configJSON, err := marshal(anomConfig, indent) + if err != nil { + return "", err + } + + return doOnJSON(string(configJSON)), nil +} + +func doOnJSON(input string) string { + mailExp := regexp.MustCompile(`\w[-._\w]*\w@\w[-._\w]*\w\.\w{2,3}"`) + return xurls.Relaxed.ReplaceAllString(mailExp.ReplaceAllString(input, maskLarge+"\""), maskLarge) +} + +func doOnStruct(field reflect.Value) error { + switch field.Kind() { + case reflect.Ptr: + if !field.IsNil() { + if err := doOnStruct(field.Elem()); err != nil { + return err + } + } + case reflect.Struct: + for i := 0; i < field.NumField(); i++ { + fld := field.Field(i) + stField := field.Type().Field(i) + if !isExported(stField) { + continue + } + if stField.Tag.Get("export") == "true" { + if err := doOnStruct(fld); err != nil { + return err + } + } else { + if err := reset(fld, stField.Name); err != nil { + return err + } + } + } + case reflect.Map: + for _, key := range field.MapKeys() { + if err := doOnStruct(field.MapIndex(key)); err != nil { + return err + } + } + case reflect.Slice: + for j := 0; j < field.Len(); j++ { + if err := doOnStruct(field.Index(j)); err != nil { + return err + } + } + } + return nil +} + +func reset(field reflect.Value, name string) error { + if !field.CanSet() { + return fmt.Errorf("cannot reset field %s", name) + } + + switch field.Kind() { + case reflect.Ptr: + if !field.IsNil() { + field.Set(reflect.Zero(field.Type())) + } + case reflect.Struct: + if field.IsValid() { + field.Set(reflect.Zero(field.Type())) + } + case reflect.String: + if field.String() != "" { + field.Set(reflect.ValueOf(maskShort)) + } + case reflect.Map: + if field.Len() > 0 { + field.Set(reflect.MakeMap(field.Type())) + } + case reflect.Slice: + if field.Len() > 0 { + field.Set(reflect.MakeSlice(field.Type(), 0, 0)) + } + case reflect.Interface: + if !field.IsNil() { + return reset(field.Elem(), "") + } + default: + // Primitive type + field.Set(reflect.Zero(field.Type())) + } + return nil +} + +// isExported return true is a struct field is exported, else false +func isExported(f reflect.StructField) bool { + if f.PkgPath != "" && !f.Anonymous { + return false + } + return true +} + +func marshal(anomConfig interface{}, indent bool) ([]byte, error) { + if indent { + return json.MarshalIndent(anomConfig, "", " ") + } + return json.Marshal(anomConfig) +} diff --git a/cmd/traefik/anonymize/anonymize_config_test.go b/cmd/traefik/anonymize/anonymize_config_test.go new file mode 100644 index 000000000..43b0844d2 --- /dev/null +++ b/cmd/traefik/anonymize/anonymize_config_test.go @@ -0,0 +1,666 @@ +package anonymize + +import ( + "crypto/tls" + "testing" + "time" + + "github.com/containous/flaeg" + "github.com/containous/traefik/acme" + "github.com/containous/traefik/configuration" + "github.com/containous/traefik/middlewares" + "github.com/containous/traefik/provider" + "github.com/containous/traefik/provider/boltdb" + "github.com/containous/traefik/provider/consul" + "github.com/containous/traefik/provider/docker" + "github.com/containous/traefik/provider/dynamodb" + "github.com/containous/traefik/provider/ecs" + "github.com/containous/traefik/provider/etcd" + "github.com/containous/traefik/provider/eureka" + "github.com/containous/traefik/provider/file" + "github.com/containous/traefik/provider/kubernetes" + "github.com/containous/traefik/provider/kv" + "github.com/containous/traefik/provider/marathon" + "github.com/containous/traefik/provider/mesos" + "github.com/containous/traefik/provider/rancher" + "github.com/containous/traefik/provider/web" + "github.com/containous/traefik/provider/zk" + "github.com/containous/traefik/safe" + "github.com/containous/traefik/types" + thoas_stats "github.com/thoas/stats" +) + +func TestDo_globalConfiguration(t *testing.T) { + + config := &configuration.GlobalConfiguration{} + + config.GraceTimeOut = flaeg.Duration(666 * time.Second) + config.Debug = true + config.CheckNewVersion = true + config.AccessLogsFile = "AccessLogsFile" + config.AccessLog = &types.AccessLog{ + FilePath: "AccessLog FilePath", + Format: "AccessLog Format", + } + config.TraefikLogsFile = "TraefikLogsFile" + config.LogLevel = "LogLevel" + config.EntryPoints = configuration.EntryPoints{ + "foo": { + Network: "foo Network", + Address: "foo Address", + TLS: &configuration.TLS{ + MinVersion: "foo MinVersion", + CipherSuites: []string{"foo CipherSuites 1", "foo CipherSuites 2", "foo CipherSuites 3"}, + Certificates: configuration.Certificates{ + {CertFile: "CertFile 1", KeyFile: "KeyFile 1"}, + {CertFile: "CertFile 2", KeyFile: "KeyFile 2"}, + }, + ClientCAFiles: []string{"foo ClientCAFiles 1", "foo ClientCAFiles 2", "foo ClientCAFiles 3"}, + }, + Redirect: &configuration.Redirect{ + Replacement: "foo Replacement", + Regex: "foo Regex", + EntryPoint: "foo EntryPoint", + }, + Auth: &types.Auth{ + Basic: &types.Basic{ + UsersFile: "foo Basic UsersFile", + Users: types.Users{"foo Basic Users 1", "foo Basic Users 2", "foo Basic Users 3"}, + }, + Digest: &types.Digest{ + UsersFile: "foo Digest UsersFile", + Users: types.Users{"foo Digest Users 1", "foo Digest Users 2", "foo Digest Users 3"}, + }, + Forward: &types.Forward{ + Address: "foo Address", + TLS: &types.ClientTLS{ + CA: "foo CA", + Cert: "foo Cert", + Key: "foo Key", + InsecureSkipVerify: true, + }, + TrustForwardHeader: true, + }, + }, + WhitelistSourceRange: []string{"foo WhitelistSourceRange 1", "foo WhitelistSourceRange 2", "foo WhitelistSourceRange 3"}, + Compress: true, + ProxyProtocol: true, + }, + "fii": { + Network: "fii Network", + Address: "fii Address", + TLS: &configuration.TLS{ + MinVersion: "fii MinVersion", + CipherSuites: []string{"fii CipherSuites 1", "fii CipherSuites 2", "fii CipherSuites 3"}, + Certificates: configuration.Certificates{ + {CertFile: "CertFile 1", KeyFile: "KeyFile 1"}, + {CertFile: "CertFile 2", KeyFile: "KeyFile 2"}, + }, + ClientCAFiles: []string{"fii ClientCAFiles 1", "fii ClientCAFiles 2", "fii ClientCAFiles 3"}, + }, + Redirect: &configuration.Redirect{ + Replacement: "fii Replacement", + Regex: "fii Regex", + EntryPoint: "fii EntryPoint", + }, + Auth: &types.Auth{ + Basic: &types.Basic{ + UsersFile: "fii Basic UsersFile", + Users: types.Users{"fii Basic Users 1", "fii Basic Users 2", "fii Basic Users 3"}, + }, + Digest: &types.Digest{ + UsersFile: "fii Digest UsersFile", + Users: types.Users{"fii Digest Users 1", "fii Digest Users 2", "fii Digest Users 3"}, + }, + Forward: &types.Forward{ + Address: "fii Address", + TLS: &types.ClientTLS{ + CA: "fii CA", + Cert: "fii Cert", + Key: "fii Key", + InsecureSkipVerify: true, + }, + TrustForwardHeader: true, + }, + }, + WhitelistSourceRange: []string{"fii WhitelistSourceRange 1", "fii WhitelistSourceRange 2", "fii WhitelistSourceRange 3"}, + Compress: true, + ProxyProtocol: true, + }, + } + config.Cluster = &types.Cluster{ + Node: "Cluster Node", + Store: &types.Store{ + Prefix: "Cluster Store Prefix", + // ... + }, + } + config.Constraints = types.Constraints{ + { + Key: "Constraints Key 1", + Regex: "Constraints Regex 2", + MustMatch: true, + }, + { + Key: "Constraints Key 1", + Regex: "Constraints Regex 2", + MustMatch: true, + }, + } + config.ACME = &acme.ACME{ + Email: "acme Email", + Domains: []acme.Domain{ + { + Main: "Domains Main", + SANs: []string{"Domains acme SANs 1", "Domains acme SANs 2", "Domains acme SANs 3"}, + }, + }, + Storage: "Storage", + StorageFile: "StorageFile", + OnDemand: true, + OnHostRule: true, + CAServer: "CAServer", + EntryPoint: "EntryPoint", + DNSProvider: "DNSProvider", + DelayDontCheckDNS: 666, + ACMELogging: true, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + // ... + }, + } + config.DefaultEntryPoints = configuration.DefaultEntryPoints{"DefaultEntryPoints 1", "DefaultEntryPoints 2", "DefaultEntryPoints 3"} + config.ProvidersThrottleDuration = flaeg.Duration(666 * time.Second) + config.MaxIdleConnsPerHost = 666 + config.IdleTimeout = flaeg.Duration(666 * time.Second) + config.InsecureSkipVerify = true + config.RootCAs = configuration.RootCAs{"RootCAs 1", "RootCAs 2", "RootCAs 3"} + config.Retry = &configuration.Retry{ + Attempts: 666, + } + config.HealthCheck = &configuration.HealthCheckConfig{ + Interval: flaeg.Duration(666 * time.Second), + } + config.RespondingTimeouts = &configuration.RespondingTimeouts{ + ReadTimeout: flaeg.Duration(666 * time.Second), + WriteTimeout: flaeg.Duration(666 * time.Second), + IdleTimeout: flaeg.Duration(666 * time.Second), + } + config.ForwardingTimeouts = &configuration.ForwardingTimeouts{ + DialTimeout: flaeg.Duration(666 * time.Second), + ResponseHeaderTimeout: flaeg.Duration(666 * time.Second), + } + config.Docker = &docker.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "docker Filename", + Constraints: types.Constraints{ + { + Key: "docker Constraints Key 1", + Regex: "docker Constraints Regex 2", + MustMatch: true, + }, + { + Key: "docker Constraints Key 1", + Regex: "docker Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "docker Endpoint", + Domain: "docker Domain", + TLS: &types.ClientTLS{ + CA: "docker CA", + Cert: "docker Cert", + Key: "docker Key", + InsecureSkipVerify: true, + }, + ExposedByDefault: true, + UseBindPortIP: true, + SwarmMode: true, + } + config.File = &file.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "file Filename", + Constraints: types.Constraints{ + { + Key: "file Constraints Key 1", + Regex: "file Constraints Regex 2", + MustMatch: true, + }, + { + Key: "file Constraints Key 1", + Regex: "file Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Directory: "file Directory", + } + config.Web = &web.Provider{ + Address: "web Address", + CertFile: "web CertFile", + KeyFile: "web KeyFile", + ReadOnly: true, + Statistics: &types.Statistics{ + RecentErrors: 666, + }, + Metrics: &types.Metrics{ + Prometheus: &types.Prometheus{ + Buckets: types.Buckets{6.5, 6.6, 6.7}, + }, + Datadog: &types.Datadog{ + Address: "Datadog Address", + PushInterval: "Datadog PushInterval", + }, + StatsD: &types.Statsd{ + Address: "StatsD Address", + PushInterval: "StatsD PushInterval", + }, + }, + Path: "web Path", + Auth: &types.Auth{ + Basic: &types.Basic{ + UsersFile: "web Basic UsersFile", + Users: types.Users{"web Basic Users 1", "web Basic Users 2", "web Basic Users 3"}, + }, + Digest: &types.Digest{ + UsersFile: "web Digest UsersFile", + Users: types.Users{"web Digest Users 1", "web Digest Users 2", "web Digest Users 3"}, + }, + Forward: &types.Forward{ + Address: "web Address", + TLS: &types.ClientTLS{ + CA: "web CA", + Cert: "web Cert", + Key: "web Key", + InsecureSkipVerify: true, + }, + TrustForwardHeader: true, + }, + }, + Debug: true, + CurrentConfigurations: &safe.Safe{}, + Stats: &thoas_stats.Stats{ + Uptime: time.Now(), + Pid: 666, + ResponseCounts: map[string]int{"foo": 1, "fii": 2, "fuu": 3}, + TotalResponseCounts: map[string]int{"foo": 1, "fii": 2, "fuu": 3}, + TotalResponseTime: time.Now(), + }, + StatsRecorder: &middlewares.StatsRecorder{}, + } + config.Marathon = &marathon.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "marathon Filename", + Constraints: types.Constraints{ + { + Key: "marathon Constraints Key 1", + Regex: "marathon Constraints Regex 2", + MustMatch: true, + }, + { + Key: "marathon Constraints Key 1", + Regex: "marathon Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "", + Domain: "", + ExposedByDefault: true, + GroupsAsSubDomains: true, + DCOSToken: "", + MarathonLBCompatibility: true, + TLS: &types.ClientTLS{ + CA: "marathon CA", + Cert: "marathon Cert", + Key: "marathon Key", + InsecureSkipVerify: true, + }, + DialerTimeout: flaeg.Duration(666 * time.Second), + KeepAlive: flaeg.Duration(666 * time.Second), + ForceTaskHostname: true, + Basic: &marathon.Basic{ + HTTPBasicAuthUser: "marathon HTTPBasicAuthUser", + HTTPBasicPassword: "marathon HTTPBasicPassword", + }, + RespectReadinessChecks: true, + } + config.ConsulCatalog = &consul.CatalogProvider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "ConsulCatalog Filename", + Constraints: types.Constraints{ + { + Key: "ConsulCatalog Constraints Key 1", + Regex: "ConsulCatalog Constraints Regex 2", + MustMatch: true, + }, + { + Key: "ConsulCatalog Constraints Key 1", + Regex: "ConsulCatalog Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "ConsulCatalog Endpoint", + Domain: "ConsulCatalog Domain", + ExposedByDefault: true, + Prefix: "ConsulCatalog Prefix", + FrontEndRule: "ConsulCatalog FrontEndRule", + } + config.Kubernetes = &kubernetes.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "k8s Filename", + Constraints: types.Constraints{ + { + Key: "k8s Constraints Key 1", + Regex: "k8s Constraints Regex 2", + MustMatch: true, + }, + { + Key: "k8s Constraints Key 1", + Regex: "k8s Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "k8s Endpoint", + Token: "k8s Token", + CertAuthFilePath: "k8s CertAuthFilePath", + DisablePassHostHeaders: true, + Namespaces: kubernetes.Namespaces{"k8s Namespaces 1", "k8s Namespaces 2", "k8s Namespaces 3"}, + LabelSelector: "k8s LabelSelector", + } + config.Mesos = &mesos.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "mesos Filename", + Constraints: types.Constraints{ + { + Key: "mesos Constraints Key 1", + Regex: "mesos Constraints Regex 2", + MustMatch: true, + }, + { + Key: "mesos Constraints Key 1", + Regex: "mesos Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "mesos Endpoint", + Domain: "mesos Domain", + ExposedByDefault: true, + GroupsAsSubDomains: true, + ZkDetectionTimeout: 666, + RefreshSeconds: 666, + IPSources: "mesos IPSources", + StateTimeoutSecond: 666, + Masters: []string{"mesos Masters 1", "mesos Masters 2", "mesos Masters 3"}, + } + config.Eureka = &eureka.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "eureka Filename", + Constraints: types.Constraints{ + { + Key: "eureka Constraints Key 1", + Regex: "eureka Constraints Regex 2", + MustMatch: true, + }, + { + Key: "eureka Constraints Key 1", + Regex: "eureka Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "eureka Endpoint", + Delay: "eureka Delay", + } + config.ECS = &ecs.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "ecs Filename", + Constraints: types.Constraints{ + { + Key: "ecs Constraints Key 1", + Regex: "ecs Constraints Regex 2", + MustMatch: true, + }, + { + Key: "ecs Constraints Key 1", + Regex: "ecs Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Domain: "ecs Domain", + ExposedByDefault: true, + RefreshSeconds: 666, + Clusters: ecs.Clusters{"ecs Clusters 1", "ecs Clusters 2", "ecs Clusters 3"}, + Cluster: "ecs Cluster", + AutoDiscoverClusters: true, + Region: "ecs Region", + AccessKeyID: "ecs AccessKeyID", + SecretAccessKey: "ecs SecretAccessKey", + } + config.Rancher = &rancher.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "rancher Filename", + Constraints: types.Constraints{ + { + Key: "rancher Constraints Key 1", + Regex: "rancher Constraints Regex 2", + MustMatch: true, + }, + { + Key: "rancher Constraints Key 1", + Regex: "rancher Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + APIConfiguration: rancher.APIConfiguration{ + Endpoint: "rancher Endpoint", + AccessKey: "rancher AccessKey", + SecretKey: "rancher SecretKey", + }, + API: &rancher.APIConfiguration{ + Endpoint: "rancher Endpoint", + AccessKey: "rancher AccessKey", + SecretKey: "rancher SecretKey", + }, + Metadata: &rancher.MetadataConfiguration{ + IntervalPoll: true, + Prefix: "rancher Metadata Prefix", + }, + Domain: "rancher Domain", + RefreshSeconds: 666, + ExposedByDefault: true, + EnableServiceHealthFilter: true, + } + config.DynamoDB = &dynamodb.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "dynamodb Filename", + Constraints: types.Constraints{ + { + Key: "dynamodb Constraints Key 1", + Regex: "dynamodb Constraints Regex 2", + MustMatch: true, + }, + { + Key: "dynamodb Constraints Key 1", + Regex: "dynamodb Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + AccessKeyID: "dynamodb AccessKeyID", + RefreshSeconds: 666, + Region: "dynamodb Region", + SecretAccessKey: "dynamodb SecretAccessKey", + TableName: "dynamodb TableName", + Endpoint: "dynamodb Endpoint", + } + config.Etcd = &etcd.Provider{ + Provider: kv.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "etcd Filename", + Constraints: types.Constraints{ + { + Key: "etcd Constraints Key 1", + Regex: "etcd Constraints Regex 2", + MustMatch: true, + }, + { + Key: "etcd Constraints Key 1", + Regex: "etcd Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "etcd Endpoint", + Prefix: "etcd Prefix", + TLS: &types.ClientTLS{ + CA: "etcd CA", + Cert: "etcd Cert", + Key: "etcd Key", + InsecureSkipVerify: true, + }, + Username: "etcd Username", + Password: "etcd Password", + }, + } + config.Zookeeper = &zk.Provider{ + Provider: kv.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "zk Filename", + Constraints: types.Constraints{ + { + Key: "zk Constraints Key 1", + Regex: "zk Constraints Regex 2", + MustMatch: true, + }, + { + Key: "zk Constraints Key 1", + Regex: "zk Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "zk Endpoint", + Prefix: "zk Prefix", + TLS: &types.ClientTLS{ + CA: "zk CA", + Cert: "zk Cert", + Key: "zk Key", + InsecureSkipVerify: true, + }, + Username: "zk Username", + Password: "zk Password", + }, + } + config.Boltdb = &boltdb.Provider{ + Provider: kv.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "boltdb Filename", + Constraints: types.Constraints{ + { + Key: "boltdb Constraints Key 1", + Regex: "boltdb Constraints Regex 2", + MustMatch: true, + }, + { + Key: "boltdb Constraints Key 1", + Regex: "boltdb Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "boltdb Endpoint", + Prefix: "boltdb Prefix", + TLS: &types.ClientTLS{ + CA: "boltdb CA", + Cert: "boltdb Cert", + Key: "boltdb Key", + InsecureSkipVerify: true, + }, + Username: "boltdb Username", + Password: "boltdb Password", + }, + } + config.Consul = &consul.Provider{ + Provider: kv.Provider{ + BaseProvider: provider.BaseProvider{ + Watch: true, + Filename: "consul Filename", + Constraints: types.Constraints{ + { + Key: "consul Constraints Key 1", + Regex: "consul Constraints Regex 2", + MustMatch: true, + }, + { + Key: "consul Constraints Key 1", + Regex: "consul Constraints Regex 2", + MustMatch: true, + }, + }, + Trace: true, + DebugLogGeneratedTemplate: true, + }, + Endpoint: "consul Endpoint", + Prefix: "consul Prefix", + TLS: &types.ClientTLS{ + CA: "consul CA", + Cert: "consul Cert", + Key: "consul Key", + InsecureSkipVerify: true, + }, + Username: "consul Username", + Password: "consul Password", + }, + } + + cleanJSON, err := Do(config, true) + if err != nil { + t.Fatal(err, cleanJSON) + } +} diff --git a/cmd/traefik/anonymize/anonymize_doOnJSON_test.go b/cmd/traefik/anonymize/anonymize_doOnJSON_test.go new file mode 100644 index 000000000..87f43d125 --- /dev/null +++ b/cmd/traefik/anonymize/anonymize_doOnJSON_test.go @@ -0,0 +1,238 @@ +package anonymize + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_doOnJSON(t *testing.T) { + baseConfiguration := ` +{ + "GraceTimeOut": 10000000000, + "Debug": false, + "CheckNewVersion": true, + "AccessLogsFile": "", + "TraefikLogsFile": "", + "LogLevel": "ERROR", + "EntryPoints": { + "http": { + "Network": "", + "Address": ":80", + "TLS": null, + "Redirect": { + "EntryPoint": "https", + "Regex": "", + "Replacement": "" + }, + "Auth": null, + "Compress": false + }, + "https": { + "Network": "", + "Address": ":443", + "TLS": { + "MinVersion": "", + "CipherSuites": null, + "Certificates": null, + "ClientCAFiles": null + }, + "Redirect": null, + "Auth": null, + "Compress": false + } + }, + "Cluster": null, + "Constraints": [], + "ACME": { + "Email": "foo@bar.com", + "Domains": [ + { + "Main": "foo@bar.com", + "SANs": null + }, + { + "Main": "foo@bar.com", + "SANs": null + } + ], + "Storage": "", + "StorageFile": "/acme/acme.json", + "OnDemand": true, + "OnHostRule": true, + "CAServer": "", + "EntryPoint": "https", + "DNSProvider": "", + "DelayDontCheckDNS": 0, + "ACMELogging": false, + "TLSConfig": null + }, + "DefaultEntryPoints": [ + "https", + "http" + ], + "ProvidersThrottleDuration": 2000000000, + "MaxIdleConnsPerHost": 200, + "IdleTimeout": 180000000000, + "InsecureSkipVerify": false, + "Retry": null, + "HealthCheck": { + "Interval": 30000000000 + }, + "Docker": null, + "File": null, + "Web": null, + "Marathon": null, + "Consul": null, + "ConsulCatalog": null, + "Etcd": null, + "Zookeeper": null, + "Boltdb": null, + "Kubernetes": null, + "Mesos": null, + "Eureka": null, + "ECS": null, + "Rancher": null, + "DynamoDB": null, + "ConfigFile": "/etc/traefik/traefik.toml" +} +` + expectedConfiguration := ` +{ + "GraceTimeOut": 10000000000, + "Debug": false, + "CheckNewVersion": true, + "AccessLogsFile": "", + "TraefikLogsFile": "", + "LogLevel": "ERROR", + "EntryPoints": { + "http": { + "Network": "", + "Address": ":80", + "TLS": null, + "Redirect": { + "EntryPoint": "https", + "Regex": "", + "Replacement": "" + }, + "Auth": null, + "Compress": false + }, + "https": { + "Network": "", + "Address": ":443", + "TLS": { + "MinVersion": "", + "CipherSuites": null, + "Certificates": null, + "ClientCAFiles": null + }, + "Redirect": null, + "Auth": null, + "Compress": false + } + }, + "Cluster": null, + "Constraints": [], + "ACME": { + "Email": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "Domains": [ + { + "Main": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "SANs": null + }, + { + "Main": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "SANs": null + } + ], + "Storage": "", + "StorageFile": "/acme/acme.json", + "OnDemand": true, + "OnHostRule": true, + "CAServer": "", + "EntryPoint": "https", + "DNSProvider": "", + "DelayDontCheckDNS": 0, + "ACMELogging": false, + "TLSConfig": null + }, + "DefaultEntryPoints": [ + "https", + "http" + ], + "ProvidersThrottleDuration": 2000000000, + "MaxIdleConnsPerHost": 200, + "IdleTimeout": 180000000000, + "InsecureSkipVerify": false, + "Retry": null, + "HealthCheck": { + "Interval": 30000000000 + }, + "Docker": null, + "File": null, + "Web": null, + "Marathon": null, + "Consul": null, + "ConsulCatalog": null, + "Etcd": null, + "Zookeeper": null, + "Boltdb": null, + "Kubernetes": null, + "Mesos": null, + "Eureka": null, + "ECS": null, + "Rancher": null, + "DynamoDB": null, + "ConfigFile": "/etc/traefik/traefik.toml" +} +` + anomConfiguration := doOnJSON(baseConfiguration) + + if anomConfiguration != expectedConfiguration { + t.Errorf("Got %s, want %s.", anomConfiguration, expectedConfiguration) + } +} + +func Test_doOnJSON_simple(t *testing.T) { + testCases := []struct { + name string + input string + expectedOutput string + }{ + { + name: "email", + input: `{ + "email1": "goo@example.com", + "email2": "foo.bargoo@example.com", + "email3": "foo.bargoo@example.com.us" + }`, + expectedOutput: `{ + "email1": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "email2": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "email3": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }`, + }, + { + name: "url", + input: `{ + "URL": "foo domain.com foo", + "URL": "foo sub.domain.com foo", + "URL": "foo sub.sub.domain.com foo", + "URL": "foo sub.sub.sub.domain.com.us foo" + }`, + expectedOutput: `{ + "URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo", + "URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo", + "URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo", + "URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo" + }`, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + output := doOnJSON(test.input) + assert.Equal(t, test.expectedOutput, output) + }) + } +} diff --git a/cmd/traefik/anonymize/anonymize_doOnStruct_test.go b/cmd/traefik/anonymize/anonymize_doOnStruct_test.go new file mode 100644 index 000000000..3d9529682 --- /dev/null +++ b/cmd/traefik/anonymize/anonymize_doOnStruct_test.go @@ -0,0 +1,176 @@ +package anonymize + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +type Courgette struct { + Ji string + Ho string +} +type Tomate struct { + Ji string + Ho string +} + +type Carotte struct { + Name string + Value int + Courgette Courgette + ECourgette Courgette `export:"true"` + Pourgette *Courgette + EPourgette *Courgette `export:"true"` + Aubergine map[string]string + EAubergine map[string]string `export:"true"` + SAubergine map[string]Tomate + ESAubergine map[string]Tomate `export:"true"` + PSAubergine map[string]*Tomate + EPAubergine map[string]*Tomate `export:"true"` +} + +func Test_doOnStruct(t *testing.T) { + testCase := []struct { + name string + base *Carotte + expected *Carotte + hasError bool + }{ + { + name: "primitive", + base: &Carotte{ + Name: "koko", + Value: 666, + }, + expected: &Carotte{ + Name: "xxxx", + }, + }, + { + name: "struct", + base: &Carotte{ + Name: "koko", + Courgette: Courgette{ + Ji: "huu", + }, + }, + expected: &Carotte{ + Name: "xxxx", + }, + }, + { + name: "pointer", + base: &Carotte{ + Name: "koko", + Pourgette: &Courgette{ + Ji: "hoo", + }, + }, + expected: &Carotte{ + Name: "xxxx", + Pourgette: nil, + }, + }, + { + name: "export struct", + base: &Carotte{ + Name: "koko", + ECourgette: Courgette{ + Ji: "huu", + }, + }, + expected: &Carotte{ + Name: "xxxx", + ECourgette: Courgette{ + Ji: "xxxx", + }, + }, + }, + { + name: "export pointer struct", + base: &Carotte{ + Name: "koko", + ECourgette: Courgette{ + Ji: "huu", + }, + }, + expected: &Carotte{ + Name: "xxxx", + ECourgette: Courgette{ + Ji: "xxxx", + }, + }, + }, + { + name: "export map string/string", + base: &Carotte{ + Name: "koko", + EAubergine: map[string]string{ + "foo": "bar", + }, + }, + expected: &Carotte{ + Name: "xxxx", + EAubergine: map[string]string{ + "foo": "bar", + }, + }, + }, + { + name: "export map string/pointer", + base: &Carotte{ + Name: "koko", + EPAubergine: map[string]*Tomate{ + "foo": { + Ji: "fdskljf", + }, + }, + }, + expected: &Carotte{ + Name: "xxxx", + EPAubergine: map[string]*Tomate{ + "foo": { + Ji: "xxxx", + }, + }, + }, + }, + { + name: "export map string/struct (UNSAFE)", + base: &Carotte{ + Name: "koko", + ESAubergine: map[string]Tomate{ + "foo": { + Ji: "JiJiJi", + }, + }, + }, + expected: &Carotte{ + Name: "xxxx", + ESAubergine: map[string]Tomate{ + "foo": { + Ji: "JiJiJi", + }, + }, + }, + hasError: true, + }, + } + + for _, test := range testCase { + t.Run(test.name, func(t *testing.T) { + val := reflect.ValueOf(test.base).Elem() + err := doOnStruct(val) + if !test.hasError && err != nil { + t.Fatal(err) + } + if test.hasError && err == nil { + t.Fatal("Got no error but want an error.") + } + + assert.EqualValues(t, test.expected, test.base) + }) + } +} diff --git a/cmd/traefik/bug.go b/cmd/traefik/bug.go index 66913c27f..27d99d389 100644 --- a/cmd/traefik/bug.go +++ b/cmd/traefik/bug.go @@ -2,20 +2,18 @@ package main import ( "bytes" - "encoding/json" "fmt" "net/url" "os/exec" - "regexp" "runtime" "text/template" "github.com/containous/flaeg" - "github.com/mvdan/xurls" + "github.com/containous/traefik/cmd/traefik/anonymize" ) -var ( - bugtracker = "https://github.com/containous/traefik/issues/new" +const ( + bugTracker = "https://github.com/containous/traefik/issues/new" bugTemplate = `