Merge branch 'master' into feature/fix-gzip-for-websockets

CHANGELOG.md

@@ -1,5 +1,44 @@
 # Change Log
 
+## [v1.1.2](https://github.com/containous/traefik/tree/v1.1.2) (2016-12-15)
+[Full Changelog](https://github.com/containous/traefik/compare/v1.1.1...v1.1.2)
+
+**Fixed bugs:**
+
+- Problem during HTTPS redirection [\#952](https://github.com/containous/traefik/issues/952)
+- nil pointer with kubernetes ingress [\#934](https://github.com/containous/traefik/issues/934)
+- ConsulCatalog and File not working [\#903](https://github.com/containous/traefik/issues/903)
+- Traefik can not start [\#902](https://github.com/containous/traefik/issues/902)
+- Cannot connect to Kubernetes server failed to decode watch event [\#532](https://github.com/containous/traefik/issues/532)
+
+**Closed issues:**
+
+- Updating certificates with configuration file. [\#968](https://github.com/containous/traefik/issues/968)
+- Let's encrypt retrieving certificate from wrong IP [\#962](https://github.com/containous/traefik/issues/962)
+- let's encrypt and dashboard? [\#961](https://github.com/containous/traefik/issues/961)
+- Working HTTPS example for GKE? [\#960](https://github.com/containous/traefik/issues/960)
+- GKE design pattern [\#958](https://github.com/containous/traefik/issues/958)
+- Consul Catalog constraints does not seem to work [\#954](https://github.com/containous/traefik/issues/954)
+- Issue in building traefik from master [\#949](https://github.com/containous/traefik/issues/949)
+- Proxy http application to https doesn't seem to work correctly for all services [\#937](https://github.com/containous/traefik/issues/937)
+- Excessive requests to kubernetes apiserver [\#922](https://github.com/containous/traefik/issues/922)
+- I am getting a connection error while creating traefik with consul backend "dial tcp 127.0.0.1:8500: getsockopt: connection refused" [\#917](https://github.com/containous/traefik/issues/917)
+- SwarmMode - 1.13 RC2 - DNS RR - Individual IPs not retrieved [\#913](https://github.com/containous/traefik/issues/913)
+- Panic in kubernetes ingress \(traefik 1.1.0\) [\#910](https://github.com/containous/traefik/issues/910)
+- Kubernetes updating deployment image requires Ingress to be remade  [\#909](https://github.com/containous/traefik/issues/909)
+- \[ACME\] Too many currently pending authorizations [\#905](https://github.com/containous/traefik/issues/905)
+- WEB UI Authentication and Let's Encrypt : error 404 [\#754](https://github.com/containous/traefik/issues/754)
+- Traefik as ingress controller for SNI based routing in kubernetes [\#745](https://github.com/containous/traefik/issues/745)
+- Kubernetes Ingress backend: using self-signed certificates [\#486](https://github.com/containous/traefik/issues/486)
+- Kubernetes Ingress backend: can't find token and ca.crt [\#484](https://github.com/containous/traefik/issues/484)
+
+**Merged pull requests:**
+
+- Fix duplicate acme certificates [\#972](https://github.com/containous/traefik/pull/972) ([emilevauge](https://github.com/emilevauge))
+- Fix leadership panic [\#956](https://github.com/containous/traefik/pull/956) ([emilevauge](https://github.com/emilevauge))
+- Fix redirect regex [\#947](https://github.com/containous/traefik/pull/947) ([emilevauge](https://github.com/emilevauge))
+- Add operation recover [\#944](https://github.com/containous/traefik/pull/944) ([emilevauge](https://github.com/emilevauge))
+
 ## [v1.1.1](https://github.com/containous/traefik/tree/v1.1.1) (2016-11-29)
 [Full Changelog](https://github.com/containous/traefik/compare/v1.1.0...v1.1.1)
 

acme/account.go

@@ -8,6 +8,8 @@ import (
 	"crypto/x509"
 	"errors"
 	"reflect"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -107,6 +109,38 @@ type DomainsCertificates struct {
 	lock  sync.RWMutex
 }
 
+func (dc *DomainsCertificates) Len() int {
+	return len(dc.Certs)
+}
+
+func (dc *DomainsCertificates) Swap(i, j int) {
+	dc.Certs[i], dc.Certs[j] = dc.Certs[j], dc.Certs[i]
+}
+
+func (dc *DomainsCertificates) Less(i, j int) bool {
+	if reflect.DeepEqual(dc.Certs[i].Domains, dc.Certs[j].Domains) {
+		return dc.Certs[i].tlsCert.Leaf.NotAfter.After(dc.Certs[j].tlsCert.Leaf.NotAfter)
+	}
+	if dc.Certs[i].Domains.Main == dc.Certs[j].Domains.Main {
+		return strings.Join(dc.Certs[i].Domains.SANs, ",") < strings.Join(dc.Certs[j].Domains.SANs, ",")
+	}
+	return dc.Certs[i].Domains.Main < dc.Certs[j].Domains.Main
+}
+
+func (dc *DomainsCertificates) removeDuplicates() {
+	sort.Sort(dc)
+	for i := 0; i < len(dc.Certs); i++ {
+		for i2 := i + 1; i2 < len(dc.Certs); i2++ {
+			if reflect.DeepEqual(dc.Certs[i].Domains, dc.Certs[i2].Domains) {
+				// delete
+				log.Warnf("Remove duplicate cert: %+v, expiration :%s", dc.Certs[i2].Domains, dc.Certs[i2].tlsCert.Leaf.NotAfter.String())
+				dc.Certs = append(dc.Certs[:i2], dc.Certs[i2+1:]...)
+				i2--
+			}
+		}
+	}
+}
+
 // Init inits DomainsCertificates
 func (dc *DomainsCertificates) Init() error {
 	dc.lock.Lock()
@@ -117,7 +151,15 @@ func (dc *DomainsCertificates) Init() error {
 			return err
 		}
 		domainsCertificate.tlsCert = &tlsCert
+		if domainsCertificate.tlsCert.Leaf == nil {
+			leaf, err := x509.ParseCertificate(domainsCertificate.tlsCert.Certificate[0])
+			if err != nil {
+				return err
+			}
+			domainsCertificate.tlsCert.Leaf = leaf
+		}
 	}
+	dc.removeDuplicates()
 	return nil
 }
 

acme/acme.go

@@ -19,6 +19,7 @@ import (
 	"github.com/containous/traefik/log"
 	"github.com/containous/traefik/safe"
 	"github.com/containous/traefik/types"
+	"github.com/eapache/channels"
 	"github.com/xenolf/lego/acme"
 	"github.com/xenolf/lego/providers/dns"
 )
@@ -46,6 +47,7 @@ type ACME struct {
 	store               cluster.Store
 	challengeProvider   *challengeProvider
 	checkOnDemandDomain func(domain string) bool
+	jobs                *channels.InfiniteChannel
 	TLSConfig           *tls.Config `description:"TLS config in case wildcard certs are used"`
 }
 
@@ -107,6 +109,7 @@ func (a *ACME) init() error {
 		log.Warnf("ACME.StorageFile is deprecated, use ACME.Storage instead")
 		a.Storage = a.StorageFile
 	}
+	a.jobs = channels.NewInfiniteChannel()
 	return nil
 }
 
@@ -159,9 +162,7 @@ func (a *ACME) CreateClusterConfig(leadership *cluster.Leadership, tlsConfig *tl
 			case <-ctx.Done():
 				return
 			case <-ticker.C:
-				if err := a.renewCertificates(); err != nil {
+				a.renewCertificates()
-					log.Errorf("Error renewing ACME certificate: %s", err.Error())
-				}
 			}
 		}
 	})
@@ -222,12 +223,10 @@ func (a *ACME) CreateClusterConfig(leadership *cluster.Leadership, tlsConfig *tl
 			if err != nil {
 				return err
 			}
-			safe.Go(func() {
+
-				a.retrieveCertificates()
+			a.retrieveCertificates()
-				if err := a.renewCertificates(); err != nil {
+			a.renewCertificates()
-					log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
+			a.runJobs()
-				}
-			})
 		}
 		return nil
 	})
@@ -312,19 +311,14 @@ func (a *ACME) CreateLocalConfig(tlsConfig *tls.Config, checkOnDemandDomain func
 		return err
 	}
 
-	safe.Go(func() {
+	a.retrieveCertificates()
-		a.retrieveCertificates()
+	a.renewCertificates()
-		if err := a.renewCertificates(); err != nil {
+	a.runJobs()
-			log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
-		}
-	})
 
 	ticker := time.NewTicker(24 * time.Hour)
 	safe.Go(func() {
 		for range ticker.C {
-			if err := a.renewCertificates(); err != nil {
+			a.renewCertificates()
-				log.Errorf("Error renewing ACME certificate %+v: %s", account, err.Error())
-			}
 		}
 
 	})
@@ -361,83 +355,87 @@ func (a *ACME) getCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificat
 }
 
 func (a *ACME) retrieveCertificates() {
-	log.Infof("Retrieving ACME certificates...")
+	a.jobs.In() <- func() {
-	for _, domain := range a.Domains {
+		log.Infof("Retrieving ACME certificates...")
-		// check if cert isn't already loaded
+		for _, domain := range a.Domains {
-		account := a.store.Get().(*Account)
+			// check if cert isn't already loaded
-		if _, exists := account.DomainsCertificate.exists(domain); !exists {
+			account := a.store.Get().(*Account)
-			domains := []string{}
+			if _, exists := account.DomainsCertificate.exists(domain); !exists {
-			domains = append(domains, domain.Main)
+				domains := []string{}
-			domains = append(domains, domain.SANs...)
+				domains = append(domains, domain.Main)
-			certificateResource, err := a.getDomainsCertificates(domains)
+				domains = append(domains, domain.SANs...)
-			if err != nil {
+				certificateResource, err := a.getDomainsCertificates(domains)
-				log.Errorf("Error getting ACME certificate for domain %s: %s", domains, err.Error())
+				if err != nil {
-				continue
+					log.Errorf("Error getting ACME certificate for domain %s: %s", domains, err.Error())
-			}
+					continue
-			transaction, object, err := a.store.Begin()
+				}
-			if err != nil {
+				transaction, object, err := a.store.Begin()
-				log.Errorf("Error creating ACME store transaction from domain %s: %s", domain, err.Error())
+				if err != nil {
-				continue
+					log.Errorf("Error creating ACME store transaction from domain %s: %s", domain, err.Error())
-			}
+					continue
-			account = object.(*Account)
+				}
-			_, err = account.DomainsCertificate.addCertificateForDomains(certificateResource, domain)
+				account = object.(*Account)
-			if err != nil {
+				_, err = account.DomainsCertificate.addCertificateForDomains(certificateResource, domain)
-				log.Errorf("Error adding ACME certificate for domain %s: %s", domains, err.Error())
+				if err != nil {
-				continue
+					log.Errorf("Error adding ACME certificate for domain %s: %s", domains, err.Error())
-			}
+					continue
+				}
 
-			if err = transaction.Commit(account); err != nil {
+				if err = transaction.Commit(account); err != nil {
-				log.Errorf("Error Saving ACME account %+v: %s", account, err.Error())
+					log.Errorf("Error Saving ACME account %+v: %s", account, err.Error())
-				continue
+					continue
+				}
 			}
 		}
+		log.Infof("Retrieved ACME certificates")
 	}
-	log.Infof("Retrieved ACME certificates")
 }
 
-func (a *ACME) renewCertificates() error {
+func (a *ACME) renewCertificates() {
-	log.Debugf("Testing certificate renew...")
+	a.jobs.In() <- func() {
-	account := a.store.Get().(*Account)
+		log.Debugf("Testing certificate renew...")
-	for _, certificateResource := range account.DomainsCertificate.Certs {
+		account := a.store.Get().(*Account)
-		if certificateResource.needRenew() {
+		for _, certificateResource := range account.DomainsCertificate.Certs {
-			log.Debugf("Renewing certificate %+v", certificateResource.Domains)
+			if certificateResource.needRenew() {
-			renewedCert, err := a.client.RenewCertificate(acme.CertificateResource{
+				log.Debugf("Renewing certificate %+v", certificateResource.Domains)
-				Domain:        certificateResource.Certificate.Domain,
+				renewedCert, err := a.client.RenewCertificate(acme.CertificateResource{
-				CertURL:       certificateResource.Certificate.CertURL,
+					Domain:        certificateResource.Certificate.Domain,
-				CertStableURL: certificateResource.Certificate.CertStableURL,
+					CertURL:       certificateResource.Certificate.CertURL,
-				PrivateKey:    certificateResource.Certificate.PrivateKey,
+					CertStableURL: certificateResource.Certificate.CertStableURL,
-				Certificate:   certificateResource.Certificate.Certificate,
+					PrivateKey:    certificateResource.Certificate.PrivateKey,
-			}, true, OSCPMustStaple)
+					Certificate:   certificateResource.Certificate.Certificate,
-			if err != nil {
+				}, true, OSCPMustStaple)
-				log.Errorf("Error renewing certificate: %v", err)
+				if err != nil {
-				continue
+					log.Errorf("Error renewing certificate: %v", err)
-			}
+					continue
-			log.Debugf("Renewed certificate %+v", certificateResource.Domains)
+				}
-			renewedACMECert := &Certificate{
+				log.Debugf("Renewed certificate %+v", certificateResource.Domains)
-				Domain:        renewedCert.Domain,
+				renewedACMECert := &Certificate{
-				CertURL:       renewedCert.CertURL,
+					Domain:        renewedCert.Domain,
-				CertStableURL: renewedCert.CertStableURL,
+					CertURL:       renewedCert.CertURL,
-				PrivateKey:    renewedCert.PrivateKey,
+					CertStableURL: renewedCert.CertStableURL,
-				Certificate:   renewedCert.Certificate,
+					PrivateKey:    renewedCert.PrivateKey,
-			}
+					Certificate:   renewedCert.Certificate,
-			transaction, object, err := a.store.Begin()
+				}
-			if err != nil {
+				transaction, object, err := a.store.Begin()
-				return err
+				if err != nil {
-			}
+					log.Errorf("Error renewing certificate: %v", err)
-			account = object.(*Account)
+					continue
-			err = account.DomainsCertificate.renewCertificates(renewedACMECert, certificateResource.Domains)
+				}
-			if err != nil {
+				account = object.(*Account)
-				log.Errorf("Error renewing certificate: %v", err)
+				err = account.DomainsCertificate.renewCertificates(renewedACMECert, certificateResource.Domains)
-				continue
+				if err != nil {
-			}
+					log.Errorf("Error renewing certificate: %v", err)
+					continue
+				}
 
-			if err = transaction.Commit(account); err != nil {
+				if err = transaction.Commit(account); err != nil {
-				log.Errorf("Error Saving ACME account %+v: %s", account, err.Error())
+					log.Errorf("Error Saving ACME account %+v: %s", account, err.Error())
-				continue
+					continue
+				}
 			}
 		}
 	}
-	return nil
 }
 
 func dnsOverrideDelay(delay int) error {
@@ -521,8 +519,9 @@ func (a *ACME) loadCertificateOnDemand(clientHello *tls.ClientHelloInfo) (*tls.C
 
 // LoadCertificateForDomains loads certificates from ACME for given domains
 func (a *ACME) LoadCertificateForDomains(domains []string) {
-	domains = fun.Map(types.CanonicalDomain, domains).([]string)
+	a.jobs.In() <- func() {
-	safe.Go(func() {
+		log.Debugf("LoadCertificateForDomains %s...", domains)
+		domains = fun.Map(types.CanonicalDomain, domains).([]string)
 		operation := func() error {
 			if a.client == nil {
 				return fmt.Errorf("ACME client still not built")
@@ -534,7 +533,7 @@ func (a *ACME) LoadCertificateForDomains(domains []string) {
 		}
 		ebo := backoff.NewExponentialBackOff()
 		ebo.MaxElapsedTime = 30 * time.Second
-		err := backoff.RetryNotify(operation, ebo, notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify)
 		if err != nil {
 			log.Errorf("Error getting ACME client: %v", err)
 			return
@@ -576,7 +575,7 @@ func (a *ACME) LoadCertificateForDomains(domains []string) {
 			log.Errorf("Error Saving ACME account %+v: %v", account, err)
 			return
 		}
-	})
+	}
 }
 
 func (a *ACME) getDomainsCertificates(domains []string) (*Certificate, error) {
@@ -597,3 +596,12 @@ func (a *ACME) getDomainsCertificates(domains []string) (*Certificate, error) {
 		Certificate:   certificate.Certificate,
 	}, nil
 }
+
+func (a *ACME) runJobs() {
+	safe.Go(func() {
+		for job := range a.jobs.Out() {
+			function := job.(func())
+			function()
+		}
+	})
+}

acme/acme_test.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/xenolf/lego/acme"
 )
@@ -67,6 +68,8 @@ func TestDomainsSetAppend(t *testing.T) {
 }
 
 func TestCertificatesRenew(t *testing.T) {
+	foo1Cert, foo1Key, _ := generateKeyPair("foo1.com", time.Now())
+	foo2Cert, foo2Key, _ := generateKeyPair("foo2.com", time.Now())
 	domainsCertificates := DomainsCertificates{
 		lock: sync.RWMutex{},
 		Certs: []*DomainsCertificate{
@@ -78,55 +81,8 @@ func TestCertificatesRenew(t *testing.T) {
 					Domain:        "foo1.com",
 					CertURL:       "url",
 					CertStableURL: "url",
-					PrivateKey: []byte(`
+					PrivateKey:    foo1Key,
------BEGIN RSA PRIVATE KEY-----
+					Certificate:   foo1Cert,
-MIIEowIBAAKCAQEA6OqHGdwGy20+3Jcz9IgfN4IR322X2Hhwk6n8Hss/Ws7FeTZo
-PvXW8uHeI1bmQJsy9C6xo3odzO64o7prgMZl5eDw5fk1mmUij3J3nM3gwtc/Cc+8
-ADXGldauASdHBFTRvWQge0Pv/Q5U0fyL2VCHoR9mGv4CQ7nRNKPus0vYJMbXoTbO
-8z4sIbNz3Ov9o/HGMRb8D0rNPTMdC62tHSbiO1UoxLXr9dcBOGt786AsiRTJ8bq9
-GCVQgzd0Wftb8z6ddW2YuWrmExlkHdfC4oG0D5SU1QB4ldPyl7fhVWlfHwC1NX+c
-RnDSEeYkAcdvvIekdM/yH+z62XhwToM0E9TCzwIDAQABAoIBACq3EC3S50AZeeTU
-qgeXizoP1Z1HKQjfFa5PB1jSZ30M3LRdIQMi7NfASo/qmPGSROb5RUS42YxC34PP
-ZXXJbNiaxzM13/m/wHXURVFxhF3XQc1X1p+nPRMvutulS2Xk9E4qdbaFgBbFsRKN
-oUwqc6U97+jVWq72/gIManNhXnNn1n1SRLBEkn+WStMPn6ZvWRlpRMjhy0c1mpwg
-u6em92HvMvfKPQ60naUhdKp+q0rsLp2YKWjiytos9ENSYI5gAGLIDhKeqiD8f92E
-4FGPmNRipwxCE2SSvZFlM26tRloWVcBPktRN79hUejE8iopiqVS0+4h/phZ2wG0D
-18cqVpECgYEA+qmagnhm0LLvwVkUN0B2nRARQEFinZDM4Hgiv823bQvc9I8dVTqJ
-aIQm5y4Y5UA3xmyDsRoO7GUdd0oVeh9GwTONzMRCOny/mOuOC51wXPhKHhI0O22u
-sfbOHszl+bxl6ZQMUJa2/I8YIWBLU5P+fTgrfNwBEgZ3YPwUV5tyHNcCgYEA7eAv
-pjQkbJNRq/fv/67sojN7N9QoH84egN5cZFh5d8PJomnsvy5JDV4WaG1G6mJpqjdD
-YRVdFw5oZ4L8yCVdCeK9op896Uy51jqvfSe3+uKmNqE0qDHgaLubQNI8yYc5sacW
-fYJBmDR6rNIeE7Q2240w3CdKfREuXdDnhyTTEskCgYBFeAnFTP8Zqe2+hSSQJ4J4
-BwLw7u4Yww+0yja/N5E1XItRD/TOMRnx6GYrvd/ScVjD2kEpLRKju2ZOMC8BmHdw
-hgwvitjcAsTK6cWFPI3uhjVsXhkxuzUmR0Naz+iQrQEFmi1LjGmMV1AVt+1IbYSj
-SZTr1sFJMJeXPmWY3hDjIwKBgQC4H9fCJoorIL0PB5NVreishHzT8fw84ibqSTPq
-2DDtazcf6C3AresN1c4ydqN1uUdg4fXdp9OujRBzTwirQ4CIrmFrBye89g7CrBo6
-Hgxivh06G/3OUw0JBG5f9lvnAiy+Pj9CVxi+36A1NU7ioZP0zY0MW71koW/qXlFY
-YkCfQQKBgBqwND/c3mPg7iY4RMQ9XjrKfV9o6FMzA51lAinjujHlNgsBmqiR951P
-NA3kWZQ73D3IxeLEMaGHpvS7andPN3Z2qPhe+FbJKcF6ZZNTrFQkh/Fpz3wmYPo1
-GIL4+09kNgMRWapaROqI+/3+qJQ+GVJZIPfYC0poJOO6vYqifWe8
------END RSA PRIVATE KEY-----
-`),
-					Certificate: []byte(`
------BEGIN CERTIFICATE-----
-MIIC+TCCAeGgAwIBAgIJAK78ukR/Qu4rMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
-BAMMCGZvbzEuY29tMB4XDTE2MDYxOTIyMDMyM1oXDTI2MDYxNzIyMDMyM1owEzER
-MA8GA1UEAwwIZm9vMS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-AQDo6ocZ3AbLbT7clzP0iB83ghHfbZfYeHCTqfweyz9azsV5Nmg+9dby4d4jVuZA
-mzL0LrGjeh3M7rijumuAxmXl4PDl+TWaZSKPcneczeDC1z8Jz7wANcaV1q4BJ0cE
-VNG9ZCB7Q+/9DlTR/IvZUIehH2Ya/gJDudE0o+6zS9gkxtehNs7zPiwhs3Pc6/2j
-8cYxFvwPSs09Mx0Lra0dJuI7VSjEtev11wE4a3vzoCyJFMnxur0YJVCDN3RZ+1vz
-Pp11bZi5auYTGWQd18LigbQPlJTVAHiV0/KXt+FVaV8fALU1f5xGcNIR5iQBx2+8
-h6R0z/If7PrZeHBOgzQT1MLPAgMBAAGjUDBOMB0GA1UdDgQWBBRFLH1wF6BT51uq
-yWNqBnCrPFIglzAfBgNVHSMEGDAWgBRFLH1wF6BT51uqyWNqBnCrPFIglzAMBgNV
-HRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQAr7aH3Db6TeAZkg4Zd7SoF2q11
-erzv552PgQUyezMZcRBo2q1ekmUYyy2600CBiYg51G+8oUqjJKiKnBuaqbMX7pFa
-FsL7uToZCGA57cBaVejeB+p24P5bxoJGKCMeZcEBe5N93Tqu5WBxNEX7lQUo6TSs
-gSN2Olf3/grNKt5V4BduSIQZ+YHlPUWLTaz5B1MXKSUqjmabARP9lhjO14u9USvi
-dMBDFskJySQ6SUfz3fyoXELoDOVbRZETuSodpw+aFCbEtbcQCLT3A0FG+BEPayZH
-tt19zKUlr6e+YFpyjQPGZ7ZkY7iMgHEkhKrXx2DiZ1+cif3X1xfXWQr0S5+E
------END CERTIFICATE-----
-`),
 				},
 			},
 			{
@@ -137,113 +93,19 @@ tt19zKUlr6e+YFpyjQPGZ7ZkY7iMgHEkhKrXx2DiZ1+cif3X1xfXWQr0S5+E
 					Domain:        "foo2.com",
 					CertURL:       "url",
 					CertStableURL: "url",
-					PrivateKey: []byte(`
+					PrivateKey:    foo2Key,
------BEGIN RSA PRIVATE KEY-----
+					Certificate:   foo2Cert,
-MIIEogIBAAKCAQEA7rIVuSrZ3FfYXhR3qaWwfVcgiqKS//yXFzNqkJS6mz9nRCNT
-lPawvrCFIRKdR7UO7xD7A5VTcbrGOAaTvrEaH7mB/4FGL+gN4AiTbVFpKXngAYEW
-A3//zeBZ7XUSWaQ+CNC+l796JeoDvQD++KwCke4rVD1pGN1hpVEeGhwzyKOYPKLo
-4+AGVe1LFWw4U/v8Iil1/gBBehZBILuhASpXy4W132LJPl76/EbGqh0nVz2UlFqU
-HRxO+2U2ba4YIpI+0/VOQ9Cq/TzHSUdTTLfBHE/Qb+aDBfptMWTRvAngLqUglOcZ
-Fi6SAljxEkJO6z6btmoVUWsoKBpbIHDC5++dZwIDAQABAoIBAAD8rYhRfAskNdnV
-vdTuwXcTOCg6md8DHWDULpmgc9EWhwfKGZthFcQEGNjVKd9VCVXFvTP7lxe+TPmI
-VW4Rb2k4LChxUWf7TqthfbKTBptMTLfU39Ft4xHn3pdTx5qlSjhhHJimCwxDFnbe
-nS9MDsqpsHYtttSKfc/gMP6spS4sNPZ/r9zseT3eWkBEhn+FQABxJiuPcQ7q7S+Q
-uOghmr7f3FeYvizQOhBtULsLrK/hsmQIIB4amS1QlpNWKbIoiUPNPjCA5PVQyAER
-waYjuc7imBbeD98L/z8bRTlEskSKjtPSEXGVHa9OYdBU+02Ci6TjKztUp6Ho7JE9
-tcHj+eECgYEA+9Ntv6RqIdpT/4/52JYiR+pOem3U8tweCOmUqm/p/AWyfAJTykqt
-cJ8RcK1MfM+uoa5Sjm8hIcA2XPVEqH2J50PC4w04Q3xtfsz3xs7KJWXQCoha8D0D
-ZIFNroEPnld0qOuJzpIIteXTrCLhSu17ZhN+Wk+5gJ7Ewu/QMM5OPjECgYEA8qbw
-zfwSjE6jkrqO70jzqSxgi2yjo0vMqv+BNBuhxhDTBXnKQI1KsHoiS0FkSLSJ9+DS
-CT3WEescD2Lumdm2s9HXvaMmnDSKBY58NqCGsNzZifSgmj1H/yS9FX8RXfSjXcxq
-RDvTbD52/HeaCiOxHZx8JjmJEb+ZKJC4MDvjtxcCgYBM516GvgEjYXdxfliAiijh
-6W4Z+Vyk5g/ODPc3rYG5U0wUjuljx7Z7xDghPusy2oGsIn5XvRxTIE35yXU0N1Jb
-69eiWzEpeuA9bv7kGdal4RfNf6K15wwYL1y3w/YvFuorg/LLwNEkK5Ge6e//X9Ll
-c2KM1fgCjXntRitAHGDMoQKBgDnkgodioLpA+N3FDN0iNqAiKlaZcOFA8G/LzfO0
-tAAhe3dO+2YzT6KTQSNbUqXWDSTKytHRowVbZrJ1FCA4xVJZunNQPaH/Fv8EY7ZU
-zk3cIzq61qZ2AHtrNIGwc2BLQb7bSm9FJsgojxLlJidNJLC/6Q7lo0JMyCnZfVhk
-sYu5AoGAZt/MfyFTKm674UddSNgGEt86PyVYbLMnRoAXOaNB38AE12kaYHPil1tL
-FnL8OQLpbX5Qo2JGgeZRlpMJ4Jxw2zzvUKr/n+6khaLxHmtX48hMu2QM7ZvnkZCs
-Kkgz6v+Wcqm94ugtl3HSm+u9xZzVQxN6gu/jZQv3VpQiAZHjPYc=
------END RSA PRIVATE KEY-----
-`),
-					Certificate: []byte(`
------BEGIN CERTIFICATE-----
-MIIC+TCCAeGgAwIBAgIJAK25/Z9Jz6IBMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
-BAMMCGZvbzIuY29tMB4XDTE2MDYyMDA5MzUyNloXDTI2MDYxODA5MzUyNlowEzER
-MA8GA1UEAwwIZm9vMi5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-AQDushW5KtncV9heFHeppbB9VyCKopL//JcXM2qQlLqbP2dEI1OU9rC+sIUhEp1H
-tQ7vEPsDlVNxusY4BpO+sRofuYH/gUYv6A3gCJNtUWkpeeABgRYDf//N4FntdRJZ
-pD4I0L6Xv3ol6gO9AP74rAKR7itUPWkY3WGlUR4aHDPIo5g8oujj4AZV7UsVbDhT
-+/wiKXX+AEF6FkEgu6EBKlfLhbXfYsk+Xvr8RsaqHSdXPZSUWpQdHE77ZTZtrhgi
-kj7T9U5D0Kr9PMdJR1NMt8EcT9Bv5oMF+m0xZNG8CeAupSCU5xkWLpICWPESQk7r
-Ppu2ahVRaygoGlsgcMLn751nAgMBAAGjUDBOMB0GA1UdDgQWBBQ6FZWqB9qI4NN+
-2jFY6xH8uoUTnTAfBgNVHSMEGDAWgBQ6FZWqB9qI4NN+2jFY6xH8uoUTnTAMBgNV
-HRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQCRhuf2dQhIEOmSOGgtRELF2wB6
-NWXt0lCty9x4u+zCvITXV8Z0C34VQGencO3H2bgyC3ZxNpPuwZfEc2Pxe8W6bDc/
-OyLckk9WLo00Tnr2t7rDOeTjEGuhXFZkhIbJbKdAH8cEXrxKR8UXWtZgTv/b8Hv/
-g6tbeH6TzBsdMoFtUCsyWxygYwnLU+quuYvE2s9FiCegf2mdYTCh/R5J5n/51gfB
-uC+NakKMfaCvNg3mOAFSYC/0r0YcKM/5ldKGTKTCVJAMhnmBnyRc/70rKkVRFy2g
-iIjUFs+9aAgfCiL0WlyyXYAtIev2gw4FHUVlcT/xKks+x8Kgj6e5LTIrRRwW
------END CERTIFICATE-----
-`),
 				},
 			},
 		},
 	}
-
+	foo1Cert, foo1Key, _ = generateKeyPair("foo1.com", time.Now())
 	newCertificate := &Certificate{
 		Domain:        "foo1.com",
 		CertURL:       "url",
 		CertStableURL: "url",
-		PrivateKey: []byte(`
+		PrivateKey:    foo1Key,
------BEGIN RSA PRIVATE KEY-----
+		Certificate:   foo1Cert,
-MIIEowIBAAKCAQEA1OdSuXK2zeSLf0UqgrI4pjkpaqhra++pnda4Li4jXo151svi
-Sn7DSynJOoq1jbfRJAoyDhxsBC4S4RuD54U5elJ4wLPZXmHRsvb+NwiHs9VmDqwu
-It21btuqeNMebkab5cnDnC6KKufMhXRcRAlluYXyCkQe/+N+LlUQd6Js34TixMpk
-eQOX4/OVrokSyVRnIq4u+o0Ufe7z5+41WVH63tcy7Hwi7244aLUzZCs+QQa2Dw6f
-qEwjbonr974fM68UxDjTZEQy9u24yDzajhDBp1OTAAklh7U+li3g9dSyNVBFXqEu
-nW2fyBvLqeJOSTihqfcrACB/YYhYOX94vMXELQIDAQABAoIBAFYK3t3fxI1VTiMz
-WsjTKh3TgC+AvVkz1ILbojfXoae22YS7hUrCDD82NgMYx+LsZPOBw1T8m5Lc4/hh
-3F8W8nHDHtYSWUjRk6QWOgsXwXAmUEahw0uH+qlA0ZZfDC9ZDexCLHHURTat03Qj
-4J4GhjwCLB2GBlk4IWisLCmNVR7HokrpfIw4oM1aB5E21Tl7zh/x7ikRijEkUsKw
-7YhaMeLJqBnMnAdV63hhF7FaDRjl8P2s/3octz/6pqDIABrDrUW3KAkNYCZIWdhF
-Kk0wRMbZ/WrYT9GIGoJe7coQC7ezTrlrEkAFEIPGHCLkgXB/0TyuSy0yY59e4zmi
-VvHoWUECgYEA/rOL2KJ/p+TZW7+YbsUzs0+F+M+G6UCr0nWfYN9MKmNtmns3eLDG
-+pIpBMc5mjqeJR/sCCdkD8OqHC202Y8e4sr0pKSBeBofh2BmXtpyu3QQ50Pa63RS
-SK6mYUrFqPmFFDbNGpFI4sIeI+Vf6hm96FQPnyPtUTGqk39m0RbWM/UCgYEA1f04
-Nf3wbqwqIHZjYpPmymfjleyMn3hGUjpi7pmI6inXGMk3nkeG1cbOhnfPxL5BWD12
-3RqHI2B4Z4r0BMyjctDNb1TxhMIpm5+PKm5KeeKfoYA85IS0mEeq6VdMm3mL1x/O
-3LYvcUvAEVf6pWX/+ZFLMudqhF3jbTrdNOC6ZFkCgYBKpEeJdyW+CD0CvEVpwPUD
-yXxTjE3XMZKpHLtWYlop2fWW3iFFh1jouci3k8L3xdHuw0oioZibXhYOJ/7l+yFs
-CVpknakrj0xKGiAmEBKriLojbClN80rh7fzoakc+29D6OY0mCgm4GndGwcO4EU8s
-NOZXFupHbyy0CRQSloSzuQKBgQC1Z/MtIlefGuijmHlsakGuuR+gS2ZzEj1bHBAe
-gZ4mFM46PuqdjblqpR0TtaI3AarXqVOI4SJLBU9NR+jR4MF3Zjeh9/q/NvKa8Usn
-B1Svu0TkXphAiZenuKnVIqLY8tNvzZFKXlAd1b+/dDwR10SHR3rebnxINmfEg7Bf
-UVvyEQKBgAEjI5O6LSkLNpbVn1l2IO8u8D2RkFqs/Sbx78uFta3f9Gddzb4wMnt3
-jVzymghCLp4Qf1ump/zC5bcQ8L97qmnjJ+H8X9HwmkqetuI362JNnz+12YKVDIWi
-wI7SJ8BwDqYMrLw6/nE+degn39KedGDH8gz5cZcdlKTZLjbuBOfU
------END RSA PRIVATE KEY-----
-`),
-		Certificate: []byte(`
------BEGIN CERTIFICATE-----
-MIIC+TCCAeGgAwIBAgIJAPQiOiQcwYaRMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
-BAMMCGZvbzEuY29tMB4XDTE2MDYxOTIyMTE1NFoXDTI2MDYxNzIyMTE1NFowEzER
-MA8GA1UEAwwIZm9vMS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
-AQDU51K5crbN5It/RSqCsjimOSlqqGtr76md1rguLiNejXnWy+JKfsNLKck6irWN
-t9EkCjIOHGwELhLhG4PnhTl6UnjAs9leYdGy9v43CIez1WYOrC4i3bVu26p40x5u
-RpvlycOcLooq58yFdFxECWW5hfIKRB7/434uVRB3omzfhOLEymR5A5fj85WuiRLJ
-VGciri76jRR97vPn7jVZUfre1zLsfCLvbjhotTNkKz5BBrYPDp+oTCNuiev3vh8z
-rxTEONNkRDL27bjIPNqOEMGnU5MACSWHtT6WLeD11LI1UEVeoS6dbZ/IG8up4k5J
-OKGp9ysAIH9hiFg5f3i8xcQtAgMBAAGjUDBOMB0GA1UdDgQWBBQPfkS5ehpstmSb
-8CGJE7GxSCxl2DAfBgNVHSMEGDAWgBQPfkS5ehpstmSb8CGJE7GxSCxl2DAMBgNV
-HRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQA99A+itS9ImdGRGgHZ5fSusiEq
-wkK5XxGyagL1S0f3VM8e78VabSvC0o/xdD7DHVg6Az8FWxkkksH6Yd7IKfZZUzvs
-kXQhlOwWpxgmguSmAs4uZTymIoMFRVj3nG664BcXkKu4Yd9UXKNOWP59zgvrCJMM
-oIsmYiq5u0MFpM31BwfmmW3erqIcfBI9OJrmr1XDzlykPZNWtUSSfVuNQ8d4bim9
-XH8RfVLeFbqDydSTCHIFvYthH/ESbpRCiGJHoJ8QLfOkhD1k2fI0oJZn5RVtG2W8
-bZME3gHPYCk1QFZUptriMCJ5fMjCgxeOTR+FAkstb/lTRuCc4UyILJguIMar
------END CERTIFICATE-----
-`),
 	}
 
 	err := domainsCertificates.renewCertificates(
@@ -262,6 +124,97 @@ bZME3gHPYCk1QFZUptriMCJ5fMjCgxeOTR+FAkstb/lTRuCc4UyILJguIMar
 	}
 }
 
+func TestRemoveDuplicates(t *testing.T) {
+	now := time.Now()
+	fooCert, fooKey, _ := generateKeyPair("foo.com", now)
+	foo24Cert, foo24Key, _ := generateKeyPair("foo.com", now.Add(24*time.Hour))
+	foo48Cert, foo48Key, _ := generateKeyPair("foo.com", now.Add(48*time.Hour))
+	barCert, barKey, _ := generateKeyPair("bar.com", now)
+	domainsCertificates := DomainsCertificates{
+		lock: sync.RWMutex{},
+		Certs: []*DomainsCertificate{
+			{
+				Domains: Domain{
+					Main: "foo.com",
+					SANs: []string{}},
+				Certificate: &Certificate{
+					Domain:        "foo.com",
+					CertURL:       "url",
+					CertStableURL: "url",
+					PrivateKey:    foo24Key,
+					Certificate:   foo24Cert,
+				},
+			},
+			{
+				Domains: Domain{
+					Main: "foo.com",
+					SANs: []string{}},
+				Certificate: &Certificate{
+					Domain:        "foo.com",
+					CertURL:       "url",
+					CertStableURL: "url",
+					PrivateKey:    foo48Key,
+					Certificate:   foo48Cert,
+				},
+			},
+			{
+				Domains: Domain{
+					Main: "foo.com",
+					SANs: []string{}},
+				Certificate: &Certificate{
+					Domain:        "foo.com",
+					CertURL:       "url",
+					CertStableURL: "url",
+					PrivateKey:    fooKey,
+					Certificate:   fooCert,
+				},
+			},
+			{
+				Domains: Domain{
+					Main: "bar.com",
+					SANs: []string{}},
+				Certificate: &Certificate{
+					Domain:        "bar.com",
+					CertURL:       "url",
+					CertStableURL: "url",
+					PrivateKey:    barKey,
+					Certificate:   barCert,
+				},
+			},
+			{
+				Domains: Domain{
+					Main: "foo.com",
+					SANs: []string{}},
+				Certificate: &Certificate{
+					Domain:        "foo.com",
+					CertURL:       "url",
+					CertStableURL: "url",
+					PrivateKey:    foo48Key,
+					Certificate:   foo48Cert,
+				},
+			},
+		},
+	}
+	domainsCertificates.Init()
+
+	if len(domainsCertificates.Certs) != 2 {
+		t.Errorf("Expected domainsCertificates length %d %+v\nGot %+v", 2, domainsCertificates.Certs, len(domainsCertificates.Certs))
+	}
+
+	for _, cert := range domainsCertificates.Certs {
+		switch cert.Domains.Main {
+		case "bar.com":
+			continue
+		case "foo.com":
+			if !cert.tlsCert.Leaf.NotAfter.Equal(now.Add(48 * time.Hour).Truncate(1 * time.Second)) {
+				t.Errorf("Bad expiration %s date for domain %+v, now %s", cert.tlsCert.Leaf.NotAfter.String(), cert, now.Add(48*time.Hour).Truncate(1*time.Second).String())
+			}
+		default:
+			t.Errorf("Unknown domain %+v", cert)
+		}
+	}
+}
+
 func TestNoPreCheckOverride(t *testing.T) {
 	acme.PreCheckDNS = nil // Irreversable - but not expecting real calls into this during testing process
 	err := dnsOverrideDelay(0)

acme/challengeProvider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cenk/backoff"
 	"github.com/containous/traefik/cluster"
 	"github.com/containous/traefik/log"
+	"github.com/containous/traefik/safe"
 	"github.com/xenolf/lego/acme"
 )
 
@@ -49,7 +50,7 @@ func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate
 	}
 	ebo := backoff.NewExponentialBackOff()
 	ebo.MaxElapsedTime = 60 * time.Second
-	err := backoff.RetryNotify(operation, ebo, notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify)
 	if err != nil {
 		log.Errorf("Error getting cert: %v", err)
 		return nil, false

acme/crypto.go

@@ -17,34 +17,44 @@ import (
 )
 
 func generateDefaultCertificate() (*tls.Certificate, error) {
-	rsaPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return nil, err
-	}
-	rsaPrivPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rsaPrivKey)})
-
 	randomBytes := make([]byte, 100)
-	_, err = rand.Read(randomBytes)
+	_, err := rand.Read(randomBytes)
 	if err != nil {
 		return nil, err
 	}
 	zBytes := sha256.Sum256(randomBytes)
 	z := hex.EncodeToString(zBytes[:sha256.Size])
 	domain := fmt.Sprintf("%s.%s.traefik.default", z[:32], z[32:])
-	tempCertPEM, err := generatePemCert(rsaPrivKey, domain)
+
+	certPEM, keyPEM, err := generateKeyPair(domain, time.Time{})
 	if err != nil {
 		return nil, err
 	}
 
-	certificate, err := tls.X509KeyPair(tempCertPEM, rsaPrivPEM)
+	certificate, err := tls.X509KeyPair(certPEM, keyPEM)
 	if err != nil {
 		return nil, err
 	}
 
 	return &certificate, nil
 }
-func generatePemCert(privKey *rsa.PrivateKey, domain string) ([]byte, error) {
+
-	derBytes, err := generateDerCert(privKey, time.Time{}, domain)
+func generateKeyPair(domain string, expiration time.Time) ([]byte, []byte, error) {
+	rsaPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rsaPrivKey)})
+
+	certPEM, err := generatePemCert(rsaPrivKey, domain, expiration)
+	if err != nil {
+		return nil, nil, err
+	}
+	return certPEM, keyPEM, nil
+}
+
+func generatePemCert(privKey *rsa.PrivateKey, domain string, expiration time.Time) ([]byte, error) {
+	derBytes, err := generateDerCert(privKey, expiration, domain)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +103,7 @@ func TLSSNI01ChallengeCert(keyAuth string) (ChallengeCert, string, error) {
 	zBytes := sha256.Sum256([]byte(keyAuth))
 	z := hex.EncodeToString(zBytes[:sha256.Size])
 	domain := fmt.Sprintf("%s.%s.acme.invalid", z[:32], z[32:])
-	tempCertPEM, err := generatePemCert(rsaPrivKey, domain)
+	tempCertPEM, err := generatePemCert(rsaPrivKey, domain, time.Time{})
 	if err != nil {
 		return ChallengeCert{}, "", err
 	}

build.Dockerfile

@@ -6,7 +6,7 @@ RUN go get github.com/jteeuwen/go-bindata/... \
 && go get github.com/client9/misspell/cmd/misspell
 
 # Which docker version to test on
-ARG DOCKER_VERSION=v0.10.3
+ARG DOCKER_VERSION=1.10.3
 
 
 # Which glide version to test on
@@ -14,12 +14,12 @@ ARG GLIDE_VERSION=v0.12.3
 
 # Download glide
 RUN mkdir -p /usr/local/bin \
-    && curl -SL https://github.com/Masterminds/glide/releases/download/${GLIDE_VERSION}/glide-${GLIDE_VERSION}-linux-amd64.tar.gz \
+    && curl -fL https://github.com/Masterminds/glide/releases/download/${GLIDE_VERSION}/glide-${GLIDE_VERSION}-linux-amd64.tar.gz \
     | tar -xzC /usr/local/bin --transform 's#^.+/##x'
 
 # Download docker
 RUN mkdir -p /usr/local/bin \
-    && curl -SL https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz \
+    && curl -fL https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz \
     | tar -xzC /usr/local/bin --transform 's#^.+/##x'
 
 WORKDIR /go/src/github.com/containous/traefik

cluster/datastore.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containous/staert"
 	"github.com/containous/traefik/job"
 	"github.com/containous/traefik/log"
+	"github.com/containous/traefik/safe"
 	"github.com/docker/libkv/store"
 	"github.com/satori/go.uuid"
 )
@@ -109,7 +110,7 @@ func (d *Datastore) watchChanges() error {
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Error in watch datastore: %v", err)
 		}
@@ -176,7 +177,7 @@ func (d *Datastore) Begin() (Transaction, Object, error) {
 	}
 	ebo := backoff.NewExponentialBackOff()
 	ebo.MaxElapsedTime = 60 * time.Second
-	err = backoff.RetryNotify(operation, ebo, notify)
+	err = backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Datastore cannot sync: %v", err)
 	}
@@ -231,21 +232,21 @@ func (s *datastoreTransaction) Commit(object Object) error {
 	s.localLock.Lock()
 	defer s.localLock.Unlock()
 	if s.dirty {
-		return fmt.Errorf("transaction already used, please begin a new one")
+		return fmt.Errorf("Transaction already used, please begin a new one")
 	}
 	s.Datastore.meta.object = object
 	err := s.Datastore.meta.Marshall()
 	if err != nil {
-		return err
+		return fmt.Errorf("Marshall error: %s", err)
 	}
 	err = s.kv.StoreConfig(s.Datastore.meta)
 	if err != nil {
-		return err
+		return fmt.Errorf("StoreConfig error: %s", err)
 	}
 
 	err = s.remoteLock.Unlock()
 	if err != nil {
-		return err
+		return fmt.Errorf("Unlock error: %s", err)
 	}
 
 	s.dirty = true

cluster/leadership.go

@@ -16,7 +16,7 @@ type Leadership struct {
 	*safe.Pool
 	*types.Cluster
 	candidate *leadership.Candidate
-	leader    safe.Safe
+	leader    *safe.Safe
 	listeners []LeaderListener
 }
 
@@ -27,6 +27,7 @@ func NewLeadership(ctx context.Context, cluster *types.Cluster) *Leadership {
 		Cluster:   cluster,
 		candidate: leadership.NewCandidate(cluster.Store, cluster.Store.Prefix+"/leader", cluster.Node, 20*time.Second),
 		listeners: []LeaderListener{},
+		leader:    safe.New(false),
 	}
 }
 
@@ -46,7 +47,7 @@ func (l *Leadership) Participate(pool *safe.Pool) {
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Leadership election error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, backOff, notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), backOff, notify)
 		if err != nil {
 			log.Errorf("Cannot elect leadership %+v", err)
 		}

docs/toml.md

@@ -1068,6 +1068,10 @@ Annotations can be used on the Kubernetes service to override default behaviour:
 
 You can find here an example [ingress](https://raw.githubusercontent.com/containous/traefik/master/examples/k8s/cheese-ingress.yaml) and [replication controller](https://raw.githubusercontent.com/containous/traefik/master/examples/k8s/traefik.yaml).
 
+Additionally, an annotation can be used on Kubernetes services to set the [circuit breaker expression](https://docs.traefik.io/basics/#backends) for a backend.
+
+- `traefik.backend.circuitbreaker: <expression>`: set the circuit breaker expression for the backend (Default: nil).
+
 ## Consul backend
 
 Træfɪk can be configured to use Consul as a backend configuration:

docs/user-guide/kubernetes.md

@@ -1,6 +1,6 @@
 # Kubernetes Ingress Controller
 
-This guide explains how to use Træfɪk as an Ingress controller in a Kubernetes cluster. 
+This guide explains how to use Træfɪk as an Ingress controller in a Kubernetes cluster.
 If you are not familiar with Ingresses in Kubernetes you might want to read the [Kubernetes user guide](http://kubernetes.io/docs/user-guide/ingress/)
 
 The config files used in this guide can be found in the [examples directory](https://github.com/containous/traefik/tree/master/examples/k8s)
@@ -19,7 +19,6 @@ We are going to deploy Træfɪk with a
 allow you to easily roll out config changes or update the image.
 
 ```yaml
-apiVersion: v1
 kind: Deployment
 apiVersion: extensions/v1beta1
 metadata:
@@ -85,7 +84,7 @@ traefik-ingress-controller-678226159-eqseo   1/1       Running   0          7m
 ```
 
 You should see that after submitting the Deployment to Kubernetes it has launched
-a pod, and it is now running. _It might take a few moments for kubenetes to pull
+a pod, and it is now running. _It might take a few moments for kubernetes to pull
 the Træfɪk image and start the container._
 
 > You could also check the deployment with the Kubernetes dashboard, run
@@ -114,7 +113,7 @@ metadata:
   namespace: kube-system
 spec:
   selector:
-    k8s-app: traefik-ingress-lb 
+    k8s-app: traefik-ingress-lb
   ports:
   - port: 80
     targetPort: 8080
@@ -140,7 +139,7 @@ kubectl apply -f examples/k8s/ui.yaml
 ```
 
 Now lets setup an entry in our /etc/hosts file to route `traefik-ui.local`
-to our cluster. 
+to our cluster.
 
 > In production you would want to set up real dns entries.
 
@@ -300,6 +299,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: wensleydale
+  annotations:
+    traefik.backend.circuitbreaker: "NetworkErrorRatio() > 0.5"
 spec:
   ports:
   - name: http
@@ -309,6 +310,11 @@ spec:
     app: cheese
     task: wensleydale
 ```
+
+> Notice that we also set a [circuit breaker expression](https://docs.traefik.io/basics/#backends) for one of the backends
+> by setting the `traefik.backend.circuitbreaker` annotation on the service.
+
+
 [examples/k8s/cheese-services.yaml](https://github.com/containous/traefik/tree/master/examples/k8s/cheese-services.yaml)
 
 ```sh

glide.lock

@@ -1,5 +1,5 @@
-hash: 0d092f94db69882e79d229c34b9483899e1208eaa7dd0acdd5184635cb0cdaaa
+hash: ccd56edd81d054a00b23493227ff0765b020aa1de24f8a9d9ff54a05c0223885
-updated: 2017-01-12T12:31:31.35220213+01:00
+updated: 2017-02-03T09:45:05.719219148+01:00
 imports:
 - name: bitbucket.org/ww/goautoneg
   version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675
@@ -213,6 +213,10 @@ imports:
   - store/zookeeper
 - name: github.com/donovanhide/eventsource
   version: fd1de70867126402be23c306e1ce32828455d85b
+- name: github.com/eapache/channels
+  version: 47238d5aae8c0fefd518ef2bee46290909cf8263
+- name: github.com/eapache/queue
+  version: 44cc805cf13205b55f69e14bcb69867d1ae92f98
 - name: github.com/edeckers/auroradnsclient
   version: 8b777c170cfd377aa16bb4368f093017dddef3f9
   subpackages:

glide.yaml

@@ -118,3 +118,4 @@ import:
   version: v0.3.0
   subpackages:
   - metrics
+- package: github.com/eapache/channels

middlewares/retry.go

@@ -3,6 +3,7 @@ package middlewares
 import (
 	"bufio"
 	"bytes"
+	"io/ioutil"
 	"net"
 	"net/http"
 
@@ -32,6 +33,13 @@ func NewRetry(attempts int, next http.Handler) *Retry {
 }
 
 func (retry *Retry) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	// if we might make multiple attempts, swap the body for an ioutil.NopCloser
+	// cf https://github.com/containous/traefik/issues/1008
+	if retry.attempts > 1 {
+		body := r.Body
+		defer body.Close()
+		r.Body = ioutil.NopCloser(body)
+	}
 	attempts := 1
 	for {
 		recorder := NewRecorder()

provider/consul_catalog.go

@@ -334,7 +334,7 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess
 		operation := func() error {
 			return provider.watch(configurationChan, stop)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to consul server %+v", err)
 		}

provider/docker.go

@@ -229,7 +229,7 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Docker connection error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to docker server %+v", err)
 		}

provider/kubernetes.go

@@ -91,7 +91,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to Kubernetes server %+v", err)
 		}
@@ -110,6 +110,10 @@ func (provider *Kubernetes) loadIngresses(k8sClient k8s.Client) (*types.Configur
 	PassHostHeader := provider.getPassHostHeader()
 	for _, i := range ingresses {
 		for _, r := range i.Spec.Rules {
+			if r.HTTP == nil {
+				log.Warnf("Error in ingress: HTTP is nil")
+				continue
+			}
 			for _, pa := range r.HTTP.Paths {
 				if _, exists := templateObjects.Backends[r.Host+pa.Path]; !exists {
 					templateObjects.Backends[r.Host+pa.Path] = &types.Backend{
@@ -171,12 +175,18 @@ func (provider *Kubernetes) loadIngresses(k8sClient k8s.Client) (*types.Configur
 					continue
 				}
 
+				if expression := service.Annotations["traefik.backend.circuitbreaker"]; expression != "" {
+					templateObjects.Backends[r.Host+pa.Path].CircuitBreaker = &types.CircuitBreaker{
+						Expression: expression,
+					}
+				}
 				if service.Annotations["traefik.backend.loadbalancer.method"] == "drr" {
 					templateObjects.Backends[r.Host+pa.Path].LoadBalancer.Method = "drr"
 				}
 				if service.Annotations["traefik.backend.loadbalancer.sticky"] == "true" {
 					templateObjects.Backends[r.Host+pa.Path].LoadBalancer.Sticky = true
 				}
+
 				protocol := "http"
 				for _, port := range service.Spec.Ports {
 					if equalPorts(port, pa.Backend.ServicePort) {

provider/kubernetes_test.go

@@ -1288,7 +1288,7 @@ func TestHostlessIngress(t *testing.T) {
 	}
 }
 
-func TestLoadBalancerAnnotation(t *testing.T) {
+func TestServiceAnnotations(t *testing.T) {
 	ingresses := []*v1beta1.Ingress{{
 		ObjectMeta: v1.ObjectMeta{
 			Namespace: "testing",
@@ -1336,6 +1336,7 @@ func TestLoadBalancerAnnotation(t *testing.T) {
 				UID:       "1",
 				Namespace: "testing",
 				Annotations: map[string]string{
+					"traefik.backend.circuitbreaker":      "NetworkErrorRatio() > 0.5",
 					"traefik.backend.loadbalancer.method": "drr",
 				},
 			},
@@ -1354,6 +1355,7 @@ func TestLoadBalancerAnnotation(t *testing.T) {
 				UID:       "2",
 				Namespace: "testing",
 				Annotations: map[string]string{
+					"traefik.backend.circuitbreaker":      "",
 					"traefik.backend.loadbalancer.sticky": "true",
 				},
 			},
@@ -1463,7 +1465,9 @@ func TestLoadBalancerAnnotation(t *testing.T) {
 						Weight: 1,
 					},
 				},
-				CircuitBreaker: nil,
+				CircuitBreaker: &types.CircuitBreaker{
+					Expression: "NetworkErrorRatio() > 0.5",
+				},
 				LoadBalancer: &types.LoadBalancer{
 					Method: "drr",
 					Sticky: false,

provider/kv.go

@@ -76,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix
 	notify := func(err error, time time.Duration) {
 		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		return fmt.Errorf("Cannot connect to KV server: %v", err)
 	}
@@ -107,7 +107,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *
 	notify := func(err error, time time.Duration) {
 		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		return fmt.Errorf("Cannot connect to KV server: %v", err)
 	}

provider/marathon.go

@@ -120,7 +120,7 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage,
 	notify := func(err error, time time.Duration) {
 		log.Errorf("Marathon connection error %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		log.Errorf("Cannot connect to Marathon server %+v", err)
 	}
@@ -264,6 +264,9 @@ func (provider *Marathon) taskFilter(task marathon.Task, applications *marathon.
 					return false
 				}
 			}
+		} else {
+			log.Debugf("Filtering marathon task %s with defined healthcheck as no healthcheck has run yet", task.AppID)
+			return false
 		}
 	}
 	return true
@@ -505,7 +508,7 @@ func processPorts(application marathon.Application, task marathon.Task) []int {
 
 	// Using port definition if available
 	if application.PortDefinitions != nil && len(*application.PortDefinitions) > 0 {
-		ports := make([]int, 0)
+		var ports []int
 		for _, def := range *application.PortDefinitions {
 			if def.Port != nil {
 				ports = append(ports, *def.Port)
@@ -515,7 +518,7 @@ func processPorts(application marathon.Application, task marathon.Task) []int {
 	}
 	// If using IP-per-task using this port definition
 	if application.IPAddressPerTask != nil && len(*((*application.IPAddressPerTask).Discovery).Ports) > 0 {
-		ports := make([]int, 0)
+		var ports []int
 		for _, def := range *((*application.IPAddressPerTask).Discovery).Ports {
 			ports = append(ports, def.Number)
 		}

provider/marathon_test.go

@@ -629,7 +629,7 @@ func TestMarathonTaskFilter(t *testing.T) {
 					},
 				},
 			},
-			expected:         true,
+			expected:         false,
 			exposedByDefault: true,
 		},
 		{

provider/mesos.go

@@ -113,7 +113,7 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo
 	notify := func(err error, time time.Duration) {
 		log.Errorf("mesos connection error %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		log.Errorf("Cannot connect to mesos server %+v", err)
 	}

safe/routine.go

@@ -2,10 +2,11 @@ package safe
 
 import (
 	"context"
+	"fmt"
+	"github.com/cenk/backoff"
+	"github.com/containous/traefik/log"
 	"runtime/debug"
 	"sync"
-
-	"github.com/containous/traefik/log"
 )
 
 type routine struct {
@@ -145,3 +146,16 @@ func defaultRecoverGoroutine(err interface{}) {
 	log.Errorf("Error in Go routine: %s", err)
 	debug.PrintStack()
 }
+
+// OperationWithRecover wrap a backoff operation in a Recover
+func OperationWithRecover(operation backoff.Operation) backoff.Operation {
+	return func() (err error) {
+		defer func() {
+			if res := recover(); res != nil {
+				defaultRecoverGoroutine(res)
+				err = fmt.Errorf("Panic in operation: %s", err)
+			}
+		}()
+		return operation()
+	}
+}

safe/routine_test.go

@@ -0,0 +1,37 @@
+package safe
+
+import (
+	"fmt"
+	"github.com/cenk/backoff"
+	"testing"
+)
+
+func TestOperationWithRecover(t *testing.T) {
+	operation := func() error {
+		return nil
+	}
+	err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{})
+	if err != nil {
+		t.Fatalf("Error in OperationWithRecover: %s", err)
+	}
+}
+
+func TestOperationWithRecoverPanic(t *testing.T) {
+	operation := func() error {
+		panic("BOOM")
+	}
+	err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{})
+	if err == nil {
+		t.Fatalf("Error in OperationWithRecover: %s", err)
+	}
+}
+
+func TestOperationWithRecoverError(t *testing.T) {
+	operation := func() error {
+		return fmt.Errorf("ERROR")
+	}
+	err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{})
+	if err == nil {
+		t.Fatalf("Error in OperationWithRecover: %s", err)
+	}
+}

script/crossbinary

@@ -22,7 +22,7 @@ fi
 rm -f dist/traefik_*
 
 # Build 386 amd64 binaries
-OS_PLATFORM_ARG=(linux darwin windows)
+OS_PLATFORM_ARG=(linux darwin windows freebsd openbsd)
 OS_ARCH_ARG=(386 amd64)
 for OS in ${OS_PLATFORM_ARG[@]}; do
   for ARCH in ${OS_ARCH_ARG[@]}; do

server.go

@@ -764,7 +764,7 @@ func (server *Server) loadEntryPointConfig(entryPointName string, entryPoint *En
 	regex := entryPoint.Redirect.Regex
 	replacement := entryPoint.Redirect.Replacement
 	if len(entryPoint.Redirect.EntryPoint) > 0 {
-		regex = "^(?:https?:\\/\\/)?([\\da-z\\.-]+)(?::\\d+)?(.*)$"
+		regex = "^(?:https?:\\/\\/)?([\\w\\._-]+)(?::\\d+)?(.*)$"
 		if server.globalConfiguration.EntryPoints[entryPoint.Redirect.EntryPoint] == nil {
 			return nil, errors.New("Unknown entrypoint " + entryPoint.Redirect.EntryPoint)
 		}

templates/kubernetes.tmpl

@@ -1,4 +1,8 @@
 [backends]{{range $backendName, $backend := .Backends}}
+    {{if $backend.CircuitBreaker}}
+    [backends."{{$backendName}}".circuitbreaker]
+      expression = "{{$backend.CircuitBreaker.Expression}}"
+    {{end}}
     [backends."{{$backendName}}".loadbalancer]
       method = "{{$backend.LoadBalancer.Method}}"
       {{if $backend.LoadBalancer.Sticky}}