From 4128c1ac8d9fce1e7524cb7b45f32128ab4e480a Mon Sep 17 00:00:00 2001 From: Richard Shepherd Date: Fri, 26 May 2017 14:32:03 +0100 Subject: [PATCH] Allow file provider to load config from files in a directory. --- docs/toml.md | 10 +- integration/file_test.go | 20 +- integration/fixtures/file/dir/simple1.toml | 11 + integration/fixtures/file/dir/simple2.toml | 11 + integration/fixtures/file/directory.toml | 10 + provider/file/file.go | 155 ++++++++---- provider/file/file_test.go | 262 +++++++++++++++++++++ traefik.sample.toml | 8 + 8 files changed, 442 insertions(+), 45 deletions(-) create mode 100644 integration/fixtures/file/dir/simple1.toml create mode 100644 integration/fixtures/file/dir/simple2.toml create mode 100644 integration/fixtures/file/directory.toml create mode 100644 provider/file/file_test.go diff --git a/docs/toml.md b/docs/toml.md index cede939ff..b1211ea34 100644 --- a/docs/toml.md +++ b/docs/toml.md @@ -445,7 +445,7 @@ entryPoint = "https" ## File backend -Like any other reverse proxy, Træfik can be configured with a file. You have two choices: +Like any other reverse proxy, Træfik can be configured with a file. You have three choices: - simply add your configuration at the end of the global configuration file `traefik.toml`: @@ -586,6 +586,13 @@ filename = "rules.toml" rule = "Path:/test" ``` +- or you could have multiple .toml files in a directory: + +```toml +[file] +directory = "/path/to/config/" +``` + If you want Træfik to watch file changes automatically, just add: ```toml @@ -593,6 +600,7 @@ If you want Træfik to watch file changes automatically, just add: watch = true ``` + ## API backend Træfik can be configured using a RESTful api. diff --git a/integration/file_test.go b/integration/file_test.go index 6397bde72..191c86314 100644 --- a/integration/file_test.go +++ b/integration/file_test.go @@ -26,7 +26,7 @@ func (s *FileSuite) TestSimpleConfiguration(c *check.C) { defer cmd.Process.Kill() // Expected a 404 as we did not configure anything - try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) + err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) c.Assert(err, checker.IsNil) } @@ -38,6 +38,22 @@ func (s *FileSuite) TestSimpleConfigurationNoPanic(c *check.C) { defer cmd.Process.Kill() // Expected a 404 as we did not configure anything - try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) + err = try.GetRequest("http://127.0.0.1:8000/", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) + c.Assert(err, checker.IsNil) +} + +func (s *FileSuite) TestDirectoryConfiguration(c *check.C) { + cmd := exec.Command(traefikBinary, "--configFile=fixtures/file/directory.toml") + + err := cmd.Start() + c.Assert(err, checker.IsNil) + defer cmd.Process.Kill() + + // Expected a 404 as we did not configure anything at /test + err = try.GetRequest("http://127.0.0.1:8000/test", 1000*time.Millisecond, try.StatusCodeIs(http.StatusNotFound)) + c.Assert(err, checker.IsNil) + + // Expected a 502 as there is no backend server + err = try.GetRequest("http://127.0.0.1:8000/test2", 1000*time.Millisecond, try.StatusCodeIs(http.StatusBadGateway)) c.Assert(err, checker.IsNil) } diff --git a/integration/fixtures/file/dir/simple1.toml b/integration/fixtures/file/dir/simple1.toml new file mode 100644 index 000000000..05bfdd9ea --- /dev/null +++ b/integration/fixtures/file/dir/simple1.toml @@ -0,0 +1,11 @@ +# rules +[backends] + [backends.backend1] + [backends.backend1.servers.server1] + url = "http://172.17.0.2:80" + +[frontends] + [frontends.frontend1] + backend = "backend1" + [frontends.frontend1.routes.test_1] + rule = "Path:/test1" diff --git a/integration/fixtures/file/dir/simple2.toml b/integration/fixtures/file/dir/simple2.toml new file mode 100644 index 000000000..52d698b2b --- /dev/null +++ b/integration/fixtures/file/dir/simple2.toml @@ -0,0 +1,11 @@ +# rules +[backends] + [backends.backend2] + [backends.backend2.servers.server1] + url = "http://172.17.0.2:80" + +[frontends] + [frontends.frontend2] + backend = "backend2" + [frontends.frontend2.routes.test_2] + rule = "Path:/test2" diff --git a/integration/fixtures/file/directory.toml b/integration/fixtures/file/directory.toml new file mode 100644 index 000000000..15308c560 --- /dev/null +++ b/integration/fixtures/file/directory.toml @@ -0,0 +1,10 @@ +defaultEntryPoints = ["http"] + +logLevel = "DEBUG" + +[entryPoints] + [entryPoints.http] + address = ":8000" + +[file] + directory = "fixtures/file/dir/" diff --git a/provider/file/file.go b/provider/file/file.go index e6959d0df..1ca651202 100644 --- a/provider/file/file.go +++ b/provider/file/file.go @@ -1,8 +1,9 @@ package file import ( - "os" - "path/filepath" + "fmt" + "io/ioutil" + "path" "strings" "github.com/BurntSushi/toml" @@ -18,68 +19,138 @@ var _ provider.Provider = (*Provider)(nil) // Provider holds configurations of the provider. type Provider struct { provider.BaseProvider `mapstructure:",squash"` + Directory string `description:"Load configuration from one or more .toml files in a directory"` } // Provide allows the file provider to provide configurations to traefik // using the given configuration channel. func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Error("Error creating file watcher", err) - return err - } + configuration, err := p.loadConfig() - file, err := os.Open(p.Filename) if err != nil { - log.Error("Error opening file", err) return err } - defer file.Close() if p.Watch { - // Process events - pool.Go(func(stop chan bool) { - defer watcher.Close() - for { - select { - case <-stop: - return - case event := <-watcher.Events: - if strings.Contains(event.Name, file.Name()) { - log.Debug("Provider event:", event) - configuration := p.loadFileConfig(file.Name()) - if configuration != nil { - configurationChan <- types.ConfigMessage{ - ProviderName: "file", - Configuration: configuration, - } - } - } - case error := <-watcher.Errors: - log.Error("Watcher event error", error) - } - } - }) - err = watcher.Add(filepath.Dir(file.Name())) - if err != nil { - log.Error("Error adding file watcher", err) + var watchItem string + + if p.Directory != "" { + watchItem = p.Directory + } else { + watchItem = p.Filename + } + + if err := p.addWatcher(pool, watchItem, configurationChan, p.watcherCallback); err != nil { return err } } - configuration := p.loadFileConfig(file.Name()) + sendConfigToChannel(configurationChan, configuration) + return nil +} + +func (p *Provider) addWatcher(pool *safe.Pool, directory string, configurationChan chan<- types.ConfigMessage, callback func(chan<- types.ConfigMessage, fsnotify.Event)) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("error creating file watcher: %s", err) + } + + // Process events + pool.Go(func(stop chan bool) { + defer watcher.Close() + for { + select { + case <-stop: + return + case evt := <-watcher.Events: + callback(configurationChan, evt) + case err := <-watcher.Errors: + log.Errorf("Watcher event error: %s", err) + } + } + }) + err = watcher.Add(directory) + if err != nil { + return fmt.Errorf("error adding file watcher: %s", err) + } + + return nil +} + +func sendConfigToChannel(configurationChan chan<- types.ConfigMessage, configuration *types.Configuration) { configurationChan <- types.ConfigMessage{ ProviderName: "file", Configuration: configuration, } - return nil } -func (p *Provider) loadFileConfig(filename string) *types.Configuration { +func loadFileConfig(filename string) (*types.Configuration, error) { configuration := new(types.Configuration) if _, err := toml.DecodeFile(filename, configuration); err != nil { - log.Error("Error reading file:", err) - return nil + return nil, fmt.Errorf("error reading configuration file: %s", err) } - return configuration + return configuration, nil +} + +func loadFileConfigFromDirectory(directory string) (*types.Configuration, error) { + fileList, err := ioutil.ReadDir(directory) + + if err != nil { + return nil, fmt.Errorf("unable to read directory %s: %v", directory, err) + } + + configuration := &types.Configuration{ + Frontends: make(map[string]*types.Frontend), + Backends: make(map[string]*types.Backend), + } + + for _, file := range fileList { + if !strings.HasSuffix(file.Name(), ".toml") { + continue + } + + var c *types.Configuration + c, err = loadFileConfig(path.Join(directory, file.Name())) + + if err != nil { + return nil, err + } + + for backendName, backend := range c.Backends { + if _, exists := configuration.Backends[backendName]; exists { + log.Warnf("Backend %s already configured, skipping", backendName) + } else { + configuration.Backends[backendName] = backend + } + } + + for frontendName, frontend := range c.Frontends { + if _, exists := configuration.Frontends[frontendName]; exists { + log.Warnf("Frontend %s already configured, skipping", frontendName) + } else { + configuration.Frontends[frontendName] = frontend + } + } + } + + return configuration, nil +} + +func (p *Provider) watcherCallback(configurationChan chan<- types.ConfigMessage, event fsnotify.Event) { + configuration, err := p.loadConfig() + + if err != nil { + log.Errorf("Error occurred during watcher callback: %s", err) + return + } + + sendConfigToChannel(configurationChan, configuration) +} + +func (p *Provider) loadConfig() (*types.Configuration, error) { + if p.Directory != "" { + return loadFileConfigFromDirectory(p.Directory) + } + + return loadFileConfig(p.Filename) } diff --git a/provider/file/file_test.go b/provider/file/file_test.go new file mode 100644 index 000000000..33bde6724 --- /dev/null +++ b/provider/file/file_test.go @@ -0,0 +1,262 @@ +package file + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "testing" + "time" + + "github.com/containous/traefik/safe" + "github.com/containous/traefik/types" + "github.com/stretchr/testify/assert" +) + +func TestProvideSingleFileAndWatch(t *testing.T) { + tempDir := createTempDir(t, "testfile") + defer os.RemoveAll(tempDir) + + expectedNumFrontends := 2 + expectedNumBackends := 2 + + tempFile := createFile(t, + tempDir, "simple.toml", + createFrontendConfiguration(expectedNumFrontends), + createBackendConfiguration(expectedNumBackends)) + + configurationChan, signal := createConfigurationRoutine(t, &expectedNumFrontends, &expectedNumBackends) + + provide(configurationChan, watch, withFile(tempFile)) + + // Wait for initial message to be tested + err := waitForSignal(signal, 2*time.Second, "initial config") + assert.NoError(t, err) + + // Now test again with single frontend and backend + expectedNumFrontends = 1 + expectedNumBackends = 1 + + tempFile = createFile(t, + tempDir, "simple.toml", + createFrontendConfiguration(expectedNumFrontends), + createBackendConfiguration(expectedNumBackends)) + + // Must fail because we don't watch the change + err = waitForSignal(signal, 2*time.Second, "single frontend and backend") + assert.NoError(t, err) +} + +func TestProvideSingleFileAndNotWatch(t *testing.T) { + tempDir := createTempDir(t, "testfile") + defer os.RemoveAll(tempDir) + + expectedNumFrontends := 2 + expectedNumBackends := 2 + + tempFile := createFile(t, + tempDir, "simple.toml", + createFrontendConfiguration(expectedNumFrontends), + createBackendConfiguration(expectedNumBackends)) + + configurationChan, signal := createConfigurationRoutine(t, &expectedNumFrontends, &expectedNumBackends) + + provide(configurationChan, withFile(tempFile)) + + // Wait for initial message to be tested + err := waitForSignal(signal, 2*time.Second, "initial config") + assert.NoError(t, err) + + // Now test again with single frontend and backend + expectedNumFrontends = 1 + expectedNumBackends = 1 + + tempFile = createFile(t, + tempDir, "simple.toml", + createFrontendConfiguration(expectedNumFrontends), + createBackendConfiguration(expectedNumBackends)) + + // Must fail because we don't watch the changes + err = waitForSignal(signal, 2*time.Second, "single frontend and backend") + assert.Error(t, err) +} + +func TestProvideDirectoryAndWatch(t *testing.T) { + tempDir := createTempDir(t, "testdir") + defer os.RemoveAll(tempDir) + + expectedNumFrontends := 2 + expectedNumBackends := 2 + + tempFile1 := createRandomFile(t, tempDir, createFrontendConfiguration(expectedNumFrontends)) + tempFile2 := createRandomFile(t, tempDir, createBackendConfiguration(expectedNumBackends)) + + configurationChan, signal := createConfigurationRoutine(t, &expectedNumFrontends, &expectedNumBackends) + + provide(configurationChan, watch, withDirectory(tempDir)) + + // Wait for initial config message to be tested + err := waitForSignal(signal, 2*time.Second, "initial config") + assert.NoError(t, err) + + // Now remove the backends file + expectedNumFrontends = 2 + expectedNumBackends = 0 + os.Remove(tempFile2.Name()) + err = waitForSignal(signal, 2*time.Second, "remove the backends file") + assert.NoError(t, err) + + // Now remove the frontends file + expectedNumFrontends = 0 + expectedNumBackends = 0 + os.Remove(tempFile1.Name()) + err = waitForSignal(signal, 2*time.Second, "remove the frontends file") + assert.NoError(t, err) +} + +func TestProvideDirectoryAndNotWatch(t *testing.T) { + tempDir := createTempDir(t, "testdir") + defer os.RemoveAll(tempDir) + + expectedNumFrontends := 2 + expectedNumBackends := 2 + + createRandomFile(t, tempDir, createFrontendConfiguration(expectedNumFrontends)) + tempFile2 := createRandomFile(t, tempDir, createBackendConfiguration(expectedNumBackends)) + + configurationChan, signal := createConfigurationRoutine(t, &expectedNumFrontends, &expectedNumBackends) + + provide(configurationChan, withDirectory(tempDir)) + + // Wait for initial config message to be tested + err := waitForSignal(signal, 2*time.Second, "initial config") + assert.NoError(t, err) + + // Now remove the backends file + expectedNumFrontends = 2 + expectedNumBackends = 0 + os.Remove(tempFile2.Name()) + + // Must fail because we don't watch the changes + err = waitForSignal(signal, 2*time.Second, "remove the backends file") + assert.Error(t, err) + +} + +func createConfigurationRoutine(t *testing.T, expectedNumFrontends *int, expectedNumBackends *int) (chan types.ConfigMessage, chan interface{}) { + configurationChan := make(chan types.ConfigMessage) + signal := make(chan interface{}) + + go func() { + for { + data := <-configurationChan + assert.Equal(t, "file", data.ProviderName) + assert.Len(t, data.Configuration.Frontends, *expectedNumFrontends) + assert.Len(t, data.Configuration.Backends, *expectedNumBackends) + signal <- nil + } + }() + + return configurationChan, signal +} + +func waitForSignal(signal chan interface{}, timeout time.Duration, caseName string) error { + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case <-signal: + + case <-timer.C: + return fmt.Errorf("Timed out waiting for assertions to be tested: %s", caseName) + } + return nil +} + +func provide(configurationChan chan types.ConfigMessage, builders ...func(p *Provider)) { + pvd := &Provider{} + + for _, builder := range builders { + builder(pvd) + } + + pvd.Provide(configurationChan, safe.NewPool(context.Background()), nil) +} + +func watch(pvd *Provider) { + pvd.Watch = true +} + +func withDirectory(name string) func(*Provider) { + return func(pvd *Provider) { + pvd.Directory = name + } +} + +func withFile(tempFile *os.File) func(*Provider) { + return func(p *Provider) { + p.Filename = tempFile.Name() + } +} + +// createRandomFile Helper +func createRandomFile(t *testing.T, tempDir string, contents ...string) *os.File { + return createFile(t, tempDir, fmt.Sprintf("temp%d.toml", time.Now().UnixNano()), contents...) +} + +// createFile Helper +func createFile(t *testing.T, tempDir string, name string, contents ...string) *os.File { + fileName := path.Join(tempDir, name) + + tempFile, err := os.Create(fileName) + if err != nil { + t.Fatal(err) + } + + for _, content := range contents { + _, err := tempFile.WriteString(content) + if err != nil { + t.Fatal(err) + } + } + + err = tempFile.Close() + if err != nil { + t.Fatal(err) + } + + return tempFile +} + +// createTempDir Helper +func createTempDir(t *testing.T, dir string) string { + d, err := ioutil.TempDir("", dir) + if err != nil { + t.Fatal(err) + } + return d +} + +// createFrontendConfiguration Helper +func createFrontendConfiguration(n int) string { + conf := "[frontends]\n" + for i := 1; i <= n; i++ { + conf += fmt.Sprintf(` [frontends.frontend%[1]d] + backend = "backend%[1]d" +`, i) + } + return conf +} + +// createBackendConfiguration Helper +func createBackendConfiguration(n int) string { + conf := "[backends]\n" + for i := 1; i <= n; i++ { + conf += fmt.Sprintf(` [backends.backend%[1]d] + [backends.backend%[1]d.servers.server1] + url = "http://172.17.0.%[1]d:80" +`, i) + } + return conf +} diff --git a/traefik.sample.toml b/traefik.sample.toml index c86f82aec..b2e1a1db2 100644 --- a/traefik.sample.toml +++ b/traefik.sample.toml @@ -424,6 +424,14 @@ # # filename = "rules.toml" +# Rules file +# If defined, traefik will load rules from .toml files in this directory. +# +# Optional +# +# directory = "/path/to/config/" + + # Enable watch file changes # # Optional