From 630de7481e720ab9225a22e014b22078c7f37686 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 16 Nov 2022 15:34:10 +0100 Subject: [PATCH] Support SNI routing with Postgres STARTTLS connections Co-authored-by: Michael Kuhnt Co-authored-by: Julien Salleyron Co-authored-by: Mathieu Lonjaret --- docs/content/routing/routers/index.md | 48 ++++++-- pkg/server/router/tcp/postgres.go | 161 ++++++++++++++++++++++++++ pkg/server/router/tcp/router.go | 13 ++- pkg/server/router/tcp/router_test.go | 86 ++++++++++++++ 4 files changed, 295 insertions(+), 13 deletions(-) create mode 100644 pkg/server/router/tcp/postgres.go diff --git a/docs/content/routing/routers/index.md b/docs/content/routing/routers/index.md index 00bd82375..4c25f4e0d 100644 --- a/docs/content/routing/routers/index.md +++ b/docs/content/routing/routers/index.md @@ -233,18 +233,18 @@ If the rule is verified, the router becomes active, calls middlewares, and then The table below lists all the available matchers: -| Rule | Description | -|--------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| -| ```Headers(`key`, `value`)``` | Check if there is a key `key`defined in the headers, with the value `value` | -| ```HeadersRegexp(`key`, `regexp`)``` | Check if there is a key `key`defined in the headers, with a value that matches the regular expression `regexp` | -| ```Host(`example.com`, ...)``` | Check if the request domain (host header value) targets one of the given `domains`. | -| ```HostHeader(`example.com`, ...)``` | Same as `Host`, only exists for historical reasons. | -| ```HostRegexp(`example.com`, `{subdomain:[a-z]+}.example.com`, ...)``` | Match the request domain. See "Regexp Syntax" below. | -| ```Method(`GET`, ...)``` | Check if the request method is one of the given `methods` (`GET`, `POST`, `PUT`, `DELETE`, `PATCH`, `HEAD`) | -| ```Path(`/path`, `/articles/{cat:[a-z]+}/{id:[0-9]+}`, ...)``` | Match exact request path. See "Regexp Syntax" below. | -| ```PathPrefix(`/products/`, `/articles/{cat:[a-z]+}/{id:[0-9]+}`)``` | Match request prefix path. See "Regexp Syntax" below. | -| ```Query(`foo=bar`, `bar=baz`)``` | Match Query String parameters. It accepts a sequence of key=value pairs. | -| ```ClientIP(`10.0.0.0/16`, `::1`)``` | Match if the request client IP is one of the given IP/CIDR. It accepts IPv4, IPv6 and CIDR formats. | +| Rule | Description | +|------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| +| ```Headers(`key`, `value`)``` | Check if there is a key `key`defined in the headers, with the value `value` | +| ```HeadersRegexp(`key`, `regexp`)``` | Check if there is a key `key`defined in the headers, with a value that matches the regular expression `regexp` | +| ```Host(`example.com`, ...)``` | Check if the request domain (host header value) targets one of the given `domains`. | +| ```HostHeader(`example.com`, ...)``` | Same as `Host`, only exists for historical reasons. | +| ```HostRegexp(`example.com`, `{subdomain:[a-z]+}.example.com`, ...)``` | Match the request domain. See "Regexp Syntax" below. | +| ```Method(`GET`, ...)``` | Check if the request method is one of the given `methods` (`GET`, `POST`, `PUT`, `DELETE`, `PATCH`, `HEAD`) | +| ```Path(`/path`, `/articles/{cat:[a-z]+}/{id:[0-9]+}`, ...)``` | Match exact request path. See "Regexp Syntax" below. | +| ```PathPrefix(`/products/`, `/articles/{cat:[a-z]+}/{id:[0-9]+}`)``` | Match request prefix path. See "Regexp Syntax" below. | +| ```Query(`foo=bar`, `bar=baz`)``` | Match Query String parameters. It accepts a sequence of key=value pairs. | +| ```ClientIP(`10.0.0.0/16`, `::1`)``` | Match if the request client IP is one of the given IP/CIDR. It accepts IPv4, IPv6 and CIDR formats. | !!! important "Non-ASCII Domain Names" @@ -1041,6 +1041,30 @@ By default, a router with a TLS section will terminate the TLS connections, mean [tcp.routers.Router-1.tls] ``` +??? info "Postgres STARTTLS" + + Traefik supports the Postgres STARTTLS protocol, + which allows TLS routing for Postgres connections. + + To do so, Traefik reads the first bytes sent by a Postgres client, + identifies if they correspond to the message of a STARTTLS negotiation, + and, if so, acknowledges and signals the client that it can start the TLS handshake. + + Please note/remember that there are subtleties inherent to STARTTLS in whether + the connection ends up being a TLS one or not. These subtleties depend on the + `sslmode` value in the client configuration (and on the server authentication + rules). Therefore, it is recommended to use the `require` value for the + `sslmode`. + + Afterwards, the TLS handshake, and routing based on TLS, can proceed as expected. + + !!! warning "Postgres STARTTLS with TCP TLS PassThrough routers" + + As mentioned above, the `sslmode` configuration parameter does have an impact on + whether a STARTTLS session will succeed. In particular in the context of TCP TLS + PassThrough, some of the values (such as `allow`) do not even make sense. Which + is why, once more it is recommended to use the `require` value. + #### `passthrough` As seen above, a TLS router will terminate the TLS connection by default. diff --git a/pkg/server/router/tcp/postgres.go b/pkg/server/router/tcp/postgres.go new file mode 100644 index 000000000..857efb3ab --- /dev/null +++ b/pkg/server/router/tcp/postgres.go @@ -0,0 +1,161 @@ +package tcp + +import ( + "bufio" + "bytes" + "errors" + "sync" + + "github.com/traefik/traefik/v2/pkg/log" + tcpmuxer "github.com/traefik/traefik/v2/pkg/muxer/tcp" + "github.com/traefik/traefik/v2/pkg/tcp" +) + +var ( + PostgresStartTLSMsg = []byte{0, 0, 0, 8, 4, 210, 22, 47} // int32(8) + int32(80877103) + PostgresStartTLSReply = []byte{83} // S +) + +// isPostgres determines whether the buffer contains the Postgres STARTTLS message. +func isPostgres(br *bufio.Reader) (bool, error) { + // Peek the first 8 bytes individually to prevent blocking on peek + // if the underlying conn does not send enough bytes. + // It could happen if a protocol start by sending less than 8 bytes, + // and expect a response before proceeding. + for i := 1; i < len(PostgresStartTLSMsg)+1; i++ { + peeked, err := br.Peek(i) + if err != nil { + log.WithoutContext().Errorf("Error while Peeking first bytes: %s", err) + return false, err + } + + if !bytes.Equal(peeked, PostgresStartTLSMsg[:i]) { + return false, nil + } + } + return true, nil +} + +// servePostgres serves a connection with a Postgres client negotiating a STARTTLS session. +// It handles TCP TLS routing, after accepting to start the STARTTLS session. +func (r *Router) servePostgres(conn tcp.WriteCloser) { + _, err := conn.Write(PostgresStartTLSReply) + if err != nil { + conn.Close() + return + } + + br := bufio.NewReader(conn) + + b := make([]byte, len(PostgresStartTLSMsg)) + _, err = br.Read(b) + if err != nil { + conn.Close() + return + } + + hello, err := clientHelloInfo(br) + if err != nil { + conn.Close() + return + } + + if !hello.isTLS { + conn.Close() + return + } + + connData, err := tcpmuxer.NewConnData(hello.serverName, conn, hello.protos) + if err != nil { + log.WithoutContext().Errorf("Error while reading TCP connection data: %v", err) + conn.Close() + return + } + + // Contains also TCP TLS passthrough routes. + handlerTCPTLS, _ := r.muxerTCPTLS.Match(connData) + if handlerTCPTLS == nil { + conn.Close() + return + } + + // We are in TLS mode and if the handler is not TLSHandler, we are in passthrough. + proxiedConn := r.GetConn(conn, hello.peeked) + if _, ok := handlerTCPTLS.(*tcp.TLSHandler); !ok { + proxiedConn = &postgresConn{WriteCloser: proxiedConn} + } + + handlerTCPTLS.ServeTCP(proxiedConn) +} + +// postgresConn is a tcp.WriteCloser that will negotiate a TLS session (STARTTLS), +// before exchanging any data. +// It enforces that the STARTTLS negotiation with the peer is successful. +type postgresConn struct { + tcp.WriteCloser + + starttlsMsgSent bool // whether we have already sent the STARTTLS handshake to the backend. + starttlsReplyReceived bool // whether we have already received the STARTTLS handshake reply from the backend. + + // errChan makes sure that an error is returned if the first operation to ever + // happen on a postgresConn is a Write (because it should instead be a Read). + errChanMu sync.Mutex + errChan chan error +} + +// Read reads bytes from the underlying connection (tcp.WriteCloser). +// On first call, it actually only injects the PostgresStartTLSMsg, +// in order to behave as a Postgres TLS client that initiates a STARTTLS handshake. +// Read does not support concurrent calls. +func (c *postgresConn) Read(p []byte) (n int, err error) { + if c.starttlsMsgSent { + if err := <-c.errChan; err != nil { + return 0, err + } + + return c.WriteCloser.Read(p) + } + + defer func() { + c.starttlsMsgSent = true + c.errChanMu.Lock() + c.errChan = make(chan error) + c.errChanMu.Unlock() + }() + + copy(p, PostgresStartTLSMsg) + return len(PostgresStartTLSMsg), nil +} + +// Write writes bytes to the underlying connection (tcp.WriteCloser). +// On first call, it checks that the bytes to write (the ones provided by the backend) +// match the PostgresStartTLSReply, and if yes it drops them (as the STARTTLS +// handshake between the client and traefik has already taken place). Otherwise, an +// error is transmitted through c.errChan, so that the second Read call gets it and +// returns it up the stack. +// Write does not support concurrent calls. +func (c *postgresConn) Write(p []byte) (n int, err error) { + if c.starttlsReplyReceived { + return c.WriteCloser.Write(p) + } + + c.errChanMu.Lock() + if c.errChan == nil { + c.errChanMu.Unlock() + return 0, errors.New("initial read never happened") + } + c.errChanMu.Unlock() + + defer func() { + c.starttlsReplyReceived = true + }() + + if len(p) != 1 || p[0] != PostgresStartTLSReply[0] { + c.errChan <- errors.New("invalid response from Postgres server") + return len(p), nil + } + + close(c.errChan) + + return 1, nil +} diff --git a/pkg/server/router/tcp/router.go b/pkg/server/router/tcp/router.go index 0a41e80e2..841d52ae8 100644 --- a/pkg/server/router/tcp/router.go +++ b/pkg/server/router/tcp/router.go @@ -108,6 +108,17 @@ func (r *Router) ServeTCP(conn tcp.WriteCloser) { // TODO -- Check if ProxyProtocol changes the first bytes of the request br := bufio.NewReader(conn) + postgres, err := isPostgres(br) + if err != nil { + conn.Close() + return + } + + if postgres { + r.servePostgres(r.GetConn(conn, getPeeked(br))) + return + } + hello, err := clientHelloInfo(br) if err != nil { conn.Close() @@ -277,7 +288,7 @@ func (r *Router) SetHTTPSHandler(handler http.Handler, config *tls.Config) { type Conn struct { // Peeked are the bytes that have been read from Conn for the // purposes of route matching, but have not yet been consumed - // by Read calls. It set to nil by Read when fully consumed. + // by Read calls. It is set to nil by Read when fully consumed. Peeked []byte // Conn is the underlying connection. diff --git a/pkg/server/router/tcp/router_test.go b/pkg/server/router/tcp/router_test.go index 26e799837..d97646749 100644 --- a/pkg/server/router/tcp/router_test.go +++ b/pkg/server/router/tcp/router_test.go @@ -922,3 +922,89 @@ func checkHTTPSTLS10(addr string, timeout time.Duration) error { func checkHTTPSTLS12(addr string, timeout time.Duration) error { return checkHTTPS(addr, timeout, tls.VersionTLS12) } + +func TestPostgres(t *testing.T) { + router, err := NewRouter() + require.NoError(t, err) + + // This test requires to have a TLS route, but does not actually check the + // content of the handler. It would require to code a TLS handshake to + // check the SNI and content of the handlerFunc. + err = router.AddRouteTLS("HostSNI(`test.localhost`)", 0, nil, &tls.Config{}) + require.NoError(t, err) + + err = router.AddRoute("HostSNI(`*`)", 0, tcp2.HandlerFunc(func(conn tcp2.WriteCloser) { + _, _ = conn.Write([]byte("OK")) + _ = conn.Close() + })) + require.NoError(t, err) + + mockConn := NewMockConn() + go router.ServeTCP(mockConn) + + mockConn.dataRead <- PostgresStartTLSMsg + b := <-mockConn.dataWrite + require.Equal(t, PostgresStartTLSReply, b) + + mockConn = NewMockConn() + go router.ServeTCP(mockConn) + + mockConn.dataRead <- []byte("HTTP") + b = <-mockConn.dataWrite + require.Equal(t, []byte("OK"), b) +} + +func NewMockConn() *MockConn { + return &MockConn{ + dataRead: make(chan []byte), + dataWrite: make(chan []byte), + } +} + +type MockConn struct { + dataRead chan []byte + dataWrite chan []byte +} + +func (m *MockConn) Read(b []byte) (n int, err error) { + temp := <-m.dataRead + copy(b, temp) + return len(temp), nil +} + +func (m *MockConn) Write(b []byte) (n int, err error) { + m.dataWrite <- b + return len(b), nil +} + +func (m *MockConn) Close() error { + close(m.dataRead) + close(m.dataWrite) + return nil +} + +func (m *MockConn) LocalAddr() net.Addr { + return nil +} + +func (m *MockConn) RemoteAddr() net.Addr { + return &net.TCPAddr{} +} + +func (m *MockConn) SetDeadline(t time.Time) error { + return nil +} + +func (m *MockConn) SetReadDeadline(t time.Time) error { + return nil +} + +func (m *MockConn) SetWriteDeadline(t time.Time) error { + return nil +} + +func (m *MockConn) CloseWrite() error { + close(m.dataRead) + close(m.dataWrite) + return nil +}