traefik/pkg/middlewares/retry/retry_test.go

392 lines
12 KiB
Go
Raw Normal View History

2018-11-14 09:18:03 +00:00
package retry
2017-04-18 06:22:06 +00:00
import (
2018-11-14 09:18:03 +00:00
"context"
"fmt"
2021-03-04 19:08:03 +00:00
"io"
2017-04-18 06:22:06 +00:00
"net/http"
"net/http/httptest"
"net/http/httptrace"
"net/textproto"
2018-08-29 09:58:03 +00:00
"strings"
2017-04-18 06:22:06 +00:00
"testing"
"time"
2018-08-29 09:58:03 +00:00
"github.com/gorilla/websocket"
2018-08-06 18:00:03 +00:00
"github.com/stretchr/testify/assert"
2018-08-29 09:58:03 +00:00
"github.com/stretchr/testify/require"
ptypes "github.com/traefik/paerser/types"
"github.com/traefik/traefik/v2/pkg/config/dynamic"
"github.com/traefik/traefik/v2/pkg/testhelpers"
2017-04-18 06:22:06 +00:00
)
func TestRetry(t *testing.T) {
testCases := []struct {
2018-08-29 09:58:03 +00:00
desc string
config dynamic.Retry
2018-08-29 09:58:03 +00:00
wantRetryAttempts int
wantResponseStatus int
amountFaultyEndpoints int
2017-04-18 06:22:06 +00:00
}{
{
desc: "no retry on success",
2021-03-04 19:08:03 +00:00
config: dynamic.Retry{Attempts: 5},
wantRetryAttempts: 0,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 0,
},
{
desc: "no retry on success with backoff",
2021-03-04 19:08:03 +00:00
config: dynamic.Retry{Attempts: 5, InitialInterval: ptypes.Duration(time.Microsecond * 50)},
wantRetryAttempts: 0,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 0,
},
{
desc: "no retry when max request attempts is one",
config: dynamic.Retry{Attempts: 1},
wantRetryAttempts: 0,
wantResponseStatus: http.StatusBadGateway,
amountFaultyEndpoints: 1,
},
{
desc: "no retry when max request attempts is one with backoff",
config: dynamic.Retry{Attempts: 1, InitialInterval: ptypes.Duration(time.Microsecond * 50)},
wantRetryAttempts: 0,
wantResponseStatus: http.StatusBadGateway,
amountFaultyEndpoints: 1,
},
{
desc: "one retry when one server is faulty",
config: dynamic.Retry{Attempts: 2},
wantRetryAttempts: 1,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 1,
},
{
desc: "one retry when one server is faulty with backoff",
config: dynamic.Retry{Attempts: 2, InitialInterval: ptypes.Duration(time.Microsecond * 50)},
wantRetryAttempts: 1,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 1,
},
{
desc: "two retries when two servers are faulty",
config: dynamic.Retry{Attempts: 3},
wantRetryAttempts: 2,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 2,
},
{
desc: "two retries when two servers are faulty with backoff",
config: dynamic.Retry{Attempts: 3, InitialInterval: ptypes.Duration(time.Microsecond * 50)},
wantRetryAttempts: 2,
wantResponseStatus: http.StatusOK,
amountFaultyEndpoints: 2,
},
{
desc: "max attempts exhausted delivers the 5xx response",
config: dynamic.Retry{Attempts: 3},
wantRetryAttempts: 2,
wantResponseStatus: http.StatusBadGateway,
amountFaultyEndpoints: 3,
},
{
desc: "max attempts exhausted delivers the 5xx response with backoff",
config: dynamic.Retry{Attempts: 3, InitialInterval: ptypes.Duration(time.Microsecond * 50)},
wantRetryAttempts: 2,
wantResponseStatus: http.StatusBadGateway,
amountFaultyEndpoints: 3,
2017-04-18 06:22:06 +00:00
},
}
2018-08-29 09:58:03 +00:00
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
2017-04-18 06:22:06 +00:00
t.Parallel()
2022-02-21 11:40:09 +00:00
retryAttempts := 0
2021-03-04 19:08:03 +00:00
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
2022-02-21 11:40:09 +00:00
retryAttempts++
2022-02-21 11:40:09 +00:00
if retryAttempts > test.amountFaultyEndpoints {
2021-03-04 19:08:03 +00:00
// calls WroteHeaders on httptrace.
_ = r.Write(io.Discard)
2021-03-04 19:08:03 +00:00
rw.WriteHeader(http.StatusOK)
return
}
rw.WriteHeader(http.StatusBadGateway)
})
retryListener := &countingRetryListener{}
2021-03-04 19:08:03 +00:00
retry, err := New(context.Background(), next, test.config, retryListener, "traefikTest")
2018-11-14 09:18:03 +00:00
require.NoError(t, err)
2017-04-18 06:22:06 +00:00
recorder := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "http://localhost:3000/ok", nil)
retry.ServeHTTP(recorder, req)
2017-04-18 06:22:06 +00:00
2018-08-29 09:58:03 +00:00
assert.Equal(t, test.wantResponseStatus, recorder.Code)
assert.Equal(t, test.wantRetryAttempts, retryListener.timesCalled)
})
}
}
2018-11-14 09:18:03 +00:00
func TestRetryEmptyServerList(t *testing.T) {
2021-03-04 19:08:03 +00:00
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusServiceUnavailable)
})
2018-11-14 09:18:03 +00:00
retryListener := &countingRetryListener{}
retry, err := New(context.Background(), next, dynamic.Retry{Attempts: 3}, retryListener, "traefikTest")
2018-11-14 09:18:03 +00:00
require.NoError(t, err)
recorder := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "http://localhost:3000/ok", nil)
retry.ServeHTTP(recorder, req)
assert.Equal(t, http.StatusServiceUnavailable, recorder.Code)
assert.Equal(t, 0, retryListener.timesCalled)
}
func TestRetryListeners(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
retryListeners := Listeners{&countingRetryListener{}, &countingRetryListener{}}
retryListeners.Retried(req, 1)
retryListeners.Retried(req, 1)
for _, retryListener := range retryListeners {
listener := retryListener.(*countingRetryListener)
if listener.timesCalled != 2 {
t.Errorf("retry listener was called %d time(s), want %d time(s)", listener.timesCalled, 2)
}
}
}
func TestMultipleRetriesShouldNotLooseHeaders(t *testing.T) {
attempt := 0
expectedHeaderName := "X-Foo-Test-2"
expectedHeaderValue := "bar"
next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
headerName := fmt.Sprintf("X-Foo-Test-%d", attempt)
rw.Header().Add(headerName, expectedHeaderValue)
if attempt < 2 {
attempt++
return
}
// Request has been successfully written to backend
trace := httptrace.ContextClientTrace(req.Context())
trace.WroteHeaders()
// And we decide to answer to client
rw.WriteHeader(http.StatusNoContent)
})
retry, err := New(context.Background(), next, dynamic.Retry{Attempts: 3}, &countingRetryListener{}, "traefikTest")
require.NoError(t, err)
responseRecorder := httptest.NewRecorder()
retry.ServeHTTP(responseRecorder, testhelpers.MustNewRequest(http.MethodGet, "http://test", http.NoBody))
headerValue := responseRecorder.Header().Get(expectedHeaderName)
// Validate if we have the correct header
if headerValue != expectedHeaderValue {
t.Errorf("Expected to have %s for header %s, got %s", expectedHeaderValue, expectedHeaderName, headerValue)
}
// Validate that we don't have headers from previous attempts
for i := 0; i < attempt; i++ {
headerName := fmt.Sprintf("X-Foo-Test-%d", i)
headerValue = responseRecorder.Header().Get("headerName")
if headerValue != "" {
t.Errorf("Expected no value for header %s, got %s", headerName, headerValue)
}
}
}
2018-11-14 09:18:03 +00:00
// countingRetryListener is a Listener implementation to count the times the Retried fn is called.
type countingRetryListener struct {
timesCalled int
}
func (l *countingRetryListener) Retried(req *http.Request, attempt int) {
l.timesCalled++
}
func TestRetryWithFlush(t *testing.T) {
next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
2022-08-09 15:36:08 +00:00
rw.WriteHeader(http.StatusOK)
2018-11-14 09:18:03 +00:00
_, err := rw.Write([]byte("FULL "))
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
}
rw.(http.Flusher).Flush()
_, err = rw.Write([]byte("DATA"))
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
}
})
retry, err := New(context.Background(), next, dynamic.Retry{Attempts: 1}, &countingRetryListener{}, "traefikTest")
2018-11-14 09:18:03 +00:00
require.NoError(t, err)
responseRecorder := httptest.NewRecorder()
retry.ServeHTTP(responseRecorder, &http.Request{})
assert.Equal(t, "FULL DATA", responseRecorder.Body.String())
}
2018-08-29 09:58:03 +00:00
func TestRetryWebsocket(t *testing.T) {
testCases := []struct {
desc string
maxRequestAttempts int
expectedRetryAttempts int
expectedResponseStatus int
expectedError bool
amountFaultyEndpoints int
}{
{
desc: "Switching ok after 2 retries",
maxRequestAttempts: 3,
expectedRetryAttempts: 2,
amountFaultyEndpoints: 2,
expectedResponseStatus: http.StatusSwitchingProtocols,
},
{
desc: "Switching failed",
maxRequestAttempts: 2,
expectedRetryAttempts: 1,
amountFaultyEndpoints: 2,
expectedResponseStatus: http.StatusBadGateway,
expectedError: true,
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
2022-02-21 11:40:09 +00:00
retryAttempts := 0
2021-03-04 19:08:03 +00:00
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
2022-02-21 11:40:09 +00:00
retryAttempts++
2018-08-29 09:58:03 +00:00
2022-02-21 11:40:09 +00:00
if retryAttempts > test.amountFaultyEndpoints {
2021-03-04 19:08:03 +00:00
upgrader := websocket.Upgrader{}
_, err := upgrader.Upgrade(rw, r, nil)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
}
return
}
2018-08-29 09:58:03 +00:00
2021-03-04 19:08:03 +00:00
rw.WriteHeader(http.StatusBadGateway)
})
2018-08-29 09:58:03 +00:00
retryListener := &countingRetryListener{}
2021-03-04 19:08:03 +00:00
retryH, err := New(context.Background(), next, dynamic.Retry{Attempts: test.maxRequestAttempts}, retryListener, "traefikTest")
2018-11-14 09:18:03 +00:00
require.NoError(t, err)
2018-08-29 09:58:03 +00:00
2018-11-14 09:18:03 +00:00
retryServer := httptest.NewServer(retryH)
2018-08-29 09:58:03 +00:00
url := strings.Replace(retryServer.URL, "http", "ws", 1)
_, response, err := websocket.DefaultDialer.Dial(url, nil)
if !test.expectedError {
require.NoError(t, err)
2017-04-18 06:22:06 +00:00
}
2018-08-29 09:58:03 +00:00
assert.Equal(t, test.expectedResponseStatus, response.StatusCode)
assert.Equal(t, test.expectedRetryAttempts, retryListener.timesCalled)
2017-04-18 06:22:06 +00:00
})
}
}
// This test is an adapted version of net/http/httputil.Test1xxResponses test.
func Test1xxResponses(t *testing.T) {
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Add("Link", "</style.css>; rel=preload; as=style")
h.Add("Link", "</script.js>; rel=preload; as=script")
w.WriteHeader(http.StatusEarlyHints)
h.Add("Link", "</foo.js>; rel=preload; as=script")
w.WriteHeader(http.StatusProcessing)
_, _ = w.Write([]byte("Hello"))
})
retryListener := &countingRetryListener{}
retry, err := New(context.Background(), next, dynamic.Retry{Attempts: 1}, retryListener, "traefikTest")
require.NoError(t, err)
server := httptest.NewServer(retry)
t.Cleanup(server.Close)
frontendClient := server.Client()
checkLinkHeaders := func(t *testing.T, expected, got []string) {
t.Helper()
if len(expected) != len(got) {
t.Errorf("Expected %d link headers; got %d", len(expected), len(got))
}
for i := range expected {
if i >= len(got) {
t.Errorf("Expected %q link header; got nothing", expected[i])
continue
}
if expected[i] != got[i] {
t.Errorf("Expected %q link header; got %q", expected[i], got[i])
}
}
}
var respCounter uint8
trace := &httptrace.ClientTrace{
Got1xxResponse: func(code int, header textproto.MIMEHeader) error {
switch code {
case http.StatusEarlyHints:
checkLinkHeaders(t, []string{"</style.css>; rel=preload; as=style", "</script.js>; rel=preload; as=script"}, header["Link"])
case http.StatusProcessing:
checkLinkHeaders(t, []string{"</style.css>; rel=preload; as=style", "</script.js>; rel=preload; as=script", "</foo.js>; rel=preload; as=script"}, header["Link"])
default:
t.Error("Unexpected 1xx response")
}
respCounter++
return nil
},
}
req, _ := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodGet, server.URL, nil)
res, err := frontendClient.Do(req)
2023-11-17 00:50:06 +00:00
assert.NoError(t, err)
defer res.Body.Close()
if respCounter != 2 {
t.Errorf("Expected 2 1xx responses; got %d", respCounter)
}
checkLinkHeaders(t, []string{"</style.css>; rel=preload; as=style", "</script.js>; rel=preload; as=script", "</foo.js>; rel=preload; as=script"}, res.Header["Link"])
body, _ := io.ReadAll(res.Body)
if string(body) != "Hello" {
t.Errorf("Read body %q; want Hello", body)
}
assert.Equal(t, 0, retryListener.timesCalled)
}