Implements the includedContentTypes option for the compress middleware

This commit is contained in:
Robert Socha 2024-01-17 11:32:06 +01:00 committed by GitHub
parent 319517adef
commit 4e0a05406b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 469 additions and 24 deletions

View file

@ -58,7 +58,7 @@ http:
If the `Accept-Encoding` request header is absent, the response won't be encoded.
If it is present, but its value is the empty string, then compression is disabled.
* The response is not already compressed, i.e. the `Content-Encoding` response header is not already set.
* The response`Content-Type` header is not one among the [excludedContentTypes options](#excludedcontenttypes).
* The response`Content-Type` header is not one among the [excludedContentTypes options](#excludedcontenttypes), or is one among the [includedContentTypes options](#includedcontenttypes).
* The response body is larger than the [configured minimum amount of bytes](#minresponsebodybytes) (default is `1024`).
## Configuration Options
@ -73,6 +73,10 @@ The responses with content types defined in `excludedContentTypes` are not compr
Content types are compared in a case-insensitive, whitespace-ignored manner.
!!! info
The `excludedContentTypes` and `includedContentTypes` options are mutually exclusive.
!!! info "In the case of gzip"
If the `Content-Type` header is not defined, or empty, the compress middleware will automatically [detect](https://mimesniff.spec.whatwg.org/) a content type.
@ -117,6 +121,59 @@ http:
excludedContentTypes = ["text/event-stream"]
```
### `includedContentTypes`
_Optional, Default=""_
`includedContentTypes` specifies a list of content types to compare the `Content-Type` header of the responses before compressing.
The responses with content types defined in `includedContentTypes` are compressed.
Content types are compared in a case-insensitive, whitespace-ignored manner.
!!! info
The `excludedContentTypes` and `includedContentTypes` options are mutually exclusive.
```yaml tab="Docker & Swarm"
labels:
- "traefik.http.middlewares.test-compress.compress.includedcontenttypes=application/json,text/html,text/plain"
```
```yaml tab="Kubernetes"
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: test-compress
spec:
compress:
includedContentTypes:
- application/json
- text/html
- text/plain
```
```yaml tab="Consul Catalog"
- "traefik.http.middlewares.test-compress.compress.includedcontenttypes=application/json,text/html,text/plain"
```
```yaml tab="File (YAML)"
http:
middlewares:
test-compress:
compress:
includedContentTypes:
- application/json
- text/html
- text/plain
```
```toml tab="File (TOML)"
[http.middlewares]
[http.middlewares.test-compress.compress]
includedContentTypes = ["application/json","text/html","text/plain"]
```
### `minResponseBodyBytes`
_Optional, Default=1024_

View file

@ -18,6 +18,7 @@
- "traefik.http.middlewares.middleware05.circuitbreaker.recoveryduration=42s"
- "traefik.http.middlewares.middleware06.compress=true"
- "traefik.http.middlewares.middleware06.compress.excludedcontenttypes=foobar, foobar"
- "traefik.http.middlewares.middleware06.compress.includedcontenttypes=foobar, foobar"
- "traefik.http.middlewares.middleware06.compress.minresponsebodybytes=42"
- "traefik.http.middlewares.middleware07.contenttype=true"
- "traefik.http.middlewares.middleware08.digestauth.headerfield=foobar"

View file

@ -134,6 +134,7 @@
[http.middlewares.Middleware06]
[http.middlewares.Middleware06.compress]
excludedContentTypes = ["foobar", "foobar"]
includedContentTypes = ["foobar", "foobar"]
minResponseBodyBytes = 42
[http.middlewares.Middleware07]
[http.middlewares.Middleware07.contentType]

View file

@ -141,6 +141,9 @@ http:
excludedContentTypes:
- foobar
- foobar
includedContentTypes:
- foobar
- foobar
minResponseBodyBytes: 42
Middleware07:
contentType: {}

View file

@ -750,6 +750,13 @@ spec:
items:
type: string
type: array
includedContentTypes:
description: IncludedContentTypes defines the list of content
types to compare the Content-Type header of the responses before
compressing.
items:
type: string
type: array
minResponseBodyBytes:
description: 'MinResponseBodyBytes defines the minimum amount
of bytes a response body must have to be compressed. Default:

View file

@ -22,6 +22,8 @@ THIS FILE MUST NOT BE EDITED BY HAND
| `traefik/http/middlewares/Middleware05/circuitBreaker/recoveryDuration` | `42s` |
| `traefik/http/middlewares/Middleware06/compress/excludedContentTypes/0` | `foobar` |
| `traefik/http/middlewares/Middleware06/compress/excludedContentTypes/1` | `foobar` |
| `traefik/http/middlewares/Middleware06/compress/includedContentTypes/0` | `foobar` |
| `traefik/http/middlewares/Middleware06/compress/includedContentTypes/1` | `foobar` |
| `traefik/http/middlewares/Middleware06/compress/minResponseBodyBytes` | `42` |
| `traefik/http/middlewares/Middleware07/contentType` | `` |
| `traefik/http/middlewares/Middleware08/digestAuth/headerField` | `foobar` |

View file

@ -18,6 +18,7 @@
"traefik.http.middlewares.middleware05.circuitbreaker.recoveryduration": "42s",
"traefik.http.middlewares.middleware06.compress": "true",
"traefik.http.middlewares.middleware06.compress.excludedcontenttypes": "foobar, foobar",
"traefik.http.middlewares.middleware06.compress.includedcontenttypes": "foobar, foobar",
"traefik.http.middlewares.middleware06.compress.minresponsebodybytes": "42",
"traefik.http.middlewares.middleware07.contenttype": "true",
"traefik.http.middlewares.middleware08.digestauth.headerfield": "foobar",

View file

@ -175,6 +175,13 @@ spec:
items:
type: string
type: array
includedContentTypes:
description: IncludedContentTypes defines the list of content
types to compare the Content-Type header of the responses before
compressing.
items:
type: string
type: array
minResponseBodyBytes:
description: 'MinResponseBodyBytes defines the minimum amount
of bytes a response body must have to be compressed. Default:

View file

@ -750,6 +750,13 @@ spec:
items:
type: string
type: array
includedContentTypes:
description: IncludedContentTypes defines the list of content
types to compare the Content-Type header of the responses before
compressing.
items:
type: string
type: array
minResponseBodyBytes:
description: 'MinResponseBodyBytes defines the minimum amount
of bytes a response body must have to be compressed. Default:

View file

@ -159,6 +159,8 @@ type Compress struct {
// ExcludedContentTypes defines the list of content types to compare the Content-Type header of the incoming requests and responses before compressing.
// `application/grpc` is always excluded.
ExcludedContentTypes []string `json:"excludedContentTypes,omitempty" toml:"excludedContentTypes,omitempty" yaml:"excludedContentTypes,omitempty" export:"true"`
// IncludedContentTypes defines the list of content types to compare the Content-Type header of the responses before compressing.
IncludedContentTypes []string `json:"includedContentTypes,omitempty" toml:"includedContentTypes,omitempty" yaml:"includedContentTypes,omitempty" export:"true"`
// MinResponseBodyBytes defines the minimum amount of bytes a response body must have to be compressed.
// Default: 1024.
MinResponseBodyBytes int `json:"minResponseBodyBytes,omitempty" toml:"minResponseBodyBytes,omitempty" yaml:"minResponseBodyBytes,omitempty" export:"true"`

View file

@ -132,6 +132,11 @@ func (in *Compress) DeepCopyInto(out *Compress) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.IncludedContentTypes != nil {
in, out := &in.IncludedContentTypes, &out.IncludedContentTypes
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}

View file

@ -22,7 +22,11 @@ const (
// Config is the Brotli handler configuration.
type Config struct {
// ExcludedContentTypes is the list of content types for which we should not compress.
// Mutually exclusive with the IncludedContentTypes option.
ExcludedContentTypes []string
// IncludedContentTypes is the list of content types for which compression should be exclusively enabled.
// Mutually exclusive with the ExcludedContentTypes option.
IncludedContentTypes []string
// MinSize is the minimum size (in bytes) required to enable compression.
MinSize int
}
@ -33,14 +37,28 @@ func NewWrapper(cfg Config) (func(http.Handler) http.HandlerFunc, error) {
return nil, fmt.Errorf("minimum size must be greater than or equal to zero")
}
var contentTypes []parsedContentType
if len(cfg.ExcludedContentTypes) > 0 && len(cfg.IncludedContentTypes) > 0 {
return nil, fmt.Errorf("excludedContentTypes and includedContentTypes options are mutually exclusive")
}
var excludedContentTypes []parsedContentType
for _, v := range cfg.ExcludedContentTypes {
mediaType, params, err := mime.ParseMediaType(v)
if err != nil {
return nil, fmt.Errorf("parsing media type: %w", err)
return nil, fmt.Errorf("parsing excluded media type: %w", err)
}
contentTypes = append(contentTypes, parsedContentType{mediaType, params})
excludedContentTypes = append(excludedContentTypes, parsedContentType{mediaType, params})
}
var includedContentTypes []parsedContentType
for _, v := range cfg.IncludedContentTypes {
mediaType, params, err := mime.ParseMediaType(v)
if err != nil {
return nil, fmt.Errorf("parsing included media type: %w", err)
}
includedContentTypes = append(includedContentTypes, parsedContentType{mediaType, params})
}
return func(h http.Handler) http.HandlerFunc {
@ -52,7 +70,8 @@ func NewWrapper(cfg Config) (func(http.Handler) http.HandlerFunc, error) {
bw: brotli.NewWriter(rw),
minSize: cfg.MinSize,
statusCode: http.StatusOK,
excludedContentTypes: contentTypes,
excludedContentTypes: excludedContentTypes,
includedContentTypes: includedContentTypes,
}
defer brw.close()
@ -69,6 +88,7 @@ type responseWriter struct {
minSize int
excludedContentTypes []parsedContentType
includedContentTypes []parsedContentType
buf []byte
hijacked bool
@ -121,11 +141,25 @@ func (r *responseWriter) Write(p []byte) (int, error) {
return r.rw.Write(p)
}
// Disable compression according to user wishes in excludedContentTypes.
// Disable compression according to user wishes in excludedContentTypes or includedContentTypes.
if ct := r.rw.Header().Get(contentType); ct != "" {
mediaType, params, err := mime.ParseMediaType(ct)
if err != nil {
return 0, fmt.Errorf("parsing media type: %w", err)
return 0, fmt.Errorf("parsing content-type media type: %w", err)
}
if len(r.includedContentTypes) > 0 {
var found bool
for _, includedContentType := range r.includedContentTypes {
if includedContentType.equals(mediaType, params) {
found = true
break
}
}
if !found {
r.compressionDisabled = true
return r.rw.Write(p)
}
}
for _, excludedContentType := range r.excludedContentTypes {

View file

@ -291,10 +291,9 @@ func Test_ExcludedContentTypes(t *testing.T) {
expCompression bool
}{
{
desc: "Always compress when content types are empty",
contentType: "",
excludedContentTypes: []string{},
expCompression: true,
desc: "Always compress when content types are empty",
contentType: "",
expCompression: true,
},
{
desc: "MIME match",
@ -389,6 +388,111 @@ func Test_ExcludedContentTypes(t *testing.T) {
}
}
func Test_IncludedContentTypes(t *testing.T) {
testCases := []struct {
desc string
contentType string
includedContentTypes []string
expCompression bool
}{
{
desc: "Always compress when content types are empty",
contentType: "",
expCompression: true,
},
{
desc: "MIME match",
contentType: "application/json",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME no match",
contentType: "text/xml",
includedContentTypes: []string{"application/json"},
expCompression: false,
},
{
desc: "MIME match with no other directive ignores non-MIME directives",
contentType: "application/json; charset=utf-8",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME match with other directives requires all directives be equal, different charset",
contentType: "application/json; charset=ascii",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: false,
},
{
desc: "MIME match with other directives requires all directives be equal, same charset",
contentType: "application/json; charset=utf-8",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: true,
},
{
desc: "MIME match with other directives requires all directives be equal, missing charset",
contentType: "application/json",
includedContentTypes: []string{"application/json; charset=ascii"},
expCompression: false,
},
{
desc: "MIME match case insensitive",
contentType: "Application/Json",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME match ignore whitespace",
contentType: "application/json;charset=utf-8",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: true,
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
cfg := Config{
MinSize: 1024,
IncludedContentTypes: test.includedContentTypes,
}
h := mustNewWrapper(t, cfg)(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Header().Set(contentType, test.contentType)
rw.WriteHeader(http.StatusOK)
_, err := rw.Write(bigTestBody)
require.NoError(t, err)
}))
req, _ := http.NewRequest(http.MethodGet, "/whatever", nil)
req.Header.Set(acceptEncoding, "br")
rw := httptest.NewRecorder()
h.ServeHTTP(rw, req)
assert.Equal(t, http.StatusOK, rw.Code)
if test.expCompression {
assert.Equal(t, "br", rw.Header().Get(contentEncoding))
got, err := io.ReadAll(brotli.NewReader(rw.Body))
assert.NoError(t, err)
assert.Equal(t, bigTestBody, got)
} else {
assert.NotEqual(t, "br", rw.Header().Get("Content-Encoding"))
got, err := io.ReadAll(rw.Body)
assert.NoError(t, err)
assert.Equal(t, bigTestBody, got)
}
})
}
}
func Test_FlushExcludedContentTypes(t *testing.T) {
testCases := []struct {
desc string
@ -397,10 +501,9 @@ func Test_FlushExcludedContentTypes(t *testing.T) {
expCompression bool
}{
{
desc: "Always compress when content types are empty",
contentType: "",
excludedContentTypes: []string{},
expCompression: true,
desc: "Always compress when content types are empty",
contentType: "",
expCompression: true,
},
{
desc: "MIME match",
@ -509,6 +612,125 @@ func Test_FlushExcludedContentTypes(t *testing.T) {
}
}
func Test_FlushIncludedContentTypes(t *testing.T) {
testCases := []struct {
desc string
contentType string
includedContentTypes []string
expCompression bool
}{
{
desc: "Always compress when content types are empty",
contentType: "",
expCompression: true,
},
{
desc: "MIME match",
contentType: "application/json",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME no match",
contentType: "text/xml",
includedContentTypes: []string{"application/json"},
expCompression: false,
},
{
desc: "MIME match with no other directive ignores non-MIME directives",
contentType: "application/json; charset=utf-8",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME match with other directives requires all directives be equal, different charset",
contentType: "application/json; charset=ascii",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: false,
},
{
desc: "MIME match with other directives requires all directives be equal, same charset",
contentType: "application/json; charset=utf-8",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: true,
},
{
desc: "MIME match with other directives requires all directives be equal, missing charset",
contentType: "application/json",
includedContentTypes: []string{"application/json; charset=ascii"},
expCompression: false,
},
{
desc: "MIME match case insensitive",
contentType: "Application/Json",
includedContentTypes: []string{"application/json"},
expCompression: true,
},
{
desc: "MIME match ignore whitespace",
contentType: "application/json;charset=utf-8",
includedContentTypes: []string{"application/json; charset=utf-8"},
expCompression: true,
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
cfg := Config{
MinSize: 1024,
IncludedContentTypes: test.includedContentTypes,
}
h := mustNewWrapper(t, cfg)(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Header().Set(contentType, test.contentType)
rw.WriteHeader(http.StatusOK)
tb := bigTestBody
for len(tb) > 0 {
// Write 100 bytes per run
// Detection should not be affected (we send 100 bytes)
toWrite := 100
if toWrite > len(tb) {
toWrite = len(tb)
}
_, err := rw.Write(tb[:toWrite])
require.NoError(t, err)
// Flush between each write
rw.(http.Flusher).Flush()
tb = tb[toWrite:]
}
}))
req, _ := http.NewRequest(http.MethodGet, "/whatever", nil)
req.Header.Set(acceptEncoding, "br")
// This doesn't allow checking flushes, but we validate if content is correct.
rw := httptest.NewRecorder()
h.ServeHTTP(rw, req)
assert.Equal(t, http.StatusOK, rw.Code)
if test.expCompression {
assert.Equal(t, "br", rw.Header().Get(contentEncoding))
got, err := io.ReadAll(brotli.NewReader(rw.Body))
assert.NoError(t, err)
assert.Equal(t, bigTestBody, got)
} else {
assert.NotEqual(t, "br", rw.Header().Get(contentEncoding))
got, err := io.ReadAll(rw.Body)
assert.NoError(t, err)
assert.Equal(t, bigTestBody, got)
}
})
}
}
func mustNewWrapper(t *testing.T, cfg Config) func(http.Handler) http.HandlerFunc {
t.Helper()

View file

@ -26,6 +26,7 @@ type compress struct {
next http.Handler
name string
excludes []string
includes []string
minSize int
brotliHandler http.Handler
@ -36,16 +37,30 @@ type compress struct {
func New(ctx context.Context, next http.Handler, conf dynamic.Compress, name string) (http.Handler, error) {
middlewares.GetLogger(ctx, name, typeName).Debug().Msg("Creating middleware")
if len(conf.ExcludedContentTypes) > 0 && len(conf.IncludedContentTypes) > 0 {
return nil, fmt.Errorf("excludedContentTypes and includedContentTypes options are mutually exclusive")
}
excludes := []string{"application/grpc"}
for _, v := range conf.ExcludedContentTypes {
mediaType, _, err := mime.ParseMediaType(v)
if err != nil {
return nil, err
return nil, fmt.Errorf("parsing excluded media type: %w", err)
}
excludes = append(excludes, mediaType)
}
var includes []string
for _, v := range conf.IncludedContentTypes {
mediaType, _, err := mime.ParseMediaType(v)
if err != nil {
return nil, fmt.Errorf("parsing included media type: %w", err)
}
includes = append(includes, mediaType)
}
minSize := DefaultMinSize
if conf.MinResponseBodyBytes > 0 {
minSize = conf.MinResponseBodyBytes
@ -55,6 +70,7 @@ func New(ctx context.Context, next http.Handler, conf dynamic.Compress, name str
next: next,
name: name,
excludes: excludes,
includes: includes,
minSize: minSize,
}
@ -118,10 +134,21 @@ func (c *compress) GetTracingInformation() (string, string, trace.SpanKind) {
}
func (c *compress) newGzipHandler() (http.Handler, error) {
wrapper, err := gzhttp.NewWrapper(
gzhttp.ExceptContentTypes(c.excludes),
gzhttp.MinSize(c.minSize),
)
var wrapper func(http.Handler) http.HandlerFunc
var err error
if len(c.includes) > 0 {
wrapper, err = gzhttp.NewWrapper(
gzhttp.ContentTypes(c.includes),
gzhttp.MinSize(c.minSize),
)
} else {
wrapper, err = gzhttp.NewWrapper(
gzhttp.ExceptContentTypes(c.excludes),
gzhttp.MinSize(c.minSize),
)
}
if err != nil {
return nil, fmt.Errorf("new gzip wrapper: %w", err)
}
@ -130,9 +157,11 @@ func (c *compress) newGzipHandler() (http.Handler, error) {
}
func (c *compress) newBrotliHandler() (http.Handler, error) {
cfg := brotli.Config{
ExcludedContentTypes: c.excludes,
MinSize: c.minSize,
cfg := brotli.Config{MinSize: c.minSize}
if len(c.includes) > 0 {
cfg.IncludedContentTypes = c.includes
} else {
cfg.ExcludedContentTypes = c.excludes
}
wrapper, err := brotli.NewWrapper(cfg)

View file

@ -271,7 +271,28 @@ func TestShouldNotCompressWhenSpecificContentType(t *testing.T) {
respContentType: "text/event-stream",
},
{
desc: "application/grpc",
desc: "Include Response Content-Type",
conf: dynamic.Compress{
IncludedContentTypes: []string{"text/plain"},
},
respContentType: "text/html",
},
{
desc: "Ignoring application/grpc with exclude option",
conf: dynamic.Compress{
ExcludedContentTypes: []string{"application/json"},
},
reqContentType: "application/grpc",
},
{
desc: "Ignoring application/grpc with include option",
conf: dynamic.Compress{
IncludedContentTypes: []string{"application/json"},
},
reqContentType: "application/grpc",
},
{
desc: "Ignoring application/grpc with no option",
conf: dynamic.Compress{},
reqContentType: "application/grpc",
},
@ -312,6 +333,52 @@ func TestShouldNotCompressWhenSpecificContentType(t *testing.T) {
}
}
func TestShouldCompressWhenSpecificContentType(t *testing.T) {
baseBody := generateBytes(gzhttp.DefaultMinSize)
testCases := []struct {
desc string
conf dynamic.Compress
respContentType string
}{
{
desc: "Include Response Content-Type",
conf: dynamic.Compress{
IncludedContentTypes: []string{"text/html"},
},
respContentType: "text/html",
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
req := testhelpers.MustNewRequest(http.MethodGet, "http://localhost", nil)
req.Header.Add(acceptEncodingHeader, gzipValue)
next := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Set(contentTypeHeader, test.respContentType)
if _, err := rw.Write(baseBody); err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
}
})
handler, err := New(context.Background(), next, test.conf, "test")
require.NoError(t, err)
rw := httptest.NewRecorder()
handler.ServeHTTP(rw, req)
assert.Equal(t, gzipValue, rw.Header().Get(contentEncodingHeader))
assert.Equal(t, acceptEncodingHeader, rw.Header().Get(varyHeader))
assert.NotEqualValues(t, rw.Body.Bytes(), baseBody)
})
}
}
func TestIntegrationShouldNotCompress(t *testing.T) {
fakeCompressedBody := generateBytes(100000)