Commit 54c63002 authored by Mateusz Gajewski's avatar Mateusz Gajewski Committed by Matt Holt

Feature #1282 - Support serving statically compressed .gz and .br files (#1289)

* Feature #1282 - Support pre-gzipped files

* Fix broken test cases

* Support brotli encoding as well

* Fix for #1276 - support integers and floats as metadata in markdown (#1278)

* Fix for #1276

* Use strconv.Format

* Use map[string]interface{} as variables

* One more file

* Always run all tests before commit

* Get rid of DocFlags

* Fix syntax in caddy.conf

* Update to Go 1.7.4

* Add send_timeout property to fastcgi directive.

* Convert rwc field on FCGIClient from type io.ReadWriteCloser to net.Conn.
* Return HTTP 504 to the client when a timeout occurs.
* In Handler.ServeHTTP(), close the connection before returning an HTTP
502/504.
* Refactor tests and add coverage.

* Return HTTP 504 when FastCGI connect times out.

* test: add unit test for #1283 (#1288)

* After review fixes

* Limit the number of restarts with systemd

* Prevent fd leak

* Prevent fd leak

* Refactor loops

* gofmt
parent c555e953
......@@ -53,9 +53,6 @@ outer:
}
}
// Delete this header so gzipping is not repeated later in the chain
r.Header.Del("Accept-Encoding")
// gzipWriter modifies underlying writer at init,
// use a discard writer instead to leave ResponseWriter in
// original form.
......
......@@ -91,9 +91,6 @@ func nextFunc(shouldGzip bool) httpserver.Handler {
}
if shouldGzip {
if r.Header.Get("Accept-Encoding") != "" {
return 0, fmt.Errorf("Accept-Encoding header not expected")
}
if w.Header().Get("Content-Encoding") != "gzip" {
return 0, fmt.Errorf("Content-Encoding must be gzip, found %v", r.Header.Get("Content-Encoding"))
}
......
......@@ -25,6 +25,20 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool {
return l != 0 && int64(l) <= length
}
// SkipCompressedFilter is ResponseFilter that will discard already compressed responses
type SkipCompressedFilter struct{}
// ShouldCompress returns true if served file is not already compressed
// encodings via https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
func (n SkipCompressedFilter) ShouldCompress(w http.ResponseWriter) bool {
switch w.Header().Get("Content-Encoding") {
case "gzip", "compress", "deflate", "br":
return false
default:
return true
}
}
// ResponseFilterWriter validates ResponseFilters. It writes
// gzip compressed data if ResponseFilters are satisfied or
// uncompressed data otherwise.
......
......@@ -87,3 +87,26 @@ func TestResponseFilterWriter(t *testing.T) {
}
}
}
func TestResponseGzippedOutput(t *testing.T) {
server := Gzip{Configs: []Config{
{ResponseFilters: []ResponseFilter{SkipCompressedFilter{}}},
}}
server.Next = httpserver.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) {
w.Header().Set("Content-Encoding", "gzip")
w.Write([]byte("gzipped"))
return 200, nil
})
r := urlRequest("/")
r.Header.Set("Accept-Encoding", "gzip")
w := httptest.NewRecorder()
server.ServeHTTP(w, r)
resp := w.Body.String()
if resp != "gzipped" {
t.Errorf("Expected output not to be gzipped")
}
}
......@@ -106,6 +106,8 @@ func gzipParse(c *caddy.Controller) ([]Config, error) {
config.RequestFilters = append(config.RequestFilters, DefaultExtFilter())
}
config.ResponseFilters = append(config.ResponseFilters, SkipCompressedFilter{})
// Response Filters
// If min_length is specified, use it.
if int64(lengthFilter) != 0 {
......
......@@ -99,3 +99,31 @@ func TestSetup(t *testing.T) {
}
}
}
func TestShouldAddResponseFilters(t *testing.T) {
configs, err := gzipParse(caddy.NewTestController("http", `gzip { min_length 654 }`))
if err != nil {
t.Errorf("Test expected no error but found: %v", err)
}
filters := 0
for _, config := range configs {
for _, filter := range config.ResponseFilters {
switch filter.(type) {
case SkipCompressedFilter:
filters++
case LengthFilter:
filters++
if filter != LengthFilter(654) {
t.Errorf("Expected LengthFilter to have length 654, got: %v", filter)
}
}
}
if filters != 2 {
t.Errorf("Expected 2 response filters to be registered, got: %v", filters)
}
}
}
......@@ -43,6 +43,9 @@ func (fs FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, err
// serveFile writes the specified file to the HTTP response.
// name is '/'-separated, not filepath.Separator.
func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name string) (int, error) {
location := name
// Prevent absolute path access on Windows.
// TODO remove when stdlib http.Dir fixes this.
if runtime.GOOS == "windows" {
......@@ -97,17 +100,27 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri
for _, indexPage := range IndexPages {
index := strings.TrimSuffix(name, "/") + "/" + indexPage
ff, err := fs.Root.Open(index)
if err == nil {
// this defer does not leak fds because previous iterations
// of the loop must have had an err, so nothing to close
defer ff.Close()
dd, err := ff.Stat()
if err == nil {
d = dd
f = ff
break
}
if err != nil {
continue
}
// this defer does not leak fds because previous iterations
// of the loop must have had an err, so nothing to close
defer ff.Close()
dd, err := ff.Stat()
if err != nil {
ff.Close()
continue
}
// Close previous file - release fd immediately
f.Close()
d = dd
f = ff
location = index
break
}
}
......@@ -121,13 +134,48 @@ func (fs FileServer) serveFile(w http.ResponseWriter, r *http.Request, name stri
return http.StatusNotFound, nil
}
filename := d.Name()
for _, encoding := range staticEncodingPriority {
if !strings.Contains(r.Header.Get("Accept-Encoding"), encoding) {
continue
}
encodedFile, err := fs.Root.Open(location + staticEncoding[encoding])
if err != nil {
continue
}
encodedFileInfo, err := encodedFile.Stat()
if err != nil {
encodedFile.Close()
continue
}
// Close previous file - release fd
f.Close()
// Stat is needed for generating valid ETag
d = encodedFileInfo
// Encoded file will be served
f = encodedFile
w.Header().Add("Vary", "Accept-Encoding")
w.Header().Set("Content-Encoding", encoding)
defer f.Close()
break
}
// Experimental ETag header
e := fmt.Sprintf(`W/"%x-%x"`, d.ModTime().Unix(), d.Size())
w.Header().Set("ETag", e)
// Note: Errors generated by ServeContent are written immediately
// to the response. This usually only happens if seeking fails (rare).
http.ServeContent(w, r, d.Name(), d.ModTime(), f)
http.ServeContent(w, r, filename, d.ModTime(), f)
return http.StatusOK, nil
}
......@@ -168,3 +216,17 @@ var IndexPages = []string{
"default.htm",
"default.txt",
}
// staticEncoding is a map of content-encoding to a file extension.
// If client accepts given encoding (via Accept-Encoding header) and compressed file with given extensions exists
// it will be served to the client instead of original one.
var staticEncoding = map[string]string{
"gzip": ".gz",
"br": ".br",
}
// staticEncodingPriority is a list of preferred static encodings (most efficient compression to least one).
var staticEncodingPriority = []string{
"br",
"gzip",
}
......@@ -33,6 +33,12 @@ var (
var testFiles = map[string]string{
"unreachable.html": "<h1>must not leak</h1>",
filepath.Join("webroot", "file1.html"): "<h1>file1.html</h1>",
filepath.Join("webroot", "sub", "gzipped.html"): "<h1>gzipped.html</h1>",
filepath.Join("webroot", "sub", "gzipped.html.gz"): "gzipped.html.gz",
filepath.Join("webroot", "sub", "gzipped.html.gz"): "gzipped.html.gz",
filepath.Join("webroot", "sub", "brotli.html"): "brotli.html",
filepath.Join("webroot", "sub", "brotli.html.gz"): "brotli.html.gz",
filepath.Join("webroot", "sub", "brotli.html.br"): "brotli.html.br",
filepath.Join("webroot", "dirwithindex", "index.html"): "<h1>dirwithindex/index.html</h1>",
filepath.Join("webroot", "dir", "file2.html"): "<h1>dir/file2.html</h1>",
filepath.Join("webroot", "dir", "hidden.html"): "<h1>dir/hidden.html</h1>",
......@@ -72,14 +78,14 @@ func TestServeHTTP(t *testing.T) {
{
url: "https://foo/file1.html",
expectedStatus: http.StatusOK,
expectedBodyContent: testFiles["file1.html"],
expectedBodyContent: testFiles[filepath.Join("webroot", "file1.html")],
expectedEtag: `W/"1e240-13"`,
},
// Test 3 - access folder with index file with trailing slash
{
url: "https://foo/dirwithindex/",
expectedStatus: http.StatusOK,
expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")],
expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")],
expectedEtag: `W/"1e240-20"`,
},
// Test 4 - access folder with index file without trailing slash
......@@ -119,7 +125,7 @@ func TestServeHTTP(t *testing.T) {
{
url: "https://foo/dirwithindex/index.html",
expectedStatus: http.StatusOK,
expectedBodyContent: testFiles[filepath.Join("dirwithindex", "index.html")],
expectedBodyContent: testFiles[filepath.Join("webroot", "dirwithindex", "index.html")],
expectedEtag: `W/"1e240-20"`,
},
// Test 11 - send a request with query params
......@@ -158,11 +164,28 @@ func TestServeHTTP(t *testing.T) {
url: "https://foo/%2f..%2funreachable.html",
expectedStatus: http.StatusNotFound,
},
// Test 18 - try to get pre-gzipped file.
{
url: "https://foo/sub/gzipped.html",
expectedStatus: http.StatusOK,
expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "gzipped.html.gz")],
expectedEtag: `W/"1e240-f"`,
},
// Test 19 - try to get pre-brotli encoded file.
{
url: "https://foo/sub/brotli.html",
expectedStatus: http.StatusOK,
expectedBodyContent: testFiles[filepath.Join("webroot", "sub", "brotli.html.br")],
expectedEtag: `W/"1e240-e"`,
},
}
for i, test := range tests {
responseRecorder := httptest.NewRecorder()
request, err := http.NewRequest("GET", test.url, nil)
request.Header.Add("Accept-Encoding", "br,gzip")
if err != nil {
t.Errorf("Test %d: Error making request: %v", i, err)
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment