Commit 81e26970 authored by Matt Holt's avatar Matt Holt

proxy: Move handling of headers around to prevent memory usage spikes (#784)

* Move handling of headers around to prevent memory use spikes

While debugging #782, I noticed that using http2 and max_fails=0,
X-Forwarded-For grew infinitely when an upstream request failed after
refreshing the test page. This change ensures that headers are only
set once per request rather than appending in a time-terminated loop.

* Refactor some code into its own function
parent f561dc0b
...@@ -3,8 +3,10 @@ package proxy ...@@ -3,8 +3,10 @@ package proxy
import ( import (
"errors" "errors"
"net"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"sync/atomic" "sync/atomic"
"time" "time"
...@@ -75,71 +77,108 @@ var tryDuration = 60 * time.Second ...@@ -75,71 +77,108 @@ var tryDuration = 60 * time.Second
// ServeHTTP satisfies the middleware.Handler interface. // ServeHTTP satisfies the middleware.Handler interface.
func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
for _, upstream := range p.Upstreams { for _, upstream := range p.Upstreams {
if middleware.Path(r.URL.Path).Matches(upstream.From()) && upstream.AllowedPath(r.URL.Path) { if !middleware.Path(r.URL.Path).Matches(upstream.From()) ||
var replacer middleware.Replacer !upstream.AllowedPath(r.URL.Path) {
start := time.Now() continue
requestHost := r.Host }
// Since Select() should give us "up" hosts, keep retrying
// hosts until timeout (or until we get a nil host).
for time.Now().Sub(start) < tryDuration {
host := upstream.Select()
if host == nil {
return http.StatusBadGateway, errUnreachable
}
proxy := host.ReverseProxy
r.Host = host.Name
if rr, ok := w.(*middleware.ResponseRecorder); ok && rr.Replacer != nil {
rr.Replacer.Set("upstream", host.Name)
}
if baseURL, err := url.Parse(host.Name); err == nil { var replacer middleware.Replacer
r.Host = baseURL.Host start := time.Now()
if proxy == nil {
proxy = NewSingleHostReverseProxy(baseURL, host.WithoutPathPrefix) outreq := createUpstreamRequest(r)
}
} else if proxy == nil { // Since Select() should give us "up" hosts, keep retrying
return http.StatusInternalServerError, err // hosts until timeout (or until we get a nil host).
for time.Now().Sub(start) < tryDuration {
host := upstream.Select()
if host == nil {
return http.StatusBadGateway, errUnreachable
}
if rr, ok := w.(*middleware.ResponseRecorder); ok && rr.Replacer != nil {
rr.Replacer.Set("upstream", host.Name)
}
outreq.Host = host.Name
if host.ExtraHeaders != nil {
extraHeaders := make(http.Header)
if replacer == nil {
rHost := r.Host
replacer = middleware.NewReplacer(r, nil, "")
outreq.Host = rHost
} }
var extraHeaders http.Header for header, values := range host.ExtraHeaders {
if host.ExtraHeaders != nil { for _, value := range values {
extraHeaders = make(http.Header) extraHeaders.Add(header, replacer.Replace(value))
if replacer == nil { if header == "Host" {
rHost := r.Host outreq.Host = replacer.Replace(value)
r.Host = requestHost
replacer = middleware.NewReplacer(r, nil, "")
r.Host = rHost
}
for header, values := range host.ExtraHeaders {
for _, value := range values {
extraHeaders.Add(header,
replacer.Replace(value))
if header == "Host" {
r.Host = replacer.Replace(value)
}
} }
} }
} }
for k, v := range extraHeaders {
atomic.AddInt64(&host.Conns, 1) outreq.Header[k] = v
backendErr := proxy.ServeHTTP(w, r, extraHeaders)
atomic.AddInt64(&host.Conns, -1)
if backendErr == nil {
return 0, nil
} }
timeout := host.FailTimeout }
if timeout == 0 {
timeout = 10 * time.Second proxy := host.ReverseProxy
if baseURL, err := url.Parse(host.Name); err == nil {
r.Host = baseURL.Host
if proxy == nil {
proxy = NewSingleHostReverseProxy(baseURL, host.WithoutPathPrefix)
} }
atomic.AddInt32(&host.Fails, 1) } else if proxy == nil {
go func(host *UpstreamHost, timeout time.Duration) { return http.StatusInternalServerError, err
time.Sleep(timeout) }
atomic.AddInt32(&host.Fails, -1)
}(host, timeout) atomic.AddInt64(&host.Conns, 1)
backendErr := proxy.ServeHTTP(w, outreq)
atomic.AddInt64(&host.Conns, -1)
if backendErr == nil {
return 0, nil
} }
return http.StatusBadGateway, errUnreachable timeout := host.FailTimeout
if timeout == 0 {
timeout = 10 * time.Second
}
atomic.AddInt32(&host.Fails, 1)
go func(host *UpstreamHost, timeout time.Duration) {
time.Sleep(timeout)
atomic.AddInt32(&host.Fails, -1)
}(host, timeout)
} }
return http.StatusBadGateway, errUnreachable
} }
return p.Next.ServeHTTP(w, r) return p.Next.ServeHTTP(w, r)
} }
// createUpstremRequest shallow-copies r into a new request
// that can be sent upstream.
func createUpstreamRequest(r *http.Request) *http.Request {
outreq := new(http.Request)
*outreq = *r // includes shallow copies of maps, but okay
// Remove hop-by-hop headers to the backend. Especially
// important is "Connection" because we want a persistent
// connection, regardless of what the client sent to us. This
// is modifying the same underlying map from r (shallow
// copied above) so we only copy it if necessary.
for _, h := range hopHeaders {
if outreq.Header.Get(h) != "" {
outreq.Header = make(http.Header)
copyHeader(outreq.Header, r.Header)
outreq.Header.Del(h)
}
}
if clientIP, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
// If we aren't the first proxy, retain prior
// X-Forwarded-For information as a comma+space
// separated list and fold multiple headers into one.
if prior, ok := outreq.Header["X-Forwarded-For"]; ok {
clientIP = strings.Join(prior, ", ") + ", " + clientIP
}
outreq.Header.Set("X-Forwarded-For", clientIP)
}
return outreq
}
...@@ -154,54 +154,18 @@ var InsecureTransport http.RoundTripper = &http.Transport{ ...@@ -154,54 +154,18 @@ var InsecureTransport http.RoundTripper = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
} }
func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request, extraHeaders http.Header) error { func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, outreq *http.Request) error {
transport := p.Transport transport := p.Transport
if transport == nil { if transport == nil {
transport = http.DefaultTransport transport = http.DefaultTransport
} }
outreq := new(http.Request)
*outreq = *req // includes shallow copies of maps, but okay
p.Director(outreq) p.Director(outreq)
outreq.Proto = "HTTP/1.1" outreq.Proto = "HTTP/1.1"
outreq.ProtoMajor = 1 outreq.ProtoMajor = 1
outreq.ProtoMinor = 1 outreq.ProtoMinor = 1
outreq.Close = false outreq.Close = false
// Remove hop-by-hop headers to the backend. Especially
// important is "Connection" because we want a persistent
// connection, regardless of what the client sent to us. This
// is modifying the same underlying map from req (shallow
// copied above) so we only copy it if necessary.
copiedHeaders := false
for _, h := range hopHeaders {
if outreq.Header.Get(h) != "" {
if !copiedHeaders {
outreq.Header = make(http.Header)
copyHeader(outreq.Header, req.Header)
copiedHeaders = true
}
outreq.Header.Del(h)
}
}
if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
// If we aren't the first proxy retain prior
// X-Forwarded-For information as a comma+space
// separated list and fold multiple headers into one.
if prior, ok := outreq.Header["X-Forwarded-For"]; ok {
clientIP = strings.Join(prior, ", ") + ", " + clientIP
}
outreq.Header.Set("X-Forwarded-For", clientIP)
}
if extraHeaders != nil {
for k, v := range extraHeaders {
outreq.Header[k] = v
}
}
res, err := transport.RoundTrip(outreq) res, err := transport.RoundTrip(outreq)
if err != nil { if err != nil {
return err return err
...@@ -237,9 +201,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request, extr ...@@ -237,9 +201,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request, extr
for _, h := range hopHeaders { for _, h := range hopHeaders {
res.Header.Del(h) res.Header.Del(h)
} }
copyHeader(rw.Header(), res.Header) copyHeader(rw.Header(), res.Header)
rw.WriteHeader(res.StatusCode) rw.WriteHeader(res.StatusCode)
p.copyResponse(rw, res.Body) p.copyResponse(rw, res.Body)
} }
...@@ -260,7 +222,6 @@ func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader) { ...@@ -260,7 +222,6 @@ func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader) {
dst = mlw dst = mlw
} }
} }
io.Copy(dst, src) io.Copy(dst, src)
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment