transport.go 64.3 KB
Newer Older
1 2 3 4
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
5
// HTTP client implementation. See RFC 2616.
6
//
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
7 8 9
// This is the low-level Transport implementation of RoundTripper.
// The high-level interface is in client.go.

10 11 12 13
package http

import (
	"bufio"
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
14
	"compress/gzip"
15
	"container/list"
16
	"context"
17
	"crypto/tls"
18
	"errors"
19
	"fmt"
20 21
	"io"
	"log"
22
	"net"
23
	"net/http/httptrace"
24
	"net/url"
25 26 27
	"os"
	"strings"
	"sync"
28
	"sync/atomic"
29
	"time"
30

31
	"golang_org/x/net/lex/httplex"
32
	"golang_org/x/net/proxy"
33 34
)

35
// DefaultTransport is the default implementation of Transport and is
36 37 38 39
// used by DefaultClient. It establishes network connections as needed
// and caches them for reuse by subsequent calls. It uses HTTP proxies
// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
// $no_proxy) environment variables.
40 41
var DefaultTransport RoundTripper = &Transport{
	Proxy: ProxyFromEnvironment,
42
	DialContext: (&net.Dialer{
43 44
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
45
		DualStack: true,
46
	}).DialContext,
47
	MaxIdleConns:          100,
48
	IdleConnTimeout:       90 * time.Second,
49 50
	TLSHandshakeTimeout:   10 * time.Second,
	ExpectContinueTimeout: 1 * time.Second,
51
}
52

53 54 55 56
// DefaultMaxIdleConnsPerHost is the default value of Transport's
// MaxIdleConnsPerHost.
const DefaultMaxIdleConnsPerHost = 2

57 58
// Transport is an implementation of RoundTripper that supports HTTP,
// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
59 60 61 62 63 64 65 66 67 68 69
//
// By default, Transport caches connections for future re-use.
// This may leave many open connections when accessing many hosts.
// This behavior can be managed using Transport's CloseIdleConnections method
// and the MaxIdleConnsPerHost and DisableKeepAlives fields.
//
// Transports should be reused instead of created as needed.
// Transports are safe for concurrent use by multiple goroutines.
//
// A Transport is a low-level primitive for making HTTP and HTTPS requests.
// For high-level functionality, such as cookies and redirects, see Client.
70 71
//
// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
72 73 74 75
// for HTTPS URLs, depending on whether the server supports HTTP/2,
// and how the Transport is configured. The DefaultTransport supports HTTP/2.
// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2
// and call ConfigureTransport. See the package docs for more about HTTP/2.
76
type Transport struct {
77
	idleMu     sync.Mutex
78 79
	wantIdle   bool                                // user has requested to close all idle conns
	idleConn   map[connectMethodKey][]*persistConn // most recently used at end
80
	idleConnCh map[connectMethodKey]chan *persistConn
81
	idleLRU    connLRU
82

83
	reqMu       sync.Mutex
84
	reqCanceler map[*Request]func(error)
85

86 87
	altMu    sync.Mutex   // guards changing altProto only
	altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme
88

89 90 91 92
	// Proxy specifies a function to return a proxy for a given
	// Request. If the function returns a non-nil error, the
	// request is aborted with the provided error.
	// If Proxy is nil or returns a nil *URL, no proxy is used.
93
	Proxy func(*Request) (*url.URL, error)
94

95 96 97 98 99 100
	// DialContext specifies the dial function for creating unencrypted TCP connections.
	// If DialContext is nil (and the deprecated Dial below is also nil),
	// then the transport dials using package net.
	DialContext func(ctx context.Context, network, addr string) (net.Conn, error)

	// Dial specifies the dial function for creating unencrypted TCP connections.
101
	//
102 103 104
	// Deprecated: Use DialContext instead, which allows the transport
	// to cancel dials as soon as they are no longer needed.
	// If both are set, DialContext takes priority.
105
	Dial func(network, addr string) (net.Conn, error)
106

107 108 109 110 111 112 113 114 115 116 117
	// DialTLS specifies an optional dial function for creating
	// TLS connections for non-proxied HTTPS requests.
	//
	// If DialTLS is nil, Dial and TLSClientConfig are used.
	//
	// If DialTLS is set, the Dial hook is not used for HTTPS
	// requests and the TLSClientConfig and TLSHandshakeTimeout
	// are ignored. The returned net.Conn is assumed to already be
	// past the TLS handshake.
	DialTLS func(network, addr string) (net.Conn, error)

118
	// TLSClientConfig specifies the TLS configuration to use with
119 120 121
	// tls.Client.
	// If nil, the default configuration is used.
	// If non-nil, HTTP/2 support may not be enabled by default.
122 123
	TLSClientConfig *tls.Config

124 125 126 127
	// TLSHandshakeTimeout specifies the maximum amount of time waiting to
	// wait for a TLS handshake. Zero means no timeout.
	TLSHandshakeTimeout time.Duration

128 129 130 131 132 133 134 135 136 137 138 139
	// DisableKeepAlives, if true, prevents re-use of TCP connections
	// between different HTTP requests.
	DisableKeepAlives bool

	// DisableCompression, if true, prevents the Transport from
	// requesting compression with an "Accept-Encoding: gzip"
	// request header when the Request contains no existing
	// Accept-Encoding value. If the Transport requests gzip on
	// its own and gets a gzipped response, it's transparently
	// decoded in the Response.Body. However, if the user
	// explicitly requested gzip it is not automatically
	// uncompressed.
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
140
	DisableCompression bool
141

142 143 144 145
	// MaxIdleConns controls the maximum number of idle (keep-alive)
	// connections across all hosts. Zero means no limit.
	MaxIdleConns int

146
	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
147
	// (keep-alive) connections to keep per-host. If zero,
148 149
	// DefaultMaxIdleConnsPerHost is used.
	MaxIdleConnsPerHost int
150

151 152 153 154 155 156
	// IdleConnTimeout is the maximum amount of time an idle
	// (keep-alive) connection will remain idle before closing
	// itself.
	// Zero means no limit.
	IdleConnTimeout time.Duration

157 158 159 160 161
	// ResponseHeaderTimeout, if non-zero, specifies the amount of
	// time to wait for a server's response headers after fully
	// writing the request (including its body, if any). This
	// time does not include the time to read the response body.
	ResponseHeaderTimeout time.Duration
162

163 164 165
	// ExpectContinueTimeout, if non-zero, specifies the amount of
	// time to wait for a server's first response headers after fully
	// writing the request headers if the request has an
166 167 168
	// "Expect: 100-continue" header. Zero means no timeout and
	// causes the body to be sent immediately, without
	// waiting for the server to approve.
169 170 171
	// This time does not include the time to send the request header.
	ExpectContinueTimeout time.Duration

172 173
	// TLSNextProto specifies how the Transport switches to an
	// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
174
	// protocol negotiation. If Transport dials an TLS connection
175 176 177 178 179
	// with a non-empty protocol name and TLSNextProto contains a
	// map entry for that key (such as "h2"), then the func is
	// called with the request's authority (such as "example.com"
	// or "example.com:1234") and the TLS connection. The function
	// must return a RoundTripper that then handles the request.
180 181
	// If TLSNextProto is not nil, HTTP/2 support is not enabled
	// automatically.
182 183
	TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper

184 185 186 187
	// ProxyConnectHeader optionally specifies headers to send to
	// proxies during CONNECT requests.
	ProxyConnectHeader Header

188 189 190 191 192 193 194
	// MaxResponseHeaderBytes specifies a limit on how many
	// response bytes are allowed in the server's response
	// header.
	//
	// Zero means to use a default limit.
	MaxResponseHeaderBytes int64

195 196 197 198
	// nextProtoOnce guards initialization of TLSNextProto and
	// h2transport (via onceSetNextProtoDefaults)
	nextProtoOnce sync.Once
	h2transport   *http2Transport // non-nil if http2 wired up
199

200
	// TODO: tunable on max per-host TCP dials in flight (Issue 13957)
201 202
}

203 204 205
// onceSetNextProtoDefaults initializes TLSNextProto.
// It must be called via t.nextProtoOnce.Do.
func (t *Transport) onceSetNextProtoDefaults() {
206
	if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") {
207 208 209
		return
	}
	if t.TLSNextProto != nil {
210 211 212 213
		// This is the documented way to disable http2 on a
		// Transport.
		return
	}
214 215 216 217 218 219
	if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil {
		// Be conservative and don't automatically enable
		// http2 if they've specified a custom TLS config or
		// custom dialers. Let them opt-in themselves via
		// http2.ConfigureTransport so we don't surprise them
		// by modifying their tls.Config. Issue 14275.
220 221
		return
	}
222
	t2, err := http2configureTransport(t)
223 224
	if err != nil {
		log.Printf("Error enabling Transport HTTP/2 support: %v", err)
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
		return
	}
	t.h2transport = t2

	// Auto-configure the http2.Transport's MaxHeaderListSize from
	// the http.Transport's MaxResponseHeaderBytes. They don't
	// exactly mean the same thing, but they're close.
	//
	// TODO: also add this to x/net/http2.Configure Transport, behind
	// a +build go1.7 build tag:
	if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 {
		const h2max = 1<<32 - 1
		if limit1 >= h2max {
			t2.MaxHeaderListSize = h2max
		} else {
			t2.MaxHeaderListSize = uint32(limit1)
		}
242 243 244
	}
}

245 246
// ProxyFromEnvironment returns the URL of the proxy to use for a
// given request, as indicated by the environment variables
247 248 249 250 251 252 253 254
// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
// requests.
//
// The environment values may be either a complete URL or a
// "host[:port]", in which case the "http" scheme is assumed.
// An error is returned if the value is a different form.
//
255
// A nil URL and nil error are returned if no proxy is defined in the
256 257
// environment, or a proxy should not be used for the given request,
// as defined by NO_PROXY.
258 259 260
//
// As a special case, if req.URL.Host is "localhost" (with or without
// a port number), then a nil URL and nil error will be returned.
261
func ProxyFromEnvironment(req *Request) (*url.URL, error) {
262 263 264 265 266 267
	var proxy string
	if req.URL.Scheme == "https" {
		proxy = httpsProxyEnv.Get()
	}
	if proxy == "" {
		proxy = httpProxyEnv.Get()
268 269 270
		if proxy != "" && os.Getenv("REQUEST_METHOD") != "" {
			return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
		}
271
	}
272 273 274 275 276 277
	if proxy == "" {
		return nil, nil
	}
	if !useProxy(canonicalAddr(req.URL)) {
		return nil, nil
	}
Russ Cox's avatar
Russ Cox committed
278
	proxyURL, err := url.Parse(proxy)
279 280 281 282
	if err != nil ||
		(proxyURL.Scheme != "http" &&
			proxyURL.Scheme != "https" &&
			proxyURL.Scheme != "socks5") {
283 284 285 286 287
		// proxy was bogus. Try prepending "http://" to it and
		// see if that parses correctly. If not, we fall
		// through and complain about the original one.
		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
			return proxyURL, nil
288
		}
289

290
	}
Russ Cox's avatar
Russ Cox committed
291 292 293
	if err != nil {
		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
	}
294 295 296 297 298
	return proxyURL, nil
}

// ProxyURL returns a proxy function (for use in a Transport)
// that always returns the same URL.
299 300
func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
	return func(*Request) (*url.URL, error) {
Rob Pike's avatar
Rob Pike committed
301
		return fixedURL, nil
302 303 304
	}
}

305
// transportRequest is a wrapper around a *Request that adds
306 307
// optional extra headers to write and stores any error to return
// from roundTrip.
308
type transportRequest struct {
309 310 311
	*Request                        // original request, not to be mutated
	extra    Header                 // extra headers to write, or nil
	trace    *httptrace.ClientTrace // optional
312 313 314

	mu  sync.Mutex // guards err
	err error      // first setError value for mapRoundTripError to consider
315 316 317 318 319 320 321 322 323
}

func (tr *transportRequest) extraHeaders() Header {
	if tr.extra == nil {
		tr.extra = make(Header)
	}
	return tr.extra
}

324 325 326 327 328 329 330 331
func (tr *transportRequest) setError(err error) {
	tr.mu.Lock()
	if tr.err == nil {
		tr.err = err
	}
	tr.mu.Unlock()
}

332
// RoundTrip implements the RoundTripper interface.
333 334 335
//
// For higher-level HTTP client support (such as handling of cookies
// and redirects), see Get, Post, and the Client type.
336
func (t *Transport) RoundTrip(req *Request) (*Response, error) {
337
	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
338 339 340
	ctx := req.Context()
	trace := httptrace.ContextClientTrace(ctx)

341
	if req.URL == nil {
342
		req.closeBody()
343
		return nil, errors.New("http: nil Request.URL")
344
	}
345
	if req.Header == nil {
346
		req.closeBody()
347
		return nil, errors.New("http: nil Request.Header")
348
	}
349 350 351 352
	scheme := req.URL.Scheme
	isHTTP := scheme == "http" || scheme == "https"
	if isHTTP {
		for k, vv := range req.Header {
353
			if !httplex.ValidHeaderFieldName(k) {
354 355 356
				return nil, fmt.Errorf("net/http: invalid header field name %q", k)
			}
			for _, v := range vv {
357
				if !httplex.ValidHeaderFieldValue(v) {
358 359 360 361 362
					return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k)
				}
			}
		}
	}
363 364 365

	altProto, _ := t.altProto.Load().(map[string]RoundTripper)
	if altRT := altProto[scheme]; altRT != nil {
366 367
		if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
			return resp, err
368
		}
369
	}
370
	if !isHTTP {
371
		req.closeBody()
372
		return nil, &badStringError{"unsupported protocol scheme", scheme}
373
	}
374
	if req.Method != "" && !validMethod(req.Method) {
375 376
		return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
	}
377
	if req.URL.Host == "" {
378
		req.closeBody()
379 380
		return nil, errors.New("http: no Host in request URL")
	}
381

382 383
	for {
		// treq gets modified by roundTrip, so we need to recreate for each retry.
384
		treq := &transportRequest{Request: req, trace: trace}
385 386 387 388 389 390 391 392
		cm, err := t.connectMethodForRequest(treq)
		if err != nil {
			req.closeBody()
			return nil, err
		}

		// Get the cached or newly-created connection to either the
		// host (for http or https), the http proxy, or the http proxy
393
		// pre-CONNECTed to https server. In any case, we'll be ready
394
		// to send it requests.
395
		pconn, err := t.getConn(treq, cm)
396 397 398 399 400 401 402 403 404
		if err != nil {
			t.setReqCanceler(req, nil)
			req.closeBody()
			return nil, err
		}

		var resp *Response
		if pconn.alt != nil {
			// HTTP/2 path.
405
			t.setReqCanceler(req, nil) // not cancelable with CancelRequest
406 407 408 409 410 411 412
			resp, err = pconn.alt.RoundTrip(req)
		} else {
			resp, err = pconn.roundTrip(treq)
		}
		if err == nil {
			return resp, nil
		}
413
		if !pconn.shouldRetryRequest(req, err) {
414 415 416 417 418
			// Issue 16465: return underlying net.Conn.Read error from peek,
			// as we've historically done.
			if e, ok := err.(transportReadFromServerError); ok {
				err = e.err
			}
419 420
			return nil, err
		}
421
		testHookRoundTripRetried()
422
	}
423 424
}

425 426 427 428
// shouldRetryRequest reports whether we should retry sending a failed
// HTTP request on a new connection. The non-nil input error is the
// error from roundTrip.
func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool {
429 430 431 432 433 434 435 436 437
	if err == http2ErrNoCachedConn {
		// Issue 16582: if the user started a bunch of
		// requests at once, they can all pick the same conn
		// and violate the server's max concurrent streams.
		// Instead, match the HTTP/1 behavior for now and dial
		// again to get a new TCP connection, rather than failing
		// this request.
		return true
	}
438 439 440
	if err == errMissingHost {
		// User error.
		return false
441
	}
442 443 444 445 446 447 448 449 450
	if !pc.isReused() {
		// This was a fresh connection. There's no reason the server
		// should've hung up on us.
		//
		// Also, if we retried now, we could loop forever
		// creating new connections and retrying if the server
		// is just hanging up on us because it doesn't like
		// our request (as opposed to sending an error).
		return false
451
	}
452 453 454 455
	if _, ok := err.(nothingWrittenError); ok {
		// We never wrote anything, so it's safe to retry.
		return true
	}
456 457 458 459
	if !req.isReplayable() {
		// Don't retry non-idempotent requests.
		return false
	}
460
	if _, ok := err.(transportReadFromServerError); ok {
461 462 463
		// We got some non-EOF net.Conn.Read failure reading
		// the 1st response byte from the server.
		return true
464
	}
465 466 467 468
	if err == errServerClosedIdle {
		// The server replied with io.EOF while we were trying to
		// read the response. Probably an unfortunately keep-alive
		// timeout, just as the client was writing a request.
469 470 471
		return true
	}
	return false // conservatively
472 473
}

474 475 476
// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")

477 478 479 480 481 482
// RegisterProtocol registers a new protocol with scheme.
// The Transport will pass requests using the given scheme to rt.
// It is rt's responsibility to simulate HTTP request semantics.
//
// RegisterProtocol can be used by other packages to provide
// implementations of protocol schemes like "ftp" or "file".
483 484 485 486
//
// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
// handle the RoundTrip itself for that one request, as if the
// protocol were not registered.
487
func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
488 489
	t.altMu.Lock()
	defer t.altMu.Unlock()
490 491
	oldMap, _ := t.altProto.Load().(map[string]RoundTripper)
	if _, exists := oldMap[scheme]; exists {
492 493
		panic("protocol " + scheme + " already registered")
	}
494 495 496 497 498 499
	newMap := make(map[string]RoundTripper)
	for k, v := range oldMap {
		newMap[k] = v
	}
	newMap[scheme] = rt
	t.altProto.Store(newMap)
500 501
}

502 503 504 505 506
// CloseIdleConnections closes any connections which were previously
// connected from previous requests but are now sitting idle in
// a "keep-alive" state. It does not interrupt any connections currently
// in use.
func (t *Transport) CloseIdleConnections() {
507
	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
508
	t.idleMu.Lock()
509 510
	m := t.idleConn
	t.idleConn = nil
511
	t.idleConnCh = nil
512
	t.wantIdle = true
513
	t.idleLRU = connLRU{}
514
	t.idleMu.Unlock()
515
	for _, conns := range m {
516
		for _, pconn := range conns {
517
			pconn.close(errCloseIdleConns)
518 519
		}
	}
520 521 522
	if t2 := t.h2transport; t2 != nil {
		t2.CloseIdleConnections()
	}
523
}
524

525 526
// CancelRequest cancels an in-flight request by closing its connection.
// CancelRequest should only be called after RoundTrip has returned.
527
//
528 529 530
// Deprecated: Use Request.WithContext to create a request with a
// cancelable context instead. CancelRequest cannot cancel HTTP/2
// requests.
531
func (t *Transport) CancelRequest(req *Request) {
532 533 534 535 536
	t.cancelRequest(req, errRequestCanceled)
}

// Cancel an in-flight request, recording the error value.
func (t *Transport) cancelRequest(req *Request, err error) {
537
	t.reqMu.Lock()
538
	cancel := t.reqCanceler[req]
539
	delete(t.reqCanceler, req)
540
	t.reqMu.Unlock()
541
	if cancel != nil {
542
		cancel(err)
543 544 545
	}
}

546 547 548
//
// Private implementation past this point.
//
Russ Cox's avatar
Russ Cox committed
549

550 551 552
var (
	httpProxyEnv = &envOnce{
		names: []string{"HTTP_PROXY", "http_proxy"},
553
	}
554 555 556
	httpsProxyEnv = &envOnce{
		names: []string{"HTTPS_PROXY", "https_proxy"},
	}
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588
	noProxyEnv = &envOnce{
		names: []string{"NO_PROXY", "no_proxy"},
	}
)

// envOnce looks up an environment variable (optionally by multiple
// names) once. It mitigates expensive lookups on some platforms
// (e.g. Windows).
type envOnce struct {
	names []string
	once  sync.Once
	val   string
}

func (e *envOnce) Get() string {
	e.once.Do(e.init)
	return e.val
}

func (e *envOnce) init() {
	for _, n := range e.names {
		e.val = os.Getenv(n)
		if e.val != "" {
			return
		}
	}
}

// reset is used by tests
func (e *envOnce) reset() {
	e.once = sync.Once{}
	e.val = ""
589 590
}

591
func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
592 593 594
	if port := treq.URL.Port(); !validPort(port) {
		return cm, fmt.Errorf("invalid URL port %q", port)
	}
595 596
	cm.targetScheme = treq.URL.Scheme
	cm.targetAddr = canonicalAddr(treq.URL)
597
	if t.Proxy != nil {
598
		cm.proxyURL, err = t.Proxy(treq.Request)
599 600 601 602 603
		if err == nil && cm.proxyURL != nil {
			if port := cm.proxyURL.Port(); !validPort(port) {
				return cm, fmt.Errorf("invalid proxy URL port %q", port)
			}
		}
604
	}
605
	return cm, err
606 607 608 609 610 611 612 613
}

// proxyAuth returns the Proxy-Authorization header to set
// on requests, if applicable.
func (cm *connectMethod) proxyAuth() string {
	if cm.proxyURL == nil {
		return ""
	}
614
	if u := cm.proxyURL.User; u != nil {
615 616 617
		username := u.Username()
		password, _ := u.Password()
		return "Basic " + basicAuth(username, password)
618 619 620 621
	}
	return ""
}

622 623 624 625 626 627
// error values for debugging and testing, not seen by users.
var (
	errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
	errConnBroken         = errors.New("http: putIdleConn: connection is in bad state")
	errWantIdle           = errors.New("http: putIdleConn: CloseIdleConnections was called")
	errTooManyIdle        = errors.New("http: putIdleConn: too many idle connections")
628
	errTooManyIdleHost    = errors.New("http: putIdleConn: too many idle connections for host")
629 630
	errCloseIdleConns     = errors.New("http: CloseIdleConnections called")
	errReadLoopExiting    = errors.New("http: persistConn.readLoop exiting")
631
	errServerClosedIdle   = errors.New("http: server closed idle connection")
632
	errIdleConnTimeout    = errors.New("http: idle connection timeout")
633
	errNotCachingH2Conn   = errors.New("http: not caching alternate protocol's connections")
634 635
)

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
// transportReadFromServerError is used by Transport.readLoop when the
// 1 byte peek read fails and we're actually anticipating a response.
// Usually this is just due to the inherent keep-alive shut down race,
// where the server closed the connection at the same time the client
// wrote. The underlying err field is usually io.EOF or some
// ECONNRESET sort of thing which varies by platform. But it might be
// the user's custom net.Conn.Read error too, so we carry it along for
// them to return from Transport.RoundTrip.
type transportReadFromServerError struct {
	err error
}

func (e transportReadFromServerError) Error() string {
	return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err)
}

652 653 654 655 656 657
func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
	if err := t.tryPutIdleConn(pconn); err != nil {
		pconn.close(err)
	}
}

658 659 660 661 662 663 664
func (t *Transport) maxIdleConnsPerHost() int {
	if v := t.MaxIdleConnsPerHost; v != 0 {
		return v
	}
	return DefaultMaxIdleConnsPerHost
}

665
// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
666
// a new request.
667 668 669 670
// If pconn is no longer needed or not in a good state, tryPutIdleConn returns
// an error explaining why it wasn't registered.
// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
671
	if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
672
		return errKeepAlivesDisabled
673 674
	}
	if pconn.isBroken() {
675
		return errConnBroken
676
	}
677 678 679
	if pconn.alt != nil {
		return errNotCachingH2Conn
	}
680
	pconn.markReused()
681
	key := pconn.cacheKey
682

683
	t.idleMu.Lock()
684
	defer t.idleMu.Unlock()
685 686

	waitingDialer := t.idleConnCh[key]
687
	select {
688
	case waitingDialer <- pconn:
689 690 691
		// We're done with this pconn and somebody else is
		// currently waiting for a conn of this type (they're
		// actively dialing, but this conn is ready
692
		// first). Chrome calls this socket late binding. See
693
		// https://insouciant.org/tech/connection-management-in-chromium/
694
		return nil
695
	default:
696 697 698 699 700
		if waitingDialer != nil {
			// They had populated this, but their dial won
			// first, so we can clean up this map entry.
			delete(t.idleConnCh, key)
		}
701
	}
702
	if t.wantIdle {
703
		return errWantIdle
704
	}
705
	if t.idleConn == nil {
706
		t.idleConn = make(map[connectMethodKey][]*persistConn)
707
	}
708
	idles := t.idleConn[key]
709 710
	if len(idles) >= t.maxIdleConnsPerHost() {
		return errTooManyIdleHost
711
	}
712
	for _, exist := range idles {
713 714 715 716
		if exist == pconn {
			log.Fatalf("dup idle pconn %p in freelist", pconn)
		}
	}
717
	t.idleConn[key] = append(idles, pconn)
718 719 720 721 722 723
	t.idleLRU.add(pconn)
	if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns {
		oldest := t.idleLRU.removeOldest()
		oldest.close(errTooManyIdle)
		t.removeIdleConnLocked(oldest)
	}
724 725 726 727 728 729 730
	if t.IdleConnTimeout > 0 {
		if pconn.idleTimer != nil {
			pconn.idleTimer.Reset(t.IdleConnTimeout)
		} else {
			pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle)
		}
	}
731
	pconn.idleAt = time.Now()
732
	return nil
733 734
}

735 736 737
// getIdleConnCh returns a channel to receive and return idle
// persistent connection for the given connectMethod.
// It may return nil, if persistent connections are not being used.
738
func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
739 740 741
	if t.DisableKeepAlives {
		return nil
	}
742 743 744
	key := cm.key()
	t.idleMu.Lock()
	defer t.idleMu.Unlock()
745
	t.wantIdle = false
746
	if t.idleConnCh == nil {
747
		t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
748 749 750 751 752 753 754 755 756
	}
	ch, ok := t.idleConnCh[key]
	if !ok {
		ch = make(chan *persistConn)
		t.idleConnCh[key] = ch
	}
	return ch
}

757
func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) {
758
	key := cm.key()
759 760
	t.idleMu.Lock()
	defer t.idleMu.Unlock()
761 762 763
	for {
		pconns, ok := t.idleConn[key]
		if !ok {
764
			return nil, time.Time{}
765 766 767
		}
		if len(pconns) == 1 {
			pconn = pconns[0]
Russ Cox's avatar
Russ Cox committed
768
			delete(t.idleConn, key)
769
		} else {
770
			// 2 or more cached connections; use the most
771
			// recently used one at the end.
772
			pconn = pconns[len(pconns)-1]
773
			t.idleConn[key] = pconns[:len(pconns)-1]
774
		}
775
		t.idleLRU.remove(pconn)
776 777 778 779 780 781 782 783
		if pconn.isBroken() {
			// There is a tiny window where this is
			// possible, between the connecting dying and
			// the persistConn readLoop calling
			// Transport.removeIdleConn. Just skip it and
			// carry on.
			continue
		}
784 785 786 787 788 789
		if pconn.idleTimer != nil && !pconn.idleTimer.Stop() {
			// We picked this conn at the ~same time it
			// was expiring and it's trying to close
			// itself in another goroutine. Don't use it.
			continue
		}
790
		return pconn, pconn.idleAt
791 792 793 794 795 796 797
	}
}

// removeIdleConn marks pconn as dead.
func (t *Transport) removeIdleConn(pconn *persistConn) {
	t.idleMu.Lock()
	defer t.idleMu.Unlock()
798 799
	t.removeIdleConnLocked(pconn)
}
800

801 802
// t.idleMu must be held.
func (t *Transport) removeIdleConnLocked(pconn *persistConn) {
803 804 805
	if pconn.idleTimer != nil {
		pconn.idleTimer.Stop()
	}
806 807
	t.idleLRU.remove(pconn)
	key := pconn.cacheKey
808 809 810 811 812 813 814 815 816 817 818 819 820
	pconns, _ := t.idleConn[key]
	switch len(pconns) {
	case 0:
		// Nothing
	case 1:
		if pconns[0] == pconn {
			delete(t.idleConn, key)
		}
	default:
		for i, v := range pconns {
			if v != pconn {
				continue
			}
821 822 823
			// Slide down, keeping most recently-used
			// conns at the end.
			copy(pconns[i:], pconns[i+1:])
824 825
			t.idleConn[key] = pconns[:len(pconns)-1]
			break
826 827
		}
	}
828 829
}

830
func (t *Transport) setReqCanceler(r *Request, fn func(error)) {
831 832
	t.reqMu.Lock()
	defer t.reqMu.Unlock()
833
	if t.reqCanceler == nil {
834
		t.reqCanceler = make(map[*Request]func(error))
835
	}
836 837
	if fn != nil {
		t.reqCanceler[r] = fn
838
	} else {
839
		delete(t.reqCanceler, r)
840 841 842
	}
}

843 844 845 846
// replaceReqCanceler replaces an existing cancel function. If there is no cancel function
// for the request, we don't set the function and return false.
// Since CancelRequest will clear the canceler, we can use the return value to detect if
// the request was canceled since the last setReqCancel call.
847
func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool {
848 849 850 851 852 853 854 855 856 857 858 859 860 861
	t.reqMu.Lock()
	defer t.reqMu.Unlock()
	_, ok := t.reqCanceler[r]
	if !ok {
		return false
	}
	if fn != nil {
		t.reqCanceler[r] = fn
	} else {
		delete(t.reqCanceler, r)
	}
	return true
}

862 863
var zeroDialer net.Dialer

864
func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) {
865 866
	if t.DialContext != nil {
		return t.DialContext(ctx, network, addr)
867
	}
868
	if t.Dial != nil {
869 870 871 872 873
		c, err := t.Dial(network, addr)
		if c == nil && err == nil {
			err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
		}
		return c, err
874
	}
875
	return zeroDialer.DialContext(ctx, network, addr)
876 877
}

878
// getConn dials and creates a new persistConn to the target as
879
// specified in the connectMethod. This includes doing a proxy CONNECT
880 881
// and/or setting up TLS.  If this doesn't return an error, the persistConn
// is ready to write requests to.
882 883 884
func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) {
	req := treq.Request
	trace := treq.trace
885
	ctx := req.Context()
886
	if trace != nil && trace.GetConn != nil {
887 888 889
		trace.GetConn(cm.addr())
	}
	if pc, idleSince := t.getIdleConn(cm); pc != nil {
890
		if trace != nil && trace.GotConn != nil {
891 892
			trace.GotConn(pc.gotIdleConnTrace(idleSince))
		}
893 894 895
		// set request canceler to some non-nil function so we
		// can detect whether it was cleared between now and when
		// we enter roundTrip
896
		t.setReqCanceler(req, func(error) {})
897 898
		return pc, nil
	}
899

900 901 902 903 904
	type dialRes struct {
		pc  *persistConn
		err error
	}
	dialc := make(chan dialRes)
905

906 907
	// Copy these hooks so we don't race on the postPendingDial in
	// the goroutine we launch. Issue 11136.
908 909
	testHookPrePendingDial := testHookPrePendingDial
	testHookPostPendingDial := testHookPostPendingDial
910

911
	handlePendingDial := func() {
912
		testHookPrePendingDial()
913 914
		go func() {
			if v := <-dialc; v.err == nil {
915
				t.putOrCloseIdleConn(v.pc)
916
			}
917
			testHookPostPendingDial()
918
		}()
919 920
	}

921 922
	cancelc := make(chan error, 1)
	t.setReqCanceler(req, func(err error) { cancelc <- err })
923

924
	go func() {
925
		pc, err := t.dialConn(ctx, cm)
926 927 928 929 930 931 932
		dialc <- dialRes{pc, err}
	}()

	idleConnCh := t.getIdleConnCh(cm)
	select {
	case v := <-dialc:
		// Our dial finished.
933 934 935 936 937 938 939 940 941 942
		if v.pc != nil {
			if trace != nil && trace.GotConn != nil && v.pc.alt == nil {
				trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn})
			}
			return v.pc, nil
		}
		// Our dial failed. See why to return a nicer error
		// value.
		select {
		case <-req.Cancel:
943 944 945
			// It was an error due to cancelation, so prioritize that
			// error value. (Issue 16049)
			return nil, errRequestCanceledConn
946
		case <-req.Context().Done():
947 948 949 950 951 952
			return nil, req.Context().Err()
		case err := <-cancelc:
			if err == errRequestCanceled {
				err = errRequestCanceledConn
			}
			return nil, err
953 954 955 956
		default:
			// It wasn't an error due to cancelation, so
			// return the original error message:
			return nil, v.err
957
		}
958 959 960 961 962 963
	case pc := <-idleConnCh:
		// Another request finished first and its net.Conn
		// became available before our dial. Or somebody
		// else's dial that they didn't use.
		// But our dial is still going, so give it away
		// when it finishes:
964
		handlePendingDial()
965
		if trace != nil && trace.GotConn != nil {
966 967
			trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()})
		}
968
		return pc, nil
969 970
	case <-req.Cancel:
		handlePendingDial()
971
		return nil, errRequestCanceledConn
972 973
	case <-req.Context().Done():
		handlePendingDial()
974 975
		return nil, req.Context().Err()
	case err := <-cancelc:
976
		handlePendingDial()
977 978 979 980
		if err == errRequestCanceled {
			err = errRequestCanceledConn
		}
		return nil, err
981 982 983
	}
}

984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
type oneConnDialer <-chan net.Conn

func newOneConnDialer(c net.Conn) proxy.Dialer {
	ch := make(chan net.Conn, 1)
	ch <- c
	return oneConnDialer(ch)
}

func (d oneConnDialer) Dial(network, addr string) (net.Conn, error) {
	select {
	case c := <-d:
		return c, nil
	default:
		return nil, io.EOF
	}
}

1001
func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) {
1002
	pconn := &persistConn{
1003 1004 1005 1006 1007 1008 1009
		t:             t,
		cacheKey:      cm.key(),
		reqch:         make(chan requestAndChan, 1),
		writech:       make(chan writeRequest, 1),
		closech:       make(chan struct{}),
		writeErrCh:    make(chan error, 1),
		writeLoopDone: make(chan struct{}),
1010
	}
1011
	trace := httptrace.ContextClientTrace(ctx)
1012 1013 1014 1015 1016 1017 1018
	tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil
	if tlsDial {
		var err error
		pconn.conn, err = t.DialTLS("tcp", cm.addr())
		if err != nil {
			return nil, err
		}
1019 1020 1021
		if pconn.conn == nil {
			return nil, errors.New("net/http: Transport.DialTLS returned (nil, nil)")
		}
1022
		if tc, ok := pconn.conn.(*tls.Conn); ok {
1023 1024
			// Handshake here, in case DialTLS didn't. TLSNextProto below
			// depends on it for knowing the connection state.
1025 1026 1027
			if trace != nil && trace.TLSHandshakeStart != nil {
				trace.TLSHandshakeStart()
			}
1028 1029
			if err := tc.Handshake(); err != nil {
				go pconn.conn.Close()
1030 1031 1032
				if trace != nil && trace.TLSHandshakeDone != nil {
					trace.TLSHandshakeDone(tls.ConnectionState{}, err)
				}
1033 1034
				return nil, err
			}
1035
			cs := tc.ConnectionState()
1036 1037 1038
			if trace != nil && trace.TLSHandshakeDone != nil {
				trace.TLSHandshakeDone(cs, nil)
			}
1039 1040 1041
			pconn.tlsState = &cs
		}
	} else {
1042
		conn, err := t.dial(ctx, "tcp", cm.addr())
1043 1044
		if err != nil {
			if cm.proxyURL != nil {
1045 1046
				// Return a typed error, per Issue 16997:
				err = &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err}
1047 1048 1049 1050 1051
			}
			return nil, err
		}
		pconn.conn = conn
	}
1052

1053
	// Proxy setup.
1054 1055
	switch {
	case cm.proxyURL == nil:
1056
		// Do nothing. Not using a proxy.
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	case cm.proxyURL.Scheme == "socks5":
		conn := pconn.conn
		var auth *proxy.Auth
		if u := cm.proxyURL.User; u != nil {
			auth = &proxy.Auth{}
			auth.User = u.Username()
			auth.Password, _ = u.Password()
		}
		p, err := proxy.SOCKS5("", cm.addr(), auth, newOneConnDialer(conn))
		if err != nil {
			conn.Close()
			return nil, err
		}
		if _, err := p.Dial("tcp", cm.targetAddr); err != nil {
			conn.Close()
			return nil, err
		}
1074
	case cm.targetScheme == "http":
1075
		pconn.isProxy = true
1076
		if pa := cm.proxyAuth(); pa != "" {
1077 1078
			pconn.mutateHeaderFunc = func(h Header) {
				h.Set("Proxy-Authorization", pa)
1079 1080
			}
		}
1081
	case cm.targetScheme == "https":
1082
		conn := pconn.conn
1083 1084 1085 1086
		hdr := t.ProxyConnectHeader
		if hdr == nil {
			hdr = make(Header)
		}
1087 1088
		connectReq := &Request{
			Method: "CONNECT",
1089
			URL:    &url.URL{Opaque: cm.targetAddr},
1090
			Host:   cm.targetAddr,
1091
			Header: hdr,
1092
		}
1093
		if pa := cm.proxyAuth(); pa != "" {
1094
			connectReq.Header.Set("Proxy-Authorization", pa)
1095
		}
1096
		connectReq.Write(conn)
1097

1098 1099 1100 1101
		// Read response.
		// Okay to use and discard buffered reader here, because
		// TLS server will not speak until spoken to.
		br := bufio.NewReader(conn)
1102
		resp, err := ReadResponse(br, connectReq)
1103 1104 1105 1106 1107
		if err != nil {
			conn.Close()
			return nil, err
		}
		if resp.StatusCode != 200 {
1108
			f := strings.SplitN(resp.Status, " ", 2)
1109
			conn.Close()
1110
			return nil, errors.New(f[1])
1111 1112 1113
		}
	}

1114
	if cm.targetScheme == "https" && !tlsDial {
1115
		// Initiate TLS and check remote host name against certificate.
1116
		cfg := cloneTLSConfig(t.TLSClientConfig)
1117 1118
		if cfg.ServerName == "" {
			cfg.ServerName = cm.tlsHost()
1119
		}
1120
		plainConn := pconn.conn
1121 1122 1123 1124 1125 1126 1127 1128 1129
		tlsConn := tls.Client(plainConn, cfg)
		errc := make(chan error, 2)
		var timer *time.Timer // for canceling TLS handshake
		if d := t.TLSHandshakeTimeout; d != 0 {
			timer = time.AfterFunc(d, func() {
				errc <- tlsHandshakeTimeoutError{}
			})
		}
		go func() {
1130 1131 1132
			if trace != nil && trace.TLSHandshakeStart != nil {
				trace.TLSHandshakeStart()
			}
1133 1134 1135 1136 1137 1138 1139 1140
			err := tlsConn.Handshake()
			if timer != nil {
				timer.Stop()
			}
			errc <- err
		}()
		if err := <-errc; err != nil {
			plainConn.Close()
1141 1142 1143
			if trace != nil && trace.TLSHandshakeDone != nil {
				trace.TLSHandshakeDone(tls.ConnectionState{}, err)
			}
1144 1145
			return nil, err
		}
1146
		if !cfg.InsecureSkipVerify {
1147 1148
			if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
				plainConn.Close()
1149 1150
				return nil, err
			}
1151
		}
1152
		cs := tlsConn.ConnectionState()
1153 1154 1155
		if trace != nil && trace.TLSHandshakeDone != nil {
			trace.TLSHandshakeDone(cs, nil)
		}
1156
		pconn.tlsState = &cs
1157
		pconn.conn = tlsConn
1158 1159
	}

1160 1161 1162 1163 1164 1165
	if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
		if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
			return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
		}
	}

1166
	pconn.br = bufio.NewReader(pconn)
1167
	pconn.bw = bufio.NewWriter(persistConnWriter{pconn})
1168
	go pconn.readLoop()
1169
	go pconn.writeLoop()
1170 1171 1172
	return pconn, nil
}

1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
// persistConnWriter is the io.Writer written to by pc.bw.
// It accumulates the number of bytes written to the underlying conn,
// so the retry logic can determine whether any bytes made it across
// the wire.
// This is exactly 1 pointer field wide so it can go into an interface
// without allocation.
type persistConnWriter struct {
	pc *persistConn
}

func (w persistConnWriter) Write(p []byte) (n int, err error) {
	n, err = w.pc.conn.Write(p)
	w.pc.nwrite += int64(n)
	return
}

1189
// useProxy reports whether requests to addr should use a proxy,
1190
// according to the NO_PROXY or no_proxy environment variable.
1191
// addr is always a canonicalAddr with a host and port.
1192
func useProxy(addr string) bool {
1193 1194 1195
	if len(addr) == 0 {
		return true
	}
1196 1197 1198 1199 1200 1201 1202 1203
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return false
	}
	if host == "localhost" {
		return false
	}
	if ip := net.ParseIP(host); ip != nil {
1204
		if ip.IsLoopback() {
1205 1206 1207 1208
			return false
		}
	}

1209
	no_proxy := noProxyEnv.Get()
1210 1211 1212 1213 1214 1215 1216
	if no_proxy == "*" {
		return false
	}

	addr = strings.ToLower(strings.TrimSpace(addr))
	if hasPort(addr) {
		addr = addr[:strings.LastIndex(addr, ":")]
1217 1218
	}

1219
	for _, p := range strings.Split(no_proxy, ",") {
1220 1221 1222 1223 1224 1225 1226
		p = strings.ToLower(strings.TrimSpace(p))
		if len(p) == 0 {
			continue
		}
		if hasPort(p) {
			p = p[:strings.LastIndex(p, ":")]
		}
1227 1228 1229
		if addr == p {
			return false
		}
1230 1231 1232 1233
		if len(p) == 0 {
			// There is no host part, likely the entry is malformed; ignore.
			continue
		}
1234 1235 1236 1237 1238 1239
		if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
			// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
			return false
		}
		if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
			// no_proxy "foo.com" matches "bar.foo.com"
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
			return false
		}
	}
	return true
}

// connectMethod is the map key (in its String form) for keeping persistent
// TCP connections alive for subsequent HTTP requests.
//
// A connect method may be of the following types:
//
1251 1252 1253 1254 1255 1256 1257 1258
// Cache key form                    Description
// -----------------                 -------------------------
// |http|foo.com                     http directly to server, no proxy
// |https|foo.com                    https directly to server, no proxy
// http://proxy.com|https|foo.com    http to proxy, then CONNECT to foo.com
// http://proxy.com|http             http to proxy, http to anywhere after that
// socks5://proxy.com|http|foo.com   socks5 to proxy, then http to foo.com
// socks5://proxy.com|https|foo.com  socks5 to proxy, then https to foo.com
1259 1260 1261 1262
//
// Note: no support to https to the proxy yet.
//
type connectMethod struct {
Rob Pike's avatar
Rob Pike committed
1263 1264
	proxyURL     *url.URL // nil for no proxy, else full proxy URL
	targetScheme string   // "http" or "https"
1265
	targetAddr   string   // Not used if http proxy + http targetScheme (4th example in table)
1266 1267
}

1268
func (cm *connectMethod) key() connectMethodKey {
1269
	proxyStr := ""
1270 1271 1272
	targetAddr := cm.targetAddr
	if cm.proxyURL != nil {
		proxyStr = cm.proxyURL.String()
1273
		if strings.HasPrefix(cm.proxyURL.Scheme, "http") && cm.targetScheme == "http" {
1274 1275
			targetAddr = ""
		}
1276
	}
1277 1278 1279 1280 1281
	return connectMethodKey{
		proxy:  proxyStr,
		scheme: cm.targetScheme,
		addr:   targetAddr,
	}
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
}

// addr returns the first hop "host:port" to which we need to TCP connect.
func (cm *connectMethod) addr() string {
	if cm.proxyURL != nil {
		return canonicalAddr(cm.proxyURL)
	}
	return cm.targetAddr
}

// tlsHost returns the host name to match against the peer's
// TLS certificate.
func (cm *connectMethod) tlsHost() string {
	h := cm.targetAddr
	if hasPort(h) {
		h = h[:strings.LastIndex(h, ":")]
	}
	return h
}

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
// connectMethodKey is the map key version of connectMethod, with a
// stringified proxy URL (or the empty string) instead of a pointer to
// a URL.
type connectMethodKey struct {
	proxy, scheme, addr string
}

func (k connectMethodKey) String() string {
	// Only used by tests.
	return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
}

1314 1315 1316
// persistConn wraps a connection, usually a persistent one
// (but may be used for non-keep-alive requests as well)
type persistConn struct {
1317
	// alt optionally specifies the TLS NextProto RoundTripper.
1318
	// This is used for HTTP/2 today and future protocols later.
1319 1320 1321
	// If it's non-nil, the rest of the fields are unused.
	alt RoundTripper

1322 1323 1324 1325 1326 1327
	t         *Transport
	cacheKey  connectMethodKey
	conn      net.Conn
	tlsState  *tls.ConnectionState
	br        *bufio.Reader       // from conn
	bw        *bufio.Writer       // to conn
1328
	nwrite    int64               // bytes written
1329 1330 1331 1332 1333 1334
	reqch     chan requestAndChan // written by roundTrip; read by readLoop
	writech   chan writeRequest   // written by roundTrip; read by writeLoop
	closech   chan struct{}       // closed when conn closed
	isProxy   bool
	sawEOF    bool  // whether we've seen EOF from conn; owned by readLoop
	readLimit int64 // bytes allowed to be read; owned by readLoop
1335 1336 1337 1338 1339
	// writeErrCh passes the request write error (usually nil)
	// from the writeLoop goroutine to the readLoop which passes
	// it off to the res.Body reader, which then uses it to decide
	// whether or not a connection can be reused. Issue 7569.
	writeErrCh chan error
1340

1341 1342
	writeLoopDone chan struct{} // closed when write loop ends

1343 1344 1345
	// Both guarded by Transport.idleMu:
	idleAt    time.Time   // time it last become idle
	idleTimer *time.Timer // holding an AfterFunc to close it
1346

1347
	mu                   sync.Mutex // guards following fields
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1348
	numExpectedResponses int
1349
	closed               error // set non-nil when conn is closed, before closech is closed
1350
	canceledErr          error // set non-nil if conn is canceled
1351 1352
	broken               bool  // an error has happened on this connection; marked broken so it's not reused.
	reused               bool  // whether conn has had successful request/response and is being reused.
1353 1354 1355 1356
	// mutateHeaderFunc is an optional func to modify extra
	// headers on each outbound request before it's written. (the
	// original Request given to RoundTrip is not modified)
	mutateHeaderFunc func(Header)
1357 1358
}

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
func (pc *persistConn) maxHeaderResponseSize() int64 {
	if v := pc.t.MaxResponseHeaderBytes; v != 0 {
		return v
	}
	return 10 << 20 // conservative default; same as http2
}

func (pc *persistConn) Read(p []byte) (n int, err error) {
	if pc.readLimit <= 0 {
		return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize())
	}
	if int64(len(p)) > pc.readLimit {
		p = p[:pc.readLimit]
	}
	n, err = pc.conn.Read(p)
	if err == io.EOF {
		pc.sawEOF = true
	}
	pc.readLimit -= int64(n)
	return
}

1381
// isBroken reports whether this connection is in a known broken state.
1382
func (pc *persistConn) isBroken() bool {
1383
	pc.mu.Lock()
1384
	b := pc.closed != nil
1385
	pc.mu.Unlock()
1386
	return b
1387 1388
}

1389 1390 1391
// canceled returns non-nil if the connection was closed due to
// CancelRequest or due to context cancelation.
func (pc *persistConn) canceled() error {
1392 1393
	pc.mu.Lock()
	defer pc.mu.Unlock()
1394
	return pc.canceledErr
1395 1396
}

1397 1398
// isReused reports whether this connection is in a known broken state.
func (pc *persistConn) isReused() bool {
1399
	pc.mu.Lock()
1400
	r := pc.reused
1401
	pc.mu.Unlock()
1402 1403 1404
	return r
}

1405 1406 1407 1408 1409 1410
func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) {
	pc.mu.Lock()
	defer pc.mu.Unlock()
	t.Reused = pc.reused
	t.Conn = pc.conn
	t.WasIdle = true
1411 1412 1413
	if !idleAt.IsZero() {
		t.IdleTime = time.Since(idleAt)
	}
1414 1415 1416
	return
}

1417
func (pc *persistConn) cancelRequest(err error) {
1418 1419
	pc.mu.Lock()
	defer pc.mu.Unlock()
1420
	pc.canceledErr = err
1421
	pc.closeLocked(errRequestCanceled)
1422 1423
}

1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
// closeConnIfStillIdle closes the connection if it's still sitting idle.
// This is what's called by the persistConn's idleTimer, and is run in its
// own goroutine.
func (pc *persistConn) closeConnIfStillIdle() {
	t := pc.t
	t.idleMu.Lock()
	defer t.idleMu.Unlock()
	if _, ok := t.idleLRU.m[pc]; !ok {
		// Not idle.
		return
	}
	t.removeIdleConnLocked(pc)
	pc.close(errIdleConnTimeout)
}

1439 1440 1441 1442 1443
// mapRoundTripError returns the appropriate error value for
// persistConn.roundTrip.
//
// The provided err is the first error that (*persistConn).roundTrip
// happened to receive from its select statement.
1444 1445 1446
//
// The startBytesWritten value should be the value of pc.nwrite before the roundTrip
// started writing the request.
1447
func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error {
1448 1449 1450
	if err == nil {
		return nil
	}
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464

	// If the request was canceled, that's better than network
	// failures that were likely the result of tearing down the
	// connection.
	if cerr := pc.canceled(); cerr != nil {
		return cerr
	}

	// See if an error was set explicitly.
	req.mu.Lock()
	reqErr := req.err
	req.mu.Unlock()
	if reqErr != nil {
		return reqErr
1465
	}
1466

1467
	if err == errServerClosedIdle {
1468
		// Don't decorate
1469 1470
		return err
	}
1471

1472
	if _, ok := err.(transportReadFromServerError); ok {
1473
		// Don't decorate
1474 1475 1476 1477
		return err
	}
	if pc.isBroken() {
		<-pc.writeLoopDone
1478
		if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 {
1479 1480
			return nothingWrittenError{err}
		}
1481
		return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err)
1482 1483 1484 1485
	}
	return err
}

1486
func (pc *persistConn) readLoop() {
1487
	closeErr := errReadLoopExiting // default value, if not changed below
1488 1489 1490 1491
	defer func() {
		pc.close(closeErr)
		pc.t.removeIdleConn(pc)
	}()
1492

1493
	tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
1494 1495
		if err := pc.t.tryPutIdleConn(pc); err != nil {
			closeErr = err
1496 1497 1498
			if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
				trace.PutIdleConn(err)
			}
1499 1500
			return false
		}
1501 1502 1503
		if trace != nil && trace.PutIdleConn != nil {
			trace.PutIdleConn(nil)
		}
1504 1505
		return true
	}
1506 1507

	// eofc is used to block caller goroutines reading from Response.Body
1508 1509 1510 1511
	// at EOF until this goroutines has (potentially) added the connection
	// back to the idle pool.
	eofc := make(chan struct{})
	defer close(eofc) // unblock reader on errors
1512

1513 1514 1515 1516 1517
	// Read this once, before loop starts. (to avoid races in tests)
	testHookMu.Lock()
	testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
	testHookMu.Unlock()

1518
	alive := true
1519
	for alive {
1520
		pc.readLimit = pc.maxHeaderResponseSize()
1521
		_, err := pc.br.Peek(1)
1522

1523
		pc.mu.Lock()
1524
		if pc.numExpectedResponses == 0 {
1525
			pc.readLoopPeekFailLocked(err)
1526
			pc.mu.Unlock()
1527 1528
			return
		}
1529
		pc.mu.Unlock()
1530 1531

		rc := <-pc.reqch
1532
		trace := httptrace.ContextClientTrace(rc.req.Context())
1533

1534 1535
		var resp *Response
		if err == nil {
1536
			resp, err = pc.readResponse(rc, trace)
1537
		} else {
1538
			err = transportReadFromServerError{err}
1539
			closeErr = err
1540 1541
		}

1542
		if err != nil {
1543 1544 1545
			if pc.readLimit <= 0 {
				err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
			}
1546

1547 1548 1549 1550 1551 1552 1553
			// If we won't be able to retry this request later (from the
			// roundTrip goroutine), mark it as done now.
			// BEFORE the send on rc.ch, as the client might re-use the
			// same *Request pointer, and we don't want to set call
			// t.setReqCanceler from this persistConn while the Transport
			// potentially spins up a different persistConn for the
			// caller's subsequent request.
1554
			if !pc.shouldRetryRequest(rc.req, err) {
1555
				pc.t.setReqCanceler(rc.req, nil)
1556
			}
1557 1558 1559 1560 1561
			select {
			case rc.ch <- responseAndError{err: err}:
			case <-rc.callerGone:
				return
			}
1562
			return
1563
		}
1564
		pc.readLimit = maxInt64 // effictively no limit for response bodies
1565

1566
		pc.mu.Lock()
1567
		pc.numExpectedResponses--
1568
		pc.mu.Unlock()
1569 1570 1571 1572

		hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0

		if resp.Close || rc.req.Close || resp.StatusCode <= 199 {
1573 1574 1575
			// Don't do keep-alive on error if either party requested a close
			// or we get an unexpected informational (1xx) response.
			// StatusCode 100 is already handled above.
1576 1577
			alive = false
		}
1578

1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
		if !hasBody {
			pc.t.setReqCanceler(rc.req, nil)

			// Put the idle conn back into the pool before we send the response
			// so if they process it quickly and make another request, they'll
			// get this same conn. But we use the unbuffered channel 'rc'
			// to guarantee that persistConn.roundTrip got out of its select
			// potentially waiting for this persistConn to close.
			// but after
			alive = alive &&
				!pc.sawEOF &&
				pc.wroteRequest() &&
1591
				tryPutIdleConn(trace)
1592

1593 1594 1595 1596 1597
			select {
			case rc.ch <- responseAndError{res: resp}:
			case <-rc.callerGone:
				return
			}
1598 1599 1600 1601 1602 1603

			// Now that they've read from the unbuffered channel, they're safely
			// out of the select that also waits on this goroutine to die, so
			// we're allowed to exit now if needed (if alive is false)
			testHookReadLoopBeforeNextRead()
			continue
1604 1605
		}

1606
		waitForBodyRead := make(chan bool, 2)
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618
		body := &bodyEOFSignal{
			body: resp.Body,
			earlyCloseFn: func() error {
				waitForBodyRead <- false
				return nil

			},
			fn: func(err error) error {
				isEOF := err == io.EOF
				waitForBodyRead <- isEOF
				if isEOF {
					<-eofc // see comment above eofc declaration
1619 1620 1621 1622
				} else if err != nil {
					if cerr := pc.canceled(); cerr != nil {
						return cerr
					}
1623 1624 1625
				}
				return err
			},
1626
		}
1627 1628 1629 1630 1631 1632 1633

		resp.Body = body
		if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" {
			resp.Body = &gzipReader{body: body}
			resp.Header.Del("Content-Encoding")
			resp.Header.Del("Content-Length")
			resp.ContentLength = -1
1634
			resp.Uncompressed = true
1635 1636
		}

1637 1638 1639 1640 1641
		select {
		case rc.ch <- responseAndError{res: resp}:
		case <-rc.callerGone:
			return
		}
1642 1643 1644 1645 1646 1647 1648

		// Before looping back to the top of this function and peeking on
		// the bufio.Reader, wait for the caller goroutine to finish
		// reading the response body. (or for cancelation or death)
		select {
		case bodyEOF := <-waitForBodyRead:
			pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
1649
			alive = alive &&
1650
				bodyEOF &&
1651 1652
				!pc.sawEOF &&
				pc.wroteRequest() &&
1653
				tryPutIdleConn(trace)
1654 1655 1656 1657 1658 1659
			if bodyEOF {
				eofc <- struct{}{}
			}
		case <-rc.req.Cancel:
			alive = false
			pc.t.CancelRequest(rc.req)
1660 1661
		case <-rc.req.Context().Done():
			alive = false
1662
			pc.t.cancelRequest(rc.req, rc.req.Context().Err())
1663 1664
		case <-pc.closech:
			alive = false
1665
		}
1666

1667
		testHookReadLoopBeforeNextRead()
1668
	}
1669 1670 1671
}

func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
1672
	if pc.closed != nil {
1673 1674 1675 1676 1677 1678
		return
	}
	if n := pc.br.Buffered(); n > 0 {
		buf, _ := pc.br.Peek(n)
		log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
	}
1679 1680 1681 1682 1683 1684
	if peekErr == io.EOF {
		// common case.
		pc.closeLocked(errServerClosedIdle)
	} else {
		pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr))
	}
1685 1686 1687 1688
}

// readResponse reads an HTTP response (or two, in the case of "Expect:
// 100-continue") from the server. It returns the final non-100 one.
1689 1690 1691 1692 1693 1694 1695
// trace is optional.
func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) {
	if trace != nil && trace.GotFirstResponseByte != nil {
		if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 {
			trace.GotFirstResponseByte()
		}
	}
1696 1697 1698 1699 1700 1701
	resp, err = ReadResponse(pc.br, rc.req)
	if err != nil {
		return
	}
	if rc.continueCh != nil {
		if resp.StatusCode == 100 {
1702 1703 1704
			if trace != nil && trace.Got100Continue != nil {
				trace.Got100Continue()
			}
1705 1706 1707 1708 1709 1710
			rc.continueCh <- struct{}{}
		} else {
			close(rc.continueCh)
		}
	}
	if resp.StatusCode == 100 {
1711
		pc.readLimit = pc.maxHeaderResponseSize() // reset the limit
1712 1713 1714 1715 1716 1717 1718
		resp, err = ReadResponse(pc.br, rc.req)
		if err != nil {
			return
		}
	}
	resp.TLS = pc.tlsState
	return
1719 1720
}

1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742
// waitForContinue returns the function to block until
// any response, timeout or connection close. After any of them,
// the function returns a bool which indicates if the body should be sent.
func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
	if continueCh == nil {
		return nil
	}
	return func() bool {
		timer := time.NewTimer(pc.t.ExpectContinueTimeout)
		defer timer.Stop()

		select {
		case _, ok := <-continueCh:
			return ok
		case <-timer.C:
			return true
		case <-pc.closech:
			return false
		}
	}
}

1743 1744 1745 1746 1747
// nothingWrittenError wraps a write errors which ended up writing zero bytes.
type nothingWrittenError struct {
	error
}

1748
func (pc *persistConn) writeLoop() {
1749
	defer close(pc.writeLoopDone)
1750 1751 1752
	for {
		select {
		case wr := <-pc.writech:
1753
			startBytesWritten := pc.nwrite
1754
			err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765
			if bre, ok := err.(requestBodyReadError); ok {
				err = bre.error
				// Errors reading from the user's
				// Request.Body are high priority.
				// Set it here before sending on the
				// channels below or calling
				// pc.close() which tears town
				// connections and causes other
				// errors.
				wr.req.setError(err)
			}
1766 1767 1768 1769
			if err == nil {
				err = pc.bw.Flush()
			}
			if err != nil {
1770
				wr.req.Request.closeBody()
1771
				if pc.nwrite == startBytesWritten && wr.req.outgoingLength() == 0 {
1772 1773
					err = nothingWrittenError{err}
				}
1774
			}
1775 1776
			pc.writeErrCh <- err // to the body reader, which might recycle us
			wr.ch <- err         // to the roundTrip function
1777 1778 1779 1780
			if err != nil {
				pc.close(err)
				return
			}
1781 1782
		case <-pc.closech:
			return
1783 1784 1785 1786
		}
	}
}

1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814
// wroteRequest is a check before recycling a connection that the previous write
// (from writeLoop above) happened and was successful.
func (pc *persistConn) wroteRequest() bool {
	select {
	case err := <-pc.writeErrCh:
		// Common case: the write happened well before the response, so
		// avoid creating a timer.
		return err == nil
	default:
		// Rare case: the request was written in writeLoop above but
		// before it could send to pc.writeErrCh, the reader read it
		// all, processed it, and called us here. In this case, give the
		// write goroutine a bit of time to finish its send.
		//
		// Less rare case: We also get here in the legitimate case of
		// Issue 7569, where the writer is still writing (or stalled),
		// but the server has already replied. In this case, we don't
		// want to wait too long, and we want to return false so this
		// connection isn't re-used.
		select {
		case err := <-pc.writeErrCh:
			return err == nil
		case <-time.After(50 * time.Millisecond):
			return false
		}
	}
}

1815 1816
// responseAndError is how the goroutine reading from an HTTP/1 server
// communicates with the goroutine doing the RoundTrip.
1817
type responseAndError struct {
1818
	res *Response // else use this response (see res method)
1819
	err error
1820 1821 1822 1823
}

type requestAndChan struct {
	req *Request
1824
	ch  chan responseAndError // unbuffered; always send in select on callerGone
1825

1826 1827 1828
	// whether the Transport (as opposed to the user client code)
	// added the Accept-Encoding gzip header. If the Transport
	// set it, only then do we transparently decode the gzip.
1829
	addedGzip bool
1830 1831 1832 1833 1834 1835

	// Optional blocking chan for Expect: 100-continue (for send).
	// If the request has an "Expect: 100-continue" header and
	// the server responds 100 Continue, readLoop send a value
	// to writeLoop via this chan.
	continueCh chan<- struct{}
1836 1837

	callerGone <-chan struct{} // closed when roundTrip caller has returned
1838 1839
}

1840 1841 1842 1843 1844 1845 1846
// A writeRequest is sent by the readLoop's goroutine to the
// writeLoop's goroutine to write a request while the read loop
// concurrently waits on both the write response and the server's
// reply.
type writeRequest struct {
	req *transportRequest
	ch  chan<- error
1847

Shawn Smith's avatar
Shawn Smith committed
1848
	// Optional blocking chan for Expect: 100-continue (for receive).
1849 1850 1851
	// If not nil, writeLoop blocks sending request body until
	// it receives from this chan.
	continueCh <-chan struct{}
1852 1853
}

1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
type httpError struct {
	err     string
	timeout bool
}

func (e *httpError) Error() string   { return e.err }
func (e *httpError) Timeout() bool   { return e.timeout }
func (e *httpError) Temporary() bool { return true }

var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
1864
var errRequestCanceled = errors.New("net/http: request canceled")
1865
var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
1866

1867 1868 1869
func nop() {}

// testHooks. Always non-nil.
1870
var (
1871 1872 1873 1874 1875
	testHookEnterRoundTrip   = nop
	testHookWaitResLoop      = nop
	testHookRoundTripRetried = nop
	testHookPrePendingDial   = nop
	testHookPostPendingDial  = nop
1876 1877 1878

	testHookMu                     sync.Locker = fakeLocker{} // guards following
	testHookReadLoopBeforeNextRead             = nop
1879
)
1880

1881
func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
1882
	testHookEnterRoundTrip()
1883
	if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
1884
		pc.t.putOrCloseIdleConn(pc)
1885 1886
		return nil, errRequestCanceled
	}
1887
	pc.mu.Lock()
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1888 1889
	pc.numExpectedResponses++
	headerFn := pc.mutateHeaderFunc
1890
	pc.mu.Unlock()
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1891 1892 1893

	if headerFn != nil {
		headerFn(req.extraHeaders())
1894 1895
	}

Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1896
	// Ask for a compressed version if the caller didn't set their
1897
	// own value for Accept-Encoding. We only attempt to
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1898 1899 1900
	// uncompress the gzip stream if we were the layer that
	// requested it.
	requestedGzip := false
1901 1902 1903 1904
	if !pc.t.DisableCompression &&
		req.Header.Get("Accept-Encoding") == "" &&
		req.Header.Get("Range") == "" &&
		req.Method != "HEAD" {
1905
		// Request gzip only, not deflate. Deflate is ambiguous and
1906
		// not as universally supported anyway.
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1907
		// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
1908 1909 1910 1911
		//
		// Note that we don't request this for HEAD requests,
		// due to a bug in nginx:
		//   http://trac.nginx.org/nginx/ticket/358
1912
		//   https://golang.org/issue/5522
1913 1914 1915
		//
		// We don't request gzip if the request is for a range, since
		// auto-decoding a portion of a gzipped document will just fail
1916
		// anyway. See https://golang.org/issue/8923
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1917
		requestedGzip = true
1918
		req.extraHeaders().Set("Accept-Encoding", "gzip")
Brad Fitzpatrick's avatar
Brad Fitzpatrick committed
1919 1920
	}

1921 1922 1923 1924 1925
	var continueCh chan struct{}
	if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
		continueCh = make(chan struct{}, 1)
	}

1926 1927 1928 1929
	if pc.t.DisableKeepAlives {
		req.extraHeaders().Set("Connection", "close")
	}

1930 1931 1932
	gone := make(chan struct{})
	defer close(gone)

1933 1934 1935 1936 1937 1938 1939 1940
	defer func() {
		if err != nil {
			pc.t.setReqCanceler(req.Request, nil)
		}
	}()

	const debugRoundTrip = false

1941 1942 1943
	// Write the request concurrently with waiting for a response,
	// in case the server decides to reply before reading our full
	// request body.
1944
	startBytesWritten := pc.nwrite
1945
	writeErrCh := make(chan error, 1)
1946
	pc.writech <- writeRequest{req, writeErrCh, continueCh}
1947

1948 1949 1950 1951 1952 1953 1954 1955
	resc := make(chan responseAndError)
	pc.reqch <- requestAndChan{
		req:        req.Request,
		ch:         resc,
		addedGzip:  requestedGzip,
		continueCh: continueCh,
		callerGone: gone,
	}
1956

1957
	var respHeaderTimer <-chan time.Time
1958
	cancelChan := req.Request.Cancel
1959
	ctxDoneChan := req.Context().Done()
1960
	for {
1961
		testHookWaitResLoop()
1962 1963
		select {
		case err := <-writeErrCh:
1964 1965 1966
			if debugRoundTrip {
				req.logf("writeErrCh resv: %T/%#v", err, err)
			}
1967
			if err != nil {
1968
				pc.close(fmt.Errorf("write error: %v", err))
1969
				return nil, pc.mapRoundTripError(req, startBytesWritten, err)
1970
			}
1971
			if d := pc.t.ResponseHeaderTimeout; d > 0 {
1972 1973 1974
				if debugRoundTrip {
					req.logf("starting timer for %v", d)
				}
1975 1976 1977
				timer := time.NewTimer(d)
				defer timer.Stop() // prevent leaks
				respHeaderTimer = timer.C
1978
			}
1979
		case <-pc.closech:
1980 1981 1982 1983
			if debugRoundTrip {
				req.logf("closech recv: %T %#v", pc.closed, pc.closed)
			}
			return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed)
1984
		case <-respHeaderTimer:
1985 1986 1987
			if debugRoundTrip {
				req.logf("timeout waiting for response headers.")
			}
1988
			pc.close(errTimeout)
1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000
			return nil, errTimeout
		case re := <-resc:
			if (re.res == nil) == (re.err == nil) {
				panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil))
			}
			if debugRoundTrip {
				req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err)
			}
			if re.err != nil {
				return nil, pc.mapRoundTripError(req, startBytesWritten, re.err)
			}
			return re.res, nil
2001 2002 2003
		case <-cancelChan:
			pc.t.CancelRequest(req.Request)
			cancelChan = nil
2004
		case <-ctxDoneChan:
2005
			pc.t.cancelRequest(req.Request, req.Context().Err())
2006
			cancelChan = nil
2007
			ctxDoneChan = nil
2008
		}
2009
	}
2010
}
2011

2012 2013 2014 2015 2016 2017 2018
// tLogKey is a context WithValue key for test debugging contexts containing
// a t.Logf func. See export_test.go's Request.WithT method.
type tLogKey struct{}

func (r *transportRequest) logf(format string, args ...interface{}) {
	if logf, ok := r.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok {
		logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...)
2019
	}
2020 2021
}

2022 2023 2024
// markReused marks this connection as having been successfully used for a
// request and response.
func (pc *persistConn) markReused() {
2025
	pc.mu.Lock()
2026
	pc.reused = true
2027
	pc.mu.Unlock()
2028 2029
}

2030 2031 2032 2033 2034 2035
// close closes the underlying TCP connection and closes
// the pc.closech channel.
//
// The provided err is only for testing and debugging; in normal
// circumstances it should never be seen by users.
func (pc *persistConn) close(err error) {
2036 2037
	pc.mu.Lock()
	defer pc.mu.Unlock()
2038
	pc.closeLocked(err)
2039 2040
}

2041 2042 2043 2044
func (pc *persistConn) closeLocked(err error) {
	if err == nil {
		panic("nil error")
	}
2045
	pc.broken = true
2046 2047
	if pc.closed == nil {
		pc.closed = err
2048 2049 2050 2051 2052
		if pc.alt != nil {
			// Do nothing; can only get here via getConn's
			// handlePendingDial's putOrCloseIdleConn when
			// it turns out the abandoned connection in
			// flight ended up negotiating an alternate
2053
			// protocol. We don't use the connection
2054 2055 2056 2057 2058 2059
			// freelist for http2. That's done by the
			// alternate protocol's RoundTripper.
		} else {
			pc.conn.Close()
			close(pc.closech)
		}
2060
	}
2061
	pc.mutateHeaderFunc = nil
2062 2063 2064
}

var portMap = map[string]string{
2065 2066 2067
	"http":   "80",
	"https":  "443",
	"socks5": "1080",
2068 2069 2070
}

// canonicalAddr returns url.Host but always with a ":port" suffix
Rob Pike's avatar
Rob Pike committed
2071
func canonicalAddr(url *url.URL) string {
2072
	addr := url.Hostname()
2073
	if v, err := idnaASCII(addr); err == nil {
2074
		addr = v
2075
	}
2076 2077 2078 2079 2080
	port := url.Port()
	if port == "" {
		port = portMap[url.Scheme]
	}
	return net.JoinHostPort(addr, port)
2081
}
2082

2083 2084 2085 2086 2087
// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
2088
// once, right before its final (error-producing) Read or Close call
2089 2090 2091 2092 2093
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
2094
type bodyEOFSignal struct {
2095
	body         io.ReadCloser
2096 2097 2098 2099 2100
	mu           sync.Mutex        // guards following 4 fields
	closed       bool              // whether Close has been called
	rerr         error             // sticky Read error
	fn           func(error) error // err will be nil on Read io.EOF
	earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
2101 2102
}

2103 2104
var errReadOnClosedResBody = errors.New("http: read on closed response body")

2105
func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
2106 2107 2108 2109
	es.mu.Lock()
	closed, rerr := es.closed, es.rerr
	es.mu.Unlock()
	if closed {
2110
		return 0, errReadOnClosedResBody
2111
	}
2112 2113
	if rerr != nil {
		return 0, rerr
2114 2115
	}

2116 2117 2118 2119 2120 2121 2122
	n, err = es.body.Read(p)
	if err != nil {
		es.mu.Lock()
		defer es.mu.Unlock()
		if es.rerr == nil {
			es.rerr = err
		}
2123
		err = es.condfn(err)
2124
	}
2125 2126
	return
}
2127

2128 2129 2130 2131 2132
func (es *bodyEOFSignal) Close() error {
	es.mu.Lock()
	defer es.mu.Unlock()
	if es.closed {
		return nil
2133
	}
2134
	es.closed = true
2135 2136 2137
	if es.earlyCloseFn != nil && es.rerr != io.EOF {
		return es.earlyCloseFn()
	}
2138
	err := es.body.Close()
2139
	return es.condfn(err)
2140 2141
}

2142
// caller must hold es.mu.
2143
func (es *bodyEOFSignal) condfn(err error) error {
2144
	if es.fn == nil {
2145
		return err
2146
	}
2147
	err = es.fn(err)
2148
	es.fn = nil
2149
	return err
2150 2151
}

2152 2153 2154
// gzipReader wraps a response body so it can lazily
// call gzip.NewReader on the first call to Read
type gzipReader struct {
2155 2156 2157
	body *bodyEOFSignal // underlying HTTP/1 response body framing
	zr   *gzip.Reader   // lazily-initialized gzip reader
	zerr error          // any error from gzip.NewReader; sticky
2158 2159 2160 2161
}

func (gz *gzipReader) Read(p []byte) (n int, err error) {
	if gz.zr == nil {
2162 2163
		if gz.zerr == nil {
			gz.zr, gz.zerr = gzip.NewReader(gz.body)
2164
		}
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
		if gz.zerr != nil {
			return 0, gz.zerr
		}
	}

	gz.body.mu.Lock()
	if gz.body.closed {
		err = errReadOnClosedResBody
	}
	gz.body.mu.Unlock()

	if err != nil {
		return 0, err
2178 2179 2180 2181 2182 2183 2184 2185
	}
	return gz.zr.Read(p)
}

func (gz *gzipReader) Close() error {
	return gz.body.Close()
}

2186 2187
type readerAndCloser struct {
	io.Reader
2188 2189
	io.Closer
}
2190 2191 2192 2193 2194 2195

type tlsHandshakeTimeoutError struct{}

func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
func (tlsHandshakeTimeoutError) Temporary() bool { return true }
func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
2196

2197 2198 2199 2200 2201 2202 2203
// fakeLocker is a sync.Locker which does nothing. It's used to guard
// test-only fields when not under test, to avoid runtime atomic
// overhead.
type fakeLocker struct{}

func (fakeLocker) Lock()   {}
func (fakeLocker) Unlock() {}
2204

2205 2206 2207
// clneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if
// cfg is nil. This is safe to call even if cfg is in active use by a TLS
// client or server.
2208 2209 2210 2211
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
	if cfg == nil {
		return &tls.Config{}
	}
2212
	return cfg.Clone()
2213 2214
}

2215 2216 2217 2218 2219
type connLRU struct {
	ll *list.List // list.Element.Value type is of *persistConn
	m  map[*persistConn]*list.Element
}

2220
// add adds pc to the head of the linked list.
2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
func (cl *connLRU) add(pc *persistConn) {
	if cl.ll == nil {
		cl.ll = list.New()
		cl.m = make(map[*persistConn]*list.Element)
	}
	ele := cl.ll.PushFront(pc)
	if _, ok := cl.m[pc]; ok {
		panic("persistConn was already in LRU")
	}
	cl.m[pc] = ele
}

func (cl *connLRU) removeOldest() *persistConn {
	ele := cl.ll.Back()
	pc := ele.Value.(*persistConn)
	cl.ll.Remove(ele)
	delete(cl.m, pc)
	return pc
}

// remove removes pc from cl.
func (cl *connLRU) remove(pc *persistConn) {
	if ele, ok := cl.m[pc]; ok {
		cl.ll.Remove(ele)
		delete(cl.m, pc)
	}
}

// len returns the number of items in the cache.
func (cl *connLRU) len() int {
	return len(cl.m)
}
2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264

// validPort reports whether p (without the colon) is a valid port in
// a URL, per RFC 3986 Section 3.2.3, which says the port may be
// empty, or only contain digits.
func validPort(p string) bool {
	for _, r := range []byte(p) {
		if r < '0' || r > '9' {
			return false
		}
	}
	return true
}