1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// HTTP client implementation. See RFC 2616. 6// 7// This is the low-level Transport implementation of RoundTripper. 8// The high-level interface is in client.go. 9 10package http 11 12import ( 13 "bufio" 14 "compress/gzip" 15 "crypto/tls" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net" 21 "net/url" 22 "os" 23 "strings" 24 "sync" 25 "time" 26) 27 28// DefaultTransport is the default implementation of Transport and is 29// used by DefaultClient. It establishes network connections as needed 30// and caches them for reuse by subsequent calls. It uses HTTP proxies 31// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and 32// $no_proxy) environment variables. 33var DefaultTransport RoundTripper = &Transport{ 34 Proxy: ProxyFromEnvironment, 35 Dial: (&net.Dialer{ 36 Timeout: 30 * time.Second, 37 KeepAlive: 30 * time.Second, 38 }).Dial, 39 TLSHandshakeTimeout: 10 * time.Second, 40 ExpectContinueTimeout: 1 * time.Second, 41} 42 43// DefaultMaxIdleConnsPerHost is the default value of Transport's 44// MaxIdleConnsPerHost. 45const DefaultMaxIdleConnsPerHost = 2 46 47// Transport is an implementation of RoundTripper that supports HTTP, 48// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 49// 50// By default, Transport caches connections for future re-use. 51// This may leave many open connections when accessing many hosts. 52// This behavior can be managed using Transport's CloseIdleConnections method 53// and the MaxIdleConnsPerHost and DisableKeepAlives fields. 54// 55// Transports should be reused instead of created as needed. 56// Transports are safe for concurrent use by multiple goroutines. 57// 58// A Transport is a low-level primitive for making HTTP and HTTPS requests. 59// For high-level functionality, such as cookies and redirects, see Client. 60// 61// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 62// for HTTPS URLs, depending on whether the server supports HTTP/2. 63// See the package docs for more about HTTP/2. 64type Transport struct { 65 idleMu sync.Mutex 66 wantIdle bool // user has requested to close all idle conns 67 idleConn map[connectMethodKey][]*persistConn 68 idleConnCh map[connectMethodKey]chan *persistConn 69 70 reqMu sync.Mutex 71 reqCanceler map[*Request]func() 72 73 altMu sync.RWMutex 74 altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper 75 76 // Proxy specifies a function to return a proxy for a given 77 // Request. If the function returns a non-nil error, the 78 // request is aborted with the provided error. 79 // If Proxy is nil or returns a nil *URL, no proxy is used. 80 Proxy func(*Request) (*url.URL, error) 81 82 // Dial specifies the dial function for creating unencrypted 83 // TCP connections. 84 // If Dial is nil, net.Dial is used. 85 Dial func(network, addr string) (net.Conn, error) 86 87 // DialTLS specifies an optional dial function for creating 88 // TLS connections for non-proxied HTTPS requests. 89 // 90 // If DialTLS is nil, Dial and TLSClientConfig are used. 91 // 92 // If DialTLS is set, the Dial hook is not used for HTTPS 93 // requests and the TLSClientConfig and TLSHandshakeTimeout 94 // are ignored. The returned net.Conn is assumed to already be 95 // past the TLS handshake. 96 DialTLS func(network, addr string) (net.Conn, error) 97 98 // TLSClientConfig specifies the TLS configuration to use with 99 // tls.Client. If nil, the default configuration is used. 100 TLSClientConfig *tls.Config 101 102 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 103 // wait for a TLS handshake. Zero means no timeout. 104 TLSHandshakeTimeout time.Duration 105 106 // DisableKeepAlives, if true, prevents re-use of TCP connections 107 // between different HTTP requests. 108 DisableKeepAlives bool 109 110 // DisableCompression, if true, prevents the Transport from 111 // requesting compression with an "Accept-Encoding: gzip" 112 // request header when the Request contains no existing 113 // Accept-Encoding value. If the Transport requests gzip on 114 // its own and gets a gzipped response, it's transparently 115 // decoded in the Response.Body. However, if the user 116 // explicitly requested gzip it is not automatically 117 // uncompressed. 118 DisableCompression bool 119 120 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 121 // (keep-alive) to keep per-host. If zero, 122 // DefaultMaxIdleConnsPerHost is used. 123 MaxIdleConnsPerHost int 124 125 // ResponseHeaderTimeout, if non-zero, specifies the amount of 126 // time to wait for a server's response headers after fully 127 // writing the request (including its body, if any). This 128 // time does not include the time to read the response body. 129 ResponseHeaderTimeout time.Duration 130 131 // ExpectContinueTimeout, if non-zero, specifies the amount of 132 // time to wait for a server's first response headers after fully 133 // writing the request headers if the request has an 134 // "Expect: 100-continue" header. Zero means no timeout. 135 // This time does not include the time to send the request header. 136 ExpectContinueTimeout time.Duration 137 138 // TLSNextProto specifies how the Transport switches to an 139 // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN 140 // protocol negotiation. If Transport dials an TLS connection 141 // with a non-empty protocol name and TLSNextProto contains a 142 // map entry for that key (such as "h2"), then the func is 143 // called with the request's authority (such as "example.com" 144 // or "example.com:1234") and the TLS connection. The function 145 // must return a RoundTripper that then handles the request. 146 // If TLSNextProto is nil, HTTP/2 support is enabled automatically. 147 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 148 149 // nextProtoOnce guards initialization of TLSNextProto and 150 // h2transport (via onceSetNextProtoDefaults) 151 nextProtoOnce sync.Once 152 h2transport *http2Transport // non-nil if http2 wired up 153 154 // TODO: tunable on global max cached connections 155 // TODO: tunable on timeout on cached connections 156 // TODO: tunable on max per-host TCP dials in flight (Issue 13957) 157} 158 159// onceSetNextProtoDefaults initializes TLSNextProto. 160// It must be called via t.nextProtoOnce.Do. 161func (t *Transport) onceSetNextProtoDefaults() { 162 if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") { 163 return 164 } 165 if t.TLSNextProto != nil { 166 // This is the documented way to disable http2 on a 167 // Transport. 168 return 169 } 170 if t.TLSClientConfig != nil { 171 // Be conservative for now (for Go 1.6) at least and 172 // don't automatically enable http2 if they've 173 // specified a custom TLS config. Let them opt-in 174 // themselves via http2.ConfigureTransport so we don't 175 // surprise them by modifying their tls.Config. 176 // Issue 14275. 177 return 178 } 179 if t.ExpectContinueTimeout != 0 { 180 // Unsupported in http2, so disable http2 for now. 181 // Issue 13851. 182 return 183 } 184 t2, err := http2configureTransport(t) 185 if err != nil { 186 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 187 } else { 188 t.h2transport = t2 189 } 190} 191 192// ProxyFromEnvironment returns the URL of the proxy to use for a 193// given request, as indicated by the environment variables 194// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 195// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https 196// requests. 197// 198// The environment values may be either a complete URL or a 199// "host[:port]", in which case the "http" scheme is assumed. 200// An error is returned if the value is a different form. 201// 202// A nil URL and nil error are returned if no proxy is defined in the 203// environment, or a proxy should not be used for the given request, 204// as defined by NO_PROXY. 205// 206// As a special case, if req.URL.Host is "localhost" (with or without 207// a port number), then a nil URL and nil error will be returned. 208func ProxyFromEnvironment(req *Request) (*url.URL, error) { 209 var proxy string 210 if req.URL.Scheme == "https" { 211 proxy = httpsProxyEnv.Get() 212 } 213 if proxy == "" { 214 proxy = httpProxyEnv.Get() 215 } 216 if proxy == "" { 217 return nil, nil 218 } 219 if !useProxy(canonicalAddr(req.URL)) { 220 return nil, nil 221 } 222 proxyURL, err := url.Parse(proxy) 223 if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { 224 // proxy was bogus. Try prepending "http://" to it and 225 // see if that parses correctly. If not, we fall 226 // through and complain about the original one. 227 if proxyURL, err := url.Parse("http://" + proxy); err == nil { 228 return proxyURL, nil 229 } 230 } 231 if err != nil { 232 return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) 233 } 234 return proxyURL, nil 235} 236 237// ProxyURL returns a proxy function (for use in a Transport) 238// that always returns the same URL. 239func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 240 return func(*Request) (*url.URL, error) { 241 return fixedURL, nil 242 } 243} 244 245// transportRequest is a wrapper around a *Request that adds 246// optional extra headers to write. 247type transportRequest struct { 248 *Request // original request, not to be mutated 249 extra Header // extra headers to write, or nil 250} 251 252func (tr *transportRequest) extraHeaders() Header { 253 if tr.extra == nil { 254 tr.extra = make(Header) 255 } 256 return tr.extra 257} 258 259// RoundTrip implements the RoundTripper interface. 260// 261// For higher-level HTTP client support (such as handling of cookies 262// and redirects), see Get, Post, and the Client type. 263func (t *Transport) RoundTrip(req *Request) (*Response, error) { 264 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 265 if req.URL == nil { 266 req.closeBody() 267 return nil, errors.New("http: nil Request.URL") 268 } 269 if req.Header == nil { 270 req.closeBody() 271 return nil, errors.New("http: nil Request.Header") 272 } 273 // TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex 274 t.altMu.RLock() 275 altRT := t.altProto[req.URL.Scheme] 276 t.altMu.RUnlock() 277 if altRT != nil { 278 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 279 return resp, err 280 } 281 } 282 if s := req.URL.Scheme; s != "http" && s != "https" { 283 req.closeBody() 284 return nil, &badStringError{"unsupported protocol scheme", s} 285 } 286 if req.Method != "" && !validMethod(req.Method) { 287 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 288 } 289 if req.URL.Host == "" { 290 req.closeBody() 291 return nil, errors.New("http: no Host in request URL") 292 } 293 294 for { 295 // treq gets modified by roundTrip, so we need to recreate for each retry. 296 treq := &transportRequest{Request: req} 297 cm, err := t.connectMethodForRequest(treq) 298 if err != nil { 299 req.closeBody() 300 return nil, err 301 } 302 303 // Get the cached or newly-created connection to either the 304 // host (for http or https), the http proxy, or the http proxy 305 // pre-CONNECTed to https server. In any case, we'll be ready 306 // to send it requests. 307 pconn, err := t.getConn(req, cm) 308 if err != nil { 309 t.setReqCanceler(req, nil) 310 req.closeBody() 311 return nil, err 312 } 313 314 var resp *Response 315 if pconn.alt != nil { 316 // HTTP/2 path. 317 t.setReqCanceler(req, nil) // not cancelable with CancelRequest 318 resp, err = pconn.alt.RoundTrip(req) 319 } else { 320 resp, err = pconn.roundTrip(treq) 321 } 322 if err == nil { 323 return resp, nil 324 } 325 if err := checkTransportResend(err, req, pconn); err != nil { 326 return nil, err 327 } 328 testHookRoundTripRetried() 329 } 330} 331 332// checkTransportResend checks whether a failed HTTP request can be 333// resent on a new connection. The non-nil input error is the error from 334// roundTrip, which might be wrapped in a beforeRespHeaderError error. 335// 336// The return value is err or the unwrapped error inside a 337// beforeRespHeaderError. 338func checkTransportResend(err error, req *Request, pconn *persistConn) error { 339 brhErr, ok := err.(beforeRespHeaderError) 340 if !ok { 341 return err 342 } 343 err = brhErr.error // unwrap the custom error in case we return it 344 if err != errMissingHost && pconn.isReused() && req.isReplayable() { 345 // If we try to reuse a connection that the server is in the process of 346 // closing, we may end up successfully writing out our request (or a 347 // portion of our request) only to find a connection error when we try to 348 // read from (or finish writing to) the socket. 349 350 // There can be a race between the socket pool checking whether a socket 351 // is still connected, receiving the FIN, and sending/reading data on a 352 // reused socket. If we receive the FIN between the connectedness check 353 // and writing/reading from the socket, we may first learn the socket is 354 // disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most 355 // likely happen when trying to retrieve its IP address. See 356 // http://crbug.com/105824 for more details. 357 358 // We resend a request only if we reused a keep-alive connection and did 359 // not yet receive any header data. This automatically prevents an 360 // infinite resend loop because we'll run out of the cached keep-alive 361 // connections eventually. 362 return nil 363 } 364 return err 365} 366 367// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 368var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 369 370// RegisterProtocol registers a new protocol with scheme. 371// The Transport will pass requests using the given scheme to rt. 372// It is rt's responsibility to simulate HTTP request semantics. 373// 374// RegisterProtocol can be used by other packages to provide 375// implementations of protocol schemes like "ftp" or "file". 376// 377// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 378// handle the RoundTrip itself for that one request, as if the 379// protocol were not registered. 380func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 381 t.altMu.Lock() 382 defer t.altMu.Unlock() 383 if t.altProto == nil { 384 t.altProto = make(map[string]RoundTripper) 385 } 386 if _, exists := t.altProto[scheme]; exists { 387 panic("protocol " + scheme + " already registered") 388 } 389 t.altProto[scheme] = rt 390} 391 392// CloseIdleConnections closes any connections which were previously 393// connected from previous requests but are now sitting idle in 394// a "keep-alive" state. It does not interrupt any connections currently 395// in use. 396func (t *Transport) CloseIdleConnections() { 397 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 398 t.idleMu.Lock() 399 m := t.idleConn 400 t.idleConn = nil 401 t.idleConnCh = nil 402 t.wantIdle = true 403 t.idleMu.Unlock() 404 for _, conns := range m { 405 for _, pconn := range conns { 406 pconn.close(errCloseIdleConns) 407 } 408 } 409 if t2 := t.h2transport; t2 != nil { 410 t2.CloseIdleConnections() 411 } 412} 413 414// CancelRequest cancels an in-flight request by closing its connection. 415// CancelRequest should only be called after RoundTrip has returned. 416// 417// Deprecated: Use Request.Cancel instead. CancelRequest can not cancel 418// HTTP/2 requests. 419func (t *Transport) CancelRequest(req *Request) { 420 t.reqMu.Lock() 421 cancel := t.reqCanceler[req] 422 delete(t.reqCanceler, req) 423 t.reqMu.Unlock() 424 if cancel != nil { 425 cancel() 426 } 427} 428 429// 430// Private implementation past this point. 431// 432 433var ( 434 httpProxyEnv = &envOnce{ 435 names: []string{"HTTP_PROXY", "http_proxy"}, 436 } 437 httpsProxyEnv = &envOnce{ 438 names: []string{"HTTPS_PROXY", "https_proxy"}, 439 } 440 noProxyEnv = &envOnce{ 441 names: []string{"NO_PROXY", "no_proxy"}, 442 } 443) 444 445// envOnce looks up an environment variable (optionally by multiple 446// names) once. It mitigates expensive lookups on some platforms 447// (e.g. Windows). 448type envOnce struct { 449 names []string 450 once sync.Once 451 val string 452} 453 454func (e *envOnce) Get() string { 455 e.once.Do(e.init) 456 return e.val 457} 458 459func (e *envOnce) init() { 460 for _, n := range e.names { 461 e.val = os.Getenv(n) 462 if e.val != "" { 463 return 464 } 465 } 466} 467 468// reset is used by tests 469func (e *envOnce) reset() { 470 e.once = sync.Once{} 471 e.val = "" 472} 473 474func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 475 cm.targetScheme = treq.URL.Scheme 476 cm.targetAddr = canonicalAddr(treq.URL) 477 if t.Proxy != nil { 478 cm.proxyURL, err = t.Proxy(treq.Request) 479 } 480 return cm, err 481} 482 483// proxyAuth returns the Proxy-Authorization header to set 484// on requests, if applicable. 485func (cm *connectMethod) proxyAuth() string { 486 if cm.proxyURL == nil { 487 return "" 488 } 489 if u := cm.proxyURL.User; u != nil { 490 username := u.Username() 491 password, _ := u.Password() 492 return "Basic " + basicAuth(username, password) 493 } 494 return "" 495} 496 497// error values for debugging and testing, not seen by users. 498var ( 499 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 500 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 501 errWantIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 502 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 503 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 504 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 505 errServerClosedIdle = errors.New("http: server closed idle conn") 506) 507 508func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 509 if err := t.tryPutIdleConn(pconn); err != nil { 510 pconn.close(err) 511 } 512} 513 514// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 515// a new request. 516// If pconn is no longer needed or not in a good state, tryPutIdleConn returns 517// an error explaining why it wasn't registered. 518// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 519func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 520 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 521 return errKeepAlivesDisabled 522 } 523 if pconn.isBroken() { 524 return errConnBroken 525 } 526 key := pconn.cacheKey 527 max := t.MaxIdleConnsPerHost 528 if max == 0 { 529 max = DefaultMaxIdleConnsPerHost 530 } 531 pconn.markReused() 532 t.idleMu.Lock() 533 534 waitingDialer := t.idleConnCh[key] 535 select { 536 case waitingDialer <- pconn: 537 // We're done with this pconn and somebody else is 538 // currently waiting for a conn of this type (they're 539 // actively dialing, but this conn is ready 540 // first). Chrome calls this socket late binding. See 541 // https://insouciant.org/tech/connection-management-in-chromium/ 542 t.idleMu.Unlock() 543 return nil 544 default: 545 if waitingDialer != nil { 546 // They had populated this, but their dial won 547 // first, so we can clean up this map entry. 548 delete(t.idleConnCh, key) 549 } 550 } 551 if t.wantIdle { 552 t.idleMu.Unlock() 553 return errWantIdle 554 } 555 if t.idleConn == nil { 556 t.idleConn = make(map[connectMethodKey][]*persistConn) 557 } 558 if len(t.idleConn[key]) >= max { 559 t.idleMu.Unlock() 560 return errTooManyIdle 561 } 562 for _, exist := range t.idleConn[key] { 563 if exist == pconn { 564 log.Fatalf("dup idle pconn %p in freelist", pconn) 565 } 566 } 567 t.idleConn[key] = append(t.idleConn[key], pconn) 568 t.idleMu.Unlock() 569 return nil 570} 571 572// getIdleConnCh returns a channel to receive and return idle 573// persistent connection for the given connectMethod. 574// It may return nil, if persistent connections are not being used. 575func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { 576 if t.DisableKeepAlives { 577 return nil 578 } 579 key := cm.key() 580 t.idleMu.Lock() 581 defer t.idleMu.Unlock() 582 t.wantIdle = false 583 if t.idleConnCh == nil { 584 t.idleConnCh = make(map[connectMethodKey]chan *persistConn) 585 } 586 ch, ok := t.idleConnCh[key] 587 if !ok { 588 ch = make(chan *persistConn) 589 t.idleConnCh[key] = ch 590 } 591 return ch 592} 593 594func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { 595 key := cm.key() 596 t.idleMu.Lock() 597 defer t.idleMu.Unlock() 598 if t.idleConn == nil { 599 return nil 600 } 601 for { 602 pconns, ok := t.idleConn[key] 603 if !ok { 604 return nil 605 } 606 if len(pconns) == 1 { 607 pconn = pconns[0] 608 delete(t.idleConn, key) 609 } else { 610 // 2 or more cached connections; pop last 611 // TODO: queue? 612 pconn = pconns[len(pconns)-1] 613 t.idleConn[key] = pconns[:len(pconns)-1] 614 } 615 if !pconn.isBroken() { 616 return 617 } 618 } 619} 620 621func (t *Transport) setReqCanceler(r *Request, fn func()) { 622 t.reqMu.Lock() 623 defer t.reqMu.Unlock() 624 if t.reqCanceler == nil { 625 t.reqCanceler = make(map[*Request]func()) 626 } 627 if fn != nil { 628 t.reqCanceler[r] = fn 629 } else { 630 delete(t.reqCanceler, r) 631 } 632} 633 634// replaceReqCanceler replaces an existing cancel function. If there is no cancel function 635// for the request, we don't set the function and return false. 636// Since CancelRequest will clear the canceler, we can use the return value to detect if 637// the request was canceled since the last setReqCancel call. 638func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { 639 t.reqMu.Lock() 640 defer t.reqMu.Unlock() 641 _, ok := t.reqCanceler[r] 642 if !ok { 643 return false 644 } 645 if fn != nil { 646 t.reqCanceler[r] = fn 647 } else { 648 delete(t.reqCanceler, r) 649 } 650 return true 651} 652 653func (t *Transport) dial(network, addr string) (net.Conn, error) { 654 if t.Dial != nil { 655 c, err := t.Dial(network, addr) 656 if c == nil && err == nil { 657 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 658 } 659 return c, err 660 } 661 return net.Dial(network, addr) 662} 663 664// getConn dials and creates a new persistConn to the target as 665// specified in the connectMethod. This includes doing a proxy CONNECT 666// and/or setting up TLS. If this doesn't return an error, the persistConn 667// is ready to write requests to. 668func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { 669 if pc := t.getIdleConn(cm); pc != nil { 670 // set request canceler to some non-nil function so we 671 // can detect whether it was cleared between now and when 672 // we enter roundTrip 673 t.setReqCanceler(req, func() {}) 674 return pc, nil 675 } 676 677 type dialRes struct { 678 pc *persistConn 679 err error 680 } 681 dialc := make(chan dialRes) 682 683 // Copy these hooks so we don't race on the postPendingDial in 684 // the goroutine we launch. Issue 11136. 685 testHookPrePendingDial := testHookPrePendingDial 686 testHookPostPendingDial := testHookPostPendingDial 687 688 handlePendingDial := func() { 689 testHookPrePendingDial() 690 go func() { 691 if v := <-dialc; v.err == nil { 692 t.putOrCloseIdleConn(v.pc) 693 } 694 testHookPostPendingDial() 695 }() 696 } 697 698 cancelc := make(chan struct{}) 699 t.setReqCanceler(req, func() { close(cancelc) }) 700 701 go func() { 702 pc, err := t.dialConn(cm) 703 dialc <- dialRes{pc, err} 704 }() 705 706 idleConnCh := t.getIdleConnCh(cm) 707 select { 708 case v := <-dialc: 709 // Our dial finished. 710 return v.pc, v.err 711 case pc := <-idleConnCh: 712 // Another request finished first and its net.Conn 713 // became available before our dial. Or somebody 714 // else's dial that they didn't use. 715 // But our dial is still going, so give it away 716 // when it finishes: 717 handlePendingDial() 718 return pc, nil 719 case <-req.Cancel: 720 handlePendingDial() 721 return nil, errRequestCanceledConn 722 case <-cancelc: 723 handlePendingDial() 724 return nil, errRequestCanceledConn 725 } 726} 727 728func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { 729 pconn := &persistConn{ 730 t: t, 731 cacheKey: cm.key(), 732 reqch: make(chan requestAndChan, 1), 733 writech: make(chan writeRequest, 1), 734 closech: make(chan struct{}), 735 writeErrCh: make(chan error, 1), 736 } 737 tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil 738 if tlsDial { 739 var err error 740 pconn.conn, err = t.DialTLS("tcp", cm.addr()) 741 if err != nil { 742 return nil, err 743 } 744 if pconn.conn == nil { 745 return nil, errors.New("net/http: Transport.DialTLS returned (nil, nil)") 746 } 747 if tc, ok := pconn.conn.(*tls.Conn); ok { 748 // Handshake here, in case DialTLS didn't. TLSNextProto below 749 // depends on it for knowing the connection state. 750 if err := tc.Handshake(); err != nil { 751 go pconn.conn.Close() 752 return nil, err 753 } 754 cs := tc.ConnectionState() 755 pconn.tlsState = &cs 756 } 757 } else { 758 conn, err := t.dial("tcp", cm.addr()) 759 if err != nil { 760 if cm.proxyURL != nil { 761 err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) 762 } 763 return nil, err 764 } 765 pconn.conn = conn 766 } 767 768 // Proxy setup. 769 switch { 770 case cm.proxyURL == nil: 771 // Do nothing. Not using a proxy. 772 case cm.targetScheme == "http": 773 pconn.isProxy = true 774 if pa := cm.proxyAuth(); pa != "" { 775 pconn.mutateHeaderFunc = func(h Header) { 776 h.Set("Proxy-Authorization", pa) 777 } 778 } 779 case cm.targetScheme == "https": 780 conn := pconn.conn 781 connectReq := &Request{ 782 Method: "CONNECT", 783 URL: &url.URL{Opaque: cm.targetAddr}, 784 Host: cm.targetAddr, 785 Header: make(Header), 786 } 787 if pa := cm.proxyAuth(); pa != "" { 788 connectReq.Header.Set("Proxy-Authorization", pa) 789 } 790 connectReq.Write(conn) 791 792 // Read response. 793 // Okay to use and discard buffered reader here, because 794 // TLS server will not speak until spoken to. 795 br := bufio.NewReader(conn) 796 resp, err := ReadResponse(br, connectReq) 797 if err != nil { 798 conn.Close() 799 return nil, err 800 } 801 if resp.StatusCode != 200 { 802 f := strings.SplitN(resp.Status, " ", 2) 803 conn.Close() 804 return nil, errors.New(f[1]) 805 } 806 } 807 808 if cm.targetScheme == "https" && !tlsDial { 809 // Initiate TLS and check remote host name against certificate. 810 cfg := cloneTLSClientConfig(t.TLSClientConfig) 811 if cfg.ServerName == "" { 812 cfg.ServerName = cm.tlsHost() 813 } 814 plainConn := pconn.conn 815 tlsConn := tls.Client(plainConn, cfg) 816 errc := make(chan error, 2) 817 var timer *time.Timer // for canceling TLS handshake 818 if d := t.TLSHandshakeTimeout; d != 0 { 819 timer = time.AfterFunc(d, func() { 820 errc <- tlsHandshakeTimeoutError{} 821 }) 822 } 823 go func() { 824 err := tlsConn.Handshake() 825 if timer != nil { 826 timer.Stop() 827 } 828 errc <- err 829 }() 830 if err := <-errc; err != nil { 831 plainConn.Close() 832 return nil, err 833 } 834 if !cfg.InsecureSkipVerify { 835 if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { 836 plainConn.Close() 837 return nil, err 838 } 839 } 840 cs := tlsConn.ConnectionState() 841 pconn.tlsState = &cs 842 pconn.conn = tlsConn 843 } 844 845 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 846 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 847 return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil 848 } 849 } 850 851 pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) 852 pconn.bw = bufio.NewWriter(pconn.conn) 853 go pconn.readLoop() 854 go pconn.writeLoop() 855 return pconn, nil 856} 857 858// useProxy reports whether requests to addr should use a proxy, 859// according to the NO_PROXY or no_proxy environment variable. 860// addr is always a canonicalAddr with a host and port. 861func useProxy(addr string) bool { 862 if len(addr) == 0 { 863 return true 864 } 865 host, _, err := net.SplitHostPort(addr) 866 if err != nil { 867 return false 868 } 869 if host == "localhost" { 870 return false 871 } 872 if ip := net.ParseIP(host); ip != nil { 873 if ip.IsLoopback() { 874 return false 875 } 876 } 877 878 no_proxy := noProxyEnv.Get() 879 if no_proxy == "*" { 880 return false 881 } 882 883 addr = strings.ToLower(strings.TrimSpace(addr)) 884 if hasPort(addr) { 885 addr = addr[:strings.LastIndex(addr, ":")] 886 } 887 888 for _, p := range strings.Split(no_proxy, ",") { 889 p = strings.ToLower(strings.TrimSpace(p)) 890 if len(p) == 0 { 891 continue 892 } 893 if hasPort(p) { 894 p = p[:strings.LastIndex(p, ":")] 895 } 896 if addr == p { 897 return false 898 } 899 if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { 900 // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" 901 return false 902 } 903 if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { 904 // no_proxy "foo.com" matches "bar.foo.com" 905 return false 906 } 907 } 908 return true 909} 910 911// connectMethod is the map key (in its String form) for keeping persistent 912// TCP connections alive for subsequent HTTP requests. 913// 914// A connect method may be of the following types: 915// 916// Cache key form Description 917// ----------------- ------------------------- 918// |http|foo.com http directly to server, no proxy 919// |https|foo.com https directly to server, no proxy 920// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 921// http://proxy.com|http http to proxy, http to anywhere after that 922// 923// Note: no support to https to the proxy yet. 924// 925type connectMethod struct { 926 proxyURL *url.URL // nil for no proxy, else full proxy URL 927 targetScheme string // "http" or "https" 928 targetAddr string // Not used if proxy + http targetScheme (4th example in table) 929} 930 931func (cm *connectMethod) key() connectMethodKey { 932 proxyStr := "" 933 targetAddr := cm.targetAddr 934 if cm.proxyURL != nil { 935 proxyStr = cm.proxyURL.String() 936 if cm.targetScheme == "http" { 937 targetAddr = "" 938 } 939 } 940 return connectMethodKey{ 941 proxy: proxyStr, 942 scheme: cm.targetScheme, 943 addr: targetAddr, 944 } 945} 946 947// addr returns the first hop "host:port" to which we need to TCP connect. 948func (cm *connectMethod) addr() string { 949 if cm.proxyURL != nil { 950 return canonicalAddr(cm.proxyURL) 951 } 952 return cm.targetAddr 953} 954 955// tlsHost returns the host name to match against the peer's 956// TLS certificate. 957func (cm *connectMethod) tlsHost() string { 958 h := cm.targetAddr 959 if hasPort(h) { 960 h = h[:strings.LastIndex(h, ":")] 961 } 962 return h 963} 964 965// connectMethodKey is the map key version of connectMethod, with a 966// stringified proxy URL (or the empty string) instead of a pointer to 967// a URL. 968type connectMethodKey struct { 969 proxy, scheme, addr string 970} 971 972func (k connectMethodKey) String() string { 973 // Only used by tests. 974 return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr) 975} 976 977// persistConn wraps a connection, usually a persistent one 978// (but may be used for non-keep-alive requests as well) 979type persistConn struct { 980 // alt optionally specifies the TLS NextProto RoundTripper. 981 // This is used for HTTP/2 today and future protocol laters. 982 // If it's non-nil, the rest of the fields are unused. 983 alt RoundTripper 984 985 t *Transport 986 cacheKey connectMethodKey 987 conn net.Conn 988 tlsState *tls.ConnectionState 989 br *bufio.Reader // from conn 990 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 991 bw *bufio.Writer // to conn 992 reqch chan requestAndChan // written by roundTrip; read by readLoop 993 writech chan writeRequest // written by roundTrip; read by writeLoop 994 closech chan struct{} // closed when conn closed 995 isProxy bool 996 // writeErrCh passes the request write error (usually nil) 997 // from the writeLoop goroutine to the readLoop which passes 998 // it off to the res.Body reader, which then uses it to decide 999 // whether or not a connection can be reused. Issue 7569. 1000 writeErrCh chan error 1001 1002 lk sync.Mutex // guards following fields 1003 numExpectedResponses int 1004 closed error // set non-nil when conn is closed, before closech is closed 1005 broken bool // an error has happened on this connection; marked broken so it's not reused. 1006 canceled bool // whether this conn was broken due a CancelRequest 1007 reused bool // whether conn has had successful request/response and is being reused. 1008 // mutateHeaderFunc is an optional func to modify extra 1009 // headers on each outbound request before it's written. (the 1010 // original Request given to RoundTrip is not modified) 1011 mutateHeaderFunc func(Header) 1012} 1013 1014// isBroken reports whether this connection is in a known broken state. 1015func (pc *persistConn) isBroken() bool { 1016 pc.lk.Lock() 1017 b := pc.broken 1018 pc.lk.Unlock() 1019 return b 1020} 1021 1022// isCanceled reports whether this connection was closed due to CancelRequest. 1023func (pc *persistConn) isCanceled() bool { 1024 pc.lk.Lock() 1025 defer pc.lk.Unlock() 1026 return pc.canceled 1027} 1028 1029// isReused reports whether this connection is in a known broken state. 1030func (pc *persistConn) isReused() bool { 1031 pc.lk.Lock() 1032 r := pc.reused 1033 pc.lk.Unlock() 1034 return r 1035} 1036 1037func (pc *persistConn) cancelRequest() { 1038 pc.lk.Lock() 1039 defer pc.lk.Unlock() 1040 pc.canceled = true 1041 pc.closeLocked(errRequestCanceled) 1042} 1043 1044func (pc *persistConn) readLoop() { 1045 closeErr := errReadLoopExiting // default value, if not changed below 1046 defer func() { pc.close(closeErr) }() 1047 1048 tryPutIdleConn := func() bool { 1049 if err := pc.t.tryPutIdleConn(pc); err != nil { 1050 closeErr = err 1051 return false 1052 } 1053 return true 1054 } 1055 1056 // eofc is used to block caller goroutines reading from Response.Body 1057 // at EOF until this goroutines has (potentially) added the connection 1058 // back to the idle pool. 1059 eofc := make(chan struct{}) 1060 defer close(eofc) // unblock reader on errors 1061 1062 // Read this once, before loop starts. (to avoid races in tests) 1063 testHookMu.Lock() 1064 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 1065 testHookMu.Unlock() 1066 1067 alive := true 1068 for alive { 1069 _, err := pc.br.Peek(1) 1070 if err != nil { 1071 err = beforeRespHeaderError{err} 1072 } 1073 1074 pc.lk.Lock() 1075 if pc.numExpectedResponses == 0 { 1076 pc.readLoopPeekFailLocked(err) 1077 pc.lk.Unlock() 1078 return 1079 } 1080 pc.lk.Unlock() 1081 1082 rc := <-pc.reqch 1083 1084 var resp *Response 1085 if err == nil { 1086 resp, err = pc.readResponse(rc) 1087 } 1088 1089 if err != nil { 1090 // If we won't be able to retry this request later (from the 1091 // roundTrip goroutine), mark it as done now. 1092 // BEFORE the send on rc.ch, as the client might re-use the 1093 // same *Request pointer, and we don't want to set call 1094 // t.setReqCanceler from this persistConn while the Transport 1095 // potentially spins up a different persistConn for the 1096 // caller's subsequent request. 1097 if checkTransportResend(err, rc.req, pc) != nil { 1098 pc.t.setReqCanceler(rc.req, nil) 1099 } 1100 select { 1101 case rc.ch <- responseAndError{err: err}: 1102 case <-rc.callerGone: 1103 return 1104 } 1105 return 1106 } 1107 1108 pc.lk.Lock() 1109 pc.numExpectedResponses-- 1110 pc.lk.Unlock() 1111 1112 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 1113 1114 if resp.Close || rc.req.Close || resp.StatusCode <= 199 { 1115 // Don't do keep-alive on error if either party requested a close 1116 // or we get an unexpected informational (1xx) response. 1117 // StatusCode 100 is already handled above. 1118 alive = false 1119 } 1120 1121 if !hasBody { 1122 pc.t.setReqCanceler(rc.req, nil) 1123 1124 // Put the idle conn back into the pool before we send the response 1125 // so if they process it quickly and make another request, they'll 1126 // get this same conn. But we use the unbuffered channel 'rc' 1127 // to guarantee that persistConn.roundTrip got out of its select 1128 // potentially waiting for this persistConn to close. 1129 // but after 1130 alive = alive && 1131 !pc.sawEOF && 1132 pc.wroteRequest() && 1133 tryPutIdleConn() 1134 1135 select { 1136 case rc.ch <- responseAndError{res: resp}: 1137 case <-rc.callerGone: 1138 return 1139 } 1140 1141 // Now that they've read from the unbuffered channel, they're safely 1142 // out of the select that also waits on this goroutine to die, so 1143 // we're allowed to exit now if needed (if alive is false) 1144 testHookReadLoopBeforeNextRead() 1145 continue 1146 } 1147 1148 if rc.addedGzip { 1149 maybeUngzipResponse(resp) 1150 } 1151 resp.Body = &bodyEOFSignal{body: resp.Body} 1152 1153 waitForBodyRead := make(chan bool, 2) 1154 resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { 1155 waitForBodyRead <- false 1156 return nil 1157 } 1158 resp.Body.(*bodyEOFSignal).fn = func(err error) error { 1159 isEOF := err == io.EOF 1160 waitForBodyRead <- isEOF 1161 if isEOF { 1162 <-eofc // see comment above eofc declaration 1163 } else if err != nil && pc.isCanceled() { 1164 return errRequestCanceled 1165 } 1166 return err 1167 } 1168 1169 select { 1170 case rc.ch <- responseAndError{res: resp}: 1171 case <-rc.callerGone: 1172 return 1173 } 1174 1175 // Before looping back to the top of this function and peeking on 1176 // the bufio.Reader, wait for the caller goroutine to finish 1177 // reading the response body. (or for cancelation or death) 1178 select { 1179 case bodyEOF := <-waitForBodyRead: 1180 pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool 1181 alive = alive && 1182 bodyEOF && 1183 !pc.sawEOF && 1184 pc.wroteRequest() && 1185 tryPutIdleConn() 1186 if bodyEOF { 1187 eofc <- struct{}{} 1188 } 1189 case <-rc.req.Cancel: 1190 alive = false 1191 pc.t.CancelRequest(rc.req) 1192 case <-pc.closech: 1193 alive = false 1194 } 1195 1196 testHookReadLoopBeforeNextRead() 1197 } 1198} 1199 1200func maybeUngzipResponse(resp *Response) { 1201 if resp.Header.Get("Content-Encoding") == "gzip" { 1202 resp.Header.Del("Content-Encoding") 1203 resp.Header.Del("Content-Length") 1204 resp.ContentLength = -1 1205 resp.Body = &gzipReader{body: resp.Body} 1206 } 1207} 1208 1209func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 1210 if pc.closed != nil { 1211 return 1212 } 1213 if n := pc.br.Buffered(); n > 0 { 1214 buf, _ := pc.br.Peek(n) 1215 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 1216 } 1217 if peekErr == io.EOF { 1218 // common case. 1219 pc.closeLocked(errServerClosedIdle) 1220 } else { 1221 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr)) 1222 } 1223} 1224 1225// readResponse reads an HTTP response (or two, in the case of "Expect: 1226// 100-continue") from the server. It returns the final non-100 one. 1227func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) { 1228 resp, err = ReadResponse(pc.br, rc.req) 1229 if err != nil { 1230 return 1231 } 1232 if rc.continueCh != nil { 1233 if resp.StatusCode == 100 { 1234 rc.continueCh <- struct{}{} 1235 } else { 1236 close(rc.continueCh) 1237 } 1238 } 1239 if resp.StatusCode == 100 { 1240 resp, err = ReadResponse(pc.br, rc.req) 1241 if err != nil { 1242 return 1243 } 1244 } 1245 resp.TLS = pc.tlsState 1246 return 1247} 1248 1249// waitForContinue returns the function to block until 1250// any response, timeout or connection close. After any of them, 1251// the function returns a bool which indicates if the body should be sent. 1252func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 1253 if continueCh == nil { 1254 return nil 1255 } 1256 return func() bool { 1257 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 1258 defer timer.Stop() 1259 1260 select { 1261 case _, ok := <-continueCh: 1262 return ok 1263 case <-timer.C: 1264 return true 1265 case <-pc.closech: 1266 return false 1267 } 1268 } 1269} 1270 1271func (pc *persistConn) writeLoop() { 1272 for { 1273 select { 1274 case wr := <-pc.writech: 1275 if pc.isBroken() { 1276 wr.ch <- errors.New("http: can't write HTTP request on broken connection") 1277 continue 1278 } 1279 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 1280 if err == nil { 1281 err = pc.bw.Flush() 1282 } 1283 if err != nil { 1284 pc.markBroken() 1285 wr.req.Request.closeBody() 1286 } 1287 pc.writeErrCh <- err // to the body reader, which might recycle us 1288 wr.ch <- err // to the roundTrip function 1289 case <-pc.closech: 1290 return 1291 } 1292 } 1293} 1294 1295// wroteRequest is a check before recycling a connection that the previous write 1296// (from writeLoop above) happened and was successful. 1297func (pc *persistConn) wroteRequest() bool { 1298 select { 1299 case err := <-pc.writeErrCh: 1300 // Common case: the write happened well before the response, so 1301 // avoid creating a timer. 1302 return err == nil 1303 default: 1304 // Rare case: the request was written in writeLoop above but 1305 // before it could send to pc.writeErrCh, the reader read it 1306 // all, processed it, and called us here. In this case, give the 1307 // write goroutine a bit of time to finish its send. 1308 // 1309 // Less rare case: We also get here in the legitimate case of 1310 // Issue 7569, where the writer is still writing (or stalled), 1311 // but the server has already replied. In this case, we don't 1312 // want to wait too long, and we want to return false so this 1313 // connection isn't re-used. 1314 select { 1315 case err := <-pc.writeErrCh: 1316 return err == nil 1317 case <-time.After(50 * time.Millisecond): 1318 return false 1319 } 1320 } 1321} 1322 1323// responseAndError is how the goroutine reading from an HTTP/1 server 1324// communicates with the goroutine doing the RoundTrip. 1325type responseAndError struct { 1326 res *Response // else use this response (see res method) 1327 err error 1328} 1329 1330type requestAndChan struct { 1331 req *Request 1332 ch chan responseAndError // unbuffered; always send in select on callerGone 1333 1334 // did the Transport (as opposed to the client code) add an 1335 // Accept-Encoding gzip header? only if it we set it do 1336 // we transparently decode the gzip. 1337 addedGzip bool 1338 1339 // Optional blocking chan for Expect: 100-continue (for send). 1340 // If the request has an "Expect: 100-continue" header and 1341 // the server responds 100 Continue, readLoop send a value 1342 // to writeLoop via this chan. 1343 continueCh chan<- struct{} 1344 1345 callerGone <-chan struct{} // closed when roundTrip caller has returned 1346} 1347 1348// A writeRequest is sent by the readLoop's goroutine to the 1349// writeLoop's goroutine to write a request while the read loop 1350// concurrently waits on both the write response and the server's 1351// reply. 1352type writeRequest struct { 1353 req *transportRequest 1354 ch chan<- error 1355 1356 // Optional blocking chan for Expect: 100-continue (for recieve). 1357 // If not nil, writeLoop blocks sending request body until 1358 // it receives from this chan. 1359 continueCh <-chan struct{} 1360} 1361 1362type httpError struct { 1363 err string 1364 timeout bool 1365} 1366 1367func (e *httpError) Error() string { return e.err } 1368func (e *httpError) Timeout() bool { return e.timeout } 1369func (e *httpError) Temporary() bool { return true } 1370 1371var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 1372var errClosed error = &httpError{err: "net/http: server closed connection before response was received"} 1373var errRequestCanceled = errors.New("net/http: request canceled") 1374var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 1375 1376func nop() {} 1377 1378// testHooks. Always non-nil. 1379var ( 1380 testHookEnterRoundTrip = nop 1381 testHookWaitResLoop = nop 1382 testHookRoundTripRetried = nop 1383 testHookPrePendingDial = nop 1384 testHookPostPendingDial = nop 1385 1386 testHookMu sync.Locker = fakeLocker{} // guards following 1387 testHookReadLoopBeforeNextRead = nop 1388) 1389 1390// beforeRespHeaderError is used to indicate when an IO error has occurred before 1391// any header data was received. 1392type beforeRespHeaderError struct { 1393 error 1394} 1395 1396func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 1397 testHookEnterRoundTrip() 1398 if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { 1399 pc.t.putOrCloseIdleConn(pc) 1400 return nil, errRequestCanceled 1401 } 1402 pc.lk.Lock() 1403 pc.numExpectedResponses++ 1404 headerFn := pc.mutateHeaderFunc 1405 pc.lk.Unlock() 1406 1407 if headerFn != nil { 1408 headerFn(req.extraHeaders()) 1409 } 1410 1411 // Ask for a compressed version if the caller didn't set their 1412 // own value for Accept-Encoding. We only attempt to 1413 // uncompress the gzip stream if we were the layer that 1414 // requested it. 1415 requestedGzip := false 1416 if !pc.t.DisableCompression && 1417 req.Header.Get("Accept-Encoding") == "" && 1418 req.Header.Get("Range") == "" && 1419 req.Method != "HEAD" { 1420 // Request gzip only, not deflate. Deflate is ambiguous and 1421 // not as universally supported anyway. 1422 // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 1423 // 1424 // Note that we don't request this for HEAD requests, 1425 // due to a bug in nginx: 1426 // http://trac.nginx.org/nginx/ticket/358 1427 // https://golang.org/issue/5522 1428 // 1429 // We don't request gzip if the request is for a range, since 1430 // auto-decoding a portion of a gzipped document will just fail 1431 // anyway. See https://golang.org/issue/8923 1432 requestedGzip = true 1433 req.extraHeaders().Set("Accept-Encoding", "gzip") 1434 } 1435 1436 var continueCh chan struct{} 1437 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 1438 continueCh = make(chan struct{}, 1) 1439 } 1440 1441 if pc.t.DisableKeepAlives { 1442 req.extraHeaders().Set("Connection", "close") 1443 } 1444 1445 gone := make(chan struct{}) 1446 defer close(gone) 1447 1448 // Write the request concurrently with waiting for a response, 1449 // in case the server decides to reply before reading our full 1450 // request body. 1451 writeErrCh := make(chan error, 1) 1452 pc.writech <- writeRequest{req, writeErrCh, continueCh} 1453 1454 resc := make(chan responseAndError) 1455 pc.reqch <- requestAndChan{ 1456 req: req.Request, 1457 ch: resc, 1458 addedGzip: requestedGzip, 1459 continueCh: continueCh, 1460 callerGone: gone, 1461 } 1462 1463 var re responseAndError 1464 var respHeaderTimer <-chan time.Time 1465 cancelChan := req.Request.Cancel 1466WaitResponse: 1467 for { 1468 testHookWaitResLoop() 1469 select { 1470 case err := <-writeErrCh: 1471 if err != nil { 1472 if pc.isCanceled() { 1473 err = errRequestCanceled 1474 } 1475 re = responseAndError{err: beforeRespHeaderError{err}} 1476 pc.close(fmt.Errorf("write error: %v", err)) 1477 break WaitResponse 1478 } 1479 if d := pc.t.ResponseHeaderTimeout; d > 0 { 1480 timer := time.NewTimer(d) 1481 defer timer.Stop() // prevent leaks 1482 respHeaderTimer = timer.C 1483 } 1484 case <-pc.closech: 1485 var err error 1486 if pc.isCanceled() { 1487 err = errRequestCanceled 1488 } else { 1489 err = beforeRespHeaderError{fmt.Errorf("net/http: HTTP/1 transport connection broken: %v", pc.closed)} 1490 } 1491 re = responseAndError{err: err} 1492 break WaitResponse 1493 case <-respHeaderTimer: 1494 pc.close(errTimeout) 1495 re = responseAndError{err: errTimeout} 1496 break WaitResponse 1497 case re = <-resc: 1498 if re.err != nil && pc.isCanceled() { 1499 re.err = errRequestCanceled 1500 } 1501 break WaitResponse 1502 case <-cancelChan: 1503 pc.t.CancelRequest(req.Request) 1504 cancelChan = nil 1505 } 1506 } 1507 1508 if re.err != nil { 1509 pc.t.setReqCanceler(req.Request, nil) 1510 } 1511 if (re.res == nil) == (re.err == nil) { 1512 panic("internal error: exactly one of res or err should be set") 1513 } 1514 return re.res, re.err 1515} 1516 1517// markBroken marks a connection as broken (so it's not reused). 1518// It differs from close in that it doesn't close the underlying 1519// connection for use when it's still being read. 1520func (pc *persistConn) markBroken() { 1521 pc.lk.Lock() 1522 defer pc.lk.Unlock() 1523 pc.broken = true 1524} 1525 1526// markReused marks this connection as having been successfully used for a 1527// request and response. 1528func (pc *persistConn) markReused() { 1529 pc.lk.Lock() 1530 pc.reused = true 1531 pc.lk.Unlock() 1532} 1533 1534// close closes the underlying TCP connection and closes 1535// the pc.closech channel. 1536// 1537// The provided err is only for testing and debugging; in normal 1538// circumstances it should never be seen by users. 1539func (pc *persistConn) close(err error) { 1540 pc.lk.Lock() 1541 defer pc.lk.Unlock() 1542 pc.closeLocked(err) 1543} 1544 1545func (pc *persistConn) closeLocked(err error) { 1546 if err == nil { 1547 panic("nil error") 1548 } 1549 pc.broken = true 1550 if pc.closed == nil { 1551 pc.closed = err 1552 if pc.alt != nil { 1553 // Do nothing; can only get here via getConn's 1554 // handlePendingDial's putOrCloseIdleConn when 1555 // it turns out the abandoned connection in 1556 // flight ended up negotiating an alternate 1557 // protocol. We don't use the connection 1558 // freelist for http2. That's done by the 1559 // alternate protocol's RoundTripper. 1560 } else { 1561 pc.conn.Close() 1562 close(pc.closech) 1563 } 1564 } 1565 pc.mutateHeaderFunc = nil 1566} 1567 1568var portMap = map[string]string{ 1569 "http": "80", 1570 "https": "443", 1571} 1572 1573// canonicalAddr returns url.Host but always with a ":port" suffix 1574func canonicalAddr(url *url.URL) string { 1575 addr := url.Host 1576 if !hasPort(addr) { 1577 return addr + ":" + portMap[url.Scheme] 1578 } 1579 return addr 1580} 1581 1582// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most 1583// once, right before its final (error-producing) Read or Close call 1584// returns. fn should return the new error to return from Read or Close. 1585// 1586// If earlyCloseFn is non-nil and Close is called before io.EOF is 1587// seen, earlyCloseFn is called instead of fn, and its return value is 1588// the return value from Close. 1589type bodyEOFSignal struct { 1590 body io.ReadCloser 1591 mu sync.Mutex // guards following 4 fields 1592 closed bool // whether Close has been called 1593 rerr error // sticky Read error 1594 fn func(error) error // err will be nil on Read io.EOF 1595 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 1596} 1597 1598func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 1599 es.mu.Lock() 1600 closed, rerr := es.closed, es.rerr 1601 es.mu.Unlock() 1602 if closed { 1603 return 0, errors.New("http: read on closed response body") 1604 } 1605 if rerr != nil { 1606 return 0, rerr 1607 } 1608 1609 n, err = es.body.Read(p) 1610 if err != nil { 1611 es.mu.Lock() 1612 defer es.mu.Unlock() 1613 if es.rerr == nil { 1614 es.rerr = err 1615 } 1616 err = es.condfn(err) 1617 } 1618 return 1619} 1620 1621func (es *bodyEOFSignal) Close() error { 1622 es.mu.Lock() 1623 defer es.mu.Unlock() 1624 if es.closed { 1625 return nil 1626 } 1627 es.closed = true 1628 if es.earlyCloseFn != nil && es.rerr != io.EOF { 1629 return es.earlyCloseFn() 1630 } 1631 err := es.body.Close() 1632 return es.condfn(err) 1633} 1634 1635// caller must hold es.mu. 1636func (es *bodyEOFSignal) condfn(err error) error { 1637 if es.fn == nil { 1638 return err 1639 } 1640 err = es.fn(err) 1641 es.fn = nil 1642 return err 1643} 1644 1645// gzipReader wraps a response body so it can lazily 1646// call gzip.NewReader on the first call to Read 1647type gzipReader struct { 1648 body io.ReadCloser // underlying Response.Body 1649 zr io.Reader // lazily-initialized gzip reader 1650} 1651 1652func (gz *gzipReader) Read(p []byte) (n int, err error) { 1653 if gz.zr == nil { 1654 gz.zr, err = gzip.NewReader(gz.body) 1655 if err != nil { 1656 return 0, err 1657 } 1658 } 1659 return gz.zr.Read(p) 1660} 1661 1662func (gz *gzipReader) Close() error { 1663 return gz.body.Close() 1664} 1665 1666type readerAndCloser struct { 1667 io.Reader 1668 io.Closer 1669} 1670 1671type tlsHandshakeTimeoutError struct{} 1672 1673func (tlsHandshakeTimeoutError) Timeout() bool { return true } 1674func (tlsHandshakeTimeoutError) Temporary() bool { return true } 1675func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 1676 1677type noteEOFReader struct { 1678 r io.Reader 1679 sawEOF *bool 1680} 1681 1682func (nr noteEOFReader) Read(p []byte) (n int, err error) { 1683 n, err = nr.r.Read(p) 1684 if err == io.EOF { 1685 *nr.sawEOF = true 1686 } 1687 return 1688} 1689 1690// fakeLocker is a sync.Locker which does nothing. It's used to guard 1691// test-only fields when not under test, to avoid runtime atomic 1692// overhead. 1693type fakeLocker struct{} 1694 1695func (fakeLocker) Lock() {} 1696func (fakeLocker) Unlock() {} 1697 1698func isNetWriteError(err error) bool { 1699 switch e := err.(type) { 1700 case *url.Error: 1701 return isNetWriteError(e.Err) 1702 case *net.OpError: 1703 return e.Op == "write" 1704 default: 1705 return false 1706 } 1707} 1708 1709// cloneTLSConfig returns a shallow clone of the exported 1710// fields of cfg, ignoring the unexported sync.Once, which 1711// contains a mutex and must not be copied. 1712// 1713// The cfg must not be in active use by tls.Server, or else 1714// there can still be a race with tls.Server updating SessionTicketKey 1715// and our copying it, and also a race with the server setting 1716// SessionTicketsDisabled=false on failure to set the random 1717// ticket key. 1718// 1719// If cfg is nil, a new zero tls.Config is returned. 1720func cloneTLSConfig(cfg *tls.Config) *tls.Config { 1721 if cfg == nil { 1722 return &tls.Config{} 1723 } 1724 return &tls.Config{ 1725 Rand: cfg.Rand, 1726 Time: cfg.Time, 1727 Certificates: cfg.Certificates, 1728 NameToCertificate: cfg.NameToCertificate, 1729 GetCertificate: cfg.GetCertificate, 1730 RootCAs: cfg.RootCAs, 1731 NextProtos: cfg.NextProtos, 1732 ServerName: cfg.ServerName, 1733 ClientAuth: cfg.ClientAuth, 1734 ClientCAs: cfg.ClientCAs, 1735 InsecureSkipVerify: cfg.InsecureSkipVerify, 1736 CipherSuites: cfg.CipherSuites, 1737 PreferServerCipherSuites: cfg.PreferServerCipherSuites, 1738 SessionTicketsDisabled: cfg.SessionTicketsDisabled, 1739 SessionTicketKey: cfg.SessionTicketKey, 1740 ClientSessionCache: cfg.ClientSessionCache, 1741 MinVersion: cfg.MinVersion, 1742 MaxVersion: cfg.MaxVersion, 1743 CurvePreferences: cfg.CurvePreferences, 1744 } 1745} 1746 1747// cloneTLSClientConfig is like cloneTLSConfig but omits 1748// the fields SessionTicketsDisabled and SessionTicketKey. 1749// This makes it safe to call cloneTLSClientConfig on a config 1750// in active use by a server. 1751func cloneTLSClientConfig(cfg *tls.Config) *tls.Config { 1752 if cfg == nil { 1753 return &tls.Config{} 1754 } 1755 return &tls.Config{ 1756 Rand: cfg.Rand, 1757 Time: cfg.Time, 1758 Certificates: cfg.Certificates, 1759 NameToCertificate: cfg.NameToCertificate, 1760 GetCertificate: cfg.GetCertificate, 1761 RootCAs: cfg.RootCAs, 1762 NextProtos: cfg.NextProtos, 1763 ServerName: cfg.ServerName, 1764 ClientAuth: cfg.ClientAuth, 1765 ClientCAs: cfg.ClientCAs, 1766 InsecureSkipVerify: cfg.InsecureSkipVerify, 1767 CipherSuites: cfg.CipherSuites, 1768 PreferServerCipherSuites: cfg.PreferServerCipherSuites, 1769 ClientSessionCache: cfg.ClientSessionCache, 1770 MinVersion: cfg.MinVersion, 1771 MaxVersion: cfg.MaxVersion, 1772 CurvePreferences: cfg.CurvePreferences, 1773 } 1774} 1775