1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// HTTP client implementation. See RFC 7230 through 7235. 6// 7// This is the low-level Transport implementation of RoundTripper. 8// The high-level interface is in client.go. 9 10package http 11 12import ( 13 "bufio" 14 "compress/gzip" 15 "container/list" 16 "context" 17 "crypto/tls" 18 "errors" 19 "fmt" 20 "internal/godebug" 21 "io" 22 "log" 23 "net" 24 "net/http/httptrace" 25 "net/http/internal/ascii" 26 "net/textproto" 27 "net/url" 28 "reflect" 29 "strings" 30 "sync" 31 "sync/atomic" 32 "time" 33 34 "golang.org/x/net/http/httpguts" 35 "golang.org/x/net/http/httpproxy" 36) 37 38// DefaultTransport is the default implementation of Transport and is 39// used by DefaultClient. It establishes network connections as needed 40// and caches them for reuse by subsequent calls. It uses HTTP proxies 41// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and 42// $no_proxy) environment variables. 43var DefaultTransport RoundTripper = &Transport{ 44 Proxy: ProxyFromEnvironment, 45 DialContext: defaultTransportDialContext(&net.Dialer{ 46 Timeout: 30 * time.Second, 47 KeepAlive: 30 * time.Second, 48 }), 49 ForceAttemptHTTP2: true, 50 MaxIdleConns: 100, 51 IdleConnTimeout: 90 * time.Second, 52 TLSHandshakeTimeout: 10 * time.Second, 53 ExpectContinueTimeout: 1 * time.Second, 54} 55 56// DefaultMaxIdleConnsPerHost is the default value of Transport's 57// MaxIdleConnsPerHost. 58const DefaultMaxIdleConnsPerHost = 2 59 60// Transport is an implementation of RoundTripper that supports HTTP, 61// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 62// 63// By default, Transport caches connections for future re-use. 64// This may leave many open connections when accessing many hosts. 65// This behavior can be managed using Transport's CloseIdleConnections method 66// and the MaxIdleConnsPerHost and DisableKeepAlives fields. 67// 68// Transports should be reused instead of created as needed. 69// Transports are safe for concurrent use by multiple goroutines. 70// 71// A Transport is a low-level primitive for making HTTP and HTTPS requests. 72// For high-level functionality, such as cookies and redirects, see Client. 73// 74// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 75// for HTTPS URLs, depending on whether the server supports HTTP/2, 76// and how the Transport is configured. The DefaultTransport supports HTTP/2. 77// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 78// and call ConfigureTransport. See the package docs for more about HTTP/2. 79// 80// Responses with status codes in the 1xx range are either handled 81// automatically (100 expect-continue) or ignored. The one 82// exception is HTTP status code 101 (Switching Protocols), which is 83// considered a terminal status and returned by RoundTrip. To see the 84// ignored 1xx responses, use the httptrace trace package's 85// ClientTrace.Got1xxResponse. 86// 87// Transport only retries a request upon encountering a network error 88// if the request is idempotent and either has no body or has its 89// Request.GetBody defined. HTTP requests are considered idempotent if 90// they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their 91// Header map contains an "Idempotency-Key" or "X-Idempotency-Key" 92// entry. If the idempotency key value is a zero-length slice, the 93// request is treated as idempotent but the header is not sent on the 94// wire. 95type Transport struct { 96 idleMu sync.Mutex 97 closeIdle bool // user has requested to close all idle conns 98 idleConn map[connectMethodKey][]*persistConn // most recently used at end 99 idleConnWait map[connectMethodKey]wantConnQueue // waiting getConns 100 idleLRU connLRU 101 102 reqMu sync.Mutex 103 reqCanceler map[cancelKey]func(error) 104 105 altMu sync.Mutex // guards changing altProto only 106 altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme 107 108 connsPerHostMu sync.Mutex 109 connsPerHost map[connectMethodKey]int 110 connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns 111 112 // Proxy specifies a function to return a proxy for a given 113 // Request. If the function returns a non-nil error, the 114 // request is aborted with the provided error. 115 // 116 // The proxy type is determined by the URL scheme. "http", 117 // "https", and "socks5" are supported. If the scheme is empty, 118 // "http" is assumed. 119 // 120 // If Proxy is nil or returns a nil *URL, no proxy is used. 121 Proxy func(*Request) (*url.URL, error) 122 123 // DialContext specifies the dial function for creating unencrypted TCP connections. 124 // If DialContext is nil (and the deprecated Dial below is also nil), 125 // then the transport dials using package net. 126 // 127 // DialContext runs concurrently with calls to RoundTrip. 128 // A RoundTrip call that initiates a dial may end up using 129 // a connection dialed previously when the earlier connection 130 // becomes idle before the later DialContext completes. 131 DialContext func(ctx context.Context, network, addr string) (net.Conn, error) 132 133 // Dial specifies the dial function for creating unencrypted TCP connections. 134 // 135 // Dial runs concurrently with calls to RoundTrip. 136 // A RoundTrip call that initiates a dial may end up using 137 // a connection dialed previously when the earlier connection 138 // becomes idle before the later Dial completes. 139 // 140 // Deprecated: Use DialContext instead, which allows the transport 141 // to cancel dials as soon as they are no longer needed. 142 // If both are set, DialContext takes priority. 143 Dial func(network, addr string) (net.Conn, error) 144 145 // DialTLSContext specifies an optional dial function for creating 146 // TLS connections for non-proxied HTTPS requests. 147 // 148 // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), 149 // DialContext and TLSClientConfig are used. 150 // 151 // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS 152 // requests and the TLSClientConfig and TLSHandshakeTimeout 153 // are ignored. The returned net.Conn is assumed to already be 154 // past the TLS handshake. 155 DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) 156 157 // DialTLS specifies an optional dial function for creating 158 // TLS connections for non-proxied HTTPS requests. 159 // 160 // Deprecated: Use DialTLSContext instead, which allows the transport 161 // to cancel dials as soon as they are no longer needed. 162 // If both are set, DialTLSContext takes priority. 163 DialTLS func(network, addr string) (net.Conn, error) 164 165 // TLSClientConfig specifies the TLS configuration to use with 166 // tls.Client. 167 // If nil, the default configuration is used. 168 // If non-nil, HTTP/2 support may not be enabled by default. 169 TLSClientConfig *tls.Config 170 171 // TLSHandshakeTimeout specifies the maximum amount of time waiting to 172 // wait for a TLS handshake. Zero means no timeout. 173 TLSHandshakeTimeout time.Duration 174 175 // DisableKeepAlives, if true, disables HTTP keep-alives and 176 // will only use the connection to the server for a single 177 // HTTP request. 178 // 179 // This is unrelated to the similarly named TCP keep-alives. 180 DisableKeepAlives bool 181 182 // DisableCompression, if true, prevents the Transport from 183 // requesting compression with an "Accept-Encoding: gzip" 184 // request header when the Request contains no existing 185 // Accept-Encoding value. If the Transport requests gzip on 186 // its own and gets a gzipped response, it's transparently 187 // decoded in the Response.Body. However, if the user 188 // explicitly requested gzip it is not automatically 189 // uncompressed. 190 DisableCompression bool 191 192 // MaxIdleConns controls the maximum number of idle (keep-alive) 193 // connections across all hosts. Zero means no limit. 194 MaxIdleConns int 195 196 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 197 // (keep-alive) connections to keep per-host. If zero, 198 // DefaultMaxIdleConnsPerHost is used. 199 MaxIdleConnsPerHost int 200 201 // MaxConnsPerHost optionally limits the total number of 202 // connections per host, including connections in the dialing, 203 // active, and idle states. On limit violation, dials will block. 204 // 205 // Zero means no limit. 206 MaxConnsPerHost int 207 208 // IdleConnTimeout is the maximum amount of time an idle 209 // (keep-alive) connection will remain idle before closing 210 // itself. 211 // Zero means no limit. 212 IdleConnTimeout time.Duration 213 214 // ResponseHeaderTimeout, if non-zero, specifies the amount of 215 // time to wait for a server's response headers after fully 216 // writing the request (including its body, if any). This 217 // time does not include the time to read the response body. 218 ResponseHeaderTimeout time.Duration 219 220 // ExpectContinueTimeout, if non-zero, specifies the amount of 221 // time to wait for a server's first response headers after fully 222 // writing the request headers if the request has an 223 // "Expect: 100-continue" header. Zero means no timeout and 224 // causes the body to be sent immediately, without 225 // waiting for the server to approve. 226 // This time does not include the time to send the request header. 227 ExpectContinueTimeout time.Duration 228 229 // TLSNextProto specifies how the Transport switches to an 230 // alternate protocol (such as HTTP/2) after a TLS ALPN 231 // protocol negotiation. If Transport dials an TLS connection 232 // with a non-empty protocol name and TLSNextProto contains a 233 // map entry for that key (such as "h2"), then the func is 234 // called with the request's authority (such as "example.com" 235 // or "example.com:1234") and the TLS connection. The function 236 // must return a RoundTripper that then handles the request. 237 // If TLSNextProto is not nil, HTTP/2 support is not enabled 238 // automatically. 239 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 240 241 // ProxyConnectHeader optionally specifies headers to send to 242 // proxies during CONNECT requests. 243 // To set the header dynamically, see GetProxyConnectHeader. 244 ProxyConnectHeader Header 245 246 // GetProxyConnectHeader optionally specifies a func to return 247 // headers to send to proxyURL during a CONNECT request to the 248 // ip:port target. 249 // If it returns an error, the Transport's RoundTrip fails with 250 // that error. It can return (nil, nil) to not add headers. 251 // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is 252 // ignored. 253 GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (Header, error) 254 255 // MaxResponseHeaderBytes specifies a limit on how many 256 // response bytes are allowed in the server's response 257 // header. 258 // 259 // Zero means to use a default limit. 260 MaxResponseHeaderBytes int64 261 262 // WriteBufferSize specifies the size of the write buffer used 263 // when writing to the transport. 264 // If zero, a default (currently 4KB) is used. 265 WriteBufferSize int 266 267 // ReadBufferSize specifies the size of the read buffer used 268 // when reading from the transport. 269 // If zero, a default (currently 4KB) is used. 270 ReadBufferSize int 271 272 // nextProtoOnce guards initialization of TLSNextProto and 273 // h2transport (via onceSetNextProtoDefaults) 274 nextProtoOnce sync.Once 275 h2transport h2Transport // non-nil if http2 wired up 276 tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired 277 278 // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero 279 // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. 280 // By default, use of any those fields conservatively disables HTTP/2. 281 // To use a custom dialer or TLS config and still attempt HTTP/2 282 // upgrades, set this to true. 283 ForceAttemptHTTP2 bool 284} 285 286// A cancelKey is the key of the reqCanceler map. 287// We wrap the *Request in this type since we want to use the original request, 288// not any transient one created by roundTrip. 289type cancelKey struct { 290 req *Request 291} 292 293func (t *Transport) writeBufferSize() int { 294 if t.WriteBufferSize > 0 { 295 return t.WriteBufferSize 296 } 297 return 4 << 10 298} 299 300func (t *Transport) readBufferSize() int { 301 if t.ReadBufferSize > 0 { 302 return t.ReadBufferSize 303 } 304 return 4 << 10 305} 306 307// Clone returns a deep copy of t's exported fields. 308func (t *Transport) Clone() *Transport { 309 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 310 t2 := &Transport{ 311 Proxy: t.Proxy, 312 DialContext: t.DialContext, 313 Dial: t.Dial, 314 DialTLS: t.DialTLS, 315 DialTLSContext: t.DialTLSContext, 316 TLSHandshakeTimeout: t.TLSHandshakeTimeout, 317 DisableKeepAlives: t.DisableKeepAlives, 318 DisableCompression: t.DisableCompression, 319 MaxIdleConns: t.MaxIdleConns, 320 MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, 321 MaxConnsPerHost: t.MaxConnsPerHost, 322 IdleConnTimeout: t.IdleConnTimeout, 323 ResponseHeaderTimeout: t.ResponseHeaderTimeout, 324 ExpectContinueTimeout: t.ExpectContinueTimeout, 325 ProxyConnectHeader: t.ProxyConnectHeader.Clone(), 326 GetProxyConnectHeader: t.GetProxyConnectHeader, 327 MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, 328 ForceAttemptHTTP2: t.ForceAttemptHTTP2, 329 WriteBufferSize: t.WriteBufferSize, 330 ReadBufferSize: t.ReadBufferSize, 331 } 332 if t.TLSClientConfig != nil { 333 t2.TLSClientConfig = t.TLSClientConfig.Clone() 334 } 335 if !t.tlsNextProtoWasNil { 336 npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} 337 for k, v := range t.TLSNextProto { 338 npm[k] = v 339 } 340 t2.TLSNextProto = npm 341 } 342 return t2 343} 344 345// h2Transport is the interface we expect to be able to call from 346// net/http against an *http2.Transport that's either bundled into 347// h2_bundle.go or supplied by the user via x/net/http2. 348// 349// We name it with the "h2" prefix to stay out of the "http2" prefix 350// namespace used by x/tools/cmd/bundle for h2_bundle.go. 351type h2Transport interface { 352 CloseIdleConnections() 353} 354 355func (t *Transport) hasCustomTLSDialer() bool { 356 return t.DialTLS != nil || t.DialTLSContext != nil 357} 358 359// onceSetNextProtoDefaults initializes TLSNextProto. 360// It must be called via t.nextProtoOnce.Do. 361func (t *Transport) onceSetNextProtoDefaults() { 362 t.tlsNextProtoWasNil = (t.TLSNextProto == nil) 363 if godebug.Get("http2client") == "0" { 364 return 365 } 366 367 // If they've already configured http2 with 368 // golang.org/x/net/http2 instead of the bundled copy, try to 369 // get at its http2.Transport value (via the "https" 370 // altproto map) so we can call CloseIdleConnections on it if 371 // requested. (Issue 22891) 372 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 373 if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { 374 if v := rv.Field(0); v.CanInterface() { 375 if h2i, ok := v.Interface().(h2Transport); ok { 376 t.h2transport = h2i 377 return 378 } 379 } 380 } 381 382 if t.TLSNextProto != nil { 383 // This is the documented way to disable http2 on a 384 // Transport. 385 return 386 } 387 if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { 388 // Be conservative and don't automatically enable 389 // http2 if they've specified a custom TLS config or 390 // custom dialers. Let them opt-in themselves via 391 // http2.ConfigureTransport so we don't surprise them 392 // by modifying their tls.Config. Issue 14275. 393 // However, if ForceAttemptHTTP2 is true, it overrides the above checks. 394 return 395 } 396 if omitBundledHTTP2 { 397 return 398 } 399 t2, err := http2configureTransports(t) 400 if err != nil { 401 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 402 return 403 } 404 t.h2transport = t2 405 406 // Auto-configure the http2.Transport's MaxHeaderListSize from 407 // the http.Transport's MaxResponseHeaderBytes. They don't 408 // exactly mean the same thing, but they're close. 409 // 410 // TODO: also add this to x/net/http2.Configure Transport, behind 411 // a +build go1.7 build tag: 412 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { 413 const h2max = 1<<32 - 1 414 if limit1 >= h2max { 415 t2.MaxHeaderListSize = h2max 416 } else { 417 t2.MaxHeaderListSize = uint32(limit1) 418 } 419 } 420} 421 422// ProxyFromEnvironment returns the URL of the proxy to use for a 423// given request, as indicated by the environment variables 424// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 425// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https 426// requests. 427// 428// The environment values may be either a complete URL or a 429// "host[:port]", in which case the "http" scheme is assumed. 430// The schemes "http", "https", and "socks5" are supported. 431// An error is returned if the value is a different form. 432// 433// A nil URL and nil error are returned if no proxy is defined in the 434// environment, or a proxy should not be used for the given request, 435// as defined by NO_PROXY. 436// 437// As a special case, if req.URL.Host is "localhost" (with or without 438// a port number), then a nil URL and nil error will be returned. 439func ProxyFromEnvironment(req *Request) (*url.URL, error) { 440 return envProxyFunc()(req.URL) 441} 442 443// ProxyURL returns a proxy function (for use in a Transport) 444// that always returns the same URL. 445func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 446 return func(*Request) (*url.URL, error) { 447 return fixedURL, nil 448 } 449} 450 451// transportRequest is a wrapper around a *Request that adds 452// optional extra headers to write and stores any error to return 453// from roundTrip. 454type transportRequest struct { 455 *Request // original request, not to be mutated 456 extra Header // extra headers to write, or nil 457 trace *httptrace.ClientTrace // optional 458 cancelKey cancelKey 459 460 mu sync.Mutex // guards err 461 err error // first setError value for mapRoundTripError to consider 462} 463 464func (tr *transportRequest) extraHeaders() Header { 465 if tr.extra == nil { 466 tr.extra = make(Header) 467 } 468 return tr.extra 469} 470 471func (tr *transportRequest) setError(err error) { 472 tr.mu.Lock() 473 if tr.err == nil { 474 tr.err = err 475 } 476 tr.mu.Unlock() 477} 478 479// useRegisteredProtocol reports whether an alternate protocol (as registered 480// with Transport.RegisterProtocol) should be respected for this request. 481func (t *Transport) useRegisteredProtocol(req *Request) bool { 482 if req.URL.Scheme == "https" && req.requiresHTTP1() { 483 // If this request requires HTTP/1, don't use the 484 // "https" alternate protocol, which is used by the 485 // HTTP/2 code to take over requests if there's an 486 // existing cached HTTP/2 connection. 487 return false 488 } 489 return true 490} 491 492// alternateRoundTripper returns the alternate RoundTripper to use 493// for this request if the Request's URL scheme requires one, 494// or nil for the normal case of using the Transport. 495func (t *Transport) alternateRoundTripper(req *Request) RoundTripper { 496 if !t.useRegisteredProtocol(req) { 497 return nil 498 } 499 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 500 return altProto[req.URL.Scheme] 501} 502 503// roundTrip implements a RoundTripper over HTTP. 504func (t *Transport) roundTrip(req *Request) (*Response, error) { 505 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 506 ctx := req.Context() 507 trace := httptrace.ContextClientTrace(ctx) 508 509 if req.URL == nil { 510 req.closeBody() 511 return nil, errors.New("http: nil Request.URL") 512 } 513 if req.Header == nil { 514 req.closeBody() 515 return nil, errors.New("http: nil Request.Header") 516 } 517 scheme := req.URL.Scheme 518 isHTTP := scheme == "http" || scheme == "https" 519 if isHTTP { 520 for k, vv := range req.Header { 521 if !httpguts.ValidHeaderFieldName(k) { 522 req.closeBody() 523 return nil, fmt.Errorf("net/http: invalid header field name %q", k) 524 } 525 for _, v := range vv { 526 if !httpguts.ValidHeaderFieldValue(v) { 527 req.closeBody() 528 return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) 529 } 530 } 531 } 532 } 533 534 origReq := req 535 cancelKey := cancelKey{origReq} 536 req = setupRewindBody(req) 537 538 if altRT := t.alternateRoundTripper(req); altRT != nil { 539 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 540 return resp, err 541 } 542 var err error 543 req, err = rewindBody(req) 544 if err != nil { 545 return nil, err 546 } 547 } 548 if !isHTTP { 549 req.closeBody() 550 return nil, badStringError("unsupported protocol scheme", scheme) 551 } 552 if req.Method != "" && !validMethod(req.Method) { 553 req.closeBody() 554 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 555 } 556 if req.URL.Host == "" { 557 req.closeBody() 558 return nil, errors.New("http: no Host in request URL") 559 } 560 561 for { 562 select { 563 case <-ctx.Done(): 564 req.closeBody() 565 return nil, ctx.Err() 566 default: 567 } 568 569 // treq gets modified by roundTrip, so we need to recreate for each retry. 570 treq := &transportRequest{Request: req, trace: trace, cancelKey: cancelKey} 571 cm, err := t.connectMethodForRequest(treq) 572 if err != nil { 573 req.closeBody() 574 return nil, err 575 } 576 577 // Get the cached or newly-created connection to either the 578 // host (for http or https), the http proxy, or the http proxy 579 // pre-CONNECTed to https server. In any case, we'll be ready 580 // to send it requests. 581 pconn, err := t.getConn(treq, cm) 582 if err != nil { 583 t.setReqCanceler(cancelKey, nil) 584 req.closeBody() 585 return nil, err 586 } 587 588 var resp *Response 589 if pconn.alt != nil { 590 // HTTP/2 path. 591 t.setReqCanceler(cancelKey, nil) // not cancelable with CancelRequest 592 resp, err = pconn.alt.RoundTrip(req) 593 } else { 594 resp, err = pconn.roundTrip(treq) 595 } 596 if err == nil { 597 resp.Request = origReq 598 return resp, nil 599 } 600 601 // Failed. Clean up and determine whether to retry. 602 if http2isNoCachedConnError(err) { 603 if t.removeIdleConn(pconn) { 604 t.decConnsPerHost(pconn.cacheKey) 605 } 606 } else if !pconn.shouldRetryRequest(req, err) { 607 // Issue 16465: return underlying net.Conn.Read error from peek, 608 // as we've historically done. 609 if e, ok := err.(transportReadFromServerError); ok { 610 err = e.err 611 } 612 return nil, err 613 } 614 testHookRoundTripRetried() 615 616 // Rewind the body if we're able to. 617 req, err = rewindBody(req) 618 if err != nil { 619 return nil, err 620 } 621 } 622} 623 624var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") 625 626type readTrackingBody struct { 627 io.ReadCloser 628 didRead bool 629 didClose bool 630} 631 632func (r *readTrackingBody) Read(data []byte) (int, error) { 633 r.didRead = true 634 return r.ReadCloser.Read(data) 635} 636 637func (r *readTrackingBody) Close() error { 638 r.didClose = true 639 return r.ReadCloser.Close() 640} 641 642// setupRewindBody returns a new request with a custom body wrapper 643// that can report whether the body needs rewinding. 644// This lets rewindBody avoid an error result when the request 645// does not have GetBody but the body hasn't been read at all yet. 646func setupRewindBody(req *Request) *Request { 647 if req.Body == nil || req.Body == NoBody { 648 return req 649 } 650 newReq := *req 651 newReq.Body = &readTrackingBody{ReadCloser: req.Body} 652 return &newReq 653} 654 655// rewindBody returns a new request with the body rewound. 656// It returns req unmodified if the body does not need rewinding. 657// rewindBody takes care of closing req.Body when appropriate 658// (in all cases except when rewindBody returns req unmodified). 659func rewindBody(req *Request) (rewound *Request, err error) { 660 if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) { 661 return req, nil // nothing to rewind 662 } 663 if !req.Body.(*readTrackingBody).didClose { 664 req.closeBody() 665 } 666 if req.GetBody == nil { 667 return nil, errCannotRewind 668 } 669 body, err := req.GetBody() 670 if err != nil { 671 return nil, err 672 } 673 newReq := *req 674 newReq.Body = &readTrackingBody{ReadCloser: body} 675 return &newReq, nil 676} 677 678// shouldRetryRequest reports whether we should retry sending a failed 679// HTTP request on a new connection. The non-nil input error is the 680// error from roundTrip. 681func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { 682 if http2isNoCachedConnError(err) { 683 // Issue 16582: if the user started a bunch of 684 // requests at once, they can all pick the same conn 685 // and violate the server's max concurrent streams. 686 // Instead, match the HTTP/1 behavior for now and dial 687 // again to get a new TCP connection, rather than failing 688 // this request. 689 return true 690 } 691 if err == errMissingHost { 692 // User error. 693 return false 694 } 695 if !pc.isReused() { 696 // This was a fresh connection. There's no reason the server 697 // should've hung up on us. 698 // 699 // Also, if we retried now, we could loop forever 700 // creating new connections and retrying if the server 701 // is just hanging up on us because it doesn't like 702 // our request (as opposed to sending an error). 703 return false 704 } 705 if _, ok := err.(nothingWrittenError); ok { 706 // We never wrote anything, so it's safe to retry, if there's no body or we 707 // can "rewind" the body with GetBody. 708 return req.outgoingLength() == 0 || req.GetBody != nil 709 } 710 if !req.isReplayable() { 711 // Don't retry non-idempotent requests. 712 return false 713 } 714 if _, ok := err.(transportReadFromServerError); ok { 715 // We got some non-EOF net.Conn.Read failure reading 716 // the 1st response byte from the server. 717 return true 718 } 719 if err == errServerClosedIdle { 720 // The server replied with io.EOF while we were trying to 721 // read the response. Probably an unfortunately keep-alive 722 // timeout, just as the client was writing a request. 723 return true 724 } 725 return false // conservatively 726} 727 728// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 729var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 730 731// RegisterProtocol registers a new protocol with scheme. 732// The Transport will pass requests using the given scheme to rt. 733// It is rt's responsibility to simulate HTTP request semantics. 734// 735// RegisterProtocol can be used by other packages to provide 736// implementations of protocol schemes like "ftp" or "file". 737// 738// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will 739// handle the RoundTrip itself for that one request, as if the 740// protocol were not registered. 741func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 742 t.altMu.Lock() 743 defer t.altMu.Unlock() 744 oldMap, _ := t.altProto.Load().(map[string]RoundTripper) 745 if _, exists := oldMap[scheme]; exists { 746 panic("protocol " + scheme + " already registered") 747 } 748 newMap := make(map[string]RoundTripper) 749 for k, v := range oldMap { 750 newMap[k] = v 751 } 752 newMap[scheme] = rt 753 t.altProto.Store(newMap) 754} 755 756// CloseIdleConnections closes any connections which were previously 757// connected from previous requests but are now sitting idle in 758// a "keep-alive" state. It does not interrupt any connections currently 759// in use. 760func (t *Transport) CloseIdleConnections() { 761 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 762 t.idleMu.Lock() 763 m := t.idleConn 764 t.idleConn = nil 765 t.closeIdle = true // close newly idle connections 766 t.idleLRU = connLRU{} 767 t.idleMu.Unlock() 768 for _, conns := range m { 769 for _, pconn := range conns { 770 pconn.close(errCloseIdleConns) 771 } 772 } 773 if t2 := t.h2transport; t2 != nil { 774 t2.CloseIdleConnections() 775 } 776} 777 778// CancelRequest cancels an in-flight request by closing its connection. 779// CancelRequest should only be called after RoundTrip has returned. 780// 781// Deprecated: Use Request.WithContext to create a request with a 782// cancelable context instead. CancelRequest cannot cancel HTTP/2 783// requests. 784func (t *Transport) CancelRequest(req *Request) { 785 t.cancelRequest(cancelKey{req}, errRequestCanceled) 786} 787 788// Cancel an in-flight request, recording the error value. 789// Returns whether the request was canceled. 790func (t *Transport) cancelRequest(key cancelKey, err error) bool { 791 // This function must not return until the cancel func has completed. 792 // See: https://golang.org/issue/34658 793 t.reqMu.Lock() 794 defer t.reqMu.Unlock() 795 cancel := t.reqCanceler[key] 796 delete(t.reqCanceler, key) 797 if cancel != nil { 798 cancel(err) 799 } 800 801 return cancel != nil 802} 803 804// 805// Private implementation past this point. 806// 807 808var ( 809 // proxyConfigOnce guards proxyConfig 810 envProxyOnce sync.Once 811 envProxyFuncValue func(*url.URL) (*url.URL, error) 812) 813 814// defaultProxyConfig returns a ProxyConfig value looked up 815// from the environment. This mitigates expensive lookups 816// on some platforms (e.g. Windows). 817func envProxyFunc() func(*url.URL) (*url.URL, error) { 818 envProxyOnce.Do(func() { 819 envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() 820 }) 821 return envProxyFuncValue 822} 823 824// resetProxyConfig is used by tests. 825func resetProxyConfig() { 826 envProxyOnce = sync.Once{} 827 envProxyFuncValue = nil 828} 829 830func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 831 cm.targetScheme = treq.URL.Scheme 832 cm.targetAddr = canonicalAddr(treq.URL) 833 if t.Proxy != nil { 834 cm.proxyURL, err = t.Proxy(treq.Request) 835 } 836 cm.onlyH1 = treq.requiresHTTP1() 837 return cm, err 838} 839 840// proxyAuth returns the Proxy-Authorization header to set 841// on requests, if applicable. 842func (cm *connectMethod) proxyAuth() string { 843 if cm.proxyURL == nil { 844 return "" 845 } 846 if u := cm.proxyURL.User; u != nil { 847 username := u.Username() 848 password, _ := u.Password() 849 return "Basic " + basicAuth(username, password) 850 } 851 return "" 852} 853 854// error values for debugging and testing, not seen by users. 855var ( 856 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 857 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 858 errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 859 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 860 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") 861 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 862 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 863 errIdleConnTimeout = errors.New("http: idle connection timeout") 864 865 // errServerClosedIdle is not seen by users for idempotent requests, but may be 866 // seen by a user if the server shuts down an idle connection and sends its FIN 867 // in flight with already-written POST body bytes from the client. 868 // See https://github.com/golang/go/issues/19943#issuecomment-355607646 869 errServerClosedIdle = errors.New("http: server closed idle connection") 870) 871 872// transportReadFromServerError is used by Transport.readLoop when the 873// 1 byte peek read fails and we're actually anticipating a response. 874// Usually this is just due to the inherent keep-alive shut down race, 875// where the server closed the connection at the same time the client 876// wrote. The underlying err field is usually io.EOF or some 877// ECONNRESET sort of thing which varies by platform. But it might be 878// the user's custom net.Conn.Read error too, so we carry it along for 879// them to return from Transport.RoundTrip. 880type transportReadFromServerError struct { 881 err error 882} 883 884func (e transportReadFromServerError) Unwrap() error { return e.err } 885 886func (e transportReadFromServerError) Error() string { 887 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) 888} 889 890func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 891 if err := t.tryPutIdleConn(pconn); err != nil { 892 pconn.close(err) 893 } 894} 895 896func (t *Transport) maxIdleConnsPerHost() int { 897 if v := t.MaxIdleConnsPerHost; v != 0 { 898 return v 899 } 900 return DefaultMaxIdleConnsPerHost 901} 902 903// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 904// a new request. 905// If pconn is no longer needed or not in a good state, tryPutIdleConn returns 906// an error explaining why it wasn't registered. 907// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 908func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 909 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 910 return errKeepAlivesDisabled 911 } 912 if pconn.isBroken() { 913 return errConnBroken 914 } 915 pconn.markReused() 916 917 t.idleMu.Lock() 918 defer t.idleMu.Unlock() 919 920 // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list, 921 // because multiple goroutines can use them simultaneously. 922 // If this is an HTTP/2 connection being “returned,” we're done. 923 if pconn.alt != nil && t.idleLRU.m[pconn] != nil { 924 return nil 925 } 926 927 // Deliver pconn to goroutine waiting for idle connection, if any. 928 // (They may be actively dialing, but this conn is ready first. 929 // Chrome calls this socket late binding. 930 // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.) 931 key := pconn.cacheKey 932 if q, ok := t.idleConnWait[key]; ok { 933 done := false 934 if pconn.alt == nil { 935 // HTTP/1. 936 // Loop over the waiting list until we find a w that isn't done already, and hand it pconn. 937 for q.len() > 0 { 938 w := q.popFront() 939 if w.tryDeliver(pconn, nil) { 940 done = true 941 break 942 } 943 } 944 } else { 945 // HTTP/2. 946 // Can hand the same pconn to everyone in the waiting list, 947 // and we still won't be done: we want to put it in the idle 948 // list unconditionally, for any future clients too. 949 for q.len() > 0 { 950 w := q.popFront() 951 w.tryDeliver(pconn, nil) 952 } 953 } 954 if q.len() == 0 { 955 delete(t.idleConnWait, key) 956 } else { 957 t.idleConnWait[key] = q 958 } 959 if done { 960 return nil 961 } 962 } 963 964 if t.closeIdle { 965 return errCloseIdle 966 } 967 if t.idleConn == nil { 968 t.idleConn = make(map[connectMethodKey][]*persistConn) 969 } 970 idles := t.idleConn[key] 971 if len(idles) >= t.maxIdleConnsPerHost() { 972 return errTooManyIdleHost 973 } 974 for _, exist := range idles { 975 if exist == pconn { 976 log.Fatalf("dup idle pconn %p in freelist", pconn) 977 } 978 } 979 t.idleConn[key] = append(idles, pconn) 980 t.idleLRU.add(pconn) 981 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { 982 oldest := t.idleLRU.removeOldest() 983 oldest.close(errTooManyIdle) 984 t.removeIdleConnLocked(oldest) 985 } 986 987 // Set idle timer, but only for HTTP/1 (pconn.alt == nil). 988 // The HTTP/2 implementation manages the idle timer itself 989 // (see idleConnTimeout in h2_bundle.go). 990 if t.IdleConnTimeout > 0 && pconn.alt == nil { 991 if pconn.idleTimer != nil { 992 pconn.idleTimer.Reset(t.IdleConnTimeout) 993 } else { 994 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) 995 } 996 } 997 pconn.idleAt = time.Now() 998 return nil 999} 1000 1001// queueForIdleConn queues w to receive the next idle connection for w.cm. 1002// As an optimization hint to the caller, queueForIdleConn reports whether 1003// it successfully delivered an already-idle connection. 1004func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { 1005 if t.DisableKeepAlives { 1006 return false 1007 } 1008 1009 t.idleMu.Lock() 1010 defer t.idleMu.Unlock() 1011 1012 // Stop closing connections that become idle - we might want one. 1013 // (That is, undo the effect of t.CloseIdleConnections.) 1014 t.closeIdle = false 1015 1016 if w == nil { 1017 // Happens in test hook. 1018 return false 1019 } 1020 1021 // If IdleConnTimeout is set, calculate the oldest 1022 // persistConn.idleAt time we're willing to use a cached idle 1023 // conn. 1024 var oldTime time.Time 1025 if t.IdleConnTimeout > 0 { 1026 oldTime = time.Now().Add(-t.IdleConnTimeout) 1027 } 1028 1029 // Look for most recently-used idle connection. 1030 if list, ok := t.idleConn[w.key]; ok { 1031 stop := false 1032 delivered := false 1033 for len(list) > 0 && !stop { 1034 pconn := list[len(list)-1] 1035 1036 // See whether this connection has been idle too long, considering 1037 // only the wall time (the Round(0)), in case this is a laptop or VM 1038 // coming out of suspend with previously cached idle connections. 1039 tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) 1040 if tooOld { 1041 // Async cleanup. Launch in its own goroutine (as if a 1042 // time.AfterFunc called it); it acquires idleMu, which we're 1043 // holding, and does a synchronous net.Conn.Close. 1044 go pconn.closeConnIfStillIdle() 1045 } 1046 if pconn.isBroken() || tooOld { 1047 // If either persistConn.readLoop has marked the connection 1048 // broken, but Transport.removeIdleConn has not yet removed it 1049 // from the idle list, or if this persistConn is too old (it was 1050 // idle too long), then ignore it and look for another. In both 1051 // cases it's already in the process of being closed. 1052 list = list[:len(list)-1] 1053 continue 1054 } 1055 delivered = w.tryDeliver(pconn, nil) 1056 if delivered { 1057 if pconn.alt != nil { 1058 // HTTP/2: multiple clients can share pconn. 1059 // Leave it in the list. 1060 } else { 1061 // HTTP/1: only one client can use pconn. 1062 // Remove it from the list. 1063 t.idleLRU.remove(pconn) 1064 list = list[:len(list)-1] 1065 } 1066 } 1067 stop = true 1068 } 1069 if len(list) > 0 { 1070 t.idleConn[w.key] = list 1071 } else { 1072 delete(t.idleConn, w.key) 1073 } 1074 if stop { 1075 return delivered 1076 } 1077 } 1078 1079 // Register to receive next connection that becomes idle. 1080 if t.idleConnWait == nil { 1081 t.idleConnWait = make(map[connectMethodKey]wantConnQueue) 1082 } 1083 q := t.idleConnWait[w.key] 1084 q.cleanFront() 1085 q.pushBack(w) 1086 t.idleConnWait[w.key] = q 1087 return false 1088} 1089 1090// removeIdleConn marks pconn as dead. 1091func (t *Transport) removeIdleConn(pconn *persistConn) bool { 1092 t.idleMu.Lock() 1093 defer t.idleMu.Unlock() 1094 return t.removeIdleConnLocked(pconn) 1095} 1096 1097// t.idleMu must be held. 1098func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { 1099 if pconn.idleTimer != nil { 1100 pconn.idleTimer.Stop() 1101 } 1102 t.idleLRU.remove(pconn) 1103 key := pconn.cacheKey 1104 pconns := t.idleConn[key] 1105 var removed bool 1106 switch len(pconns) { 1107 case 0: 1108 // Nothing 1109 case 1: 1110 if pconns[0] == pconn { 1111 delete(t.idleConn, key) 1112 removed = true 1113 } 1114 default: 1115 for i, v := range pconns { 1116 if v != pconn { 1117 continue 1118 } 1119 // Slide down, keeping most recently-used 1120 // conns at the end. 1121 copy(pconns[i:], pconns[i+1:]) 1122 t.idleConn[key] = pconns[:len(pconns)-1] 1123 removed = true 1124 break 1125 } 1126 } 1127 return removed 1128} 1129 1130func (t *Transport) setReqCanceler(key cancelKey, fn func(error)) { 1131 t.reqMu.Lock() 1132 defer t.reqMu.Unlock() 1133 if t.reqCanceler == nil { 1134 t.reqCanceler = make(map[cancelKey]func(error)) 1135 } 1136 if fn != nil { 1137 t.reqCanceler[key] = fn 1138 } else { 1139 delete(t.reqCanceler, key) 1140 } 1141} 1142 1143// replaceReqCanceler replaces an existing cancel function. If there is no cancel function 1144// for the request, we don't set the function and return false. 1145// Since CancelRequest will clear the canceler, we can use the return value to detect if 1146// the request was canceled since the last setReqCancel call. 1147func (t *Transport) replaceReqCanceler(key cancelKey, fn func(error)) bool { 1148 t.reqMu.Lock() 1149 defer t.reqMu.Unlock() 1150 _, ok := t.reqCanceler[key] 1151 if !ok { 1152 return false 1153 } 1154 if fn != nil { 1155 t.reqCanceler[key] = fn 1156 } else { 1157 delete(t.reqCanceler, key) 1158 } 1159 return true 1160} 1161 1162var zeroDialer net.Dialer 1163 1164func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { 1165 if t.DialContext != nil { 1166 return t.DialContext(ctx, network, addr) 1167 } 1168 if t.Dial != nil { 1169 c, err := t.Dial(network, addr) 1170 if c == nil && err == nil { 1171 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 1172 } 1173 return c, err 1174 } 1175 return zeroDialer.DialContext(ctx, network, addr) 1176} 1177 1178// A wantConn records state about a wanted connection 1179// (that is, an active call to getConn). 1180// The conn may be gotten by dialing or by finding an idle connection, 1181// or a cancellation may make the conn no longer wanted. 1182// These three options are racing against each other and use 1183// wantConn to coordinate and agree about the winning outcome. 1184type wantConn struct { 1185 cm connectMethod 1186 key connectMethodKey // cm.key() 1187 ctx context.Context // context for dial 1188 ready chan struct{} // closed when pc, err pair is delivered 1189 1190 // hooks for testing to know when dials are done 1191 // beforeDial is called in the getConn goroutine when the dial is queued. 1192 // afterDial is called when the dial is completed or canceled. 1193 beforeDial func() 1194 afterDial func() 1195 1196 mu sync.Mutex // protects pc, err, close(ready) 1197 pc *persistConn 1198 err error 1199} 1200 1201// waiting reports whether w is still waiting for an answer (connection or error). 1202func (w *wantConn) waiting() bool { 1203 select { 1204 case <-w.ready: 1205 return false 1206 default: 1207 return true 1208 } 1209} 1210 1211// tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. 1212func (w *wantConn) tryDeliver(pc *persistConn, err error) bool { 1213 w.mu.Lock() 1214 defer w.mu.Unlock() 1215 1216 if w.pc != nil || w.err != nil { 1217 return false 1218 } 1219 1220 w.pc = pc 1221 w.err = err 1222 if w.pc == nil && w.err == nil { 1223 panic("net/http: internal error: misuse of tryDeliver") 1224 } 1225 close(w.ready) 1226 return true 1227} 1228 1229// cancel marks w as no longer wanting a result (for example, due to cancellation). 1230// If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. 1231func (w *wantConn) cancel(t *Transport, err error) { 1232 w.mu.Lock() 1233 if w.pc == nil && w.err == nil { 1234 close(w.ready) // catch misbehavior in future delivery 1235 } 1236 pc := w.pc 1237 w.pc = nil 1238 w.err = err 1239 w.mu.Unlock() 1240 1241 if pc != nil { 1242 t.putOrCloseIdleConn(pc) 1243 } 1244} 1245 1246// A wantConnQueue is a queue of wantConns. 1247type wantConnQueue struct { 1248 // This is a queue, not a deque. 1249 // It is split into two stages - head[headPos:] and tail. 1250 // popFront is trivial (headPos++) on the first stage, and 1251 // pushBack is trivial (append) on the second stage. 1252 // If the first stage is empty, popFront can swap the 1253 // first and second stages to remedy the situation. 1254 // 1255 // This two-stage split is analogous to the use of two lists 1256 // in Okasaki's purely functional queue but without the 1257 // overhead of reversing the list when swapping stages. 1258 head []*wantConn 1259 headPos int 1260 tail []*wantConn 1261} 1262 1263// len returns the number of items in the queue. 1264func (q *wantConnQueue) len() int { 1265 return len(q.head) - q.headPos + len(q.tail) 1266} 1267 1268// pushBack adds w to the back of the queue. 1269func (q *wantConnQueue) pushBack(w *wantConn) { 1270 q.tail = append(q.tail, w) 1271} 1272 1273// popFront removes and returns the wantConn at the front of the queue. 1274func (q *wantConnQueue) popFront() *wantConn { 1275 if q.headPos >= len(q.head) { 1276 if len(q.tail) == 0 { 1277 return nil 1278 } 1279 // Pick up tail as new head, clear tail. 1280 q.head, q.headPos, q.tail = q.tail, 0, q.head[:0] 1281 } 1282 w := q.head[q.headPos] 1283 q.head[q.headPos] = nil 1284 q.headPos++ 1285 return w 1286} 1287 1288// peekFront returns the wantConn at the front of the queue without removing it. 1289func (q *wantConnQueue) peekFront() *wantConn { 1290 if q.headPos < len(q.head) { 1291 return q.head[q.headPos] 1292 } 1293 if len(q.tail) > 0 { 1294 return q.tail[0] 1295 } 1296 return nil 1297} 1298 1299// cleanFront pops any wantConns that are no longer waiting from the head of the 1300// queue, reporting whether any were popped. 1301func (q *wantConnQueue) cleanFront() (cleaned bool) { 1302 for { 1303 w := q.peekFront() 1304 if w == nil || w.waiting() { 1305 return cleaned 1306 } 1307 q.popFront() 1308 cleaned = true 1309 } 1310} 1311 1312func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { 1313 if t.DialTLSContext != nil { 1314 conn, err = t.DialTLSContext(ctx, network, addr) 1315 } else { 1316 conn, err = t.DialTLS(network, addr) 1317 } 1318 if conn == nil && err == nil { 1319 err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") 1320 } 1321 return 1322} 1323 1324// getConn dials and creates a new persistConn to the target as 1325// specified in the connectMethod. This includes doing a proxy CONNECT 1326// and/or setting up TLS. If this doesn't return an error, the persistConn 1327// is ready to write requests to. 1328func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (pc *persistConn, err error) { 1329 req := treq.Request 1330 trace := treq.trace 1331 ctx := req.Context() 1332 if trace != nil && trace.GetConn != nil { 1333 trace.GetConn(cm.addr()) 1334 } 1335 1336 w := &wantConn{ 1337 cm: cm, 1338 key: cm.key(), 1339 ctx: ctx, 1340 ready: make(chan struct{}, 1), 1341 beforeDial: testHookPrePendingDial, 1342 afterDial: testHookPostPendingDial, 1343 } 1344 defer func() { 1345 if err != nil { 1346 w.cancel(t, err) 1347 } 1348 }() 1349 1350 // Queue for idle connection. 1351 if delivered := t.queueForIdleConn(w); delivered { 1352 pc := w.pc 1353 // Trace only for HTTP/1. 1354 // HTTP/2 calls trace.GotConn itself. 1355 if pc.alt == nil && trace != nil && trace.GotConn != nil { 1356 trace.GotConn(pc.gotIdleConnTrace(pc.idleAt)) 1357 } 1358 // set request canceler to some non-nil function so we 1359 // can detect whether it was cleared between now and when 1360 // we enter roundTrip 1361 t.setReqCanceler(treq.cancelKey, func(error) {}) 1362 return pc, nil 1363 } 1364 1365 cancelc := make(chan error, 1) 1366 t.setReqCanceler(treq.cancelKey, func(err error) { cancelc <- err }) 1367 1368 // Queue for permission to dial. 1369 t.queueForDial(w) 1370 1371 // Wait for completion or cancellation. 1372 select { 1373 case <-w.ready: 1374 // Trace success but only for HTTP/1. 1375 // HTTP/2 calls trace.GotConn itself. 1376 if w.pc != nil && w.pc.alt == nil && trace != nil && trace.GotConn != nil { 1377 trace.GotConn(httptrace.GotConnInfo{Conn: w.pc.conn, Reused: w.pc.isReused()}) 1378 } 1379 if w.err != nil { 1380 // If the request has been canceled, that's probably 1381 // what caused w.err; if so, prefer to return the 1382 // cancellation error (see golang.org/issue/16049). 1383 select { 1384 case <-req.Cancel: 1385 return nil, errRequestCanceledConn 1386 case <-req.Context().Done(): 1387 return nil, req.Context().Err() 1388 case err := <-cancelc: 1389 if err == errRequestCanceled { 1390 err = errRequestCanceledConn 1391 } 1392 return nil, err 1393 default: 1394 // return below 1395 } 1396 } 1397 return w.pc, w.err 1398 case <-req.Cancel: 1399 return nil, errRequestCanceledConn 1400 case <-req.Context().Done(): 1401 return nil, req.Context().Err() 1402 case err := <-cancelc: 1403 if err == errRequestCanceled { 1404 err = errRequestCanceledConn 1405 } 1406 return nil, err 1407 } 1408} 1409 1410// queueForDial queues w to wait for permission to begin dialing. 1411// Once w receives permission to dial, it will do so in a separate goroutine. 1412func (t *Transport) queueForDial(w *wantConn) { 1413 w.beforeDial() 1414 if t.MaxConnsPerHost <= 0 { 1415 go t.dialConnFor(w) 1416 return 1417 } 1418 1419 t.connsPerHostMu.Lock() 1420 defer t.connsPerHostMu.Unlock() 1421 1422 if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost { 1423 if t.connsPerHost == nil { 1424 t.connsPerHost = make(map[connectMethodKey]int) 1425 } 1426 t.connsPerHost[w.key] = n + 1 1427 go t.dialConnFor(w) 1428 return 1429 } 1430 1431 if t.connsPerHostWait == nil { 1432 t.connsPerHostWait = make(map[connectMethodKey]wantConnQueue) 1433 } 1434 q := t.connsPerHostWait[w.key] 1435 q.cleanFront() 1436 q.pushBack(w) 1437 t.connsPerHostWait[w.key] = q 1438} 1439 1440// dialConnFor dials on behalf of w and delivers the result to w. 1441// dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()]. 1442// If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()]. 1443func (t *Transport) dialConnFor(w *wantConn) { 1444 defer w.afterDial() 1445 1446 pc, err := t.dialConn(w.ctx, w.cm) 1447 delivered := w.tryDeliver(pc, err) 1448 if err == nil && (!delivered || pc.alt != nil) { 1449 // pconn was not passed to w, 1450 // or it is HTTP/2 and can be shared. 1451 // Add to the idle connection pool. 1452 t.putOrCloseIdleConn(pc) 1453 } 1454 if err != nil { 1455 t.decConnsPerHost(w.key) 1456 } 1457} 1458 1459// decConnsPerHost decrements the per-host connection count for key, 1460// which may in turn give a different waiting goroutine permission to dial. 1461func (t *Transport) decConnsPerHost(key connectMethodKey) { 1462 if t.MaxConnsPerHost <= 0 { 1463 return 1464 } 1465 1466 t.connsPerHostMu.Lock() 1467 defer t.connsPerHostMu.Unlock() 1468 n := t.connsPerHost[key] 1469 if n == 0 { 1470 // Shouldn't happen, but if it does, the counting is buggy and could 1471 // easily lead to a silent deadlock, so report the problem loudly. 1472 panic("net/http: internal error: connCount underflow") 1473 } 1474 1475 // Can we hand this count to a goroutine still waiting to dial? 1476 // (Some goroutines on the wait list may have timed out or 1477 // gotten a connection another way. If they're all gone, 1478 // we don't want to kick off any spurious dial operations.) 1479 if q := t.connsPerHostWait[key]; q.len() > 0 { 1480 done := false 1481 for q.len() > 0 { 1482 w := q.popFront() 1483 if w.waiting() { 1484 go t.dialConnFor(w) 1485 done = true 1486 break 1487 } 1488 } 1489 if q.len() == 0 { 1490 delete(t.connsPerHostWait, key) 1491 } else { 1492 // q is a value (like a slice), so we have to store 1493 // the updated q back into the map. 1494 t.connsPerHostWait[key] = q 1495 } 1496 if done { 1497 return 1498 } 1499 } 1500 1501 // Otherwise, decrement the recorded count. 1502 if n--; n == 0 { 1503 delete(t.connsPerHost, key) 1504 } else { 1505 t.connsPerHost[key] = n 1506 } 1507} 1508 1509// Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS 1510// tunnel, this function establishes a nested TLS session inside the encrypted channel. 1511// The remote endpoint's name may be overridden by TLSClientConfig.ServerName. 1512func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error { 1513 // Initiate TLS and check remote host name against certificate. 1514 cfg := cloneTLSConfig(pconn.t.TLSClientConfig) 1515 if cfg.ServerName == "" { 1516 cfg.ServerName = name 1517 } 1518 if pconn.cacheKey.onlyH1 { 1519 cfg.NextProtos = nil 1520 } 1521 plainConn := pconn.conn 1522 tlsConn := tls.Client(plainConn, cfg) 1523 errc := make(chan error, 2) 1524 var timer *time.Timer // for canceling TLS handshake 1525 if d := pconn.t.TLSHandshakeTimeout; d != 0 { 1526 timer = time.AfterFunc(d, func() { 1527 errc <- tlsHandshakeTimeoutError{} 1528 }) 1529 } 1530 go func() { 1531 if trace != nil && trace.TLSHandshakeStart != nil { 1532 trace.TLSHandshakeStart() 1533 } 1534 err := tlsConn.HandshakeContext(ctx) 1535 if timer != nil { 1536 timer.Stop() 1537 } 1538 errc <- err 1539 }() 1540 if err := <-errc; err != nil { 1541 plainConn.Close() 1542 if trace != nil && trace.TLSHandshakeDone != nil { 1543 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1544 } 1545 return err 1546 } 1547 cs := tlsConn.ConnectionState() 1548 if trace != nil && trace.TLSHandshakeDone != nil { 1549 trace.TLSHandshakeDone(cs, nil) 1550 } 1551 pconn.tlsState = &cs 1552 pconn.conn = tlsConn 1553 return nil 1554} 1555 1556type erringRoundTripper interface { 1557 RoundTripErr() error 1558} 1559 1560func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) { 1561 pconn = &persistConn{ 1562 t: t, 1563 cacheKey: cm.key(), 1564 reqch: make(chan requestAndChan, 1), 1565 writech: make(chan writeRequest, 1), 1566 closech: make(chan struct{}), 1567 writeErrCh: make(chan error, 1), 1568 writeLoopDone: make(chan struct{}), 1569 } 1570 trace := httptrace.ContextClientTrace(ctx) 1571 wrapErr := func(err error) error { 1572 if cm.proxyURL != nil { 1573 // Return a typed error, per Issue 16997 1574 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} 1575 } 1576 return err 1577 } 1578 if cm.scheme() == "https" && t.hasCustomTLSDialer() { 1579 var err error 1580 pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) 1581 if err != nil { 1582 return nil, wrapErr(err) 1583 } 1584 if tc, ok := pconn.conn.(*tls.Conn); ok { 1585 // Handshake here, in case DialTLS didn't. TLSNextProto below 1586 // depends on it for knowing the connection state. 1587 if trace != nil && trace.TLSHandshakeStart != nil { 1588 trace.TLSHandshakeStart() 1589 } 1590 if err := tc.HandshakeContext(ctx); err != nil { 1591 go pconn.conn.Close() 1592 if trace != nil && trace.TLSHandshakeDone != nil { 1593 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1594 } 1595 return nil, err 1596 } 1597 cs := tc.ConnectionState() 1598 if trace != nil && trace.TLSHandshakeDone != nil { 1599 trace.TLSHandshakeDone(cs, nil) 1600 } 1601 pconn.tlsState = &cs 1602 } 1603 } else { 1604 conn, err := t.dial(ctx, "tcp", cm.addr()) 1605 if err != nil { 1606 return nil, wrapErr(err) 1607 } 1608 pconn.conn = conn 1609 if cm.scheme() == "https" { 1610 var firstTLSHost string 1611 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { 1612 return nil, wrapErr(err) 1613 } 1614 if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil { 1615 return nil, wrapErr(err) 1616 } 1617 } 1618 } 1619 1620 // Proxy setup. 1621 switch { 1622 case cm.proxyURL == nil: 1623 // Do nothing. Not using a proxy. 1624 case cm.proxyURL.Scheme == "socks5": 1625 conn := pconn.conn 1626 d := socksNewDialer("tcp", conn.RemoteAddr().String()) 1627 if u := cm.proxyURL.User; u != nil { 1628 auth := &socksUsernamePassword{ 1629 Username: u.Username(), 1630 } 1631 auth.Password, _ = u.Password() 1632 d.AuthMethods = []socksAuthMethod{ 1633 socksAuthMethodNotRequired, 1634 socksAuthMethodUsernamePassword, 1635 } 1636 d.Authenticate = auth.Authenticate 1637 } 1638 if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { 1639 conn.Close() 1640 return nil, err 1641 } 1642 case cm.targetScheme == "http": 1643 pconn.isProxy = true 1644 if pa := cm.proxyAuth(); pa != "" { 1645 pconn.mutateHeaderFunc = func(h Header) { 1646 h.Set("Proxy-Authorization", pa) 1647 } 1648 } 1649 case cm.targetScheme == "https": 1650 conn := pconn.conn 1651 var hdr Header 1652 if t.GetProxyConnectHeader != nil { 1653 var err error 1654 hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr) 1655 if err != nil { 1656 conn.Close() 1657 return nil, err 1658 } 1659 } else { 1660 hdr = t.ProxyConnectHeader 1661 } 1662 if hdr == nil { 1663 hdr = make(Header) 1664 } 1665 if pa := cm.proxyAuth(); pa != "" { 1666 hdr = hdr.Clone() 1667 hdr.Set("Proxy-Authorization", pa) 1668 } 1669 connectReq := &Request{ 1670 Method: "CONNECT", 1671 URL: &url.URL{Opaque: cm.targetAddr}, 1672 Host: cm.targetAddr, 1673 Header: hdr, 1674 } 1675 1676 // If there's no done channel (no deadline or cancellation 1677 // from the caller possible), at least set some (long) 1678 // timeout here. This will make sure we don't block forever 1679 // and leak a goroutine if the connection stops replying 1680 // after the TCP connect. 1681 connectCtx := ctx 1682 if ctx.Done() == nil { 1683 newCtx, cancel := context.WithTimeout(ctx, 1*time.Minute) 1684 defer cancel() 1685 connectCtx = newCtx 1686 } 1687 1688 didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails 1689 var ( 1690 resp *Response 1691 err error // write or read error 1692 ) 1693 // Write the CONNECT request & read the response. 1694 go func() { 1695 defer close(didReadResponse) 1696 err = connectReq.Write(conn) 1697 if err != nil { 1698 return 1699 } 1700 // Okay to use and discard buffered reader here, because 1701 // TLS server will not speak until spoken to. 1702 br := bufio.NewReader(conn) 1703 resp, err = ReadResponse(br, connectReq) 1704 }() 1705 select { 1706 case <-connectCtx.Done(): 1707 conn.Close() 1708 <-didReadResponse 1709 return nil, connectCtx.Err() 1710 case <-didReadResponse: 1711 // resp or err now set 1712 } 1713 if err != nil { 1714 conn.Close() 1715 return nil, err 1716 } 1717 if resp.StatusCode != 200 { 1718 _, text, ok := strings.Cut(resp.Status, " ") 1719 conn.Close() 1720 if !ok { 1721 return nil, errors.New("unknown status code") 1722 } 1723 return nil, errors.New(text) 1724 } 1725 } 1726 1727 if cm.proxyURL != nil && cm.targetScheme == "https" { 1728 if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil { 1729 return nil, err 1730 } 1731 } 1732 1733 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 1734 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 1735 alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) 1736 if e, ok := alt.(erringRoundTripper); ok { 1737 // pconn.conn was closed by next (http2configureTransports.upgradeFn). 1738 return nil, e.RoundTripErr() 1739 } 1740 return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil 1741 } 1742 } 1743 1744 pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) 1745 pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize()) 1746 1747 go pconn.readLoop() 1748 go pconn.writeLoop() 1749 return pconn, nil 1750} 1751 1752// persistConnWriter is the io.Writer written to by pc.bw. 1753// It accumulates the number of bytes written to the underlying conn, 1754// so the retry logic can determine whether any bytes made it across 1755// the wire. 1756// This is exactly 1 pointer field wide so it can go into an interface 1757// without allocation. 1758type persistConnWriter struct { 1759 pc *persistConn 1760} 1761 1762func (w persistConnWriter) Write(p []byte) (n int, err error) { 1763 n, err = w.pc.conn.Write(p) 1764 w.pc.nwrite += int64(n) 1765 return 1766} 1767 1768// ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if 1769// the Conn implements io.ReaderFrom, it can take advantage of optimizations 1770// such as sendfile. 1771func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { 1772 n, err = io.Copy(w.pc.conn, r) 1773 w.pc.nwrite += n 1774 return 1775} 1776 1777var _ io.ReaderFrom = (*persistConnWriter)(nil) 1778 1779// connectMethod is the map key (in its String form) for keeping persistent 1780// TCP connections alive for subsequent HTTP requests. 1781// 1782// A connect method may be of the following types: 1783// 1784// connectMethod.key().String() Description 1785// ------------------------------ ------------------------- 1786// |http|foo.com http directly to server, no proxy 1787// |https|foo.com https directly to server, no proxy 1788// |https,h1|foo.com https directly to server w/o HTTP/2, no proxy 1789// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 1790// http://proxy.com|http http to proxy, http to anywhere after that 1791// socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com 1792// socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com 1793// https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com 1794// https://proxy.com|http https to proxy, http to anywhere after that 1795// 1796type connectMethod struct { 1797 _ incomparable 1798 proxyURL *url.URL // nil for no proxy, else full proxy URL 1799 targetScheme string // "http" or "https" 1800 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), 1801 // then targetAddr is not included in the connect method key, because the socket can 1802 // be reused for different targetAddr values. 1803 targetAddr string 1804 onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 1805} 1806 1807func (cm *connectMethod) key() connectMethodKey { 1808 proxyStr := "" 1809 targetAddr := cm.targetAddr 1810 if cm.proxyURL != nil { 1811 proxyStr = cm.proxyURL.String() 1812 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { 1813 targetAddr = "" 1814 } 1815 } 1816 return connectMethodKey{ 1817 proxy: proxyStr, 1818 scheme: cm.targetScheme, 1819 addr: targetAddr, 1820 onlyH1: cm.onlyH1, 1821 } 1822} 1823 1824// scheme returns the first hop scheme: http, https, or socks5 1825func (cm *connectMethod) scheme() string { 1826 if cm.proxyURL != nil { 1827 return cm.proxyURL.Scheme 1828 } 1829 return cm.targetScheme 1830} 1831 1832// addr returns the first hop "host:port" to which we need to TCP connect. 1833func (cm *connectMethod) addr() string { 1834 if cm.proxyURL != nil { 1835 return canonicalAddr(cm.proxyURL) 1836 } 1837 return cm.targetAddr 1838} 1839 1840// tlsHost returns the host name to match against the peer's 1841// TLS certificate. 1842func (cm *connectMethod) tlsHost() string { 1843 h := cm.targetAddr 1844 if hasPort(h) { 1845 h = h[:strings.LastIndex(h, ":")] 1846 } 1847 return h 1848} 1849 1850// connectMethodKey is the map key version of connectMethod, with a 1851// stringified proxy URL (or the empty string) instead of a pointer to 1852// a URL. 1853type connectMethodKey struct { 1854 proxy, scheme, addr string 1855 onlyH1 bool 1856} 1857 1858func (k connectMethodKey) String() string { 1859 // Only used by tests. 1860 var h1 string 1861 if k.onlyH1 { 1862 h1 = ",h1" 1863 } 1864 return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) 1865} 1866 1867// persistConn wraps a connection, usually a persistent one 1868// (but may be used for non-keep-alive requests as well) 1869type persistConn struct { 1870 // alt optionally specifies the TLS NextProto RoundTripper. 1871 // This is used for HTTP/2 today and future protocols later. 1872 // If it's non-nil, the rest of the fields are unused. 1873 alt RoundTripper 1874 1875 t *Transport 1876 cacheKey connectMethodKey 1877 conn net.Conn 1878 tlsState *tls.ConnectionState 1879 br *bufio.Reader // from conn 1880 bw *bufio.Writer // to conn 1881 nwrite int64 // bytes written 1882 reqch chan requestAndChan // written by roundTrip; read by readLoop 1883 writech chan writeRequest // written by roundTrip; read by writeLoop 1884 closech chan struct{} // closed when conn closed 1885 isProxy bool 1886 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 1887 readLimit int64 // bytes allowed to be read; owned by readLoop 1888 // writeErrCh passes the request write error (usually nil) 1889 // from the writeLoop goroutine to the readLoop which passes 1890 // it off to the res.Body reader, which then uses it to decide 1891 // whether or not a connection can be reused. Issue 7569. 1892 writeErrCh chan error 1893 1894 writeLoopDone chan struct{} // closed when write loop ends 1895 1896 // Both guarded by Transport.idleMu: 1897 idleAt time.Time // time it last become idle 1898 idleTimer *time.Timer // holding an AfterFunc to close it 1899 1900 mu sync.Mutex // guards following fields 1901 numExpectedResponses int 1902 closed error // set non-nil when conn is closed, before closech is closed 1903 canceledErr error // set non-nil if conn is canceled 1904 broken bool // an error has happened on this connection; marked broken so it's not reused. 1905 reused bool // whether conn has had successful request/response and is being reused. 1906 // mutateHeaderFunc is an optional func to modify extra 1907 // headers on each outbound request before it's written. (the 1908 // original Request given to RoundTrip is not modified) 1909 mutateHeaderFunc func(Header) 1910} 1911 1912func (pc *persistConn) maxHeaderResponseSize() int64 { 1913 if v := pc.t.MaxResponseHeaderBytes; v != 0 { 1914 return v 1915 } 1916 return 10 << 20 // conservative default; same as http2 1917} 1918 1919func (pc *persistConn) Read(p []byte) (n int, err error) { 1920 if pc.readLimit <= 0 { 1921 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) 1922 } 1923 if int64(len(p)) > pc.readLimit { 1924 p = p[:pc.readLimit] 1925 } 1926 n, err = pc.conn.Read(p) 1927 if err == io.EOF { 1928 pc.sawEOF = true 1929 } 1930 pc.readLimit -= int64(n) 1931 return 1932} 1933 1934// isBroken reports whether this connection is in a known broken state. 1935func (pc *persistConn) isBroken() bool { 1936 pc.mu.Lock() 1937 b := pc.closed != nil 1938 pc.mu.Unlock() 1939 return b 1940} 1941 1942// canceled returns non-nil if the connection was closed due to 1943// CancelRequest or due to context cancellation. 1944func (pc *persistConn) canceled() error { 1945 pc.mu.Lock() 1946 defer pc.mu.Unlock() 1947 return pc.canceledErr 1948} 1949 1950// isReused reports whether this connection has been used before. 1951func (pc *persistConn) isReused() bool { 1952 pc.mu.Lock() 1953 r := pc.reused 1954 pc.mu.Unlock() 1955 return r 1956} 1957 1958func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { 1959 pc.mu.Lock() 1960 defer pc.mu.Unlock() 1961 t.Reused = pc.reused 1962 t.Conn = pc.conn 1963 t.WasIdle = true 1964 if !idleAt.IsZero() { 1965 t.IdleTime = time.Since(idleAt) 1966 } 1967 return 1968} 1969 1970func (pc *persistConn) cancelRequest(err error) { 1971 pc.mu.Lock() 1972 defer pc.mu.Unlock() 1973 pc.canceledErr = err 1974 pc.closeLocked(errRequestCanceled) 1975} 1976 1977// closeConnIfStillIdle closes the connection if it's still sitting idle. 1978// This is what's called by the persistConn's idleTimer, and is run in its 1979// own goroutine. 1980func (pc *persistConn) closeConnIfStillIdle() { 1981 t := pc.t 1982 t.idleMu.Lock() 1983 defer t.idleMu.Unlock() 1984 if _, ok := t.idleLRU.m[pc]; !ok { 1985 // Not idle. 1986 return 1987 } 1988 t.removeIdleConnLocked(pc) 1989 pc.close(errIdleConnTimeout) 1990} 1991 1992// mapRoundTripError returns the appropriate error value for 1993// persistConn.roundTrip. 1994// 1995// The provided err is the first error that (*persistConn).roundTrip 1996// happened to receive from its select statement. 1997// 1998// The startBytesWritten value should be the value of pc.nwrite before the roundTrip 1999// started writing the request. 2000func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { 2001 if err == nil { 2002 return nil 2003 } 2004 2005 // Wait for the writeLoop goroutine to terminate to avoid data 2006 // races on callers who mutate the request on failure. 2007 // 2008 // When resc in pc.roundTrip and hence rc.ch receives a responseAndError 2009 // with a non-nil error it implies that the persistConn is either closed 2010 // or closing. Waiting on pc.writeLoopDone is hence safe as all callers 2011 // close closech which in turn ensures writeLoop returns. 2012 <-pc.writeLoopDone 2013 2014 // If the request was canceled, that's better than network 2015 // failures that were likely the result of tearing down the 2016 // connection. 2017 if cerr := pc.canceled(); cerr != nil { 2018 return cerr 2019 } 2020 2021 // See if an error was set explicitly. 2022 req.mu.Lock() 2023 reqErr := req.err 2024 req.mu.Unlock() 2025 if reqErr != nil { 2026 return reqErr 2027 } 2028 2029 if err == errServerClosedIdle { 2030 // Don't decorate 2031 return err 2032 } 2033 2034 if _, ok := err.(transportReadFromServerError); ok { 2035 // Don't decorate 2036 return err 2037 } 2038 if pc.isBroken() { 2039 if pc.nwrite == startBytesWritten { 2040 return nothingWrittenError{err} 2041 } 2042 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) 2043 } 2044 return err 2045} 2046 2047// errCallerOwnsConn is an internal sentinel error used when we hand 2048// off a writable response.Body to the caller. We use this to prevent 2049// closing a net.Conn that is now owned by the caller. 2050var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") 2051 2052func (pc *persistConn) readLoop() { 2053 closeErr := errReadLoopExiting // default value, if not changed below 2054 defer func() { 2055 pc.close(closeErr) 2056 pc.t.removeIdleConn(pc) 2057 }() 2058 2059 tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { 2060 if err := pc.t.tryPutIdleConn(pc); err != nil { 2061 closeErr = err 2062 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { 2063 trace.PutIdleConn(err) 2064 } 2065 return false 2066 } 2067 if trace != nil && trace.PutIdleConn != nil { 2068 trace.PutIdleConn(nil) 2069 } 2070 return true 2071 } 2072 2073 // eofc is used to block caller goroutines reading from Response.Body 2074 // at EOF until this goroutines has (potentially) added the connection 2075 // back to the idle pool. 2076 eofc := make(chan struct{}) 2077 defer close(eofc) // unblock reader on errors 2078 2079 // Read this once, before loop starts. (to avoid races in tests) 2080 testHookMu.Lock() 2081 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 2082 testHookMu.Unlock() 2083 2084 alive := true 2085 for alive { 2086 pc.readLimit = pc.maxHeaderResponseSize() 2087 _, err := pc.br.Peek(1) 2088 2089 pc.mu.Lock() 2090 if pc.numExpectedResponses == 0 { 2091 pc.readLoopPeekFailLocked(err) 2092 pc.mu.Unlock() 2093 return 2094 } 2095 pc.mu.Unlock() 2096 2097 rc := <-pc.reqch 2098 trace := httptrace.ContextClientTrace(rc.req.Context()) 2099 2100 var resp *Response 2101 if err == nil { 2102 resp, err = pc.readResponse(rc, trace) 2103 } else { 2104 err = transportReadFromServerError{err} 2105 closeErr = err 2106 } 2107 2108 if err != nil { 2109 if pc.readLimit <= 0 { 2110 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) 2111 } 2112 2113 select { 2114 case rc.ch <- responseAndError{err: err}: 2115 case <-rc.callerGone: 2116 return 2117 } 2118 return 2119 } 2120 pc.readLimit = maxInt64 // effectively no limit for response bodies 2121 2122 pc.mu.Lock() 2123 pc.numExpectedResponses-- 2124 pc.mu.Unlock() 2125 2126 bodyWritable := resp.bodyIsWritable() 2127 hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 2128 2129 if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable { 2130 // Don't do keep-alive on error if either party requested a close 2131 // or we get an unexpected informational (1xx) response. 2132 // StatusCode 100 is already handled above. 2133 alive = false 2134 } 2135 2136 if !hasBody || bodyWritable { 2137 replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) 2138 2139 // Put the idle conn back into the pool before we send the response 2140 // so if they process it quickly and make another request, they'll 2141 // get this same conn. But we use the unbuffered channel 'rc' 2142 // to guarantee that persistConn.roundTrip got out of its select 2143 // potentially waiting for this persistConn to close. 2144 alive = alive && 2145 !pc.sawEOF && 2146 pc.wroteRequest() && 2147 replaced && tryPutIdleConn(trace) 2148 2149 if bodyWritable { 2150 closeErr = errCallerOwnsConn 2151 } 2152 2153 select { 2154 case rc.ch <- responseAndError{res: resp}: 2155 case <-rc.callerGone: 2156 return 2157 } 2158 2159 // Now that they've read from the unbuffered channel, they're safely 2160 // out of the select that also waits on this goroutine to die, so 2161 // we're allowed to exit now if needed (if alive is false) 2162 testHookReadLoopBeforeNextRead() 2163 continue 2164 } 2165 2166 waitForBodyRead := make(chan bool, 2) 2167 body := &bodyEOFSignal{ 2168 body: resp.Body, 2169 earlyCloseFn: func() error { 2170 waitForBodyRead <- false 2171 <-eofc // will be closed by deferred call at the end of the function 2172 return nil 2173 2174 }, 2175 fn: func(err error) error { 2176 isEOF := err == io.EOF 2177 waitForBodyRead <- isEOF 2178 if isEOF { 2179 <-eofc // see comment above eofc declaration 2180 } else if err != nil { 2181 if cerr := pc.canceled(); cerr != nil { 2182 return cerr 2183 } 2184 } 2185 return err 2186 }, 2187 } 2188 2189 resp.Body = body 2190 if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { 2191 resp.Body = &gzipReader{body: body} 2192 resp.Header.Del("Content-Encoding") 2193 resp.Header.Del("Content-Length") 2194 resp.ContentLength = -1 2195 resp.Uncompressed = true 2196 } 2197 2198 select { 2199 case rc.ch <- responseAndError{res: resp}: 2200 case <-rc.callerGone: 2201 return 2202 } 2203 2204 // Before looping back to the top of this function and peeking on 2205 // the bufio.Reader, wait for the caller goroutine to finish 2206 // reading the response body. (or for cancellation or death) 2207 select { 2208 case bodyEOF := <-waitForBodyRead: 2209 replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool 2210 alive = alive && 2211 bodyEOF && 2212 !pc.sawEOF && 2213 pc.wroteRequest() && 2214 replaced && tryPutIdleConn(trace) 2215 if bodyEOF { 2216 eofc <- struct{}{} 2217 } 2218 case <-rc.req.Cancel: 2219 alive = false 2220 pc.t.CancelRequest(rc.req) 2221 case <-rc.req.Context().Done(): 2222 alive = false 2223 pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) 2224 case <-pc.closech: 2225 alive = false 2226 } 2227 2228 testHookReadLoopBeforeNextRead() 2229 } 2230} 2231 2232func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 2233 if pc.closed != nil { 2234 return 2235 } 2236 if n := pc.br.Buffered(); n > 0 { 2237 buf, _ := pc.br.Peek(n) 2238 if is408Message(buf) { 2239 pc.closeLocked(errServerClosedIdle) 2240 return 2241 } else { 2242 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 2243 } 2244 } 2245 if peekErr == io.EOF { 2246 // common case. 2247 pc.closeLocked(errServerClosedIdle) 2248 } else { 2249 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr)) 2250 } 2251} 2252 2253// is408Message reports whether buf has the prefix of an 2254// HTTP 408 Request Timeout response. 2255// See golang.org/issue/32310. 2256func is408Message(buf []byte) bool { 2257 if len(buf) < len("HTTP/1.x 408") { 2258 return false 2259 } 2260 if string(buf[:7]) != "HTTP/1." { 2261 return false 2262 } 2263 return string(buf[8:12]) == " 408" 2264} 2265 2266// readResponse reads an HTTP response (or two, in the case of "Expect: 2267// 100-continue") from the server. It returns the final non-100 one. 2268// trace is optional. 2269func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { 2270 if trace != nil && trace.GotFirstResponseByte != nil { 2271 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { 2272 trace.GotFirstResponseByte() 2273 } 2274 } 2275 num1xx := 0 // number of informational 1xx headers received 2276 const max1xxResponses = 5 // arbitrary bound on number of informational responses 2277 2278 continueCh := rc.continueCh 2279 for { 2280 resp, err = ReadResponse(pc.br, rc.req) 2281 if err != nil { 2282 return 2283 } 2284 resCode := resp.StatusCode 2285 if continueCh != nil { 2286 if resCode == 100 { 2287 if trace != nil && trace.Got100Continue != nil { 2288 trace.Got100Continue() 2289 } 2290 continueCh <- struct{}{} 2291 continueCh = nil 2292 } else if resCode >= 200 { 2293 close(continueCh) 2294 continueCh = nil 2295 } 2296 } 2297 is1xx := 100 <= resCode && resCode <= 199 2298 // treat 101 as a terminal status, see issue 26161 2299 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols 2300 if is1xxNonTerminal { 2301 num1xx++ 2302 if num1xx > max1xxResponses { 2303 return nil, errors.New("net/http: too many 1xx informational responses") 2304 } 2305 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit 2306 if trace != nil && trace.Got1xxResponse != nil { 2307 if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { 2308 return nil, err 2309 } 2310 } 2311 continue 2312 } 2313 break 2314 } 2315 if resp.isProtocolSwitch() { 2316 resp.Body = newReadWriteCloserBody(pc.br, pc.conn) 2317 } 2318 2319 resp.TLS = pc.tlsState 2320 return 2321} 2322 2323// waitForContinue returns the function to block until 2324// any response, timeout or connection close. After any of them, 2325// the function returns a bool which indicates if the body should be sent. 2326func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 2327 if continueCh == nil { 2328 return nil 2329 } 2330 return func() bool { 2331 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 2332 defer timer.Stop() 2333 2334 select { 2335 case _, ok := <-continueCh: 2336 return ok 2337 case <-timer.C: 2338 return true 2339 case <-pc.closech: 2340 return false 2341 } 2342 } 2343} 2344 2345func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { 2346 body := &readWriteCloserBody{ReadWriteCloser: rwc} 2347 if br.Buffered() != 0 { 2348 body.br = br 2349 } 2350 return body 2351} 2352 2353// readWriteCloserBody is the Response.Body type used when we want to 2354// give users write access to the Body through the underlying 2355// connection (TCP, unless using custom dialers). This is then 2356// the concrete type for a Response.Body on the 101 Switching 2357// Protocols response, as used by WebSockets, h2c, etc. 2358type readWriteCloserBody struct { 2359 _ incomparable 2360 br *bufio.Reader // used until empty 2361 io.ReadWriteCloser 2362} 2363 2364func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { 2365 if b.br != nil { 2366 if n := b.br.Buffered(); len(p) > n { 2367 p = p[:n] 2368 } 2369 n, err = b.br.Read(p) 2370 if b.br.Buffered() == 0 { 2371 b.br = nil 2372 } 2373 return n, err 2374 } 2375 return b.ReadWriteCloser.Read(p) 2376} 2377 2378// nothingWrittenError wraps a write errors which ended up writing zero bytes. 2379type nothingWrittenError struct { 2380 error 2381} 2382 2383func (pc *persistConn) writeLoop() { 2384 defer close(pc.writeLoopDone) 2385 for { 2386 select { 2387 case wr := <-pc.writech: 2388 startBytesWritten := pc.nwrite 2389 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 2390 if bre, ok := err.(requestBodyReadError); ok { 2391 err = bre.error 2392 // Errors reading from the user's 2393 // Request.Body are high priority. 2394 // Set it here before sending on the 2395 // channels below or calling 2396 // pc.close() which tears down 2397 // connections and causes other 2398 // errors. 2399 wr.req.setError(err) 2400 } 2401 if err == nil { 2402 err = pc.bw.Flush() 2403 } 2404 if err != nil { 2405 if pc.nwrite == startBytesWritten { 2406 err = nothingWrittenError{err} 2407 } 2408 } 2409 pc.writeErrCh <- err // to the body reader, which might recycle us 2410 wr.ch <- err // to the roundTrip function 2411 if err != nil { 2412 pc.close(err) 2413 return 2414 } 2415 case <-pc.closech: 2416 return 2417 } 2418 } 2419} 2420 2421// maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip 2422// will wait to see the Request's Body.Write result after getting a 2423// response from the server. See comments in (*persistConn).wroteRequest. 2424const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond 2425 2426// wroteRequest is a check before recycling a connection that the previous write 2427// (from writeLoop above) happened and was successful. 2428func (pc *persistConn) wroteRequest() bool { 2429 select { 2430 case err := <-pc.writeErrCh: 2431 // Common case: the write happened well before the response, so 2432 // avoid creating a timer. 2433 return err == nil 2434 default: 2435 // Rare case: the request was written in writeLoop above but 2436 // before it could send to pc.writeErrCh, the reader read it 2437 // all, processed it, and called us here. In this case, give the 2438 // write goroutine a bit of time to finish its send. 2439 // 2440 // Less rare case: We also get here in the legitimate case of 2441 // Issue 7569, where the writer is still writing (or stalled), 2442 // but the server has already replied. In this case, we don't 2443 // want to wait too long, and we want to return false so this 2444 // connection isn't re-used. 2445 t := time.NewTimer(maxWriteWaitBeforeConnReuse) 2446 defer t.Stop() 2447 select { 2448 case err := <-pc.writeErrCh: 2449 return err == nil 2450 case <-t.C: 2451 return false 2452 } 2453 } 2454} 2455 2456// responseAndError is how the goroutine reading from an HTTP/1 server 2457// communicates with the goroutine doing the RoundTrip. 2458type responseAndError struct { 2459 _ incomparable 2460 res *Response // else use this response (see res method) 2461 err error 2462} 2463 2464type requestAndChan struct { 2465 _ incomparable 2466 req *Request 2467 cancelKey cancelKey 2468 ch chan responseAndError // unbuffered; always send in select on callerGone 2469 2470 // whether the Transport (as opposed to the user client code) 2471 // added the Accept-Encoding gzip header. If the Transport 2472 // set it, only then do we transparently decode the gzip. 2473 addedGzip bool 2474 2475 // Optional blocking chan for Expect: 100-continue (for send). 2476 // If the request has an "Expect: 100-continue" header and 2477 // the server responds 100 Continue, readLoop send a value 2478 // to writeLoop via this chan. 2479 continueCh chan<- struct{} 2480 2481 callerGone <-chan struct{} // closed when roundTrip caller has returned 2482} 2483 2484// A writeRequest is sent by the caller's goroutine to the 2485// writeLoop's goroutine to write a request while the read loop 2486// concurrently waits on both the write response and the server's 2487// reply. 2488type writeRequest struct { 2489 req *transportRequest 2490 ch chan<- error 2491 2492 // Optional blocking chan for Expect: 100-continue (for receive). 2493 // If not nil, writeLoop blocks sending request body until 2494 // it receives from this chan. 2495 continueCh <-chan struct{} 2496} 2497 2498type httpError struct { 2499 err string 2500 timeout bool 2501} 2502 2503func (e *httpError) Error() string { return e.err } 2504func (e *httpError) Timeout() bool { return e.timeout } 2505func (e *httpError) Temporary() bool { return true } 2506 2507var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} 2508 2509// errRequestCanceled is set to be identical to the one from h2 to facilitate 2510// testing. 2511var errRequestCanceled = http2errRequestCanceled 2512var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 2513 2514func nop() {} 2515 2516// testHooks. Always non-nil. 2517var ( 2518 testHookEnterRoundTrip = nop 2519 testHookWaitResLoop = nop 2520 testHookRoundTripRetried = nop 2521 testHookPrePendingDial = nop 2522 testHookPostPendingDial = nop 2523 2524 testHookMu sync.Locker = fakeLocker{} // guards following 2525 testHookReadLoopBeforeNextRead = nop 2526) 2527 2528func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 2529 testHookEnterRoundTrip() 2530 if !pc.t.replaceReqCanceler(req.cancelKey, pc.cancelRequest) { 2531 pc.t.putOrCloseIdleConn(pc) 2532 return nil, errRequestCanceled 2533 } 2534 pc.mu.Lock() 2535 pc.numExpectedResponses++ 2536 headerFn := pc.mutateHeaderFunc 2537 pc.mu.Unlock() 2538 2539 if headerFn != nil { 2540 headerFn(req.extraHeaders()) 2541 } 2542 2543 // Ask for a compressed version if the caller didn't set their 2544 // own value for Accept-Encoding. We only attempt to 2545 // uncompress the gzip stream if we were the layer that 2546 // requested it. 2547 requestedGzip := false 2548 if !pc.t.DisableCompression && 2549 req.Header.Get("Accept-Encoding") == "" && 2550 req.Header.Get("Range") == "" && 2551 req.Method != "HEAD" { 2552 // Request gzip only, not deflate. Deflate is ambiguous and 2553 // not as universally supported anyway. 2554 // See: https://zlib.net/zlib_faq.html#faq39 2555 // 2556 // Note that we don't request this for HEAD requests, 2557 // due to a bug in nginx: 2558 // https://trac.nginx.org/nginx/ticket/358 2559 // https://golang.org/issue/5522 2560 // 2561 // We don't request gzip if the request is for a range, since 2562 // auto-decoding a portion of a gzipped document will just fail 2563 // anyway. See https://golang.org/issue/8923 2564 requestedGzip = true 2565 req.extraHeaders().Set("Accept-Encoding", "gzip") 2566 } 2567 2568 var continueCh chan struct{} 2569 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 2570 continueCh = make(chan struct{}, 1) 2571 } 2572 2573 if pc.t.DisableKeepAlives && 2574 !req.wantsClose() && 2575 !isProtocolSwitchHeader(req.Header) { 2576 req.extraHeaders().Set("Connection", "close") 2577 } 2578 2579 gone := make(chan struct{}) 2580 defer close(gone) 2581 2582 defer func() { 2583 if err != nil { 2584 pc.t.setReqCanceler(req.cancelKey, nil) 2585 } 2586 }() 2587 2588 const debugRoundTrip = false 2589 2590 // Write the request concurrently with waiting for a response, 2591 // in case the server decides to reply before reading our full 2592 // request body. 2593 startBytesWritten := pc.nwrite 2594 writeErrCh := make(chan error, 1) 2595 pc.writech <- writeRequest{req, writeErrCh, continueCh} 2596 2597 resc := make(chan responseAndError) 2598 pc.reqch <- requestAndChan{ 2599 req: req.Request, 2600 cancelKey: req.cancelKey, 2601 ch: resc, 2602 addedGzip: requestedGzip, 2603 continueCh: continueCh, 2604 callerGone: gone, 2605 } 2606 2607 var respHeaderTimer <-chan time.Time 2608 cancelChan := req.Request.Cancel 2609 ctxDoneChan := req.Context().Done() 2610 pcClosed := pc.closech 2611 canceled := false 2612 for { 2613 testHookWaitResLoop() 2614 select { 2615 case err := <-writeErrCh: 2616 if debugRoundTrip { 2617 req.logf("writeErrCh resv: %T/%#v", err, err) 2618 } 2619 if err != nil { 2620 pc.close(fmt.Errorf("write error: %v", err)) 2621 return nil, pc.mapRoundTripError(req, startBytesWritten, err) 2622 } 2623 if d := pc.t.ResponseHeaderTimeout; d > 0 { 2624 if debugRoundTrip { 2625 req.logf("starting timer for %v", d) 2626 } 2627 timer := time.NewTimer(d) 2628 defer timer.Stop() // prevent leaks 2629 respHeaderTimer = timer.C 2630 } 2631 case <-pcClosed: 2632 pcClosed = nil 2633 if canceled || pc.t.replaceReqCanceler(req.cancelKey, nil) { 2634 if debugRoundTrip { 2635 req.logf("closech recv: %T %#v", pc.closed, pc.closed) 2636 } 2637 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) 2638 } 2639 case <-respHeaderTimer: 2640 if debugRoundTrip { 2641 req.logf("timeout waiting for response headers.") 2642 } 2643 pc.close(errTimeout) 2644 return nil, errTimeout 2645 case re := <-resc: 2646 if (re.res == nil) == (re.err == nil) { 2647 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) 2648 } 2649 if debugRoundTrip { 2650 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) 2651 } 2652 if re.err != nil { 2653 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) 2654 } 2655 return re.res, nil 2656 case <-cancelChan: 2657 canceled = pc.t.cancelRequest(req.cancelKey, errRequestCanceled) 2658 cancelChan = nil 2659 case <-ctxDoneChan: 2660 canceled = pc.t.cancelRequest(req.cancelKey, req.Context().Err()) 2661 cancelChan = nil 2662 ctxDoneChan = nil 2663 } 2664 } 2665} 2666 2667// tLogKey is a context WithValue key for test debugging contexts containing 2668// a t.Logf func. See export_test.go's Request.WithT method. 2669type tLogKey struct{} 2670 2671func (tr *transportRequest) logf(format string, args ...any) { 2672 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...any)); ok { 2673 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) 2674 } 2675} 2676 2677// markReused marks this connection as having been successfully used for a 2678// request and response. 2679func (pc *persistConn) markReused() { 2680 pc.mu.Lock() 2681 pc.reused = true 2682 pc.mu.Unlock() 2683} 2684 2685// close closes the underlying TCP connection and closes 2686// the pc.closech channel. 2687// 2688// The provided err is only for testing and debugging; in normal 2689// circumstances it should never be seen by users. 2690func (pc *persistConn) close(err error) { 2691 pc.mu.Lock() 2692 defer pc.mu.Unlock() 2693 pc.closeLocked(err) 2694} 2695 2696func (pc *persistConn) closeLocked(err error) { 2697 if err == nil { 2698 panic("nil error") 2699 } 2700 pc.broken = true 2701 if pc.closed == nil { 2702 pc.closed = err 2703 pc.t.decConnsPerHost(pc.cacheKey) 2704 // Close HTTP/1 (pc.alt == nil) connection. 2705 // HTTP/2 closes its connection itself. 2706 if pc.alt == nil { 2707 if err != errCallerOwnsConn { 2708 pc.conn.Close() 2709 } 2710 close(pc.closech) 2711 } 2712 } 2713 pc.mutateHeaderFunc = nil 2714} 2715 2716var portMap = map[string]string{ 2717 "http": "80", 2718 "https": "443", 2719 "socks5": "1080", 2720} 2721 2722// canonicalAddr returns url.Host but always with a ":port" suffix 2723func canonicalAddr(url *url.URL) string { 2724 addr := url.Hostname() 2725 if v, err := idnaASCII(addr); err == nil { 2726 addr = v 2727 } 2728 port := url.Port() 2729 if port == "" { 2730 port = portMap[url.Scheme] 2731 } 2732 return net.JoinHostPort(addr, port) 2733} 2734 2735// bodyEOFSignal is used by the HTTP/1 transport when reading response 2736// bodies to make sure we see the end of a response body before 2737// proceeding and reading on the connection again. 2738// 2739// It wraps a ReadCloser but runs fn (if non-nil) at most 2740// once, right before its final (error-producing) Read or Close call 2741// returns. fn should return the new error to return from Read or Close. 2742// 2743// If earlyCloseFn is non-nil and Close is called before io.EOF is 2744// seen, earlyCloseFn is called instead of fn, and its return value is 2745// the return value from Close. 2746type bodyEOFSignal struct { 2747 body io.ReadCloser 2748 mu sync.Mutex // guards following 4 fields 2749 closed bool // whether Close has been called 2750 rerr error // sticky Read error 2751 fn func(error) error // err will be nil on Read io.EOF 2752 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 2753} 2754 2755var errReadOnClosedResBody = errors.New("http: read on closed response body") 2756 2757func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 2758 es.mu.Lock() 2759 closed, rerr := es.closed, es.rerr 2760 es.mu.Unlock() 2761 if closed { 2762 return 0, errReadOnClosedResBody 2763 } 2764 if rerr != nil { 2765 return 0, rerr 2766 } 2767 2768 n, err = es.body.Read(p) 2769 if err != nil { 2770 es.mu.Lock() 2771 defer es.mu.Unlock() 2772 if es.rerr == nil { 2773 es.rerr = err 2774 } 2775 err = es.condfn(err) 2776 } 2777 return 2778} 2779 2780func (es *bodyEOFSignal) Close() error { 2781 es.mu.Lock() 2782 defer es.mu.Unlock() 2783 if es.closed { 2784 return nil 2785 } 2786 es.closed = true 2787 if es.earlyCloseFn != nil && es.rerr != io.EOF { 2788 return es.earlyCloseFn() 2789 } 2790 err := es.body.Close() 2791 return es.condfn(err) 2792} 2793 2794// caller must hold es.mu. 2795func (es *bodyEOFSignal) condfn(err error) error { 2796 if es.fn == nil { 2797 return err 2798 } 2799 err = es.fn(err) 2800 es.fn = nil 2801 return err 2802} 2803 2804// gzipReader wraps a response body so it can lazily 2805// call gzip.NewReader on the first call to Read 2806type gzipReader struct { 2807 _ incomparable 2808 body *bodyEOFSignal // underlying HTTP/1 response body framing 2809 zr *gzip.Reader // lazily-initialized gzip reader 2810 zerr error // any error from gzip.NewReader; sticky 2811} 2812 2813func (gz *gzipReader) Read(p []byte) (n int, err error) { 2814 if gz.zr == nil { 2815 if gz.zerr == nil { 2816 gz.zr, gz.zerr = gzip.NewReader(gz.body) 2817 } 2818 if gz.zerr != nil { 2819 return 0, gz.zerr 2820 } 2821 } 2822 2823 gz.body.mu.Lock() 2824 if gz.body.closed { 2825 err = errReadOnClosedResBody 2826 } 2827 gz.body.mu.Unlock() 2828 2829 if err != nil { 2830 return 0, err 2831 } 2832 return gz.zr.Read(p) 2833} 2834 2835func (gz *gzipReader) Close() error { 2836 return gz.body.Close() 2837} 2838 2839type tlsHandshakeTimeoutError struct{} 2840 2841func (tlsHandshakeTimeoutError) Timeout() bool { return true } 2842func (tlsHandshakeTimeoutError) Temporary() bool { return true } 2843func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 2844 2845// fakeLocker is a sync.Locker which does nothing. It's used to guard 2846// test-only fields when not under test, to avoid runtime atomic 2847// overhead. 2848type fakeLocker struct{} 2849 2850func (fakeLocker) Lock() {} 2851func (fakeLocker) Unlock() {} 2852 2853// cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if 2854// cfg is nil. This is safe to call even if cfg is in active use by a TLS 2855// client or server. 2856func cloneTLSConfig(cfg *tls.Config) *tls.Config { 2857 if cfg == nil { 2858 return &tls.Config{} 2859 } 2860 return cfg.Clone() 2861} 2862 2863type connLRU struct { 2864 ll *list.List // list.Element.Value type is of *persistConn 2865 m map[*persistConn]*list.Element 2866} 2867 2868// add adds pc to the head of the linked list. 2869func (cl *connLRU) add(pc *persistConn) { 2870 if cl.ll == nil { 2871 cl.ll = list.New() 2872 cl.m = make(map[*persistConn]*list.Element) 2873 } 2874 ele := cl.ll.PushFront(pc) 2875 if _, ok := cl.m[pc]; ok { 2876 panic("persistConn was already in LRU") 2877 } 2878 cl.m[pc] = ele 2879} 2880 2881func (cl *connLRU) removeOldest() *persistConn { 2882 ele := cl.ll.Back() 2883 pc := ele.Value.(*persistConn) 2884 cl.ll.Remove(ele) 2885 delete(cl.m, pc) 2886 return pc 2887} 2888 2889// remove removes pc from cl. 2890func (cl *connLRU) remove(pc *persistConn) { 2891 if ele, ok := cl.m[pc]; ok { 2892 cl.ll.Remove(ele) 2893 delete(cl.m, pc) 2894 } 2895} 2896 2897// len returns the number of items in the cache. 2898func (cl *connLRU) len() int { 2899 return len(cl.m) 2900} 2901