1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// HTTP client. See RFC 2616. 6// 7// This is the high-level Client interface. 8// The low-level implementation is in transport.go. 9 10package http 11 12import ( 13 "crypto/tls" 14 "encoding/base64" 15 "errors" 16 "fmt" 17 "io" 18 "io/ioutil" 19 "log" 20 "net/url" 21 "strings" 22 "sync" 23 "time" 24) 25 26// A Client is an HTTP client. Its zero value (DefaultClient) is a 27// usable client that uses DefaultTransport. 28// 29// The Client's Transport typically has internal state (cached TCP 30// connections), so Clients should be reused instead of created as 31// needed. Clients are safe for concurrent use by multiple goroutines. 32// 33// A Client is higher-level than a RoundTripper (such as Transport) 34// and additionally handles HTTP details such as cookies and 35// redirects. 36type Client struct { 37 // Transport specifies the mechanism by which individual 38 // HTTP requests are made. 39 // If nil, DefaultTransport is used. 40 Transport RoundTripper 41 42 // CheckRedirect specifies the policy for handling redirects. 43 // If CheckRedirect is not nil, the client calls it before 44 // following an HTTP redirect. The arguments req and via are 45 // the upcoming request and the requests made already, oldest 46 // first. If CheckRedirect returns an error, the Client's Get 47 // method returns both the previous Response and 48 // CheckRedirect's error (wrapped in a url.Error) instead of 49 // issuing the Request req. 50 // 51 // If CheckRedirect is nil, the Client uses its default policy, 52 // which is to stop after 10 consecutive requests. 53 CheckRedirect func(req *Request, via []*Request) error 54 55 // Jar specifies the cookie jar. 56 // If Jar is nil, cookies are not sent in requests and ignored 57 // in responses. 58 Jar CookieJar 59 60 // Timeout specifies a time limit for requests made by this 61 // Client. The timeout includes connection time, any 62 // redirects, and reading the response body. The timer remains 63 // running after Get, Head, Post, or Do return and will 64 // interrupt reading of the Response.Body. 65 // 66 // A Timeout of zero means no timeout. 67 // 68 // The Client cancels requests to the underlying Transport 69 // using the Request.Cancel mechanism. Requests passed 70 // to Client.Do may still set Request.Cancel; both will 71 // cancel the request. 72 // 73 // For compatibility, the Client will also use the deprecated 74 // CancelRequest method on Transport if found. New 75 // RoundTripper implementations should use Request.Cancel 76 // instead of implementing CancelRequest. 77 Timeout time.Duration 78} 79 80// DefaultClient is the default Client and is used by Get, Head, and Post. 81var DefaultClient = &Client{} 82 83// RoundTripper is an interface representing the ability to execute a 84// single HTTP transaction, obtaining the Response for a given Request. 85// 86// A RoundTripper must be safe for concurrent use by multiple 87// goroutines. 88type RoundTripper interface { 89 // RoundTrip executes a single HTTP transaction, returning 90 // a Response for the provided Request. 91 // 92 // RoundTrip should not attempt to interpret the response. In 93 // particular, RoundTrip must return err == nil if it obtained 94 // a response, regardless of the response's HTTP status code. 95 // A non-nil err should be reserved for failure to obtain a 96 // response. Similarly, RoundTrip should not attempt to 97 // handle higher-level protocol details such as redirects, 98 // authentication, or cookies. 99 // 100 // RoundTrip should not modify the request, except for 101 // consuming and closing the Request's Body. 102 // 103 // RoundTrip must always close the body, including on errors, 104 // but depending on the implementation may do so in a separate 105 // goroutine even after RoundTrip returns. This means that 106 // callers wanting to reuse the body for subsequent requests 107 // must arrange to wait for the Close call before doing so. 108 // 109 // The Request's URL and Header fields must be initialized. 110 RoundTrip(*Request) (*Response, error) 111} 112 113// Given a string of the form "host", "host:port", or "[ipv6::address]:port", 114// return true if the string includes a port. 115func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } 116 117// refererForURL returns a referer without any authentication info or 118// an empty string if lastReq scheme is https and newReq scheme is http. 119func refererForURL(lastReq, newReq *url.URL) string { 120 // https://tools.ietf.org/html/rfc7231#section-5.5.2 121 // "Clients SHOULD NOT include a Referer header field in a 122 // (non-secure) HTTP request if the referring page was 123 // transferred with a secure protocol." 124 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 125 return "" 126 } 127 referer := lastReq.String() 128 if lastReq.User != nil { 129 // This is not very efficient, but is the best we can 130 // do without: 131 // - introducing a new method on URL 132 // - creating a race condition 133 // - copying the URL struct manually, which would cause 134 // maintenance problems down the line 135 auth := lastReq.User.String() + "@" 136 referer = strings.Replace(referer, auth, "", 1) 137 } 138 return referer 139} 140 141// Used in Send to implement io.ReadCloser by bundling together the 142// bufio.Reader through which we read the response, and the underlying 143// network connection. 144type readClose struct { 145 io.Reader 146 io.Closer 147} 148 149func (c *Client) send(req *Request, deadline time.Time) (*Response, error) { 150 if c.Jar != nil { 151 for _, cookie := range c.Jar.Cookies(req.URL) { 152 req.AddCookie(cookie) 153 } 154 } 155 resp, err := send(req, c.transport(), deadline) 156 if err != nil { 157 return nil, err 158 } 159 if c.Jar != nil { 160 if rc := resp.Cookies(); len(rc) > 0 { 161 c.Jar.SetCookies(req.URL, rc) 162 } 163 } 164 return resp, err 165} 166 167// Do sends an HTTP request and returns an HTTP response, following 168// policy (e.g. redirects, cookies, auth) as configured on the client. 169// 170// An error is returned if caused by client policy (such as 171// CheckRedirect), or if there was an HTTP protocol error. 172// A non-2xx response doesn't cause an error. 173// 174// When err is nil, resp always contains a non-nil resp.Body. 175// 176// Callers should close resp.Body when done reading from it. If 177// resp.Body is not closed, the Client's underlying RoundTripper 178// (typically Transport) may not be able to re-use a persistent TCP 179// connection to the server for a subsequent "keep-alive" request. 180// 181// The request Body, if non-nil, will be closed by the underlying 182// Transport, even on errors. 183// 184// Generally Get, Post, or PostForm will be used instead of Do. 185func (c *Client) Do(req *Request) (resp *Response, err error) { 186 method := valueOrDefault(req.Method, "GET") 187 if method == "GET" || method == "HEAD" { 188 return c.doFollowingRedirects(req, shouldRedirectGet) 189 } 190 if method == "POST" || method == "PUT" { 191 return c.doFollowingRedirects(req, shouldRedirectPost) 192 } 193 return c.send(req, c.deadline()) 194} 195 196func (c *Client) deadline() time.Time { 197 if c.Timeout > 0 { 198 return time.Now().Add(c.Timeout) 199 } 200 return time.Time{} 201} 202 203func (c *Client) transport() RoundTripper { 204 if c.Transport != nil { 205 return c.Transport 206 } 207 return DefaultTransport 208} 209 210// send issues an HTTP request. 211// Caller should close resp.Body when done reading from it. 212func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) { 213 req := ireq // req is either the original request, or a modified fork 214 215 if rt == nil { 216 req.closeBody() 217 return nil, errors.New("http: no Client.Transport or DefaultTransport") 218 } 219 220 if req.URL == nil { 221 req.closeBody() 222 return nil, errors.New("http: nil Request.URL") 223 } 224 225 if req.RequestURI != "" { 226 req.closeBody() 227 return nil, errors.New("http: Request.RequestURI can't be set in client requests.") 228 } 229 230 // forkReq forks req into a shallow clone of ireq the first 231 // time it's called. 232 forkReq := func() { 233 if ireq == req { 234 req = new(Request) 235 *req = *ireq // shallow clone 236 } 237 } 238 239 // Most the callers of send (Get, Post, et al) don't need 240 // Headers, leaving it uninitialized. We guarantee to the 241 // Transport that this has been initialized, though. 242 if req.Header == nil { 243 forkReq() 244 req.Header = make(Header) 245 } 246 247 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 248 username := u.Username() 249 password, _ := u.Password() 250 forkReq() 251 req.Header = cloneHeader(ireq.Header) 252 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 253 } 254 255 if !deadline.IsZero() { 256 forkReq() 257 } 258 stopTimer, wasCanceled := setRequestCancel(req, rt, deadline) 259 260 resp, err := rt.RoundTrip(req) 261 if err != nil { 262 stopTimer() 263 if resp != nil { 264 log.Printf("RoundTripper returned a response & error; ignoring response") 265 } 266 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 267 // If we get a bad TLS record header, check to see if the 268 // response looks like HTTP and give a more helpful error. 269 // See golang.org/issue/11111. 270 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 271 err = errors.New("http: server gave HTTP response to HTTPS client") 272 } 273 } 274 return nil, err 275 } 276 if !deadline.IsZero() { 277 resp.Body = &cancelTimerBody{ 278 stop: stopTimer, 279 rc: resp.Body, 280 reqWasCanceled: wasCanceled, 281 } 282 } 283 return resp, nil 284} 285 286// setRequestCancel sets the Cancel field of req, if deadline is 287// non-zero. The RoundTripper's type is used to determine whether the legacy 288// CancelRequest behavior should be used. 289func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), wasCanceled func() bool) { 290 if deadline.IsZero() { 291 return nop, alwaysFalse 292 } 293 294 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 295 296 cancel := make(chan struct{}) 297 req.Cancel = cancel 298 299 wasCanceled = func() bool { 300 select { 301 case <-cancel: 302 return true 303 default: 304 return false 305 } 306 } 307 308 doCancel := func() { 309 // The new way: 310 close(cancel) 311 312 // The legacy compatibility way, used only 313 // for RoundTripper implementations written 314 // before Go 1.5 or Go 1.6. 315 type canceler interface { 316 CancelRequest(*Request) 317 } 318 switch v := rt.(type) { 319 case *Transport, *http2Transport: 320 // Do nothing. The net/http package's transports 321 // support the new Request.Cancel channel 322 case canceler: 323 v.CancelRequest(req) 324 } 325 } 326 327 stopTimerCh := make(chan struct{}) 328 var once sync.Once 329 stopTimer = func() { once.Do(func() { close(stopTimerCh) }) } 330 331 timer := time.NewTimer(deadline.Sub(time.Now())) 332 go func() { 333 select { 334 case <-initialReqCancel: 335 doCancel() 336 case <-timer.C: 337 doCancel() 338 case <-stopTimerCh: 339 timer.Stop() 340 } 341 }() 342 343 return stopTimer, wasCanceled 344} 345 346// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt 347// "To receive authorization, the client sends the userid and password, 348// separated by a single colon (":") character, within a base64 349// encoded string in the credentials." 350// It is not meant to be urlencoded. 351func basicAuth(username, password string) string { 352 auth := username + ":" + password 353 return base64.StdEncoding.EncodeToString([]byte(auth)) 354} 355 356// True if the specified HTTP status code is one for which the Get utility should 357// automatically redirect. 358func shouldRedirectGet(statusCode int) bool { 359 switch statusCode { 360 case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect: 361 return true 362 } 363 return false 364} 365 366// True if the specified HTTP status code is one for which the Post utility should 367// automatically redirect. 368func shouldRedirectPost(statusCode int) bool { 369 switch statusCode { 370 case StatusFound, StatusSeeOther: 371 return true 372 } 373 return false 374} 375 376// Get issues a GET to the specified URL. If the response is one of 377// the following redirect codes, Get follows the redirect, up to a 378// maximum of 10 redirects: 379// 380// 301 (Moved Permanently) 381// 302 (Found) 382// 303 (See Other) 383// 307 (Temporary Redirect) 384// 385// An error is returned if there were too many redirects or if there 386// was an HTTP protocol error. A non-2xx response doesn't cause an 387// error. 388// 389// When err is nil, resp always contains a non-nil resp.Body. 390// Caller should close resp.Body when done reading from it. 391// 392// Get is a wrapper around DefaultClient.Get. 393// 394// To make a request with custom headers, use NewRequest and 395// DefaultClient.Do. 396func Get(url string) (resp *Response, err error) { 397 return DefaultClient.Get(url) 398} 399 400// Get issues a GET to the specified URL. If the response is one of the 401// following redirect codes, Get follows the redirect after calling the 402// Client's CheckRedirect function: 403// 404// 301 (Moved Permanently) 405// 302 (Found) 406// 303 (See Other) 407// 307 (Temporary Redirect) 408// 409// An error is returned if the Client's CheckRedirect function fails 410// or if there was an HTTP protocol error. A non-2xx response doesn't 411// cause an error. 412// 413// When err is nil, resp always contains a non-nil resp.Body. 414// Caller should close resp.Body when done reading from it. 415// 416// To make a request with custom headers, use NewRequest and Client.Do. 417func (c *Client) Get(url string) (resp *Response, err error) { 418 req, err := NewRequest("GET", url, nil) 419 if err != nil { 420 return nil, err 421 } 422 return c.doFollowingRedirects(req, shouldRedirectGet) 423} 424 425func alwaysFalse() bool { return false } 426 427func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) { 428 var base *url.URL 429 redirectChecker := c.CheckRedirect 430 if redirectChecker == nil { 431 redirectChecker = defaultCheckRedirect 432 } 433 var via []*Request 434 435 if ireq.URL == nil { 436 ireq.closeBody() 437 return nil, errors.New("http: nil Request.URL") 438 } 439 440 req := ireq 441 deadline := c.deadline() 442 443 urlStr := "" // next relative or absolute URL to fetch (after first request) 444 redirectFailed := false 445 for redirect := 0; ; redirect++ { 446 if redirect != 0 { 447 nreq := new(Request) 448 nreq.Cancel = ireq.Cancel 449 nreq.Method = ireq.Method 450 if ireq.Method == "POST" || ireq.Method == "PUT" { 451 nreq.Method = "GET" 452 } 453 nreq.Header = make(Header) 454 nreq.URL, err = base.Parse(urlStr) 455 if err != nil { 456 break 457 } 458 if len(via) > 0 { 459 // Add the Referer header. 460 lastReq := via[len(via)-1] 461 if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" { 462 nreq.Header.Set("Referer", ref) 463 } 464 465 err = redirectChecker(nreq, via) 466 if err != nil { 467 redirectFailed = true 468 break 469 } 470 } 471 req = nreq 472 } 473 474 urlStr = req.URL.String() 475 if resp, err = c.send(req, deadline); err != nil { 476 if !deadline.IsZero() && !time.Now().Before(deadline) { 477 err = &httpError{ 478 err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", 479 timeout: true, 480 } 481 } 482 break 483 } 484 485 if shouldRedirect(resp.StatusCode) { 486 // Read the body if small so underlying TCP connection will be re-used. 487 // No need to check for errors: if it fails, Transport won't reuse it anyway. 488 const maxBodySlurpSize = 2 << 10 489 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 490 io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize) 491 } 492 resp.Body.Close() 493 if urlStr = resp.Header.Get("Location"); urlStr == "" { 494 err = fmt.Errorf("%d response missing Location header", resp.StatusCode) 495 break 496 } 497 base = req.URL 498 via = append(via, req) 499 continue 500 } 501 return resp, nil 502 } 503 504 method := valueOrDefault(ireq.Method, "GET") 505 urlErr := &url.Error{ 506 Op: method[:1] + strings.ToLower(method[1:]), 507 URL: urlStr, 508 Err: err, 509 } 510 511 if redirectFailed { 512 // Special case for Go 1 compatibility: return both the response 513 // and an error if the CheckRedirect function failed. 514 // See https://golang.org/issue/3795 515 return resp, urlErr 516 } 517 518 if resp != nil { 519 resp.Body.Close() 520 } 521 return nil, urlErr 522} 523 524func defaultCheckRedirect(req *Request, via []*Request) error { 525 if len(via) >= 10 { 526 return errors.New("stopped after 10 redirects") 527 } 528 return nil 529} 530 531// Post issues a POST to the specified URL. 532// 533// Caller should close resp.Body when done reading from it. 534// 535// If the provided body is an io.Closer, it is closed after the 536// request. 537// 538// Post is a wrapper around DefaultClient.Post. 539// 540// To set custom headers, use NewRequest and DefaultClient.Do. 541func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { 542 return DefaultClient.Post(url, bodyType, body) 543} 544 545// Post issues a POST to the specified URL. 546// 547// Caller should close resp.Body when done reading from it. 548// 549// If the provided body is an io.Closer, it is closed after the 550// request. 551// 552// To set custom headers, use NewRequest and Client.Do. 553func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { 554 req, err := NewRequest("POST", url, body) 555 if err != nil { 556 return nil, err 557 } 558 req.Header.Set("Content-Type", bodyType) 559 return c.doFollowingRedirects(req, shouldRedirectPost) 560} 561 562// PostForm issues a POST to the specified URL, with data's keys and 563// values URL-encoded as the request body. 564// 565// The Content-Type header is set to application/x-www-form-urlencoded. 566// To set other headers, use NewRequest and DefaultClient.Do. 567// 568// When err is nil, resp always contains a non-nil resp.Body. 569// Caller should close resp.Body when done reading from it. 570// 571// PostForm is a wrapper around DefaultClient.PostForm. 572func PostForm(url string, data url.Values) (resp *Response, err error) { 573 return DefaultClient.PostForm(url, data) 574} 575 576// PostForm issues a POST to the specified URL, 577// with data's keys and values URL-encoded as the request body. 578// 579// The Content-Type header is set to application/x-www-form-urlencoded. 580// To set other headers, use NewRequest and DefaultClient.Do. 581// 582// When err is nil, resp always contains a non-nil resp.Body. 583// Caller should close resp.Body when done reading from it. 584func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 585 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 586} 587 588// Head issues a HEAD to the specified URL. If the response is one of 589// the following redirect codes, Head follows the redirect, up to a 590// maximum of 10 redirects: 591// 592// 301 (Moved Permanently) 593// 302 (Found) 594// 303 (See Other) 595// 307 (Temporary Redirect) 596// 597// Head is a wrapper around DefaultClient.Head 598func Head(url string) (resp *Response, err error) { 599 return DefaultClient.Head(url) 600} 601 602// Head issues a HEAD to the specified URL. If the response is one of the 603// following redirect codes, Head follows the redirect after calling the 604// Client's CheckRedirect function: 605// 606// 301 (Moved Permanently) 607// 302 (Found) 608// 303 (See Other) 609// 307 (Temporary Redirect) 610func (c *Client) Head(url string) (resp *Response, err error) { 611 req, err := NewRequest("HEAD", url, nil) 612 if err != nil { 613 return nil, err 614 } 615 return c.doFollowingRedirects(req, shouldRedirectGet) 616} 617 618// cancelTimerBody is an io.ReadCloser that wraps rc with two features: 619// 1) on Read error or close, the stop func is called. 620// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and 621// marked as net.Error that hit its timeout. 622type cancelTimerBody struct { 623 stop func() // stops the time.Timer waiting to cancel the request 624 rc io.ReadCloser 625 reqWasCanceled func() bool 626} 627 628func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 629 n, err = b.rc.Read(p) 630 if err == nil { 631 return n, nil 632 } 633 b.stop() 634 if err == io.EOF { 635 return n, err 636 } 637 if b.reqWasCanceled() { 638 err = &httpError{ 639 err: err.Error() + " (Client.Timeout exceeded while reading body)", 640 timeout: true, 641 } 642 } 643 return n, err 644} 645 646func (b *cancelTimerBody) Close() error { 647 err := b.rc.Close() 648 b.stop() 649 return err 650} 651