1// Copyright 2016 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package gensupport 6 7import ( 8 "bytes" 9 "fmt" 10 "io" 11 "io/ioutil" 12 "mime" 13 "mime/multipart" 14 "net/http" 15 "net/textproto" 16 "strings" 17 "sync" 18 19 "google.golang.org/api/googleapi" 20) 21 22const sniffBuffSize = 512 23 24func newContentSniffer(r io.Reader) *contentSniffer { 25 return &contentSniffer{r: r} 26} 27 28// contentSniffer wraps a Reader, and reports the content type determined by sniffing up to 512 bytes from the Reader. 29type contentSniffer struct { 30 r io.Reader 31 start []byte // buffer for the sniffed bytes. 32 err error // set to any error encountered while reading bytes to be sniffed. 33 34 ctype string // set on first sniff. 35 sniffed bool // set to true on first sniff. 36} 37 38func (cs *contentSniffer) Read(p []byte) (n int, err error) { 39 // Ensure that the content type is sniffed before any data is consumed from Reader. 40 _, _ = cs.ContentType() 41 42 if len(cs.start) > 0 { 43 n := copy(p, cs.start) 44 cs.start = cs.start[n:] 45 return n, nil 46 } 47 48 // We may have read some bytes into start while sniffing, even if the read ended in an error. 49 // We should first return those bytes, then the error. 50 if cs.err != nil { 51 return 0, cs.err 52 } 53 54 // Now we have handled all bytes that were buffered while sniffing. Now just delegate to the underlying reader. 55 return cs.r.Read(p) 56} 57 58// ContentType returns the sniffed content type, and whether the content type was successfully sniffed. 59func (cs *contentSniffer) ContentType() (string, bool) { 60 if cs.sniffed { 61 return cs.ctype, cs.ctype != "" 62 } 63 cs.sniffed = true 64 // If ReadAll hits EOF, it returns err==nil. 65 cs.start, cs.err = ioutil.ReadAll(io.LimitReader(cs.r, sniffBuffSize)) 66 67 // Don't try to detect the content type based on possibly incomplete data. 68 if cs.err != nil { 69 return "", false 70 } 71 72 cs.ctype = http.DetectContentType(cs.start) 73 return cs.ctype, true 74} 75 76// DetermineContentType determines the content type of the supplied reader. 77// If the content type is already known, it can be specified via ctype. 78// Otherwise, the content of media will be sniffed to determine the content type. 79// If media implements googleapi.ContentTyper (deprecated), this will be used 80// instead of sniffing the content. 81// After calling DetectContentType the caller must not perform further reads on 82// media, but rather read from the Reader that is returned. 83func DetermineContentType(media io.Reader, ctype string) (io.Reader, string) { 84 // Note: callers could avoid calling DetectContentType if ctype != "", 85 // but doing the check inside this function reduces the amount of 86 // generated code. 87 if ctype != "" { 88 return media, ctype 89 } 90 91 // For backwards compatibility, allow clients to set content 92 // type by providing a ContentTyper for media. 93 if typer, ok := media.(googleapi.ContentTyper); ok { 94 return media, typer.ContentType() 95 } 96 97 sniffer := newContentSniffer(media) 98 if ctype, ok := sniffer.ContentType(); ok { 99 return sniffer, ctype 100 } 101 // If content type could not be sniffed, reads from sniffer will eventually fail with an error. 102 return sniffer, "" 103} 104 105type typeReader struct { 106 io.Reader 107 typ string 108} 109 110// multipartReader combines the contents of multiple readers to create a multipart/related HTTP body. 111// Close must be called if reads from the multipartReader are abandoned before reaching EOF. 112type multipartReader struct { 113 pr *io.PipeReader 114 ctype string 115 mu sync.Mutex 116 pipeOpen bool 117} 118 119// boundary optionally specifies the MIME boundary 120func newMultipartReader(parts []typeReader, boundary string) *multipartReader { 121 mp := &multipartReader{pipeOpen: true} 122 var pw *io.PipeWriter 123 mp.pr, pw = io.Pipe() 124 mpw := multipart.NewWriter(pw) 125 if boundary != "" { 126 mpw.SetBoundary(boundary) 127 } 128 mp.ctype = "multipart/related; boundary=" + mpw.Boundary() 129 go func() { 130 for _, part := range parts { 131 w, err := mpw.CreatePart(typeHeader(part.typ)) 132 if err != nil { 133 mpw.Close() 134 pw.CloseWithError(fmt.Errorf("googleapi: CreatePart failed: %v", err)) 135 return 136 } 137 _, err = io.Copy(w, part.Reader) 138 if err != nil { 139 mpw.Close() 140 pw.CloseWithError(fmt.Errorf("googleapi: Copy failed: %v", err)) 141 return 142 } 143 } 144 145 mpw.Close() 146 pw.Close() 147 }() 148 return mp 149} 150 151func (mp *multipartReader) Read(data []byte) (n int, err error) { 152 return mp.pr.Read(data) 153} 154 155func (mp *multipartReader) Close() error { 156 mp.mu.Lock() 157 if !mp.pipeOpen { 158 mp.mu.Unlock() 159 return nil 160 } 161 mp.pipeOpen = false 162 mp.mu.Unlock() 163 return mp.pr.Close() 164} 165 166// CombineBodyMedia combines a json body with media content to create a multipart/related HTTP body. 167// It returns a ReadCloser containing the combined body, and the overall "multipart/related" content type, with random boundary. 168// 169// The caller must call Close on the returned ReadCloser if reads are abandoned before reaching EOF. 170func CombineBodyMedia(body io.Reader, bodyContentType string, media io.Reader, mediaContentType string) (io.ReadCloser, string) { 171 return combineBodyMedia(body, bodyContentType, media, mediaContentType, "") 172} 173 174// combineBodyMedia is CombineBodyMedia but with an optional mimeBoundary field. 175func combineBodyMedia(body io.Reader, bodyContentType string, media io.Reader, mediaContentType, mimeBoundary string) (io.ReadCloser, string) { 176 mp := newMultipartReader([]typeReader{ 177 {body, bodyContentType}, 178 {media, mediaContentType}, 179 }, mimeBoundary) 180 return mp, mp.ctype 181} 182 183func typeHeader(contentType string) textproto.MIMEHeader { 184 h := make(textproto.MIMEHeader) 185 if contentType != "" { 186 h.Set("Content-Type", contentType) 187 } 188 return h 189} 190 191// PrepareUpload determines whether the data in the supplied reader should be 192// uploaded in a single request, or in sequential chunks. 193// chunkSize is the size of the chunk that media should be split into. 194// 195// If chunkSize is zero, media is returned as the first value, and the other 196// two return values are nil, true. 197// 198// Otherwise, a MediaBuffer is returned, along with a bool indicating whether the 199// contents of media fit in a single chunk. 200// 201// After PrepareUpload has been called, media should no longer be used: the 202// media content should be accessed via one of the return values. 203func PrepareUpload(media io.Reader, chunkSize int) (r io.Reader, mb *MediaBuffer, singleChunk bool) { 204 if chunkSize == 0 { // do not chunk 205 return media, nil, true 206 } 207 mb = NewMediaBuffer(media, chunkSize) 208 _, _, _, err := mb.Chunk() 209 // If err is io.EOF, we can upload this in a single request. Otherwise, err is 210 // either nil or a non-EOF error. If it is the latter, then the next call to 211 // mb.Chunk will return the same error. Returning a MediaBuffer ensures that this 212 // error will be handled at some point. 213 return nil, mb, err == io.EOF 214} 215 216// MediaInfo holds information for media uploads. It is intended for use by generated 217// code only. 218type MediaInfo struct { 219 // At most one of Media and MediaBuffer will be set. 220 media io.Reader 221 buffer *MediaBuffer 222 singleChunk bool 223 mType string 224 size int64 // mediaSize, if known. Used only for calls to progressUpdater_. 225 progressUpdater googleapi.ProgressUpdater 226} 227 228// NewInfoFromMedia should be invoked from the Media method of a call. It returns a 229// MediaInfo populated with chunk size and content type, and a reader or MediaBuffer 230// if needed. 231func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo { 232 mi := &MediaInfo{} 233 opts := googleapi.ProcessMediaOptions(options) 234 if !opts.ForceEmptyContentType { 235 r, mi.mType = DetermineContentType(r, opts.ContentType) 236 } 237 mi.media, mi.buffer, mi.singleChunk = PrepareUpload(r, opts.ChunkSize) 238 return mi 239} 240 241// NewInfoFromResumableMedia should be invoked from the ResumableMedia method of a 242// call. It returns a MediaInfo using the given reader, size and media type. 243func NewInfoFromResumableMedia(r io.ReaderAt, size int64, mediaType string) *MediaInfo { 244 rdr := ReaderAtToReader(r, size) 245 rdr, mType := DetermineContentType(rdr, mediaType) 246 return &MediaInfo{ 247 size: size, 248 mType: mType, 249 buffer: NewMediaBuffer(rdr, googleapi.DefaultUploadChunkSize), 250 media: nil, 251 singleChunk: false, 252 } 253} 254 255// SetProgressUpdater sets the progress updater for the media info. 256func (mi *MediaInfo) SetProgressUpdater(pu googleapi.ProgressUpdater) { 257 if mi != nil { 258 mi.progressUpdater = pu 259 } 260} 261 262// UploadType determines the type of upload: a single request, or a resumable 263// series of requests. 264func (mi *MediaInfo) UploadType() string { 265 if mi.singleChunk { 266 return "multipart" 267 } 268 return "resumable" 269} 270 271// UploadRequest sets up an HTTP request for media upload. It adds headers 272// as necessary, and returns a replacement for the body and a function for http.Request.GetBody. 273func (mi *MediaInfo) UploadRequest(reqHeaders http.Header, body io.Reader) (newBody io.Reader, getBody func() (io.ReadCloser, error), cleanup func()) { 274 cleanup = func() {} 275 if mi == nil { 276 return body, nil, cleanup 277 } 278 var media io.Reader 279 if mi.media != nil { 280 // This only happens when the caller has turned off chunking. In that 281 // case, we write all of media in a single non-retryable request. 282 media = mi.media 283 } else if mi.singleChunk { 284 // The data fits in a single chunk, which has now been read into the MediaBuffer. 285 // We obtain that chunk so we can write it in a single request. The request can 286 // be retried because the data is stored in the MediaBuffer. 287 media, _, _, _ = mi.buffer.Chunk() 288 } 289 if media != nil { 290 fb := readerFunc(body) 291 fm := readerFunc(media) 292 combined, ctype := CombineBodyMedia(body, "application/json", media, mi.mType) 293 toCleanup := []io.Closer{ 294 combined, 295 } 296 if fb != nil && fm != nil { 297 getBody = func() (io.ReadCloser, error) { 298 rb := ioutil.NopCloser(fb()) 299 rm := ioutil.NopCloser(fm()) 300 var mimeBoundary string 301 if _, params, err := mime.ParseMediaType(ctype); err == nil { 302 mimeBoundary = params["boundary"] 303 } 304 r, _ := combineBodyMedia(rb, "application/json", rm, mi.mType, mimeBoundary) 305 toCleanup = append(toCleanup, r) 306 return r, nil 307 } 308 } 309 cleanup = func() { 310 for _, closer := range toCleanup { 311 _ = closer.Close() 312 } 313 314 } 315 reqHeaders.Set("Content-Type", ctype) 316 body = combined 317 } 318 if mi.buffer != nil && mi.mType != "" && !mi.singleChunk { 319 reqHeaders.Set("X-Upload-Content-Type", mi.mType) 320 } 321 return body, getBody, cleanup 322} 323 324// readerFunc returns a function that always returns an io.Reader that has the same 325// contents as r, provided that can be done without consuming r. Otherwise, it 326// returns nil. 327// See http.NewRequest (in net/http/request.go). 328func readerFunc(r io.Reader) func() io.Reader { 329 switch r := r.(type) { 330 case *bytes.Buffer: 331 buf := r.Bytes() 332 return func() io.Reader { return bytes.NewReader(buf) } 333 case *bytes.Reader: 334 snapshot := *r 335 return func() io.Reader { r := snapshot; return &r } 336 case *strings.Reader: 337 snapshot := *r 338 return func() io.Reader { r := snapshot; return &r } 339 default: 340 return nil 341 } 342} 343 344// ResumableUpload returns an appropriately configured ResumableUpload value if the 345// upload is resumable, or nil otherwise. 346func (mi *MediaInfo) ResumableUpload(locURI string) *ResumableUpload { 347 if mi == nil || mi.singleChunk { 348 return nil 349 } 350 return &ResumableUpload{ 351 URI: locURI, 352 Media: mi.buffer, 353 MediaType: mi.mType, 354 Callback: func(curr int64) { 355 if mi.progressUpdater != nil { 356 mi.progressUpdater(curr, mi.size) 357 } 358 }, 359 } 360} 361 362// SetGetBody sets the GetBody field of req to f. This was once needed 363// to gracefully support Go 1.7 and earlier which didn't have that 364// field. 365// 366// Deprecated: the code generator no longer uses this as of 367// 2019-02-19. Nothing else should be calling this anyway, but we 368// won't delete this immediately; it will be deleted in as early as 6 369// months. 370func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) { 371 req.GetBody = f 372} 373