1// Copyright 2011 Google Inc. All rights reserved. 2// Use of this source code is governed by the Apache 2.0 3// license that can be found in the LICENSE file. 4 5// Package blobstore provides a client for App Engine's persistent blob 6// storage service. 7package blobstore // import "google.golang.org/appengine/blobstore" 8 9import ( 10 "bufio" 11 "bytes" 12 "encoding/base64" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "mime" 17 "mime/multipart" 18 "net/http" 19 "net/textproto" 20 "net/url" 21 "strconv" 22 "strings" 23 "time" 24 25 "github.com/golang/protobuf/proto" 26 "golang.org/x/net/context" 27 "golang.org/x/text/encoding/htmlindex" 28 29 "google.golang.org/appengine" 30 "google.golang.org/appengine/datastore" 31 "google.golang.org/appengine/internal" 32 33 basepb "google.golang.org/appengine/internal/base" 34 blobpb "google.golang.org/appengine/internal/blobstore" 35) 36 37const ( 38 blobInfoKind = "__BlobInfo__" 39 blobFileIndexKind = "__BlobFileIndex__" 40 zeroKey = appengine.BlobKey("") 41) 42 43// BlobInfo is the blob metadata that is stored in the datastore. 44// Filename may be empty. 45type BlobInfo struct { 46 BlobKey appengine.BlobKey 47 ContentType string `datastore:"content_type"` 48 CreationTime time.Time `datastore:"creation"` 49 Filename string `datastore:"filename"` 50 Size int64 `datastore:"size"` 51 MD5 string `datastore:"md5_hash"` 52 53 // ObjectName is the Google Cloud Storage name for this blob. 54 ObjectName string `datastore:"gs_object_name"` 55} 56 57// isErrFieldMismatch returns whether err is a datastore.ErrFieldMismatch. 58// 59// The blobstore stores blob metadata in the datastore. When loading that 60// metadata, it may contain fields that we don't care about. datastore.Get will 61// return datastore.ErrFieldMismatch in that case, so we ignore that specific 62// error. 63func isErrFieldMismatch(err error) bool { 64 _, ok := err.(*datastore.ErrFieldMismatch) 65 return ok 66} 67 68// Stat returns the BlobInfo for a provided blobKey. If no blob was found for 69// that key, Stat returns datastore.ErrNoSuchEntity. 70func Stat(c context.Context, blobKey appengine.BlobKey) (*BlobInfo, error) { 71 c, _ = appengine.Namespace(c, "") // Blobstore is always in the empty string namespace 72 dskey := datastore.NewKey(c, blobInfoKind, string(blobKey), 0, nil) 73 bi := &BlobInfo{ 74 BlobKey: blobKey, 75 } 76 if err := datastore.Get(c, dskey, bi); err != nil && !isErrFieldMismatch(err) { 77 return nil, err 78 } 79 return bi, nil 80} 81 82// Send sets the headers on response to instruct App Engine to send a blob as 83// the response body. This is more efficient than reading and writing it out 84// manually and isn't subject to normal response size limits. 85func Send(response http.ResponseWriter, blobKey appengine.BlobKey) { 86 hdr := response.Header() 87 hdr.Set("X-AppEngine-BlobKey", string(blobKey)) 88 89 if hdr.Get("Content-Type") == "" { 90 // This value is known to dev_appserver to mean automatic. 91 // In production this is remapped to the empty value which 92 // means automatic. 93 hdr.Set("Content-Type", "application/vnd.google.appengine.auto") 94 } 95} 96 97// UploadURL creates an upload URL for the form that the user will 98// fill out, passing the application path to load when the POST of the 99// form is completed. These URLs expire and should not be reused. The 100// opts parameter may be nil. 101func UploadURL(c context.Context, successPath string, opts *UploadURLOptions) (*url.URL, error) { 102 req := &blobpb.CreateUploadURLRequest{ 103 SuccessPath: proto.String(successPath), 104 } 105 if opts != nil { 106 if n := opts.MaxUploadBytes; n != 0 { 107 req.MaxUploadSizeBytes = &n 108 } 109 if n := opts.MaxUploadBytesPerBlob; n != 0 { 110 req.MaxUploadSizePerBlobBytes = &n 111 } 112 if s := opts.StorageBucket; s != "" { 113 req.GsBucketName = &s 114 } 115 } 116 res := &blobpb.CreateUploadURLResponse{} 117 if err := internal.Call(c, "blobstore", "CreateUploadURL", req, res); err != nil { 118 return nil, err 119 } 120 return url.Parse(*res.Url) 121} 122 123// UploadURLOptions are the options to create an upload URL. 124type UploadURLOptions struct { 125 MaxUploadBytes int64 // optional 126 MaxUploadBytesPerBlob int64 // optional 127 128 // StorageBucket specifies the Google Cloud Storage bucket in which 129 // to store the blob. 130 // This is required if you use Cloud Storage instead of Blobstore. 131 // Your application must have permission to write to the bucket. 132 // You may optionally specify a bucket name and path in the format 133 // "bucket_name/path", in which case the included path will be the 134 // prefix of the uploaded object's name. 135 StorageBucket string 136} 137 138// Delete deletes a blob. 139func Delete(c context.Context, blobKey appengine.BlobKey) error { 140 return DeleteMulti(c, []appengine.BlobKey{blobKey}) 141} 142 143// DeleteMulti deletes multiple blobs. 144func DeleteMulti(c context.Context, blobKey []appengine.BlobKey) error { 145 s := make([]string, len(blobKey)) 146 for i, b := range blobKey { 147 s[i] = string(b) 148 } 149 req := &blobpb.DeleteBlobRequest{ 150 BlobKey: s, 151 } 152 res := &basepb.VoidProto{} 153 if err := internal.Call(c, "blobstore", "DeleteBlob", req, res); err != nil { 154 return err 155 } 156 return nil 157} 158 159func errorf(format string, args ...interface{}) error { 160 return fmt.Errorf("blobstore: "+format, args...) 161} 162 163// ParseUpload parses the synthetic POST request that your app gets from 164// App Engine after a user's successful upload of blobs. Given the request, 165// ParseUpload returns a map of the blobs received (keyed by HTML form 166// element name) and other non-blob POST parameters. 167func ParseUpload(req *http.Request) (blobs map[string][]*BlobInfo, other url.Values, err error) { 168 _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")) 169 if err != nil { 170 return nil, nil, err 171 } 172 boundary := params["boundary"] 173 if boundary == "" { 174 return nil, nil, errorf("did not find MIME multipart boundary") 175 } 176 177 blobs = make(map[string][]*BlobInfo) 178 other = make(url.Values) 179 180 mreader := multipart.NewReader(io.MultiReader(req.Body, strings.NewReader("\r\n\r\n")), boundary) 181 for { 182 part, perr := mreader.NextPart() 183 if perr == io.EOF { 184 break 185 } 186 if perr != nil { 187 return nil, nil, errorf("error reading next mime part with boundary %q (len=%d): %v", 188 boundary, len(boundary), perr) 189 } 190 191 bi := &BlobInfo{} 192 ctype, params, err := mime.ParseMediaType(part.Header.Get("Content-Disposition")) 193 if err != nil { 194 return nil, nil, err 195 } 196 bi.Filename = params["filename"] 197 formKey := params["name"] 198 199 ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Type")) 200 if err != nil { 201 return nil, nil, err 202 } 203 bi.BlobKey = appengine.BlobKey(params["blob-key"]) 204 charset := params["charset"] 205 206 if ctype != "message/external-body" || bi.BlobKey == "" { 207 if formKey != "" { 208 slurp, serr := ioutil.ReadAll(part) 209 if serr != nil { 210 return nil, nil, errorf("error reading %q MIME part", formKey) 211 } 212 213 // Handle base64 content transfer encoding. multipart.Part transparently 214 // handles quoted-printable, and no special handling is required for 215 // 7bit, 8bit, or binary. 216 ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Transfer-Encoding")) 217 if err == nil && ctype == "base64" { 218 slurp, serr = ioutil.ReadAll(base64.NewDecoder( 219 base64.StdEncoding, bytes.NewReader(slurp))) 220 if serr != nil { 221 return nil, nil, errorf("error %s decoding %q MIME part", ctype, formKey) 222 } 223 } 224 225 // Handle charset 226 if charset != "" { 227 encoding, err := htmlindex.Get(charset) 228 if err != nil { 229 return nil, nil, errorf("error getting decoder for charset %q", charset) 230 } 231 232 slurp, err = encoding.NewDecoder().Bytes(slurp) 233 if err != nil { 234 return nil, nil, errorf("error decoding from charset %q", charset) 235 } 236 } 237 238 other[formKey] = append(other[formKey], string(slurp)) 239 } 240 continue 241 } 242 243 // App Engine sends a MIME header as the body of each MIME part. 244 tp := textproto.NewReader(bufio.NewReader(part)) 245 header, mimeerr := tp.ReadMIMEHeader() 246 if mimeerr != nil { 247 return nil, nil, mimeerr 248 } 249 bi.Size, err = strconv.ParseInt(header.Get("Content-Length"), 10, 64) 250 if err != nil { 251 return nil, nil, err 252 } 253 bi.ContentType = header.Get("Content-Type") 254 255 // Parse the time from the MIME header like: 256 // X-AppEngine-Upload-Creation: 2011-03-15 21:38:34.712136 257 createDate := header.Get("X-AppEngine-Upload-Creation") 258 if createDate == "" { 259 return nil, nil, errorf("expected to find an X-AppEngine-Upload-Creation header") 260 } 261 bi.CreationTime, err = time.Parse("2006-01-02 15:04:05.000000", createDate) 262 if err != nil { 263 return nil, nil, errorf("error parsing X-AppEngine-Upload-Creation: %s", err) 264 } 265 266 if hdr := header.Get("Content-MD5"); hdr != "" { 267 md5, err := base64.URLEncoding.DecodeString(hdr) 268 if err != nil { 269 return nil, nil, errorf("bad Content-MD5 %q: %v", hdr, err) 270 } 271 bi.MD5 = string(md5) 272 } 273 274 // If the GCS object name was provided, record it. 275 bi.ObjectName = header.Get("X-AppEngine-Cloud-Storage-Object") 276 277 blobs[formKey] = append(blobs[formKey], bi) 278 } 279 return 280} 281 282// Reader is a blob reader. 283type Reader interface { 284 io.Reader 285 io.ReaderAt 286 io.Seeker 287} 288 289// NewReader returns a reader for a blob. It always succeeds; if the blob does 290// not exist then an error will be reported upon first read. 291func NewReader(c context.Context, blobKey appengine.BlobKey) Reader { 292 return openBlob(c, blobKey) 293} 294 295// BlobKeyForFile returns a BlobKey for a Google Storage file. 296// The filename should be of the form "/gs/bucket_name/object_name". 297func BlobKeyForFile(c context.Context, filename string) (appengine.BlobKey, error) { 298 req := &blobpb.CreateEncodedGoogleStorageKeyRequest{ 299 Filename: &filename, 300 } 301 res := &blobpb.CreateEncodedGoogleStorageKeyResponse{} 302 if err := internal.Call(c, "blobstore", "CreateEncodedGoogleStorageKey", req, res); err != nil { 303 return "", err 304 } 305 return appengine.BlobKey(*res.BlobKey), nil 306} 307