1// Copyright 2011 Google Inc. All rights reserved. 2// Use of this source code is governed by the Apache 2.0 3// license that can be found in the LICENSE file. 4 5package blobstore 6 7import ( 8 "bytes" 9 "encoding/base64" 10 "fmt" 11 "io" 12 "mime/multipart" 13 "mime/quotedprintable" 14 "net/http" 15 "net/textproto" 16 "os" 17 "strconv" 18 "strings" 19 "testing" 20 21 "golang.org/x/text/encoding/htmlindex" 22 23 "google.golang.org/appengine" 24 "google.golang.org/appengine/internal/aetesting" 25 26 pb "google.golang.org/appengine/internal/blobstore" 27) 28 29const rbs = readBufferSize 30 31const charsetUTF8 = "utf-8" 32const charsetISO2022JP = "iso-2022-jp" 33const nonASCIIStr = "Hello, 世界" 34 35func min(x, y int) int { 36 if x < y { 37 return x 38 } 39 return y 40} 41 42func fakeFetchData(req *pb.FetchDataRequest, res *pb.FetchDataResponse) error { 43 i0 := int(*req.StartIndex) 44 i1 := int(*req.EndIndex + 1) // Blobstore's end-indices are inclusive; Go's are exclusive. 45 bk := *req.BlobKey 46 if i := strings.Index(bk, "."); i != -1 { 47 // Strip everything past the ".". 48 bk = bk[:i] 49 } 50 switch bk { 51 case "a14p": 52 const s = "abcdefghijklmnop" 53 i0 := min(len(s), i0) 54 i1 := min(len(s), i1) 55 res.Data = []byte(s[i0:i1]) 56 case "longBlob": 57 res.Data = make([]byte, i1-i0) 58 for i := range res.Data { 59 res.Data[i] = 'A' + uint8(i0/rbs) 60 i0++ 61 } 62 } 63 return nil 64} 65 66// step is one step of a readerTest. 67// It consists of a Reader method to call, the method arguments 68// (lenp, offset, whence) and the expected results. 69type step struct { 70 method string 71 lenp int 72 offset int64 73 whence int 74 want string 75 wantErr error 76} 77 78var readerTest = []struct { 79 blobKey string 80 step []step 81}{ 82 {"noSuchBlobKey", []step{ 83 {"Read", 8, 0, 0, "", io.EOF}, 84 }}, 85 {"a14p.0", []step{ 86 // Test basic reads. 87 {"Read", 1, 0, 0, "a", nil}, 88 {"Read", 3, 0, 0, "bcd", nil}, 89 {"Read", 1, 0, 0, "e", nil}, 90 {"Read", 2, 0, 0, "fg", nil}, 91 // Test Seek. 92 {"Seek", 0, 2, os.SEEK_SET, "2", nil}, 93 {"Read", 5, 0, 0, "cdefg", nil}, 94 {"Seek", 0, 2, os.SEEK_CUR, "9", nil}, 95 {"Read", 1, 0, 0, "j", nil}, 96 // Test reads up to and past EOF. 97 {"Read", 5, 0, 0, "klmno", nil}, 98 {"Read", 5, 0, 0, "p", nil}, 99 {"Read", 5, 0, 0, "", io.EOF}, 100 // Test ReadAt. 101 {"ReadAt", 4, 0, 0, "abcd", nil}, 102 {"ReadAt", 4, 3, 0, "defg", nil}, 103 {"ReadAt", 4, 12, 0, "mnop", nil}, 104 {"ReadAt", 4, 13, 0, "nop", io.EOF}, 105 {"ReadAt", 4, 99, 0, "", io.EOF}, 106 }}, 107 {"a14p.1", []step{ 108 // Test Seek before any reads. 109 {"Seek", 0, 2, os.SEEK_SET, "2", nil}, 110 {"Read", 1, 0, 0, "c", nil}, 111 // Test that ReadAt doesn't affect the Read offset. 112 {"ReadAt", 3, 9, 0, "jkl", nil}, 113 {"Read", 3, 0, 0, "def", nil}, 114 }}, 115 {"a14p.2", []step{ 116 // Test ReadAt before any reads or seeks. 117 {"ReadAt", 2, 14, 0, "op", nil}, 118 }}, 119 {"longBlob.0", []step{ 120 // Test basic read. 121 {"Read", 1, 0, 0, "A", nil}, 122 // Test that Read returns early when the buffer is exhausted. 123 {"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil}, 124 {"Read", 5, 0, 0, "AA", nil}, 125 {"Read", 3, 0, 0, "BBB", nil}, 126 // Test that what we just read is still in the buffer. 127 {"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil}, 128 {"Read", 5, 0, 0, "AABBB", nil}, 129 // Test ReadAt. 130 {"ReadAt", 3, rbs - 4, 0, "AAA", nil}, 131 {"ReadAt", 6, rbs - 4, 0, "AAAABB", nil}, 132 {"ReadAt", 8, rbs - 4, 0, "AAAABBBB", nil}, 133 {"ReadAt", 5, rbs - 4, 0, "AAAAB", nil}, 134 {"ReadAt", 2, rbs - 4, 0, "AA", nil}, 135 // Test seeking backwards from the Read offset. 136 {"Seek", 0, 2*rbs - 8, os.SEEK_SET, strconv.Itoa(2*rbs - 8), nil}, 137 {"Read", 1, 0, 0, "B", nil}, 138 {"Read", 1, 0, 0, "B", nil}, 139 {"Read", 1, 0, 0, "B", nil}, 140 {"Read", 1, 0, 0, "B", nil}, 141 {"Read", 8, 0, 0, "BBBBCCCC", nil}, 142 }}, 143 {"longBlob.1", []step{ 144 // Test ReadAt with a slice larger than the buffer size. 145 {"LargeReadAt", 2*rbs - 2, 0, 0, strconv.Itoa(2*rbs - 2), nil}, 146 {"LargeReadAt", 2*rbs - 1, 0, 0, strconv.Itoa(2*rbs - 1), nil}, 147 {"LargeReadAt", 2*rbs + 0, 0, 0, strconv.Itoa(2*rbs + 0), nil}, 148 {"LargeReadAt", 2*rbs + 1, 0, 0, strconv.Itoa(2*rbs + 1), nil}, 149 {"LargeReadAt", 2*rbs + 2, 0, 0, strconv.Itoa(2*rbs + 2), nil}, 150 {"LargeReadAt", 2*rbs - 2, 1, 0, strconv.Itoa(2*rbs - 2), nil}, 151 {"LargeReadAt", 2*rbs - 1, 1, 0, strconv.Itoa(2*rbs - 1), nil}, 152 {"LargeReadAt", 2*rbs + 0, 1, 0, strconv.Itoa(2*rbs + 0), nil}, 153 {"LargeReadAt", 2*rbs + 1, 1, 0, strconv.Itoa(2*rbs + 1), nil}, 154 {"LargeReadAt", 2*rbs + 2, 1, 0, strconv.Itoa(2*rbs + 2), nil}, 155 }}, 156} 157 158func TestReader(t *testing.T) { 159 for _, rt := range readerTest { 160 c := aetesting.FakeSingleContext(t, "blobstore", "FetchData", fakeFetchData) 161 r := NewReader(c, appengine.BlobKey(rt.blobKey)) 162 for i, step := range rt.step { 163 var ( 164 got string 165 gotErr error 166 n int 167 offset int64 168 ) 169 switch step.method { 170 case "LargeReadAt": 171 p := make([]byte, step.lenp) 172 n, gotErr = r.ReadAt(p, step.offset) 173 got = strconv.Itoa(n) 174 case "Read": 175 p := make([]byte, step.lenp) 176 n, gotErr = r.Read(p) 177 got = string(p[:n]) 178 case "ReadAt": 179 p := make([]byte, step.lenp) 180 n, gotErr = r.ReadAt(p, step.offset) 181 got = string(p[:n]) 182 case "Seek": 183 offset, gotErr = r.Seek(step.offset, step.whence) 184 got = strconv.FormatInt(offset, 10) 185 default: 186 t.Fatalf("unknown method: %s", step.method) 187 } 188 if gotErr != step.wantErr { 189 t.Fatalf("%s step %d: got error %v want %v", rt.blobKey, i, gotErr, step.wantErr) 190 } 191 if got != step.want { 192 t.Fatalf("%s step %d: got %q want %q", rt.blobKey, i, got, step.want) 193 } 194 } 195 } 196} 197 198// doPlainTextParseUploadTest tests ParseUpload's decoding of non-file form fields. 199// It ensures that MIME multipart parts with Content-Type not equal to 200// "message/external-body" (i.e. form fields that are not file uploads) are decoded 201// correctly according to the value of their Content-Transfer-Encoding header field. 202// If charset is not the empty string it will be set in the request's Content-Type 203// header field, and if encoding is not the empty string then the Content-Transfer-Encoding 204// header field will be set. 205func doPlainTextParseUploadTest(t *testing.T, charset string, encoding string, 206 rawContent string, encodedContent string) { 207 bodyBuf := &bytes.Buffer{} 208 w := multipart.NewWriter(bodyBuf) 209 210 fieldName := "foo" 211 hdr := textproto.MIMEHeader{} 212 hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", fieldName)) 213 214 if charset != "" { 215 hdr.Set("Content-Type", fmt.Sprintf("text/plain; charset=%q", charset)) 216 } else { 217 hdr.Set("Content-Type", "text/plain") 218 } 219 220 if encoding != "" { 221 hdr.Set("Content-Transfer-Encoding", encoding) 222 } 223 224 pw, err := w.CreatePart(hdr) 225 if err != nil { 226 t.Fatalf("error creating part: %v", err) 227 } 228 pw.Write([]byte(encodedContent)) 229 230 if err := w.Close(); err != nil { 231 t.Fatalf("error closing multipart writer: %v\n", err) 232 } 233 234 req, err := http.NewRequest("POST", "/upload", bodyBuf) 235 if err != nil { 236 t.Fatalf("error creating request: %v", err) 237 } 238 239 req.Header.Set("Content-Type", w.FormDataContentType()) 240 _, other, err := ParseUpload(req) 241 if err != nil { 242 t.Fatalf("error parsing upload: %v", err) 243 } 244 245 if other[fieldName][0] != rawContent { 246 t.Errorf("got %q expected %q", other[fieldName][0], rawContent) 247 } 248} 249 250func TestParseUploadUTF8Base64Encoding(t *testing.T) { 251 encoded := base64.StdEncoding.EncodeToString([]byte(nonASCIIStr)) 252 doPlainTextParseUploadTest(t, charsetUTF8, "base64", nonASCIIStr, encoded) 253} 254 255func TestParseUploadUTF8Base64EncodingMultiline(t *testing.T) { 256 testStr := "words words words words words words words words words words words words" 257 encoded := "d29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29y\r\nZHMgd29yZHMgd29yZHM=" 258 doPlainTextParseUploadTest(t, charsetUTF8, "base64", testStr, encoded) 259} 260 261func TestParseUploadUTF8QuotedPrintableEncoding(t *testing.T) { 262 var encoded bytes.Buffer 263 writer := quotedprintable.NewWriter(&encoded) 264 writer.Write([]byte(nonASCIIStr)) 265 writer.Close() 266 267 doPlainTextParseUploadTest(t, charsetUTF8, "quoted-printable", nonASCIIStr, 268 encoded.String()) 269} 270 271func TestParseUploadISO2022JPBase64Encoding(t *testing.T) { 272 testStr := "こんにちは" 273 encoding, err := htmlindex.Get(charsetISO2022JP) 274 if err != nil { 275 t.Fatalf("error getting encoding: %v", err) 276 } 277 278 charsetEncoded, err := encoding.NewEncoder().String(testStr) 279 if err != nil { 280 t.Fatalf("error encoding string: %v", err) 281 } 282 283 base64Encoded := base64.StdEncoding.EncodeToString([]byte(charsetEncoded)) 284 doPlainTextParseUploadTest(t, charsetISO2022JP, "base64", testStr, base64Encoded) 285} 286 287func TestParseUploadNoEncoding(t *testing.T) { 288 doPlainTextParseUploadTest(t, "", "", "Hello", "Hello") 289} 290