1package upload 2 3import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io/ioutil" 8 "mime/multipart" 9 "net/http" 10 "net/http/httptest" 11 "os" 12 "regexp" 13 "strconv" 14 "strings" 15 "testing" 16 "time" 17 18 "github.com/stretchr/testify/require" 19 20 "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" 21 "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" 22 "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" 23 "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" 24 "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" 25 "gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper" 26) 27 28var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) 29 30type testFormProcessor struct{ SavedFileTracker } 31 32func (a *testFormProcessor) ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error { 33 if formName != "token" && !strings.HasPrefix(formName, "file.") && !strings.HasPrefix(formName, "other.") { 34 return fmt.Errorf("illegal field: %v", formName) 35 } 36 return nil 37} 38 39func (a *testFormProcessor) Finalize(ctx context.Context) error { 40 return nil 41} 42 43func TestUploadTempPathRequirement(t *testing.T) { 44 apiResponse := &api.Response{} 45 preparer := &DefaultPreparer{} 46 _, _, err := preparer.Prepare(apiResponse) 47 require.Error(t, err) 48} 49 50func TestUploadHandlerForwardingRawData(t *testing.T) { 51 ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) { 52 require.Equal(t, "PATCH", r.Method, "method") 53 54 body, err := ioutil.ReadAll(r.Body) 55 require.NoError(t, err) 56 require.Equal(t, "REQUEST", string(body), "request body") 57 58 w.WriteHeader(202) 59 fmt.Fprint(w, "RESPONSE") 60 }) 61 defer ts.Close() 62 63 httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST")) 64 require.NoError(t, err) 65 66 tempPath, err := ioutil.TempDir("", "uploads") 67 require.NoError(t, err) 68 defer os.RemoveAll(tempPath) 69 70 response := httptest.NewRecorder() 71 72 handler := newProxy(ts.URL) 73 apiResponse := &api.Response{TempPath: tempPath} 74 preparer := &DefaultPreparer{} 75 opts, _, err := preparer.Prepare(apiResponse) 76 require.NoError(t, err) 77 78 HandleFileUploads(response, httpRequest, handler, apiResponse, nil, opts) 79 80 require.Equal(t, 202, response.Code) 81 require.Equal(t, "RESPONSE", response.Body.String(), "response body") 82} 83 84func TestUploadHandlerRewritingMultiPartData(t *testing.T) { 85 var filePath string 86 87 tempPath, err := ioutil.TempDir("", "uploads") 88 require.NoError(t, err) 89 defer os.RemoveAll(tempPath) 90 91 ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) { 92 require.Equal(t, "PUT", r.Method, "method") 93 require.NoError(t, r.ParseMultipartForm(100000)) 94 95 require.Empty(t, r.MultipartForm.File, "Expected to not receive any files") 96 require.Equal(t, "test", r.FormValue("token"), "Expected to receive token") 97 require.Equal(t, "my.file", r.FormValue("file.name"), "Expected to receive a filename") 98 99 filePath = r.FormValue("file.path") 100 require.True(t, strings.HasPrefix(filePath, tempPath), "Expected to the file to be in tempPath") 101 102 require.Empty(t, r.FormValue("file.remote_url"), "Expected to receive empty remote_url") 103 require.Empty(t, r.FormValue("file.remote_id"), "Expected to receive empty remote_id") 104 require.Equal(t, "4", r.FormValue("file.size"), "Expected to receive the file size") 105 106 hashes := map[string]string{ 107 "md5": "098f6bcd4621d373cade4e832627b4f6", 108 "sha1": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", 109 "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", 110 "sha512": "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff", 111 } 112 113 for algo, hash := range hashes { 114 require.Equal(t, hash, r.FormValue("file."+algo), "file hash %s", algo) 115 } 116 117 require.Len(t, r.MultipartForm.Value, 12, "multipart form values") 118 119 w.WriteHeader(202) 120 fmt.Fprint(w, "RESPONSE") 121 }) 122 123 var buffer bytes.Buffer 124 125 writer := multipart.NewWriter(&buffer) 126 writer.WriteField("token", "test") 127 file, err := writer.CreateFormFile("file", "my.file") 128 require.NoError(t, err) 129 fmt.Fprint(file, "test") 130 writer.Close() 131 132 httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil) 133 require.NoError(t, err) 134 135 ctx, cancel := context.WithCancel(context.Background()) 136 httpRequest = httpRequest.WithContext(ctx) 137 httpRequest.Body = ioutil.NopCloser(&buffer) 138 httpRequest.ContentLength = int64(buffer.Len()) 139 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 140 response := httptest.NewRecorder() 141 142 handler := newProxy(ts.URL) 143 144 apiResponse := &api.Response{TempPath: tempPath} 145 preparer := &DefaultPreparer{} 146 opts, _, err := preparer.Prepare(apiResponse) 147 require.NoError(t, err) 148 149 HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) 150 require.Equal(t, 202, response.Code) 151 152 cancel() // this will trigger an async cleanup 153 waitUntilDeleted(t, filePath) 154} 155 156func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) { 157 var filePath string 158 159 tempPath, err := ioutil.TempDir("", "uploads") 160 require.NoError(t, err) 161 defer os.RemoveAll(tempPath) 162 163 tests := []struct { 164 name string 165 field string 166 response int 167 }{ 168 { 169 name: "injected file.path", 170 field: "file.path", 171 response: 400, 172 }, 173 { 174 name: "injected file.remote_id", 175 field: "file.remote_id", 176 response: 400, 177 }, 178 { 179 name: "field with other prefix", 180 field: "other.path", 181 response: 202, 182 }, 183 } 184 185 for _, test := range tests { 186 t.Run(test.name, func(t *testing.T) { 187 ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) { 188 require.Equal(t, "PUT", r.Method, "method") 189 190 w.WriteHeader(202) 191 fmt.Fprint(w, "RESPONSE") 192 }) 193 194 var buffer bytes.Buffer 195 196 writer := multipart.NewWriter(&buffer) 197 file, err := writer.CreateFormFile("file", "my.file") 198 require.NoError(t, err) 199 fmt.Fprint(file, "test") 200 201 writer.WriteField(test.field, "value") 202 writer.Close() 203 204 httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", &buffer) 205 require.NoError(t, err) 206 207 ctx, cancel := context.WithCancel(context.Background()) 208 httpRequest = httpRequest.WithContext(ctx) 209 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 210 response := httptest.NewRecorder() 211 212 handler := newProxy(ts.URL) 213 apiResponse := &api.Response{TempPath: tempPath} 214 preparer := &DefaultPreparer{} 215 opts, _, err := preparer.Prepare(apiResponse) 216 require.NoError(t, err) 217 218 HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) 219 require.Equal(t, test.response, response.Code) 220 221 cancel() // this will trigger an async cleanup 222 waitUntilDeleted(t, filePath) 223 }) 224 } 225} 226 227func TestUploadProcessingField(t *testing.T) { 228 tempPath, err := ioutil.TempDir("", "uploads") 229 require.NoError(t, err) 230 defer os.RemoveAll(tempPath) 231 232 var buffer bytes.Buffer 233 234 writer := multipart.NewWriter(&buffer) 235 writer.WriteField("token2", "test") 236 writer.Close() 237 238 httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer) 239 require.NoError(t, err) 240 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 241 242 response := httptest.NewRecorder() 243 apiResponse := &api.Response{TempPath: tempPath} 244 preparer := &DefaultPreparer{} 245 opts, _, err := preparer.Prepare(apiResponse) 246 require.NoError(t, err) 247 248 HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) 249 250 require.Equal(t, 500, response.Code) 251} 252 253func TestUploadingMultipleFiles(t *testing.T) { 254 testhelper.ConfigureSecret() 255 256 tempPath, err := ioutil.TempDir("", "uploads") 257 require.NoError(t, err) 258 defer os.RemoveAll(tempPath) 259 260 var buffer bytes.Buffer 261 262 writer := multipart.NewWriter(&buffer) 263 for i := 0; i < 11; i++ { 264 _, err = writer.CreateFormFile(fmt.Sprintf("file %v", i), "my.file") 265 require.NoError(t, err) 266 } 267 require.NoError(t, writer.Close()) 268 269 httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer) 270 require.NoError(t, err) 271 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 272 273 response := httptest.NewRecorder() 274 apiResponse := &api.Response{TempPath: tempPath} 275 preparer := &DefaultPreparer{} 276 opts, _, err := preparer.Prepare(apiResponse) 277 require.NoError(t, err) 278 279 HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) 280 281 require.Equal(t, 400, response.Code) 282 require.Equal(t, "upload request contains more than 10 files\n", response.Body.String()) 283} 284 285func TestUploadProcessingFile(t *testing.T) { 286 tempPath, err := ioutil.TempDir("", "uploads") 287 require.NoError(t, err) 288 defer os.RemoveAll(tempPath) 289 290 _, testServer := test.StartObjectStore() 291 defer testServer.Close() 292 293 storeUrl := testServer.URL + test.ObjectPath 294 295 tests := []struct { 296 name string 297 preauth *api.Response 298 }{ 299 { 300 name: "FileStore Upload", 301 preauth: &api.Response{TempPath: tempPath}, 302 }, 303 { 304 name: "ObjectStore Upload", 305 preauth: &api.Response{RemoteObject: api.RemoteObject{StoreURL: storeUrl}}, 306 }, 307 { 308 name: "ObjectStore and FileStore Upload", 309 preauth: &api.Response{ 310 TempPath: tempPath, 311 RemoteObject: api.RemoteObject{StoreURL: storeUrl}, 312 }, 313 }, 314 } 315 316 for _, test := range tests { 317 t.Run(test.name, func(t *testing.T) { 318 var buffer bytes.Buffer 319 writer := multipart.NewWriter(&buffer) 320 file, err := writer.CreateFormFile("file", "my.file") 321 require.NoError(t, err) 322 fmt.Fprint(file, "test") 323 writer.Close() 324 325 httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer) 326 require.NoError(t, err) 327 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 328 329 response := httptest.NewRecorder() 330 apiResponse := &api.Response{TempPath: tempPath} 331 preparer := &DefaultPreparer{} 332 opts, _, err := preparer.Prepare(apiResponse) 333 require.NoError(t, err) 334 335 HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) 336 337 require.Equal(t, 200, response.Code) 338 }) 339 } 340 341} 342 343func TestInvalidFileNames(t *testing.T) { 344 testhelper.ConfigureSecret() 345 346 tempPath, err := ioutil.TempDir("", "uploads") 347 require.NoError(t, err) 348 defer os.RemoveAll(tempPath) 349 350 for _, testCase := range []struct { 351 filename string 352 code int 353 expectedPrefix string 354 }{ 355 {"foobar", 200, "foobar"}, // sanity check for test setup below 356 {"foo/bar", 200, "bar"}, 357 {"foo/bar/baz", 200, "baz"}, 358 {"/../../foobar", 200, "foobar"}, 359 {".", 500, ""}, 360 {"..", 500, ""}, 361 {"./", 500, ""}, 362 } { 363 buffer := &bytes.Buffer{} 364 365 writer := multipart.NewWriter(buffer) 366 file, err := writer.CreateFormFile("file", testCase.filename) 367 require.NoError(t, err) 368 fmt.Fprint(file, "test") 369 writer.Close() 370 371 httpRequest, err := http.NewRequest("POST", "/example", buffer) 372 require.NoError(t, err) 373 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 374 375 response := httptest.NewRecorder() 376 apiResponse := &api.Response{TempPath: tempPath} 377 preparer := &DefaultPreparer{} 378 opts, _, err := preparer.Prepare(apiResponse) 379 require.NoError(t, err) 380 381 HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) 382 require.Equal(t, testCase.code, response.Code) 383 require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix) 384 } 385} 386 387func TestUploadHandlerRemovingExif(t *testing.T) { 388 content, err := ioutil.ReadFile("exif/testdata/sample_exif.jpg") 389 require.NoError(t, err) 390 391 runUploadTest(t, content, "sample_exif.jpg", 200, func(w http.ResponseWriter, r *http.Request) { 392 err := r.ParseMultipartForm(100000) 393 require.NoError(t, err) 394 395 size, err := strconv.Atoi(r.FormValue("file.size")) 396 require.NoError(t, err) 397 require.True(t, size < len(content), "Expected the file to be smaller after removal of exif") 398 require.True(t, size > 0, "Expected to receive not empty file") 399 400 w.WriteHeader(200) 401 fmt.Fprint(w, "RESPONSE") 402 }) 403} 404 405func TestUploadHandlerRemovingExifTiff(t *testing.T) { 406 content, err := ioutil.ReadFile("exif/testdata/sample_exif.tiff") 407 require.NoError(t, err) 408 409 runUploadTest(t, content, "sample_exif.tiff", 200, func(w http.ResponseWriter, r *http.Request) { 410 err := r.ParseMultipartForm(100000) 411 require.NoError(t, err) 412 413 size, err := strconv.Atoi(r.FormValue("file.size")) 414 require.NoError(t, err) 415 require.True(t, size < len(content), "Expected the file to be smaller after removal of exif") 416 require.True(t, size > 0, "Expected to receive not empty file") 417 418 w.WriteHeader(200) 419 fmt.Fprint(w, "RESPONSE") 420 }) 421} 422 423func TestUploadHandlerRemovingExifInvalidContentType(t *testing.T) { 424 content, err := ioutil.ReadFile("exif/testdata/sample_exif_invalid.jpg") 425 require.NoError(t, err) 426 427 runUploadTest(t, content, "sample_exif_invalid.jpg", 200, func(w http.ResponseWriter, r *http.Request) { 428 err := r.ParseMultipartForm(100000) 429 require.NoError(t, err) 430 431 output, err := ioutil.ReadFile(r.FormValue("file.path")) 432 require.NoError(t, err) 433 require.Equal(t, content, output, "Expected the file to be same as before") 434 435 w.WriteHeader(200) 436 fmt.Fprint(w, "RESPONSE") 437 }) 438} 439 440func TestUploadHandlerRemovingExifCorruptedFile(t *testing.T) { 441 content, err := ioutil.ReadFile("exif/testdata/sample_exif_corrupted.jpg") 442 require.NoError(t, err) 443 444 runUploadTest(t, content, "sample_exif_corrupted.jpg", 422, func(w http.ResponseWriter, r *http.Request) { 445 err := r.ParseMultipartForm(100000) 446 require.Error(t, err) 447 }) 448} 449 450func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, tsHandler func(http.ResponseWriter, *http.Request)) { 451 tempPath, err := ioutil.TempDir("", "uploads") 452 require.NoError(t, err) 453 defer os.RemoveAll(tempPath) 454 455 var buffer bytes.Buffer 456 457 writer := multipart.NewWriter(&buffer) 458 file, err := writer.CreateFormFile("file", filename) 459 require.NoError(t, err) 460 461 _, err = file.Write(image) 462 require.NoError(t, err) 463 464 err = writer.Close() 465 require.NoError(t, err) 466 467 ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), tsHandler) 468 defer ts.Close() 469 470 httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", &buffer) 471 require.NoError(t, err) 472 473 ctx, cancel := context.WithCancel(context.Background()) 474 defer cancel() 475 476 httpRequest = httpRequest.WithContext(ctx) 477 httpRequest.ContentLength = int64(buffer.Len()) 478 httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) 479 response := httptest.NewRecorder() 480 481 handler := newProxy(ts.URL) 482 apiResponse := &api.Response{TempPath: tempPath} 483 preparer := &DefaultPreparer{} 484 opts, _, err := preparer.Prepare(apiResponse) 485 require.NoError(t, err) 486 487 HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) 488 require.Equal(t, httpCode, response.Code) 489} 490 491func newProxy(url string) *proxy.Proxy { 492 parsedURL := helper.URLMustParse(url) 493 return proxy.NewProxy(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL)) 494} 495 496func waitUntilDeleted(t *testing.T, path string) { 497 var err error 498 499 // Poll because the file removal is async 500 for i := 0; i < 100; i++ { 501 _, err = os.Stat(path) 502 if err != nil { 503 break 504 } 505 time.Sleep(100 * time.Millisecond) 506 } 507 508 require.True(t, os.IsNotExist(err), "expected the file to be deleted") 509} 510