1package upload
2
3import (
4	"bytes"
5	"context"
6	"fmt"
7	"io/ioutil"
8	"mime/multipart"
9	"net/http"
10	"net/http/httptest"
11	"os"
12	"regexp"
13	"strconv"
14	"strings"
15	"testing"
16	"time"
17
18	"github.com/stretchr/testify/require"
19
20	"gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
21	"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
22	"gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test"
23	"gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
24	"gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper"
25	"gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper"
26)
27
28var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
29
30type testFormProcessor struct{ SavedFileTracker }
31
32func (a *testFormProcessor) ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error {
33	if formName != "token" && !strings.HasPrefix(formName, "file.") && !strings.HasPrefix(formName, "other.") {
34		return fmt.Errorf("illegal field: %v", formName)
35	}
36	return nil
37}
38
39func (a *testFormProcessor) Finalize(ctx context.Context) error {
40	return nil
41}
42
43func TestUploadTempPathRequirement(t *testing.T) {
44	apiResponse := &api.Response{}
45	preparer := &DefaultPreparer{}
46	_, _, err := preparer.Prepare(apiResponse)
47	require.Error(t, err)
48}
49
50func TestUploadHandlerForwardingRawData(t *testing.T) {
51	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
52		require.Equal(t, "PATCH", r.Method, "method")
53
54		body, err := ioutil.ReadAll(r.Body)
55		require.NoError(t, err)
56		require.Equal(t, "REQUEST", string(body), "request body")
57
58		w.WriteHeader(202)
59		fmt.Fprint(w, "RESPONSE")
60	})
61	defer ts.Close()
62
63	httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
64	require.NoError(t, err)
65
66	tempPath, err := ioutil.TempDir("", "uploads")
67	require.NoError(t, err)
68	defer os.RemoveAll(tempPath)
69
70	response := httptest.NewRecorder()
71
72	handler := newProxy(ts.URL)
73	apiResponse := &api.Response{TempPath: tempPath}
74	preparer := &DefaultPreparer{}
75	opts, _, err := preparer.Prepare(apiResponse)
76	require.NoError(t, err)
77
78	HandleFileUploads(response, httpRequest, handler, apiResponse, nil, opts)
79
80	require.Equal(t, 202, response.Code)
81	require.Equal(t, "RESPONSE", response.Body.String(), "response body")
82}
83
84func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
85	var filePath string
86
87	tempPath, err := ioutil.TempDir("", "uploads")
88	require.NoError(t, err)
89	defer os.RemoveAll(tempPath)
90
91	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
92		require.Equal(t, "PUT", r.Method, "method")
93		require.NoError(t, r.ParseMultipartForm(100000))
94
95		require.Empty(t, r.MultipartForm.File, "Expected to not receive any files")
96		require.Equal(t, "test", r.FormValue("token"), "Expected to receive token")
97		require.Equal(t, "my.file", r.FormValue("file.name"), "Expected to receive a filename")
98
99		filePath = r.FormValue("file.path")
100		require.True(t, strings.HasPrefix(filePath, tempPath), "Expected to the file to be in tempPath")
101
102		require.Empty(t, r.FormValue("file.remote_url"), "Expected to receive empty remote_url")
103		require.Empty(t, r.FormValue("file.remote_id"), "Expected to receive empty remote_id")
104		require.Equal(t, "4", r.FormValue("file.size"), "Expected to receive the file size")
105
106		hashes := map[string]string{
107			"md5":    "098f6bcd4621d373cade4e832627b4f6",
108			"sha1":   "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
109			"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
110			"sha512": "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff",
111		}
112
113		for algo, hash := range hashes {
114			require.Equal(t, hash, r.FormValue("file."+algo), "file hash %s", algo)
115		}
116
117		require.Len(t, r.MultipartForm.Value, 12, "multipart form values")
118
119		w.WriteHeader(202)
120		fmt.Fprint(w, "RESPONSE")
121	})
122
123	var buffer bytes.Buffer
124
125	writer := multipart.NewWriter(&buffer)
126	writer.WriteField("token", "test")
127	file, err := writer.CreateFormFile("file", "my.file")
128	require.NoError(t, err)
129	fmt.Fprint(file, "test")
130	writer.Close()
131
132	httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil)
133	require.NoError(t, err)
134
135	ctx, cancel := context.WithCancel(context.Background())
136	httpRequest = httpRequest.WithContext(ctx)
137	httpRequest.Body = ioutil.NopCloser(&buffer)
138	httpRequest.ContentLength = int64(buffer.Len())
139	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
140	response := httptest.NewRecorder()
141
142	handler := newProxy(ts.URL)
143
144	apiResponse := &api.Response{TempPath: tempPath}
145	preparer := &DefaultPreparer{}
146	opts, _, err := preparer.Prepare(apiResponse)
147	require.NoError(t, err)
148
149	HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
150	require.Equal(t, 202, response.Code)
151
152	cancel() // this will trigger an async cleanup
153	waitUntilDeleted(t, filePath)
154}
155
156func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) {
157	var filePath string
158
159	tempPath, err := ioutil.TempDir("", "uploads")
160	require.NoError(t, err)
161	defer os.RemoveAll(tempPath)
162
163	tests := []struct {
164		name     string
165		field    string
166		response int
167	}{
168		{
169			name:     "injected file.path",
170			field:    "file.path",
171			response: 400,
172		},
173		{
174			name:     "injected file.remote_id",
175			field:    "file.remote_id",
176			response: 400,
177		},
178		{
179			name:     "field with other prefix",
180			field:    "other.path",
181			response: 202,
182		},
183	}
184
185	for _, test := range tests {
186		t.Run(test.name, func(t *testing.T) {
187			ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
188				require.Equal(t, "PUT", r.Method, "method")
189
190				w.WriteHeader(202)
191				fmt.Fprint(w, "RESPONSE")
192			})
193
194			var buffer bytes.Buffer
195
196			writer := multipart.NewWriter(&buffer)
197			file, err := writer.CreateFormFile("file", "my.file")
198			require.NoError(t, err)
199			fmt.Fprint(file, "test")
200
201			writer.WriteField(test.field, "value")
202			writer.Close()
203
204			httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", &buffer)
205			require.NoError(t, err)
206
207			ctx, cancel := context.WithCancel(context.Background())
208			httpRequest = httpRequest.WithContext(ctx)
209			httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
210			response := httptest.NewRecorder()
211
212			handler := newProxy(ts.URL)
213			apiResponse := &api.Response{TempPath: tempPath}
214			preparer := &DefaultPreparer{}
215			opts, _, err := preparer.Prepare(apiResponse)
216			require.NoError(t, err)
217
218			HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
219			require.Equal(t, test.response, response.Code)
220
221			cancel() // this will trigger an async cleanup
222			waitUntilDeleted(t, filePath)
223		})
224	}
225}
226
227func TestUploadProcessingField(t *testing.T) {
228	tempPath, err := ioutil.TempDir("", "uploads")
229	require.NoError(t, err)
230	defer os.RemoveAll(tempPath)
231
232	var buffer bytes.Buffer
233
234	writer := multipart.NewWriter(&buffer)
235	writer.WriteField("token2", "test")
236	writer.Close()
237
238	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
239	require.NoError(t, err)
240	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
241
242	response := httptest.NewRecorder()
243	apiResponse := &api.Response{TempPath: tempPath}
244	preparer := &DefaultPreparer{}
245	opts, _, err := preparer.Prepare(apiResponse)
246	require.NoError(t, err)
247
248	HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
249
250	require.Equal(t, 500, response.Code)
251}
252
253func TestUploadingMultipleFiles(t *testing.T) {
254	testhelper.ConfigureSecret()
255
256	tempPath, err := ioutil.TempDir("", "uploads")
257	require.NoError(t, err)
258	defer os.RemoveAll(tempPath)
259
260	var buffer bytes.Buffer
261
262	writer := multipart.NewWriter(&buffer)
263	for i := 0; i < 11; i++ {
264		_, err = writer.CreateFormFile(fmt.Sprintf("file %v", i), "my.file")
265		require.NoError(t, err)
266	}
267	require.NoError(t, writer.Close())
268
269	httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
270	require.NoError(t, err)
271	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
272
273	response := httptest.NewRecorder()
274	apiResponse := &api.Response{TempPath: tempPath}
275	preparer := &DefaultPreparer{}
276	opts, _, err := preparer.Prepare(apiResponse)
277	require.NoError(t, err)
278
279	HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
280
281	require.Equal(t, 400, response.Code)
282	require.Equal(t, "upload request contains more than 10 files\n", response.Body.String())
283}
284
285func TestUploadProcessingFile(t *testing.T) {
286	tempPath, err := ioutil.TempDir("", "uploads")
287	require.NoError(t, err)
288	defer os.RemoveAll(tempPath)
289
290	_, testServer := test.StartObjectStore()
291	defer testServer.Close()
292
293	storeUrl := testServer.URL + test.ObjectPath
294
295	tests := []struct {
296		name    string
297		preauth *api.Response
298	}{
299		{
300			name:    "FileStore Upload",
301			preauth: &api.Response{TempPath: tempPath},
302		},
303		{
304			name:    "ObjectStore Upload",
305			preauth: &api.Response{RemoteObject: api.RemoteObject{StoreURL: storeUrl}},
306		},
307		{
308			name: "ObjectStore and FileStore Upload",
309			preauth: &api.Response{
310				TempPath:     tempPath,
311				RemoteObject: api.RemoteObject{StoreURL: storeUrl},
312			},
313		},
314	}
315
316	for _, test := range tests {
317		t.Run(test.name, func(t *testing.T) {
318			var buffer bytes.Buffer
319			writer := multipart.NewWriter(&buffer)
320			file, err := writer.CreateFormFile("file", "my.file")
321			require.NoError(t, err)
322			fmt.Fprint(file, "test")
323			writer.Close()
324
325			httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
326			require.NoError(t, err)
327			httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
328
329			response := httptest.NewRecorder()
330			apiResponse := &api.Response{TempPath: tempPath}
331			preparer := &DefaultPreparer{}
332			opts, _, err := preparer.Prepare(apiResponse)
333			require.NoError(t, err)
334
335			HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
336
337			require.Equal(t, 200, response.Code)
338		})
339	}
340
341}
342
343func TestInvalidFileNames(t *testing.T) {
344	testhelper.ConfigureSecret()
345
346	tempPath, err := ioutil.TempDir("", "uploads")
347	require.NoError(t, err)
348	defer os.RemoveAll(tempPath)
349
350	for _, testCase := range []struct {
351		filename       string
352		code           int
353		expectedPrefix string
354	}{
355		{"foobar", 200, "foobar"}, // sanity check for test setup below
356		{"foo/bar", 200, "bar"},
357		{"foo/bar/baz", 200, "baz"},
358		{"/../../foobar", 200, "foobar"},
359		{".", 500, ""},
360		{"..", 500, ""},
361		{"./", 500, ""},
362	} {
363		buffer := &bytes.Buffer{}
364
365		writer := multipart.NewWriter(buffer)
366		file, err := writer.CreateFormFile("file", testCase.filename)
367		require.NoError(t, err)
368		fmt.Fprint(file, "test")
369		writer.Close()
370
371		httpRequest, err := http.NewRequest("POST", "/example", buffer)
372		require.NoError(t, err)
373		httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
374
375		response := httptest.NewRecorder()
376		apiResponse := &api.Response{TempPath: tempPath}
377		preparer := &DefaultPreparer{}
378		opts, _, err := preparer.Prepare(apiResponse)
379		require.NoError(t, err)
380
381		HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts)
382		require.Equal(t, testCase.code, response.Code)
383		require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix)
384	}
385}
386
387func TestUploadHandlerRemovingExif(t *testing.T) {
388	content, err := ioutil.ReadFile("exif/testdata/sample_exif.jpg")
389	require.NoError(t, err)
390
391	runUploadTest(t, content, "sample_exif.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
392		err := r.ParseMultipartForm(100000)
393		require.NoError(t, err)
394
395		size, err := strconv.Atoi(r.FormValue("file.size"))
396		require.NoError(t, err)
397		require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
398		require.True(t, size > 0, "Expected to receive not empty file")
399
400		w.WriteHeader(200)
401		fmt.Fprint(w, "RESPONSE")
402	})
403}
404
405func TestUploadHandlerRemovingExifTiff(t *testing.T) {
406	content, err := ioutil.ReadFile("exif/testdata/sample_exif.tiff")
407	require.NoError(t, err)
408
409	runUploadTest(t, content, "sample_exif.tiff", 200, func(w http.ResponseWriter, r *http.Request) {
410		err := r.ParseMultipartForm(100000)
411		require.NoError(t, err)
412
413		size, err := strconv.Atoi(r.FormValue("file.size"))
414		require.NoError(t, err)
415		require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
416		require.True(t, size > 0, "Expected to receive not empty file")
417
418		w.WriteHeader(200)
419		fmt.Fprint(w, "RESPONSE")
420	})
421}
422
423func TestUploadHandlerRemovingExifInvalidContentType(t *testing.T) {
424	content, err := ioutil.ReadFile("exif/testdata/sample_exif_invalid.jpg")
425	require.NoError(t, err)
426
427	runUploadTest(t, content, "sample_exif_invalid.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
428		err := r.ParseMultipartForm(100000)
429		require.NoError(t, err)
430
431		output, err := ioutil.ReadFile(r.FormValue("file.path"))
432		require.NoError(t, err)
433		require.Equal(t, content, output, "Expected the file to be same as before")
434
435		w.WriteHeader(200)
436		fmt.Fprint(w, "RESPONSE")
437	})
438}
439
440func TestUploadHandlerRemovingExifCorruptedFile(t *testing.T) {
441	content, err := ioutil.ReadFile("exif/testdata/sample_exif_corrupted.jpg")
442	require.NoError(t, err)
443
444	runUploadTest(t, content, "sample_exif_corrupted.jpg", 422, func(w http.ResponseWriter, r *http.Request) {
445		err := r.ParseMultipartForm(100000)
446		require.Error(t, err)
447	})
448}
449
450func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, tsHandler func(http.ResponseWriter, *http.Request)) {
451	tempPath, err := ioutil.TempDir("", "uploads")
452	require.NoError(t, err)
453	defer os.RemoveAll(tempPath)
454
455	var buffer bytes.Buffer
456
457	writer := multipart.NewWriter(&buffer)
458	file, err := writer.CreateFormFile("file", filename)
459	require.NoError(t, err)
460
461	_, err = file.Write(image)
462	require.NoError(t, err)
463
464	err = writer.Close()
465	require.NoError(t, err)
466
467	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), tsHandler)
468	defer ts.Close()
469
470	httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", &buffer)
471	require.NoError(t, err)
472
473	ctx, cancel := context.WithCancel(context.Background())
474	defer cancel()
475
476	httpRequest = httpRequest.WithContext(ctx)
477	httpRequest.ContentLength = int64(buffer.Len())
478	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
479	response := httptest.NewRecorder()
480
481	handler := newProxy(ts.URL)
482	apiResponse := &api.Response{TempPath: tempPath}
483	preparer := &DefaultPreparer{}
484	opts, _, err := preparer.Prepare(apiResponse)
485	require.NoError(t, err)
486
487	HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
488	require.Equal(t, httpCode, response.Code)
489}
490
491func newProxy(url string) *proxy.Proxy {
492	parsedURL := helper.URLMustParse(url)
493	return proxy.NewProxy(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
494}
495
496func waitUntilDeleted(t *testing.T, path string) {
497	var err error
498
499	// Poll because the file removal is async
500	for i := 0; i < 100; i++ {
501		_, err = os.Stat(path)
502		if err != nil {
503			break
504		}
505		time.Sleep(100 * time.Millisecond)
506	}
507
508	require.True(t, os.IsNotExist(err), "expected the file to be deleted")
509}
510