1package acceptance_test
2
3import (
4	"bytes"
5	"context"
6	"crypto/tls"
7	"crypto/x509"
8	"encoding/json"
9	"errors"
10	"fmt"
11	"io"
12	"io/fs"
13	"net"
14	"net/http"
15	"net/http/httptest"
16	"os"
17	"os/exec"
18	"path"
19	"strings"
20	"sync"
21	"testing"
22	"time"
23
24	"github.com/gorilla/mux"
25	"github.com/pires/go-proxyproto"
26	"github.com/stretchr/testify/require"
27	"golang.org/x/net/nettest"
28
29	"gitlab.com/gitlab-org/gitlab-pages/internal/request"
30	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
31	"gitlab.com/gitlab-org/gitlab-pages/test/acceptance/testdata"
32)
33
34// The HTTPS certificate isn't signed by anyone. This http client is set up
35// so it can talk to servers using it.
36var (
37	// The HTTPS certificate isn't signed by anyone. This http client is set up
38	// so it can talk to servers using it.
39	TestHTTPSClient = &http.Client{
40		Transport: &http.Transport{
41			TLSClientConfig: &tls.Config{RootCAs: TestCertPool},
42		},
43	}
44
45	// Use HTTP with a very short timeout to repeatedly check for the server to be
46	// up. Again, ignore HTTP
47	QuickTimeoutHTTPSClient = &http.Client{
48		Transport: &http.Transport{
49			TLSClientConfig:       &tls.Config{RootCAs: TestCertPool},
50			ResponseHeaderTimeout: 100 * time.Millisecond,
51		},
52	}
53
54	// Proxyv2 client
55	TestProxyv2Client = &http.Client{
56		Transport: &http.Transport{
57			DialContext:     Proxyv2DialContext,
58			TLSClientConfig: &tls.Config{RootCAs: TestCertPool},
59		},
60	}
61
62	QuickTimeoutProxyv2Client = &http.Client{
63		Transport: &http.Transport{
64			DialContext:           Proxyv2DialContext,
65			TLSClientConfig:       &tls.Config{RootCAs: TestCertPool},
66			ResponseHeaderTimeout: 100 * time.Millisecond,
67		},
68	}
69
70	TestCertPool = x509.NewCertPool()
71
72	// Proxyv2 will create a dummy request with src 10.1.1.1:1000
73	// and dst 20.2.2.2:2000
74	Proxyv2DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
75		var d net.Dialer
76
77		conn, err := d.DialContext(ctx, network, addr)
78		if err != nil {
79			return nil, err
80		}
81
82		header := &proxyproto.Header{
83			Version:            2,
84			Command:            proxyproto.PROXY,
85			TransportProtocol:  proxyproto.TCPv4,
86			SourceAddress:      net.ParseIP("10.1.1.1"),
87			SourcePort:         1000,
88			DestinationAddress: net.ParseIP("20.2.2.2"),
89			DestinationPort:    2000,
90		}
91
92		_, err = header.WriteTo(conn)
93
94		return conn, err
95	}
96)
97
98type tWriter struct {
99	t *testing.T
100}
101
102func (t *tWriter) Write(b []byte) (int, error) {
103	t.t.Log(string(bytes.TrimRight(b, "\r\n")))
104
105	return len(b), nil
106}
107
108type LogCaptureBuffer struct {
109	b bytes.Buffer
110	m sync.Mutex
111}
112
113func (b *LogCaptureBuffer) Read(p []byte) (n int, err error) {
114	b.m.Lock()
115	defer b.m.Unlock()
116
117	return b.b.Read(p)
118}
119func (b *LogCaptureBuffer) Write(p []byte) (n int, err error) {
120	b.m.Lock()
121	defer b.m.Unlock()
122
123	return b.b.Write(p)
124}
125func (b *LogCaptureBuffer) String() string {
126	b.m.Lock()
127	defer b.m.Unlock()
128
129	return b.b.String()
130}
131func (b *LogCaptureBuffer) Reset() {
132	b.m.Lock()
133	defer b.m.Unlock()
134
135	b.b.Reset()
136}
137
138// ListenSpec is used to point at a gitlab-pages http server, preserving the
139// type of port it is (http, https, proxy)
140type ListenSpec struct {
141	Type string
142	Host string
143	Port string
144}
145
146func supportedListeners() []ListenSpec {
147	if !nettest.SupportsIPv6() {
148		return ipv4Listeners
149	}
150
151	return listeners
152}
153
154func (l ListenSpec) URL(suffix string) string {
155	scheme := request.SchemeHTTP
156	if l.Type == request.SchemeHTTPS || l.Type == "https-proxyv2" {
157		scheme = request.SchemeHTTPS
158	}
159
160	suffix = strings.TrimPrefix(suffix, "/")
161
162	return fmt.Sprintf("%s://%s/%s", scheme, l.JoinHostPort(), suffix)
163}
164
165// Returns only once this spec points at a working TCP server
166func (l ListenSpec) WaitUntilRequestSucceeds(done chan struct{}) error {
167	timeout := 5 * time.Second
168	for start := time.Now(); time.Since(start) < timeout; {
169		select {
170		case <-done:
171			return fmt.Errorf("server has shut down already")
172		default:
173		}
174
175		req, err := http.NewRequest("GET", l.URL("/"), nil)
176		if err != nil {
177			return err
178		}
179
180		client := QuickTimeoutHTTPSClient
181		if l.Type == "https-proxyv2" {
182			client = QuickTimeoutProxyv2Client
183		}
184
185		response, err := client.Transport.RoundTrip(req)
186		if err != nil {
187			time.Sleep(100 * time.Millisecond)
188			continue
189		}
190		response.Body.Close()
191
192		if code := response.StatusCode; code >= 200 && code < 500 {
193			return nil
194		}
195
196		time.Sleep(100 * time.Millisecond)
197	}
198
199	return fmt.Errorf("timed out after %v waiting for listener %v", timeout, l)
200}
201
202func (l ListenSpec) JoinHostPort() string {
203	return net.JoinHostPort(l.Host, l.Port)
204}
205
206func RunPagesProcess(t *testing.T, opts ...processOption) *LogCaptureBuffer {
207	chdir := false
208	chdirCleanup := testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)
209
210	wd, err := os.Getwd()
211	require.NoError(t, err)
212
213	processCfg := defaultProcessConfig
214
215	for _, opt := range opts {
216		opt(&processCfg)
217	}
218
219	if processCfg.gitlabStubOpts.pagesRoot == "" {
220		processCfg.gitlabStubOpts.pagesRoot = wd
221	}
222
223	source := NewGitlabDomainsSourceStub(t, processCfg.gitlabStubOpts)
224
225	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
226	processCfg.extraArgs = append(
227		processCfg.extraArgs,
228		"-pages-root", wd,
229		"-internal-gitlab-server", source.URL,
230		"-api-secret-key", gitLabAPISecretKey,
231	)
232
233	logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...)
234
235	t.Cleanup(func() {
236		source.Close()
237		chdirCleanup()
238		cleanup()
239	})
240
241	return logBuf
242}
243
244func RunPagesProcessWithSSLCertFile(t *testing.T, listeners []ListenSpec, sslCertFile string) {
245	RunPagesProcess(t,
246		withListeners(listeners),
247		withArguments([]string{
248			"-config=" + defaultAuthConfig(t),
249		}),
250		withEnv([]string{"SSL_CERT_FILE=" + sslCertFile}),
251	)
252}
253
254func RunPagesProcessWithSSLCertDir(t *testing.T, listeners []ListenSpec, sslCertFile string) {
255	// Create temporary cert dir
256	sslCertDir, err := os.MkdirTemp("", "pages-test-SSL_CERT_DIR")
257	require.NoError(t, err)
258
259	// Copy sslCertFile into temp cert dir
260	err = copyFile(sslCertDir+"/"+path.Base(sslCertFile), sslCertFile)
261	require.NoError(t, err)
262
263	RunPagesProcess(t,
264		withListeners(listeners),
265		withArguments([]string{
266			"-config=" + defaultAuthConfig(t),
267		}),
268		withEnv([]string{"SSL_CERT_DIR=" + sslCertDir}),
269	)
270
271	t.Cleanup(func() {
272		os.RemoveAll(sslCertDir)
273	})
274}
275
276func runPagesProcess(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, promPort string, extraEnv []string, extraArgs ...string) (*LogCaptureBuffer, func()) {
277	t.Helper()
278
279	_, err := os.Stat(pagesBinary)
280	require.NoError(t, err)
281
282	logBuf := &LogCaptureBuffer{}
283	out := io.MultiWriter(&tWriter{t}, logBuf)
284
285	args, tempfiles := getPagesArgs(t, listeners, promPort, extraArgs)
286	cmd := exec.Command(pagesBinary, args...)
287	cmd.Env = append(os.Environ(), extraEnv...)
288	cmd.Stdout = out
289	cmd.Stderr = out
290	require.NoError(t, cmd.Start())
291	t.Logf("Running %s %v", pagesBinary, args)
292
293	waitCh := make(chan struct{})
294	go func() {
295		cmd.Wait()
296		for _, tempfile := range tempfiles {
297			os.Remove(tempfile)
298		}
299		close(waitCh)
300	}()
301
302	cleanup := func() {
303		cmd.Process.Signal(os.Interrupt)
304		<-waitCh
305	}
306
307	if wait {
308		for _, spec := range listeners {
309			if err := spec.WaitUntilRequestSucceeds(waitCh); err != nil {
310				cleanup()
311				t.Fatal(err)
312			}
313		}
314	}
315
316	return logBuf, cleanup
317}
318
319func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraArgs []string) (args, tempfiles []string) {
320	var hasHTTPS bool
321	args = append(args, "-log-verbose=true")
322
323	for _, spec := range listeners {
324		args = append(args, "-listen-"+spec.Type, spec.JoinHostPort())
325
326		if strings.Contains(spec.Type, request.SchemeHTTPS) {
327			hasHTTPS = true
328		}
329	}
330
331	if hasHTTPS {
332		key, cert := CreateHTTPSFixtureFiles(t)
333		tempfiles = []string{key, cert}
334		args = append(args, "-root-key", key, "-root-cert", cert)
335	}
336
337	if !contains(extraArgs, "-pages-root") {
338		args = append(args, "-pages-root", "../../shared/pages")
339	}
340
341	// default resolver configuration to execute tests faster
342	if !contains(extraArgs, "-gitlab-retrieval-") {
343		args = append(args, "-gitlab-retrieval-timeout", "50ms",
344			"-gitlab-retrieval-interval", "10ms",
345			"-gitlab-retrieval-retries", "1")
346	}
347
348	if promPort != "" {
349		args = append(args, "-metrics-address", promPort)
350	}
351
352	args = append(args, extraArgs...)
353
354	return
355}
356
357func contains(slice []string, s string) bool {
358	for _, e := range slice {
359		if strings.Contains(e, s) {
360			return true
361		}
362	}
363	return false
364}
365
366// Does a HTTP(S) GET against the listener specified, setting a fake
367// Host: and constructing the URL from the listener and the URL suffix.
368func GetPageFromListener(t *testing.T, spec ListenSpec, host, urlsuffix string) (*http.Response, error) {
369	return GetPageFromListenerWithHeaders(t, spec, host, urlsuffix, http.Header{})
370}
371
372func GetPageFromListenerWithHeaders(t *testing.T, spec ListenSpec, host, urlSuffix string, header http.Header) (*http.Response, error) {
373	t.Helper()
374
375	url := spec.URL(urlSuffix)
376	req, err := http.NewRequest("GET", url, nil)
377	if err != nil {
378		return nil, err
379	}
380
381	req.Host = host
382	req.Header = header
383
384	return DoPagesRequest(t, spec, req)
385}
386
387func DoPagesRequest(t *testing.T, spec ListenSpec, req *http.Request) (*http.Response, error) {
388	t.Logf("curl -X %s -H'Host: %s' %s", req.Method, req.Host, req.URL)
389
390	if spec.Type == "https-proxyv2" {
391		return TestProxyv2Client.Do(req)
392	}
393
394	return TestHTTPSClient.Do(req)
395}
396
397func GetRedirectPage(t *testing.T, spec ListenSpec, host, urlsuffix string) (*http.Response, error) {
398	return GetRedirectPageWithCookie(t, spec, host, urlsuffix, "")
399}
400
401func GetProxyRedirectPageWithCookie(t *testing.T, spec ListenSpec, host string, urlsuffix string, cookie string, https bool) (*http.Response, error) {
402	schema := request.SchemeHTTP
403	if https {
404		schema = request.SchemeHTTPS
405	}
406	header := http.Header{
407		"X-Forwarded-Proto": []string{schema},
408		"X-Forwarded-Host":  []string{host},
409		"cookie":            []string{cookie},
410	}
411
412	return GetRedirectPageWithHeaders(t, spec, host, urlsuffix, header)
413}
414
415func GetRedirectPageWithCookie(t *testing.T, spec ListenSpec, host, urlsuffix string, cookie string) (*http.Response, error) {
416	return GetRedirectPageWithHeaders(t, spec, host, urlsuffix, http.Header{"cookie": []string{cookie}})
417}
418
419func GetRedirectPageWithHeaders(t *testing.T, spec ListenSpec, host, urlsuffix string, header http.Header) (*http.Response, error) {
420	url := spec.URL(urlsuffix)
421	req, err := http.NewRequest("GET", url, nil)
422	if err != nil {
423		return nil, err
424	}
425	req.Header = header
426
427	req.Host = host
428
429	if spec.Type == "https-proxyv2" {
430		return TestProxyv2Client.Transport.RoundTrip(req)
431	}
432
433	return TestHTTPSClient.Transport.RoundTrip(req)
434}
435
436func ClientWithConfig(tlsConfig *tls.Config) (*http.Client, func()) {
437	tlsConfig.RootCAs = TestCertPool
438	tr := &http.Transport{TLSClientConfig: tlsConfig}
439	client := &http.Client{Transport: tr}
440
441	return client, tr.CloseIdleConnections
442}
443
444type stubOpts struct {
445	m                   sync.RWMutex
446	apiCalled           bool
447	authHandler         http.HandlerFunc
448	userHandler         http.HandlerFunc
449	pagesHandler        http.HandlerFunc
450	pagesStatusResponse int
451	pagesRoot           string
452	delay               time.Duration
453}
454
455func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
456	t.Helper()
457	require.NotNil(t, opts)
458
459	router := mux.NewRouter()
460
461	pagesHandler := defaultAPIHandler(t, opts)
462	if opts.pagesHandler != nil {
463		pagesHandler = opts.pagesHandler
464	}
465
466	router.HandleFunc("/api/v4/internal/pages", pagesHandler)
467
468	authHandler := defaultAuthHandler(t)
469	if opts.authHandler != nil {
470		authHandler = opts.authHandler
471	}
472
473	router.HandleFunc("/oauth/token", authHandler)
474
475	userHandler := defaultUserHandler(t)
476	if opts.userHandler != nil {
477		userHandler = opts.userHandler
478	}
479
480	router.HandleFunc("/api/v4/user", userHandler)
481
482	router.HandleFunc("/api/v4/projects/{project_id:[0-9]+}/pages_access", func(w http.ResponseWriter, r *http.Request) {
483		if handleAccessControlArtifactRequests(t, w, r) {
484			return
485		}
486
487		handleAccessControlRequests(t, w, r)
488	})
489
490	return httptest.NewServer(router)
491}
492
493func (o *stubOpts) setAPICalled(v bool) {
494	o.m.Lock()
495	defer o.m.Unlock()
496
497	o.apiCalled = v
498}
499
500func (o *stubOpts) getAPICalled() bool {
501	o.m.RLock()
502	defer o.m.RUnlock()
503
504	return o.apiCalled
505}
506
507func lookupFromFile(t *testing.T, domain string, w http.ResponseWriter) {
508	fixture, err := os.Open("../../shared/lookups/" + domain + ".json")
509	if errors.Is(err, fs.ErrNotExist) {
510		w.WriteHeader(http.StatusNoContent)
511
512		t.Logf("GitLab domain %s source stub served 204", domain)
513		return
514	}
515
516	defer fixture.Close()
517	require.NoError(t, err)
518
519	_, err = io.Copy(w, fixture)
520	require.NoError(t, err)
521
522	t.Logf("GitLab domain %s source stub served lookup", domain)
523}
524
525func defaultAPIHandler(t *testing.T, opts *stubOpts) http.HandlerFunc {
526	return func(w http.ResponseWriter, r *http.Request) {
527		domain := r.URL.Query().Get("host")
528		if domain == "127.0.0.1" {
529			// shortcut for healthy checkup done by WaitUntilRequestSucceeds
530			w.WriteHeader(http.StatusNoContent)
531			return
532		}
533		// to test slow responses from the API
534		if opts.delay > 0 {
535			time.Sleep(opts.delay)
536		}
537
538		opts.setAPICalled(true)
539
540		if opts.pagesStatusResponse != 0 {
541			w.WriteHeader(opts.pagesStatusResponse)
542			return
543		}
544
545		// check if predefined response exists
546		if responseFn, ok := testdata.DomainResponses[domain]; ok {
547			err := json.NewEncoder(w).Encode(responseFn(t, opts.pagesRoot))
548			require.NoError(t, err)
549			return
550		}
551
552		// serve lookup from files
553		lookupFromFile(t, domain, w)
554	}
555}
556
557func defaultAuthHandler(t *testing.T) http.HandlerFunc {
558	return func(w http.ResponseWriter, r *http.Request) {
559		require.Equal(t, "POST", r.Method)
560		err := json.NewEncoder(w).Encode(struct {
561			AccessToken string `json:"access_token"`
562		}{
563			AccessToken: "abc",
564		})
565		require.NoError(t, err)
566	}
567}
568
569func defaultUserHandler(t *testing.T) http.HandlerFunc {
570	return func(w http.ResponseWriter, r *http.Request) {
571		require.Equal(t, "Bearer abc", r.Header.Get("Authorization"))
572		w.WriteHeader(http.StatusOK)
573	}
574}
575
576func newConfigFile(t *testing.T, configs ...string) string {
577	t.Helper()
578
579	f, err := os.CreateTemp(os.TempDir(), "gitlab-pages-config")
580	require.NoError(t, err)
581	defer f.Close()
582
583	for _, config := range configs {
584		_, err := fmt.Fprintf(f, "%s\n", config)
585		require.NoError(t, err)
586	}
587
588	return f.Name()
589}
590
591func defaultConfigFileWith(t *testing.T, configs ...string) string {
592	t.Helper()
593
594	configs = append(configs, "auth-client-id=clientID",
595		"auth-client-secret=clientSecret",
596		"auth-secret=authSecret",
597		"auth-scope=authScope",
598	)
599
600	name := newConfigFile(t, configs...)
601
602	t.Cleanup(func() {
603		err := os.Remove(name)
604		require.NoError(t, err)
605	})
606
607	return name
608}
609
610func copyFile(dest, src string) error {
611	srcFile, err := os.Open(src)
612	if err != nil {
613		return err
614	}
615	defer srcFile.Close()
616
617	srcInfo, err := srcFile.Stat()
618	if err != nil {
619		return err
620	}
621
622	destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, srcInfo.Mode())
623	if err != nil {
624		return err
625	}
626	defer destFile.Close()
627
628	_, err = io.Copy(destFile, srcFile)
629	return err
630}
631
632func setupTransport(t *testing.T) {
633	t.Helper()
634
635	transport := (TestHTTPSClient.Transport).(*http.Transport)
636	defer func(t time.Duration) {
637		transport.ResponseHeaderTimeout = t
638	}(transport.ResponseHeaderTimeout)
639	transport.ResponseHeaderTimeout = 5 * time.Second
640}
641