1/*
2Copyright 2015 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package server
18
19import (
20	"context"
21	"encoding/json"
22	"fmt"
23	"io"
24	"io/ioutil"
25	"net"
26	"net/http"
27	"net/http/httptest"
28	goruntime "runtime"
29	"strconv"
30	"sync"
31	"testing"
32	"time"
33
34	"github.com/stretchr/testify/assert"
35
36	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
37	"k8s.io/apimachinery/pkg/runtime"
38	"k8s.io/apimachinery/pkg/runtime/schema"
39	"k8s.io/apimachinery/pkg/runtime/serializer"
40	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
41	"k8s.io/apimachinery/pkg/util/sets"
42	"k8s.io/apimachinery/pkg/util/wait"
43	"k8s.io/apimachinery/pkg/version"
44	"k8s.io/apiserver/pkg/apis/example"
45	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
46	"k8s.io/apiserver/pkg/authorization/authorizer"
47	"k8s.io/apiserver/pkg/endpoints/discovery"
48	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
49	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
50	"k8s.io/apiserver/pkg/registry/rest"
51	genericfilters "k8s.io/apiserver/pkg/server/filters"
52	"k8s.io/client-go/informers"
53	"k8s.io/client-go/kubernetes/fake"
54	restclient "k8s.io/client-go/rest"
55	kubeopenapi "k8s.io/kube-openapi/pkg/common"
56	"k8s.io/kube-openapi/pkg/validation/spec"
57)
58
59const (
60	extensionsGroupName = "extensions"
61)
62
63var (
64	v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"}
65
66	scheme         = runtime.NewScheme()
67	codecs         = serializer.NewCodecFactory(scheme)
68	parameterCodec = runtime.NewParameterCodec(scheme)
69)
70
71func init() {
72	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
73	scheme.AddUnversionedTypes(v1GroupVersion,
74		&metav1.Status{},
75		&metav1.APIVersions{},
76		&metav1.APIGroupList{},
77		&metav1.APIGroup{},
78		&metav1.APIResourceList{},
79	)
80	utilruntime.Must(example.AddToScheme(scheme))
81	utilruntime.Must(examplev1.AddToScheme(scheme))
82}
83
84func buildTestOpenAPIDefinition() kubeopenapi.OpenAPIDefinition {
85	return kubeopenapi.OpenAPIDefinition{
86		Schema: spec.Schema{
87			SchemaProps: spec.SchemaProps{
88				Description: "Description",
89				Properties:  map[string]spec.Schema{},
90			},
91			VendorExtensible: spec.VendorExtensible{
92				Extensions: spec.Extensions{
93					"x-kubernetes-group-version-kind": []interface{}{
94						map[string]interface{}{
95							"group":   "",
96							"version": "v1",
97							"kind":    "Getter",
98						},
99						map[string]interface{}{
100							"group":   "batch",
101							"version": "v1",
102							"kind":    "Getter",
103						},
104						map[string]interface{}{
105							"group":   "extensions",
106							"version": "v1",
107							"kind":    "Getter",
108						},
109					},
110				},
111			},
112		},
113	}
114}
115
116func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeopenapi.OpenAPIDefinition {
117	return map[string]kubeopenapi.OpenAPIDefinition{
118		"k8s.io/apimachinery/pkg/apis/meta/v1.Status":          {},
119		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":     {},
120		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList":    {},
121		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup":        buildTestOpenAPIDefinition(),
122		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList": {},
123	}
124}
125
126// setUp is a convience function for setting up for (most) tests.
127func setUp(t *testing.T) (Config, *assert.Assertions) {
128	config := NewConfig(codecs)
129	config.ExternalAddress = "192.168.10.4:443"
130	config.PublicAddress = net.ParseIP("192.168.10.4")
131	config.LegacyAPIGroupPrefixes = sets.NewString("/api")
132	config.LoopbackClientConfig = &restclient.Config{}
133
134	clientset := fake.NewSimpleClientset()
135	if clientset == nil {
136		t.Fatal("unable to create fake client set")
137	}
138
139	config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
140	config.OpenAPIConfig.Info.Version = "unversioned"
141	sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout)
142	config.Complete(sharedInformers)
143
144	return *config, assert.New(t)
145}
146
147func newMaster(t *testing.T) (*GenericAPIServer, Config, *assert.Assertions) {
148	config, assert := setUp(t)
149
150	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
151	if err != nil {
152		t.Fatalf("Error in bringing up the server: %v", err)
153	}
154	return s, config, assert
155}
156
157// TestNew verifies that the New function returns a GenericAPIServer
158// using the configuration properly.
159func TestNew(t *testing.T) {
160	s, config, assert := newMaster(t)
161
162	// Verify many of the variables match their config counterparts
163	assert.Equal(s.legacyAPIGroupPrefixes, config.LegacyAPIGroupPrefixes)
164	assert.Equal(s.admissionControl, config.AdmissionControl)
165}
166
167// Verifies that AddGroupVersions works as expected.
168func TestInstallAPIGroups(t *testing.T) {
169	config, assert := setUp(t)
170
171	config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
172	config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"}
173
174	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
175	if err != nil {
176		t.Fatalf("Error in bringing up the server: %v", err)
177	}
178
179	testAPI := func(gv schema.GroupVersion) APIGroupInfo {
180		getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{}
181
182		scheme := runtime.NewScheme()
183		scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New())
184		scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New())
185		scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{})
186		metav1.AddToGroupVersion(scheme, v1GroupVersion)
187
188		return APIGroupInfo{
189			PrioritizedVersions: []schema.GroupVersion{gv},
190			VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
191				gv.Version: {
192					"getter":  &testGetterStorage{Version: gv.Version},
193					"noverbs": &testNoVerbsStorage{Version: gv.Version},
194				},
195			},
196			OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
197			ParameterCodec:         parameterCodec,
198			NegotiatedSerializer:   codecs,
199			Scheme:                 scheme,
200		}
201	}
202
203	apis := []APIGroupInfo{
204		testAPI(schema.GroupVersion{Group: "", Version: "v1"}),
205		testAPI(schema.GroupVersion{Group: extensionsGroupName, Version: "v1"}),
206		testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}),
207	}
208
209	err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0])
210	assert.NoError(err)
211	groupPaths := []string{
212		config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix
213	}
214	for _, api := range apis[1:] {
215		err = s.InstallAPIGroup(&api)
216		assert.NoError(err)
217		groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/<group>
218	}
219
220	server := httptest.NewServer(s.Handler)
221	defer server.Close()
222
223	for i := range apis {
224		// should serve APIGroup at group path
225		info := &apis[i]
226		path := groupPaths[i]
227		resp, err := http.Get(server.URL + path)
228		if err != nil {
229			t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
230			continue
231		}
232
233		body, err := ioutil.ReadAll(resp.Body)
234		if err != nil {
235			t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
236			continue
237		}
238
239		t.Logf("[%d] json at %s: %s", i, path, string(body))
240
241		if i == 0 {
242			// legacy API returns APIVersions
243			group := metav1.APIVersions{}
244			err = json.Unmarshal(body, &group)
245			if err != nil {
246				t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
247				continue
248			}
249		} else {
250			// API groups return APIGroup
251			group := metav1.APIGroup{}
252			err = json.Unmarshal(body, &group)
253			if err != nil {
254				t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
255				continue
256			}
257
258			if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected {
259				t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected)
260				continue
261			}
262
263			if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected {
264				t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected)
265				continue
266			}
267		}
268
269		// should serve APIResourceList at group path + /<group-version>
270		path = path + "/" + info.PrioritizedVersions[0].Version
271		resp, err = http.Get(server.URL + path)
272		if err != nil {
273			t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
274			continue
275		}
276
277		body, err = ioutil.ReadAll(resp.Body)
278		if err != nil {
279			t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
280			continue
281		}
282
283		t.Logf("[%d] json at %s: %s", i, path, string(body))
284
285		resources := metav1.APIResourceList{}
286		err = json.Unmarshal(body, &resources)
287		if err != nil {
288			t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
289			continue
290		}
291
292		if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected {
293			t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected)
294			continue
295		}
296
297		// the verbs should match the features of resources
298		for _, r := range resources.APIResources {
299			switch r.Name {
300			case "getter":
301				if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) {
302					t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
303				}
304			case "noverbs":
305				if r.Verbs == nil {
306					t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i)
307				}
308				if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) {
309					t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
310				}
311			}
312		}
313	}
314}
315
316func TestPrepareRun(t *testing.T) {
317	s, config, assert := newMaster(t)
318
319	assert.NotNil(config.OpenAPIConfig)
320
321	server := httptest.NewServer(s.Handler.Director)
322	defer server.Close()
323	done := make(chan struct{})
324	defer close(done)
325
326	s.PrepareRun()
327	s.RunPostStartHooks(done)
328
329	// openapi is installed in PrepareRun
330	resp, err := http.Get(server.URL + "/openapi/v2")
331	assert.NoError(err)
332	assert.Equal(http.StatusOK, resp.StatusCode)
333
334	// wait for health (max-in-flight-filter is initialized asynchronously, can take a few milliseconds to initialize)
335	assert.NoError(wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
336		// healthz checks are installed in PrepareRun
337		resp, err = http.Get(server.URL + "/healthz")
338		assert.NoError(err)
339		data, _ := ioutil.ReadAll(resp.Body)
340		if http.StatusOK != resp.StatusCode {
341			t.Logf("got %d", resp.StatusCode)
342			t.Log(string(data))
343			return false, nil
344		}
345		return true, nil
346	}))
347	resp, err = http.Get(server.URL + "/healthz/ping")
348	assert.NoError(err)
349	assert.Equal(http.StatusOK, resp.StatusCode)
350}
351
352func TestUpdateOpenAPISpec(t *testing.T) {
353	s, _, assert := newMaster(t)
354	s.PrepareRun()
355	s.RunPostStartHooks(make(chan struct{}))
356
357	server := httptest.NewServer(s.Handler.Director)
358	defer server.Close()
359
360	// verify the static spec in record is what we currently serve
361	oldSpec, err := json.Marshal(s.StaticOpenAPISpec)
362	assert.NoError(err)
363
364	resp, err := http.Get(server.URL + "/openapi/v2")
365	assert.NoError(err)
366	assert.Equal(http.StatusOK, resp.StatusCode)
367
368	body, err := ioutil.ReadAll(resp.Body)
369	assert.NoError(err)
370	assert.Equal(oldSpec, body)
371	resp.Body.Close()
372
373	// verify we are able to update the served spec using the exposed service
374	newSpec := []byte(`{"swagger":"2.0","info":{"title":"Test Updated Generic API Server Swagger","version":"v0.1.0"},"paths":null}`)
375	swagger := new(spec.Swagger)
376	err = json.Unmarshal(newSpec, swagger)
377	assert.NoError(err)
378
379	err = s.OpenAPIVersionedService.UpdateSpec(swagger)
380	assert.NoError(err)
381
382	resp, err = http.Get(server.URL + "/openapi/v2")
383	assert.NoError(err)
384	defer resp.Body.Close()
385	assert.Equal(http.StatusOK, resp.StatusCode)
386
387	body, err = ioutil.ReadAll(resp.Body)
388	assert.NoError(err)
389	assert.Equal(newSpec, body)
390}
391
392// TestCustomHandlerChain verifies the handler chain with custom handler chain builder functions.
393func TestCustomHandlerChain(t *testing.T) {
394	config, _ := setUp(t)
395
396	var protected, called bool
397
398	config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler {
399		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
400			protected = true
401			apiHandler.ServeHTTP(w, req)
402		})
403	}
404	handler := http.HandlerFunc(func(r http.ResponseWriter, req *http.Request) {
405		called = true
406	})
407
408	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
409	if err != nil {
410		t.Fatalf("Error in bringing up the server: %v", err)
411	}
412
413	s.Handler.NonGoRestfulMux.Handle("/nonswagger", handler)
414	s.Handler.NonGoRestfulMux.Handle("/secret", handler)
415
416	type Test struct {
417		handler   http.Handler
418		path      string
419		protected bool
420	}
421	for i, test := range []Test{
422		{s.Handler, "/nonswagger", true},
423		{s.Handler, "/secret", true},
424	} {
425		protected, called = false, false
426
427		var w io.Reader
428		req, err := http.NewRequest("GET", test.path, w)
429		if err != nil {
430			t.Errorf("%d: Unexpected http error: %v", i, err)
431			continue
432		}
433
434		test.handler.ServeHTTP(httptest.NewRecorder(), req)
435
436		if !called {
437			t.Errorf("%d: Expected handler to be called.", i)
438		}
439		if test.protected != protected {
440			t.Errorf("%d: Expected protected=%v, got protected=%v.", i, test.protected, protected)
441		}
442	}
443}
444
445// TestNotRestRoutesHaveAuth checks that special non-routes are behind authz/authn.
446func TestNotRestRoutesHaveAuth(t *testing.T) {
447	config, _ := setUp(t)
448
449	authz := mockAuthorizer{}
450
451	config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
452	config.Authorization.Authorizer = &authz
453
454	config.EnableIndex = true
455	config.EnableProfiling = true
456
457	kubeVersion := fakeVersion()
458	config.Version = &kubeVersion
459
460	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
461	if err != nil {
462		t.Fatalf("Error in bringing up the server: %v", err)
463	}
464
465	for _, test := range []struct {
466		route string
467	}{
468		{"/"},
469		{"/debug/pprof/"},
470		{"/debug/flags/"},
471		{"/version"},
472	} {
473		resp := httptest.NewRecorder()
474		req, _ := http.NewRequest("GET", test.route, nil)
475		s.Handler.ServeHTTP(resp, req)
476		if resp.Code != 200 {
477			t.Errorf("route %q expected to work: code %d", test.route, resp.Code)
478			continue
479		}
480
481		if authz.lastURI != test.route {
482			t.Errorf("route %q expected to go through authorization, last route did: %q", test.route, authz.lastURI)
483		}
484	}
485}
486
487type mockAuthorizer struct {
488	lastURI string
489}
490
491func (authz *mockAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) {
492	authz.lastURI = a.GetPath()
493	return authorizer.DecisionAllow, "", nil
494}
495
496type testGetterStorage struct {
497	Version string
498}
499
500func (p *testGetterStorage) NamespaceScoped() bool {
501	return true
502}
503
504func (p *testGetterStorage) New() runtime.Object {
505	return &metav1.APIGroup{
506		TypeMeta: metav1.TypeMeta{
507			Kind:       "Getter",
508			APIVersion: p.Version,
509		},
510	}
511}
512
513func (p *testGetterStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
514	return nil, nil
515}
516
517type testNoVerbsStorage struct {
518	Version string
519}
520
521func (p *testNoVerbsStorage) NamespaceScoped() bool {
522	return true
523}
524
525func (p *testNoVerbsStorage) New() runtime.Object {
526	return &metav1.APIGroup{
527		TypeMeta: metav1.TypeMeta{
528			Kind:       "NoVerbs",
529			APIVersion: p.Version,
530		},
531	}
532}
533
534func fakeVersion() version.Info {
535	return version.Info{
536		Major:        "42",
537		Minor:        "42",
538		GitVersion:   "42",
539		GitCommit:    "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
540		GitTreeState: "Dirty",
541		BuildDate:    time.Now().String(),
542		GoVersion:    goruntime.Version(),
543		Compiler:     goruntime.Compiler,
544		Platform:     fmt.Sprintf("%s/%s", goruntime.GOOS, goruntime.GOARCH),
545	}
546}
547
548// TestGracefulShutdown verifies server shutdown after request handler finish.
549func TestGracefulShutdown(t *testing.T) {
550	config, _ := setUp(t)
551
552	var graceShutdown bool
553	wg := sync.WaitGroup{}
554	wg.Add(1)
555
556	config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler {
557		handler := genericfilters.WithWaitGroup(apiHandler, c.LongRunningFunc, c.HandlerChainWaitGroup)
558		handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
559		return handler
560	}
561
562	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
563	if err != nil {
564		t.Fatalf("Error in bringing up the server: %v", err)
565	}
566
567	twoSecondHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
568		wg.Done()
569		time.Sleep(2 * time.Second)
570		w.WriteHeader(http.StatusOK)
571		graceShutdown = true
572	})
573	okHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
574		w.WriteHeader(http.StatusOK)
575	})
576
577	s.Handler.NonGoRestfulMux.Handle("/test", twoSecondHandler)
578	s.Handler.NonGoRestfulMux.Handle("/200", okHandler)
579
580	insecureServer := &http.Server{
581		Addr:    "0.0.0.0:0",
582		Handler: s.Handler,
583	}
584	stopCh := make(chan struct{})
585
586	ln, err := net.Listen("tcp", insecureServer.Addr)
587	if err != nil {
588		t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err)
589	}
590
591	// get port
592	serverPort := ln.Addr().(*net.TCPAddr).Port
593	stoppedCh, _, err := RunServer(insecureServer, ln, 10*time.Second, stopCh)
594	if err != nil {
595		t.Fatalf("RunServer err: %v", err)
596	}
597
598	graceCh := make(chan struct{})
599	// mock a client request
600	go func() {
601		resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/test")
602		if err != nil {
603			t.Errorf("Unexpected http error: %v", err)
604		}
605		if resp.StatusCode != http.StatusOK {
606			t.Errorf("Unexpected http status code: %v", resp.StatusCode)
607		}
608		close(graceCh)
609	}()
610
611	// close stopCh after request sent to server to guarantee request handler is running.
612	wg.Wait()
613	close(stopCh)
614
615	time.Sleep(500 * time.Millisecond)
616	if _, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/200"); err == nil {
617		t.Errorf("Unexpected http success after stopCh was closed")
618	}
619
620	// wait for wait group handler finish
621	s.HandlerChainWaitGroup.Wait()
622	<-stoppedCh
623
624	// check server all handlers finished.
625	if !graceShutdown {
626		t.Errorf("server shutdown not gracefully.")
627	}
628	// check client to make sure receive response.
629	select {
630	case <-graceCh:
631		t.Logf("server shutdown gracefully.")
632	case <-time.After(30 * time.Second):
633		t.Errorf("Timed out waiting for response.")
634	}
635}
636