1/*
2Copyright 2015 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package server
18
19import (
20	"context"
21	"encoding/json"
22	"fmt"
23	"io"
24	"io/ioutil"
25	"net"
26	"net/http"
27	"net/http/httptest"
28	goruntime "runtime"
29	"strconv"
30	"sync"
31	"testing"
32	"time"
33
34	openapi "github.com/go-openapi/spec"
35	"github.com/stretchr/testify/assert"
36
37	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
38	"k8s.io/apimachinery/pkg/runtime"
39	"k8s.io/apimachinery/pkg/runtime/schema"
40	"k8s.io/apimachinery/pkg/runtime/serializer"
41	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
42	"k8s.io/apimachinery/pkg/util/sets"
43	"k8s.io/apimachinery/pkg/version"
44	"k8s.io/apiserver/pkg/apis/example"
45	examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
46	"k8s.io/apiserver/pkg/authorization/authorizer"
47	"k8s.io/apiserver/pkg/endpoints/discovery"
48	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
49	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
50	"k8s.io/apiserver/pkg/registry/rest"
51	genericfilters "k8s.io/apiserver/pkg/server/filters"
52	"k8s.io/client-go/informers"
53	"k8s.io/client-go/kubernetes/fake"
54	restclient "k8s.io/client-go/rest"
55	kubeopenapi "k8s.io/kube-openapi/pkg/common"
56)
57
58const (
59	extensionsGroupName = "extensions"
60)
61
62var (
63	v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"}
64
65	scheme         = runtime.NewScheme()
66	codecs         = serializer.NewCodecFactory(scheme)
67	parameterCodec = runtime.NewParameterCodec(scheme)
68)
69
70func init() {
71	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
72	scheme.AddUnversionedTypes(v1GroupVersion,
73		&metav1.Status{},
74		&metav1.APIVersions{},
75		&metav1.APIGroupList{},
76		&metav1.APIGroup{},
77		&metav1.APIResourceList{},
78	)
79	utilruntime.Must(example.AddToScheme(scheme))
80	utilruntime.Must(examplev1.AddToScheme(scheme))
81}
82
83func buildTestOpenAPIDefinition() kubeopenapi.OpenAPIDefinition {
84	return kubeopenapi.OpenAPIDefinition{
85		Schema: openapi.Schema{
86			SchemaProps: openapi.SchemaProps{
87				Description: "Description",
88				Properties:  map[string]openapi.Schema{},
89			},
90			VendorExtensible: openapi.VendorExtensible{
91				Extensions: openapi.Extensions{
92					"x-kubernetes-group-version-kind": []interface{}{
93						map[string]interface{}{
94							"group":   "",
95							"version": "v1",
96							"kind":    "Getter",
97						},
98						map[string]interface{}{
99							"group":   "batch",
100							"version": "v1",
101							"kind":    "Getter",
102						},
103						map[string]interface{}{
104							"group":   "extensions",
105							"version": "v1",
106							"kind":    "Getter",
107						},
108					},
109				},
110			},
111		},
112	}
113}
114
115func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeopenapi.OpenAPIDefinition {
116	return map[string]kubeopenapi.OpenAPIDefinition{
117		"k8s.io/apimachinery/pkg/apis/meta/v1.Status":          {},
118		"k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions":     {},
119		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList":    {},
120		"k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup":        buildTestOpenAPIDefinition(),
121		"k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList": {},
122	}
123}
124
125// setUp is a convience function for setting up for (most) tests.
126func setUp(t *testing.T) (Config, *assert.Assertions) {
127	config := NewConfig(codecs)
128	config.ExternalAddress = "192.168.10.4:443"
129	config.PublicAddress = net.ParseIP("192.168.10.4")
130	config.LegacyAPIGroupPrefixes = sets.NewString("/api")
131	config.LoopbackClientConfig = &restclient.Config{}
132
133	clientset := fake.NewSimpleClientset()
134	if clientset == nil {
135		t.Fatal("unable to create fake client set")
136	}
137
138	config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
139	config.OpenAPIConfig.Info.Version = "unversioned"
140	sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout)
141	config.Complete(sharedInformers)
142
143	return *config, assert.New(t)
144}
145
146func newMaster(t *testing.T) (*GenericAPIServer, Config, *assert.Assertions) {
147	config, assert := setUp(t)
148
149	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
150	if err != nil {
151		t.Fatalf("Error in bringing up the server: %v", err)
152	}
153	return s, config, assert
154}
155
156// TestNew verifies that the New function returns a GenericAPIServer
157// using the configuration properly.
158func TestNew(t *testing.T) {
159	s, config, assert := newMaster(t)
160
161	// Verify many of the variables match their config counterparts
162	assert.Equal(s.legacyAPIGroupPrefixes, config.LegacyAPIGroupPrefixes)
163	assert.Equal(s.admissionControl, config.AdmissionControl)
164}
165
166// Verifies that AddGroupVersions works as expected.
167func TestInstallAPIGroups(t *testing.T) {
168	config, assert := setUp(t)
169
170	config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
171	config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"}
172
173	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
174	if err != nil {
175		t.Fatalf("Error in bringing up the server: %v", err)
176	}
177
178	testAPI := func(gv schema.GroupVersion) APIGroupInfo {
179		getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{}
180
181		scheme := runtime.NewScheme()
182		scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New())
183		scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New())
184		scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{})
185		metav1.AddToGroupVersion(scheme, v1GroupVersion)
186
187		return APIGroupInfo{
188			PrioritizedVersions: []schema.GroupVersion{gv},
189			VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
190				gv.Version: {
191					"getter":  &testGetterStorage{Version: gv.Version},
192					"noverbs": &testNoVerbsStorage{Version: gv.Version},
193				},
194			},
195			OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
196			ParameterCodec:         parameterCodec,
197			NegotiatedSerializer:   codecs,
198			Scheme:                 scheme,
199		}
200	}
201
202	apis := []APIGroupInfo{
203		testAPI(schema.GroupVersion{Group: "", Version: "v1"}),
204		testAPI(schema.GroupVersion{Group: extensionsGroupName, Version: "v1"}),
205		testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}),
206	}
207
208	err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0])
209	assert.NoError(err)
210	groupPaths := []string{
211		config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix
212	}
213	for _, api := range apis[1:] {
214		err = s.InstallAPIGroup(&api)
215		assert.NoError(err)
216		groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/<group>
217	}
218
219	server := httptest.NewServer(s.Handler)
220	defer server.Close()
221
222	for i := range apis {
223		// should serve APIGroup at group path
224		info := &apis[i]
225		path := groupPaths[i]
226		resp, err := http.Get(server.URL + path)
227		if err != nil {
228			t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
229			continue
230		}
231
232		body, err := ioutil.ReadAll(resp.Body)
233		if err != nil {
234			t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
235			continue
236		}
237
238		t.Logf("[%d] json at %s: %s", i, path, string(body))
239
240		if i == 0 {
241			// legacy API returns APIVersions
242			group := metav1.APIVersions{}
243			err = json.Unmarshal(body, &group)
244			if err != nil {
245				t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
246				continue
247			}
248		} else {
249			// API groups return APIGroup
250			group := metav1.APIGroup{}
251			err = json.Unmarshal(body, &group)
252			if err != nil {
253				t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
254				continue
255			}
256
257			if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected {
258				t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected)
259				continue
260			}
261
262			if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected {
263				t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected)
264				continue
265			}
266		}
267
268		// should serve APIResourceList at group path + /<group-version>
269		path = path + "/" + info.PrioritizedVersions[0].Version
270		resp, err = http.Get(server.URL + path)
271		if err != nil {
272			t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err)
273			continue
274		}
275
276		body, err = ioutil.ReadAll(resp.Body)
277		if err != nil {
278			t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err)
279			continue
280		}
281
282		t.Logf("[%d] json at %s: %s", i, path, string(body))
283
284		resources := metav1.APIResourceList{}
285		err = json.Unmarshal(body, &resources)
286		if err != nil {
287			t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err)
288			continue
289		}
290
291		if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected {
292			t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected)
293			continue
294		}
295
296		// the verbs should match the features of resources
297		for _, r := range resources.APIResources {
298			switch r.Name {
299			case "getter":
300				if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) {
301					t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
302				}
303			case "noverbs":
304				if r.Verbs == nil {
305					t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i)
306				}
307				if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) {
308					t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected)
309				}
310			}
311		}
312	}
313}
314
315func TestPrepareRun(t *testing.T) {
316	s, config, assert := newMaster(t)
317
318	assert.NotNil(config.OpenAPIConfig)
319
320	server := httptest.NewServer(s.Handler.Director)
321	defer server.Close()
322	done := make(chan struct{})
323
324	s.PrepareRun()
325	s.RunPostStartHooks(done)
326
327	// openapi is installed in PrepareRun
328	resp, err := http.Get(server.URL + "/openapi/v2")
329	assert.NoError(err)
330	assert.Equal(http.StatusOK, resp.StatusCode)
331
332	// healthz checks are installed in PrepareRun
333	resp, err = http.Get(server.URL + "/healthz")
334	assert.NoError(err)
335	assert.Equal(http.StatusOK, resp.StatusCode)
336	resp, err = http.Get(server.URL + "/healthz/ping")
337	assert.NoError(err)
338	assert.Equal(http.StatusOK, resp.StatusCode)
339}
340
341func TestUpdateOpenAPISpec(t *testing.T) {
342	s, _, assert := newMaster(t)
343	s.PrepareRun()
344	s.RunPostStartHooks(make(chan struct{}))
345
346	server := httptest.NewServer(s.Handler.Director)
347	defer server.Close()
348
349	// verify the static spec in record is what we currently serve
350	oldSpec, err := json.Marshal(s.StaticOpenAPISpec)
351	assert.NoError(err)
352
353	resp, err := http.Get(server.URL + "/openapi/v2")
354	assert.NoError(err)
355	assert.Equal(http.StatusOK, resp.StatusCode)
356
357	body, err := ioutil.ReadAll(resp.Body)
358	assert.NoError(err)
359	assert.Equal(oldSpec, body)
360	resp.Body.Close()
361
362	// verify we are able to update the served spec using the exposed service
363	newSpec := []byte(`{"swagger":"2.0","info":{"title":"Test Updated Generic API Server Swagger","version":"v0.1.0"},"paths":null}`)
364	swagger := new(openapi.Swagger)
365	err = json.Unmarshal(newSpec, swagger)
366	assert.NoError(err)
367
368	err = s.OpenAPIVersionedService.UpdateSpec(swagger)
369	assert.NoError(err)
370
371	resp, err = http.Get(server.URL + "/openapi/v2")
372	assert.NoError(err)
373	defer resp.Body.Close()
374	assert.Equal(http.StatusOK, resp.StatusCode)
375
376	body, err = ioutil.ReadAll(resp.Body)
377	assert.NoError(err)
378	assert.Equal(newSpec, body)
379}
380
381// TestCustomHandlerChain verifies the handler chain with custom handler chain builder functions.
382func TestCustomHandlerChain(t *testing.T) {
383	config, _ := setUp(t)
384
385	var protected, called bool
386
387	config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler {
388		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
389			protected = true
390			apiHandler.ServeHTTP(w, req)
391		})
392	}
393	handler := http.HandlerFunc(func(r http.ResponseWriter, req *http.Request) {
394		called = true
395	})
396
397	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
398	if err != nil {
399		t.Fatalf("Error in bringing up the server: %v", err)
400	}
401
402	s.Handler.NonGoRestfulMux.Handle("/nonswagger", handler)
403	s.Handler.NonGoRestfulMux.Handle("/secret", handler)
404
405	type Test struct {
406		handler   http.Handler
407		path      string
408		protected bool
409	}
410	for i, test := range []Test{
411		{s.Handler, "/nonswagger", true},
412		{s.Handler, "/secret", true},
413	} {
414		protected, called = false, false
415
416		var w io.Reader
417		req, err := http.NewRequest("GET", test.path, w)
418		if err != nil {
419			t.Errorf("%d: Unexpected http error: %v", i, err)
420			continue
421		}
422
423		test.handler.ServeHTTP(httptest.NewRecorder(), req)
424
425		if !called {
426			t.Errorf("%d: Expected handler to be called.", i)
427		}
428		if test.protected != protected {
429			t.Errorf("%d: Expected protected=%v, got protected=%v.", i, test.protected, protected)
430		}
431	}
432}
433
434// TestNotRestRoutesHaveAuth checks that special non-routes are behind authz/authn.
435func TestNotRestRoutesHaveAuth(t *testing.T) {
436	config, _ := setUp(t)
437
438	authz := mockAuthorizer{}
439
440	config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix")
441	config.Authorization.Authorizer = &authz
442
443	config.EnableIndex = true
444	config.EnableProfiling = true
445
446	kubeVersion := fakeVersion()
447	config.Version = &kubeVersion
448
449	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
450	if err != nil {
451		t.Fatalf("Error in bringing up the server: %v", err)
452	}
453
454	for _, test := range []struct {
455		route string
456	}{
457		{"/"},
458		{"/debug/pprof/"},
459		{"/debug/flags/"},
460		{"/version"},
461	} {
462		resp := httptest.NewRecorder()
463		req, _ := http.NewRequest("GET", test.route, nil)
464		s.Handler.ServeHTTP(resp, req)
465		if resp.Code != 200 {
466			t.Errorf("route %q expected to work: code %d", test.route, resp.Code)
467			continue
468		}
469
470		if authz.lastURI != test.route {
471			t.Errorf("route %q expected to go through authorization, last route did: %q", test.route, authz.lastURI)
472		}
473	}
474}
475
476type mockAuthorizer struct {
477	lastURI string
478}
479
480func (authz *mockAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) {
481	authz.lastURI = a.GetPath()
482	return authorizer.DecisionAllow, "", nil
483}
484
485type testGetterStorage struct {
486	Version string
487}
488
489func (p *testGetterStorage) NamespaceScoped() bool {
490	return true
491}
492
493func (p *testGetterStorage) New() runtime.Object {
494	return &metav1.APIGroup{
495		TypeMeta: metav1.TypeMeta{
496			Kind:       "Getter",
497			APIVersion: p.Version,
498		},
499	}
500}
501
502func (p *testGetterStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
503	return nil, nil
504}
505
506type testNoVerbsStorage struct {
507	Version string
508}
509
510func (p *testNoVerbsStorage) NamespaceScoped() bool {
511	return true
512}
513
514func (p *testNoVerbsStorage) New() runtime.Object {
515	return &metav1.APIGroup{
516		TypeMeta: metav1.TypeMeta{
517			Kind:       "NoVerbs",
518			APIVersion: p.Version,
519		},
520	}
521}
522
523func fakeVersion() version.Info {
524	return version.Info{
525		Major:        "42",
526		Minor:        "42",
527		GitVersion:   "42",
528		GitCommit:    "34973274ccef6ab4dfaaf86599792fa9c3fe4689",
529		GitTreeState: "Dirty",
530		BuildDate:    time.Now().String(),
531		GoVersion:    goruntime.Version(),
532		Compiler:     goruntime.Compiler,
533		Platform:     fmt.Sprintf("%s/%s", goruntime.GOOS, goruntime.GOARCH),
534	}
535}
536
537// TestGracefulShutdown verifies server shutdown after request handler finish.
538func TestGracefulShutdown(t *testing.T) {
539	config, _ := setUp(t)
540
541	var graceShutdown bool
542	wg := sync.WaitGroup{}
543	wg.Add(1)
544
545	config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler {
546		handler := genericfilters.WithWaitGroup(apiHandler, c.LongRunningFunc, c.HandlerChainWaitGroup)
547		handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
548		return handler
549	}
550
551	s, err := config.Complete(nil).New("test", NewEmptyDelegate())
552	if err != nil {
553		t.Fatalf("Error in bringing up the server: %v", err)
554	}
555
556	twoSecondHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
557		wg.Done()
558		time.Sleep(2 * time.Second)
559		w.WriteHeader(http.StatusOK)
560		graceShutdown = true
561	})
562	okHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
563		w.WriteHeader(http.StatusOK)
564	})
565
566	s.Handler.NonGoRestfulMux.Handle("/test", twoSecondHandler)
567	s.Handler.NonGoRestfulMux.Handle("/200", okHandler)
568
569	insecureServer := &http.Server{
570		Addr:    "0.0.0.0:0",
571		Handler: s.Handler,
572	}
573	stopCh := make(chan struct{})
574
575	ln, err := net.Listen("tcp", insecureServer.Addr)
576	if err != nil {
577		t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err)
578	}
579
580	// get port
581	serverPort := ln.Addr().(*net.TCPAddr).Port
582	stoppedCh, err := RunServer(insecureServer, ln, 10*time.Second, stopCh)
583	if err != nil {
584		t.Fatalf("RunServer err: %v", err)
585	}
586
587	graceCh := make(chan struct{})
588	// mock a client request
589	go func() {
590		resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/test")
591		if err != nil {
592			t.Errorf("Unexpected http error: %v", err)
593		}
594		if resp.StatusCode != http.StatusOK {
595			t.Errorf("Unexpected http status code: %v", resp.StatusCode)
596		}
597		close(graceCh)
598	}()
599
600	// close stopCh after request sent to server to guarantee request handler is running.
601	wg.Wait()
602	close(stopCh)
603
604	time.Sleep(500 * time.Millisecond)
605	if _, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/200"); err == nil {
606		t.Errorf("Unexpected http success after stopCh was closed")
607	}
608
609	// wait for wait group handler finish
610	s.HandlerChainWaitGroup.Wait()
611	<-stoppedCh
612
613	// check server all handlers finished.
614	if !graceShutdown {
615		t.Errorf("server shutdown not gracefully.")
616	}
617	// check client to make sure receive response.
618	select {
619	case <-graceCh:
620		t.Logf("server shutdown gracefully.")
621	case <-time.After(30 * time.Second):
622		t.Errorf("Timed out waiting for response.")
623	}
624}
625