1/* 2Copyright 2015 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package server 18 19import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "net" 26 "net/http" 27 "net/http/httptest" 28 goruntime "runtime" 29 "strconv" 30 "sync" 31 "testing" 32 "time" 33 34 openapi "github.com/go-openapi/spec" 35 "github.com/stretchr/testify/assert" 36 37 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 38 "k8s.io/apimachinery/pkg/runtime" 39 "k8s.io/apimachinery/pkg/runtime/schema" 40 "k8s.io/apimachinery/pkg/runtime/serializer" 41 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 42 "k8s.io/apimachinery/pkg/util/sets" 43 "k8s.io/apimachinery/pkg/version" 44 "k8s.io/apiserver/pkg/apis/example" 45 examplev1 "k8s.io/apiserver/pkg/apis/example/v1" 46 "k8s.io/apiserver/pkg/authorization/authorizer" 47 "k8s.io/apiserver/pkg/endpoints/discovery" 48 genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" 49 openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" 50 "k8s.io/apiserver/pkg/registry/rest" 51 genericfilters "k8s.io/apiserver/pkg/server/filters" 52 "k8s.io/client-go/informers" 53 "k8s.io/client-go/kubernetes/fake" 54 restclient "k8s.io/client-go/rest" 55 kubeopenapi "k8s.io/kube-openapi/pkg/common" 56) 57 58const ( 59 extensionsGroupName = "extensions" 60) 61 62var ( 63 v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"} 64 65 scheme = runtime.NewScheme() 66 codecs = serializer.NewCodecFactory(scheme) 67 parameterCodec = runtime.NewParameterCodec(scheme) 68) 69 70func init() { 71 metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) 72 scheme.AddUnversionedTypes(v1GroupVersion, 73 &metav1.Status{}, 74 &metav1.APIVersions{}, 75 &metav1.APIGroupList{}, 76 &metav1.APIGroup{}, 77 &metav1.APIResourceList{}, 78 ) 79 utilruntime.Must(example.AddToScheme(scheme)) 80 utilruntime.Must(examplev1.AddToScheme(scheme)) 81} 82 83func buildTestOpenAPIDefinition() kubeopenapi.OpenAPIDefinition { 84 return kubeopenapi.OpenAPIDefinition{ 85 Schema: openapi.Schema{ 86 SchemaProps: openapi.SchemaProps{ 87 Description: "Description", 88 Properties: map[string]openapi.Schema{}, 89 }, 90 VendorExtensible: openapi.VendorExtensible{ 91 Extensions: openapi.Extensions{ 92 "x-kubernetes-group-version-kind": []interface{}{ 93 map[string]interface{}{ 94 "group": "", 95 "version": "v1", 96 "kind": "Getter", 97 }, 98 map[string]interface{}{ 99 "group": "batch", 100 "version": "v1", 101 "kind": "Getter", 102 }, 103 map[string]interface{}{ 104 "group": "extensions", 105 "version": "v1", 106 "kind": "Getter", 107 }, 108 }, 109 }, 110 }, 111 }, 112 } 113} 114 115func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeopenapi.OpenAPIDefinition { 116 return map[string]kubeopenapi.OpenAPIDefinition{ 117 "k8s.io/apimachinery/pkg/apis/meta/v1.Status": {}, 118 "k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions": {}, 119 "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList": {}, 120 "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup": buildTestOpenAPIDefinition(), 121 "k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList": {}, 122 } 123} 124 125// setUp is a convience function for setting up for (most) tests. 126func setUp(t *testing.T) (Config, *assert.Assertions) { 127 config := NewConfig(codecs) 128 config.ExternalAddress = "192.168.10.4:443" 129 config.PublicAddress = net.ParseIP("192.168.10.4") 130 config.LegacyAPIGroupPrefixes = sets.NewString("/api") 131 config.LoopbackClientConfig = &restclient.Config{} 132 133 clientset := fake.NewSimpleClientset() 134 if clientset == nil { 135 t.Fatal("unable to create fake client set") 136 } 137 138 config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) 139 config.OpenAPIConfig.Info.Version = "unversioned" 140 sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout) 141 config.Complete(sharedInformers) 142 143 return *config, assert.New(t) 144} 145 146func newMaster(t *testing.T) (*GenericAPIServer, Config, *assert.Assertions) { 147 config, assert := setUp(t) 148 149 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 150 if err != nil { 151 t.Fatalf("Error in bringing up the server: %v", err) 152 } 153 return s, config, assert 154} 155 156// TestNew verifies that the New function returns a GenericAPIServer 157// using the configuration properly. 158func TestNew(t *testing.T) { 159 s, config, assert := newMaster(t) 160 161 // Verify many of the variables match their config counterparts 162 assert.Equal(s.legacyAPIGroupPrefixes, config.LegacyAPIGroupPrefixes) 163 assert.Equal(s.admissionControl, config.AdmissionControl) 164} 165 166// Verifies that AddGroupVersions works as expected. 167func TestInstallAPIGroups(t *testing.T) { 168 config, assert := setUp(t) 169 170 config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") 171 config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"} 172 173 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 174 if err != nil { 175 t.Fatalf("Error in bringing up the server: %v", err) 176 } 177 178 testAPI := func(gv schema.GroupVersion) APIGroupInfo { 179 getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{} 180 181 scheme := runtime.NewScheme() 182 scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New()) 183 scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New()) 184 scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{}) 185 metav1.AddToGroupVersion(scheme, v1GroupVersion) 186 187 return APIGroupInfo{ 188 PrioritizedVersions: []schema.GroupVersion{gv}, 189 VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ 190 gv.Version: { 191 "getter": &testGetterStorage{Version: gv.Version}, 192 "noverbs": &testNoVerbsStorage{Version: gv.Version}, 193 }, 194 }, 195 OptionsExternalVersion: &schema.GroupVersion{Version: "v1"}, 196 ParameterCodec: parameterCodec, 197 NegotiatedSerializer: codecs, 198 Scheme: scheme, 199 } 200 } 201 202 apis := []APIGroupInfo{ 203 testAPI(schema.GroupVersion{Group: "", Version: "v1"}), 204 testAPI(schema.GroupVersion{Group: extensionsGroupName, Version: "v1"}), 205 testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}), 206 } 207 208 err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0]) 209 assert.NoError(err) 210 groupPaths := []string{ 211 config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix 212 } 213 for _, api := range apis[1:] { 214 err = s.InstallAPIGroup(&api) 215 assert.NoError(err) 216 groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/<group> 217 } 218 219 server := httptest.NewServer(s.Handler) 220 defer server.Close() 221 222 for i := range apis { 223 // should serve APIGroup at group path 224 info := &apis[i] 225 path := groupPaths[i] 226 resp, err := http.Get(server.URL + path) 227 if err != nil { 228 t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) 229 continue 230 } 231 232 body, err := ioutil.ReadAll(resp.Body) 233 if err != nil { 234 t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) 235 continue 236 } 237 238 t.Logf("[%d] json at %s: %s", i, path, string(body)) 239 240 if i == 0 { 241 // legacy API returns APIVersions 242 group := metav1.APIVersions{} 243 err = json.Unmarshal(body, &group) 244 if err != nil { 245 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 246 continue 247 } 248 } else { 249 // API groups return APIGroup 250 group := metav1.APIGroup{} 251 err = json.Unmarshal(body, &group) 252 if err != nil { 253 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 254 continue 255 } 256 257 if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected { 258 t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected) 259 continue 260 } 261 262 if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected { 263 t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected) 264 continue 265 } 266 } 267 268 // should serve APIResourceList at group path + /<group-version> 269 path = path + "/" + info.PrioritizedVersions[0].Version 270 resp, err = http.Get(server.URL + path) 271 if err != nil { 272 t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) 273 continue 274 } 275 276 body, err = ioutil.ReadAll(resp.Body) 277 if err != nil { 278 t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) 279 continue 280 } 281 282 t.Logf("[%d] json at %s: %s", i, path, string(body)) 283 284 resources := metav1.APIResourceList{} 285 err = json.Unmarshal(body, &resources) 286 if err != nil { 287 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 288 continue 289 } 290 291 if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected { 292 t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected) 293 continue 294 } 295 296 // the verbs should match the features of resources 297 for _, r := range resources.APIResources { 298 switch r.Name { 299 case "getter": 300 if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) { 301 t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) 302 } 303 case "noverbs": 304 if r.Verbs == nil { 305 t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i) 306 } 307 if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) { 308 t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) 309 } 310 } 311 } 312 } 313} 314 315func TestPrepareRun(t *testing.T) { 316 s, config, assert := newMaster(t) 317 318 assert.NotNil(config.OpenAPIConfig) 319 320 server := httptest.NewServer(s.Handler.Director) 321 defer server.Close() 322 done := make(chan struct{}) 323 324 s.PrepareRun() 325 s.RunPostStartHooks(done) 326 327 // openapi is installed in PrepareRun 328 resp, err := http.Get(server.URL + "/openapi/v2") 329 assert.NoError(err) 330 assert.Equal(http.StatusOK, resp.StatusCode) 331 332 // healthz checks are installed in PrepareRun 333 resp, err = http.Get(server.URL + "/healthz") 334 assert.NoError(err) 335 assert.Equal(http.StatusOK, resp.StatusCode) 336 resp, err = http.Get(server.URL + "/healthz/ping") 337 assert.NoError(err) 338 assert.Equal(http.StatusOK, resp.StatusCode) 339} 340 341func TestUpdateOpenAPISpec(t *testing.T) { 342 s, _, assert := newMaster(t) 343 s.PrepareRun() 344 s.RunPostStartHooks(make(chan struct{})) 345 346 server := httptest.NewServer(s.Handler.Director) 347 defer server.Close() 348 349 // verify the static spec in record is what we currently serve 350 oldSpec, err := json.Marshal(s.StaticOpenAPISpec) 351 assert.NoError(err) 352 353 resp, err := http.Get(server.URL + "/openapi/v2") 354 assert.NoError(err) 355 assert.Equal(http.StatusOK, resp.StatusCode) 356 357 body, err := ioutil.ReadAll(resp.Body) 358 assert.NoError(err) 359 assert.Equal(oldSpec, body) 360 resp.Body.Close() 361 362 // verify we are able to update the served spec using the exposed service 363 newSpec := []byte(`{"swagger":"2.0","info":{"title":"Test Updated Generic API Server Swagger","version":"v0.1.0"},"paths":null}`) 364 swagger := new(openapi.Swagger) 365 err = json.Unmarshal(newSpec, swagger) 366 assert.NoError(err) 367 368 err = s.OpenAPIVersionedService.UpdateSpec(swagger) 369 assert.NoError(err) 370 371 resp, err = http.Get(server.URL + "/openapi/v2") 372 assert.NoError(err) 373 defer resp.Body.Close() 374 assert.Equal(http.StatusOK, resp.StatusCode) 375 376 body, err = ioutil.ReadAll(resp.Body) 377 assert.NoError(err) 378 assert.Equal(newSpec, body) 379} 380 381// TestCustomHandlerChain verifies the handler chain with custom handler chain builder functions. 382func TestCustomHandlerChain(t *testing.T) { 383 config, _ := setUp(t) 384 385 var protected, called bool 386 387 config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { 388 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 389 protected = true 390 apiHandler.ServeHTTP(w, req) 391 }) 392 } 393 handler := http.HandlerFunc(func(r http.ResponseWriter, req *http.Request) { 394 called = true 395 }) 396 397 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 398 if err != nil { 399 t.Fatalf("Error in bringing up the server: %v", err) 400 } 401 402 s.Handler.NonGoRestfulMux.Handle("/nonswagger", handler) 403 s.Handler.NonGoRestfulMux.Handle("/secret", handler) 404 405 type Test struct { 406 handler http.Handler 407 path string 408 protected bool 409 } 410 for i, test := range []Test{ 411 {s.Handler, "/nonswagger", true}, 412 {s.Handler, "/secret", true}, 413 } { 414 protected, called = false, false 415 416 var w io.Reader 417 req, err := http.NewRequest("GET", test.path, w) 418 if err != nil { 419 t.Errorf("%d: Unexpected http error: %v", i, err) 420 continue 421 } 422 423 test.handler.ServeHTTP(httptest.NewRecorder(), req) 424 425 if !called { 426 t.Errorf("%d: Expected handler to be called.", i) 427 } 428 if test.protected != protected { 429 t.Errorf("%d: Expected protected=%v, got protected=%v.", i, test.protected, protected) 430 } 431 } 432} 433 434// TestNotRestRoutesHaveAuth checks that special non-routes are behind authz/authn. 435func TestNotRestRoutesHaveAuth(t *testing.T) { 436 config, _ := setUp(t) 437 438 authz := mockAuthorizer{} 439 440 config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") 441 config.Authorization.Authorizer = &authz 442 443 config.EnableIndex = true 444 config.EnableProfiling = true 445 446 kubeVersion := fakeVersion() 447 config.Version = &kubeVersion 448 449 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 450 if err != nil { 451 t.Fatalf("Error in bringing up the server: %v", err) 452 } 453 454 for _, test := range []struct { 455 route string 456 }{ 457 {"/"}, 458 {"/debug/pprof/"}, 459 {"/debug/flags/"}, 460 {"/version"}, 461 } { 462 resp := httptest.NewRecorder() 463 req, _ := http.NewRequest("GET", test.route, nil) 464 s.Handler.ServeHTTP(resp, req) 465 if resp.Code != 200 { 466 t.Errorf("route %q expected to work: code %d", test.route, resp.Code) 467 continue 468 } 469 470 if authz.lastURI != test.route { 471 t.Errorf("route %q expected to go through authorization, last route did: %q", test.route, authz.lastURI) 472 } 473 } 474} 475 476type mockAuthorizer struct { 477 lastURI string 478} 479 480func (authz *mockAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { 481 authz.lastURI = a.GetPath() 482 return authorizer.DecisionAllow, "", nil 483} 484 485type testGetterStorage struct { 486 Version string 487} 488 489func (p *testGetterStorage) NamespaceScoped() bool { 490 return true 491} 492 493func (p *testGetterStorage) New() runtime.Object { 494 return &metav1.APIGroup{ 495 TypeMeta: metav1.TypeMeta{ 496 Kind: "Getter", 497 APIVersion: p.Version, 498 }, 499 } 500} 501 502func (p *testGetterStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { 503 return nil, nil 504} 505 506type testNoVerbsStorage struct { 507 Version string 508} 509 510func (p *testNoVerbsStorage) NamespaceScoped() bool { 511 return true 512} 513 514func (p *testNoVerbsStorage) New() runtime.Object { 515 return &metav1.APIGroup{ 516 TypeMeta: metav1.TypeMeta{ 517 Kind: "NoVerbs", 518 APIVersion: p.Version, 519 }, 520 } 521} 522 523func fakeVersion() version.Info { 524 return version.Info{ 525 Major: "42", 526 Minor: "42", 527 GitVersion: "42", 528 GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", 529 GitTreeState: "Dirty", 530 BuildDate: time.Now().String(), 531 GoVersion: goruntime.Version(), 532 Compiler: goruntime.Compiler, 533 Platform: fmt.Sprintf("%s/%s", goruntime.GOOS, goruntime.GOARCH), 534 } 535} 536 537// TestGracefulShutdown verifies server shutdown after request handler finish. 538func TestGracefulShutdown(t *testing.T) { 539 config, _ := setUp(t) 540 541 var graceShutdown bool 542 wg := sync.WaitGroup{} 543 wg.Add(1) 544 545 config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { 546 handler := genericfilters.WithWaitGroup(apiHandler, c.LongRunningFunc, c.HandlerChainWaitGroup) 547 handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) 548 return handler 549 } 550 551 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 552 if err != nil { 553 t.Fatalf("Error in bringing up the server: %v", err) 554 } 555 556 twoSecondHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 557 wg.Done() 558 time.Sleep(2 * time.Second) 559 w.WriteHeader(http.StatusOK) 560 graceShutdown = true 561 }) 562 okHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 563 w.WriteHeader(http.StatusOK) 564 }) 565 566 s.Handler.NonGoRestfulMux.Handle("/test", twoSecondHandler) 567 s.Handler.NonGoRestfulMux.Handle("/200", okHandler) 568 569 insecureServer := &http.Server{ 570 Addr: "0.0.0.0:0", 571 Handler: s.Handler, 572 } 573 stopCh := make(chan struct{}) 574 575 ln, err := net.Listen("tcp", insecureServer.Addr) 576 if err != nil { 577 t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err) 578 } 579 580 // get port 581 serverPort := ln.Addr().(*net.TCPAddr).Port 582 stoppedCh, err := RunServer(insecureServer, ln, 10*time.Second, stopCh) 583 if err != nil { 584 t.Fatalf("RunServer err: %v", err) 585 } 586 587 graceCh := make(chan struct{}) 588 // mock a client request 589 go func() { 590 resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/test") 591 if err != nil { 592 t.Errorf("Unexpected http error: %v", err) 593 } 594 if resp.StatusCode != http.StatusOK { 595 t.Errorf("Unexpected http status code: %v", resp.StatusCode) 596 } 597 close(graceCh) 598 }() 599 600 // close stopCh after request sent to server to guarantee request handler is running. 601 wg.Wait() 602 close(stopCh) 603 604 time.Sleep(500 * time.Millisecond) 605 if _, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/200"); err == nil { 606 t.Errorf("Unexpected http success after stopCh was closed") 607 } 608 609 // wait for wait group handler finish 610 s.HandlerChainWaitGroup.Wait() 611 <-stoppedCh 612 613 // check server all handlers finished. 614 if !graceShutdown { 615 t.Errorf("server shutdown not gracefully.") 616 } 617 // check client to make sure receive response. 618 select { 619 case <-graceCh: 620 t.Logf("server shutdown gracefully.") 621 case <-time.After(30 * time.Second): 622 t.Errorf("Timed out waiting for response.") 623 } 624} 625