1/* 2Copyright 2015 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package server 18 19import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "net" 26 "net/http" 27 "net/http/httptest" 28 goruntime "runtime" 29 "strconv" 30 "sync" 31 "testing" 32 "time" 33 34 "github.com/stretchr/testify/assert" 35 36 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 37 "k8s.io/apimachinery/pkg/runtime" 38 "k8s.io/apimachinery/pkg/runtime/schema" 39 "k8s.io/apimachinery/pkg/runtime/serializer" 40 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 41 "k8s.io/apimachinery/pkg/util/sets" 42 "k8s.io/apimachinery/pkg/util/wait" 43 "k8s.io/apimachinery/pkg/version" 44 "k8s.io/apiserver/pkg/apis/example" 45 examplev1 "k8s.io/apiserver/pkg/apis/example/v1" 46 "k8s.io/apiserver/pkg/authorization/authorizer" 47 "k8s.io/apiserver/pkg/endpoints/discovery" 48 genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" 49 openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" 50 "k8s.io/apiserver/pkg/registry/rest" 51 genericfilters "k8s.io/apiserver/pkg/server/filters" 52 "k8s.io/client-go/informers" 53 "k8s.io/client-go/kubernetes/fake" 54 restclient "k8s.io/client-go/rest" 55 kubeopenapi "k8s.io/kube-openapi/pkg/common" 56 "k8s.io/kube-openapi/pkg/validation/spec" 57) 58 59const ( 60 extensionsGroupName = "extensions" 61) 62 63var ( 64 v1GroupVersion = schema.GroupVersion{Group: "", Version: "v1"} 65 66 scheme = runtime.NewScheme() 67 codecs = serializer.NewCodecFactory(scheme) 68 parameterCodec = runtime.NewParameterCodec(scheme) 69) 70 71func init() { 72 metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) 73 scheme.AddUnversionedTypes(v1GroupVersion, 74 &metav1.Status{}, 75 &metav1.APIVersions{}, 76 &metav1.APIGroupList{}, 77 &metav1.APIGroup{}, 78 &metav1.APIResourceList{}, 79 ) 80 utilruntime.Must(example.AddToScheme(scheme)) 81 utilruntime.Must(examplev1.AddToScheme(scheme)) 82} 83 84func buildTestOpenAPIDefinition() kubeopenapi.OpenAPIDefinition { 85 return kubeopenapi.OpenAPIDefinition{ 86 Schema: spec.Schema{ 87 SchemaProps: spec.SchemaProps{ 88 Description: "Description", 89 Properties: map[string]spec.Schema{}, 90 }, 91 VendorExtensible: spec.VendorExtensible{ 92 Extensions: spec.Extensions{ 93 "x-kubernetes-group-version-kind": []interface{}{ 94 map[string]interface{}{ 95 "group": "", 96 "version": "v1", 97 "kind": "Getter", 98 }, 99 map[string]interface{}{ 100 "group": "batch", 101 "version": "v1", 102 "kind": "Getter", 103 }, 104 map[string]interface{}{ 105 "group": "extensions", 106 "version": "v1", 107 "kind": "Getter", 108 }, 109 }, 110 }, 111 }, 112 }, 113 } 114} 115 116func testGetOpenAPIDefinitions(_ kubeopenapi.ReferenceCallback) map[string]kubeopenapi.OpenAPIDefinition { 117 return map[string]kubeopenapi.OpenAPIDefinition{ 118 "k8s.io/apimachinery/pkg/apis/meta/v1.Status": {}, 119 "k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions": {}, 120 "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList": {}, 121 "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup": buildTestOpenAPIDefinition(), 122 "k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList": {}, 123 } 124} 125 126// setUp is a convience function for setting up for (most) tests. 127func setUp(t *testing.T) (Config, *assert.Assertions) { 128 config := NewConfig(codecs) 129 config.ExternalAddress = "192.168.10.4:443" 130 config.PublicAddress = net.ParseIP("192.168.10.4") 131 config.LegacyAPIGroupPrefixes = sets.NewString("/api") 132 config.LoopbackClientConfig = &restclient.Config{} 133 134 clientset := fake.NewSimpleClientset() 135 if clientset == nil { 136 t.Fatal("unable to create fake client set") 137 } 138 139 config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) 140 config.OpenAPIConfig.Info.Version = "unversioned" 141 sharedInformers := informers.NewSharedInformerFactory(clientset, config.LoopbackClientConfig.Timeout) 142 config.Complete(sharedInformers) 143 144 return *config, assert.New(t) 145} 146 147func newMaster(t *testing.T) (*GenericAPIServer, Config, *assert.Assertions) { 148 config, assert := setUp(t) 149 150 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 151 if err != nil { 152 t.Fatalf("Error in bringing up the server: %v", err) 153 } 154 return s, config, assert 155} 156 157// TestNew verifies that the New function returns a GenericAPIServer 158// using the configuration properly. 159func TestNew(t *testing.T) { 160 s, config, assert := newMaster(t) 161 162 // Verify many of the variables match their config counterparts 163 assert.Equal(s.legacyAPIGroupPrefixes, config.LegacyAPIGroupPrefixes) 164 assert.Equal(s.admissionControl, config.AdmissionControl) 165} 166 167// Verifies that AddGroupVersions works as expected. 168func TestInstallAPIGroups(t *testing.T) { 169 config, assert := setUp(t) 170 171 config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") 172 config.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: "ExternalAddress"} 173 174 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 175 if err != nil { 176 t.Fatalf("Error in bringing up the server: %v", err) 177 } 178 179 testAPI := func(gv schema.GroupVersion) APIGroupInfo { 180 getter, noVerbs := testGetterStorage{}, testNoVerbsStorage{} 181 182 scheme := runtime.NewScheme() 183 scheme.AddKnownTypeWithName(gv.WithKind("Getter"), getter.New()) 184 scheme.AddKnownTypeWithName(gv.WithKind("NoVerb"), noVerbs.New()) 185 scheme.AddKnownTypes(v1GroupVersion, &metav1.Status{}) 186 metav1.AddToGroupVersion(scheme, v1GroupVersion) 187 188 return APIGroupInfo{ 189 PrioritizedVersions: []schema.GroupVersion{gv}, 190 VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ 191 gv.Version: { 192 "getter": &testGetterStorage{Version: gv.Version}, 193 "noverbs": &testNoVerbsStorage{Version: gv.Version}, 194 }, 195 }, 196 OptionsExternalVersion: &schema.GroupVersion{Version: "v1"}, 197 ParameterCodec: parameterCodec, 198 NegotiatedSerializer: codecs, 199 Scheme: scheme, 200 } 201 } 202 203 apis := []APIGroupInfo{ 204 testAPI(schema.GroupVersion{Group: "", Version: "v1"}), 205 testAPI(schema.GroupVersion{Group: extensionsGroupName, Version: "v1"}), 206 testAPI(schema.GroupVersion{Group: "batch", Version: "v1"}), 207 } 208 209 err = s.InstallLegacyAPIGroup("/apiPrefix", &apis[0]) 210 assert.NoError(err) 211 groupPaths := []string{ 212 config.LegacyAPIGroupPrefixes.List()[0], // /apiPrefix 213 } 214 for _, api := range apis[1:] { 215 err = s.InstallAPIGroup(&api) 216 assert.NoError(err) 217 groupPaths = append(groupPaths, APIGroupPrefix+"/"+api.PrioritizedVersions[0].Group) // /apis/<group> 218 } 219 220 server := httptest.NewServer(s.Handler) 221 defer server.Close() 222 223 for i := range apis { 224 // should serve APIGroup at group path 225 info := &apis[i] 226 path := groupPaths[i] 227 resp, err := http.Get(server.URL + path) 228 if err != nil { 229 t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) 230 continue 231 } 232 233 body, err := ioutil.ReadAll(resp.Body) 234 if err != nil { 235 t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) 236 continue 237 } 238 239 t.Logf("[%d] json at %s: %s", i, path, string(body)) 240 241 if i == 0 { 242 // legacy API returns APIVersions 243 group := metav1.APIVersions{} 244 err = json.Unmarshal(body, &group) 245 if err != nil { 246 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 247 continue 248 } 249 } else { 250 // API groups return APIGroup 251 group := metav1.APIGroup{} 252 err = json.Unmarshal(body, &group) 253 if err != nil { 254 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 255 continue 256 } 257 258 if got, expected := group.Name, info.PrioritizedVersions[0].Group; got != expected { 259 t.Errorf("[%d] unexpected group name at path %q: got=%q expected=%q", i, path, got, expected) 260 continue 261 } 262 263 if got, expected := group.PreferredVersion.Version, info.PrioritizedVersions[0].Version; got != expected { 264 t.Errorf("[%d] unexpected group version at path %q: got=%q expected=%q", i, path, got, expected) 265 continue 266 } 267 } 268 269 // should serve APIResourceList at group path + /<group-version> 270 path = path + "/" + info.PrioritizedVersions[0].Version 271 resp, err = http.Get(server.URL + path) 272 if err != nil { 273 t.Errorf("[%d] unexpected error getting path %q path: %v", i, path, err) 274 continue 275 } 276 277 body, err = ioutil.ReadAll(resp.Body) 278 if err != nil { 279 t.Errorf("[%d] unexpected error reading body at path %q: %v", i, path, err) 280 continue 281 } 282 283 t.Logf("[%d] json at %s: %s", i, path, string(body)) 284 285 resources := metav1.APIResourceList{} 286 err = json.Unmarshal(body, &resources) 287 if err != nil { 288 t.Errorf("[%d] unexpected error parsing json body at path %q: %v", i, path, err) 289 continue 290 } 291 292 if got, expected := resources.GroupVersion, info.PrioritizedVersions[0].String(); got != expected { 293 t.Errorf("[%d] unexpected groupVersion at path %q: got=%q expected=%q", i, path, got, expected) 294 continue 295 } 296 297 // the verbs should match the features of resources 298 for _, r := range resources.APIResources { 299 switch r.Name { 300 case "getter": 301 if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString("get"); !got.Equal(expected) { 302 t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) 303 } 304 case "noverbs": 305 if r.Verbs == nil { 306 t.Errorf("[%d] unexpected nil verbs slice. Expected: []string{}", i) 307 } 308 if got, expected := sets.NewString([]string(r.Verbs)...), sets.NewString(); !got.Equal(expected) { 309 t.Errorf("[%d] unexpected verbs for resource %s/%s: got=%v expected=%v", i, resources.GroupVersion, r.Name, got, expected) 310 } 311 } 312 } 313 } 314} 315 316func TestPrepareRun(t *testing.T) { 317 s, config, assert := newMaster(t) 318 319 assert.NotNil(config.OpenAPIConfig) 320 321 server := httptest.NewServer(s.Handler.Director) 322 defer server.Close() 323 done := make(chan struct{}) 324 defer close(done) 325 326 s.PrepareRun() 327 s.RunPostStartHooks(done) 328 329 // openapi is installed in PrepareRun 330 resp, err := http.Get(server.URL + "/openapi/v2") 331 assert.NoError(err) 332 assert.Equal(http.StatusOK, resp.StatusCode) 333 334 // wait for health (max-in-flight-filter is initialized asynchronously, can take a few milliseconds to initialize) 335 assert.NoError(wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 336 // healthz checks are installed in PrepareRun 337 resp, err = http.Get(server.URL + "/healthz") 338 assert.NoError(err) 339 data, _ := ioutil.ReadAll(resp.Body) 340 if http.StatusOK != resp.StatusCode { 341 t.Logf("got %d", resp.StatusCode) 342 t.Log(string(data)) 343 return false, nil 344 } 345 return true, nil 346 })) 347 resp, err = http.Get(server.URL + "/healthz/ping") 348 assert.NoError(err) 349 assert.Equal(http.StatusOK, resp.StatusCode) 350} 351 352func TestUpdateOpenAPISpec(t *testing.T) { 353 s, _, assert := newMaster(t) 354 s.PrepareRun() 355 s.RunPostStartHooks(make(chan struct{})) 356 357 server := httptest.NewServer(s.Handler.Director) 358 defer server.Close() 359 360 // verify the static spec in record is what we currently serve 361 oldSpec, err := json.Marshal(s.StaticOpenAPISpec) 362 assert.NoError(err) 363 364 resp, err := http.Get(server.URL + "/openapi/v2") 365 assert.NoError(err) 366 assert.Equal(http.StatusOK, resp.StatusCode) 367 368 body, err := ioutil.ReadAll(resp.Body) 369 assert.NoError(err) 370 assert.Equal(oldSpec, body) 371 resp.Body.Close() 372 373 // verify we are able to update the served spec using the exposed service 374 newSpec := []byte(`{"swagger":"2.0","info":{"title":"Test Updated Generic API Server Swagger","version":"v0.1.0"},"paths":null}`) 375 swagger := new(spec.Swagger) 376 err = json.Unmarshal(newSpec, swagger) 377 assert.NoError(err) 378 379 err = s.OpenAPIVersionedService.UpdateSpec(swagger) 380 assert.NoError(err) 381 382 resp, err = http.Get(server.URL + "/openapi/v2") 383 assert.NoError(err) 384 defer resp.Body.Close() 385 assert.Equal(http.StatusOK, resp.StatusCode) 386 387 body, err = ioutil.ReadAll(resp.Body) 388 assert.NoError(err) 389 assert.Equal(newSpec, body) 390} 391 392// TestCustomHandlerChain verifies the handler chain with custom handler chain builder functions. 393func TestCustomHandlerChain(t *testing.T) { 394 config, _ := setUp(t) 395 396 var protected, called bool 397 398 config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { 399 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 400 protected = true 401 apiHandler.ServeHTTP(w, req) 402 }) 403 } 404 handler := http.HandlerFunc(func(r http.ResponseWriter, req *http.Request) { 405 called = true 406 }) 407 408 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 409 if err != nil { 410 t.Fatalf("Error in bringing up the server: %v", err) 411 } 412 413 s.Handler.NonGoRestfulMux.Handle("/nonswagger", handler) 414 s.Handler.NonGoRestfulMux.Handle("/secret", handler) 415 416 type Test struct { 417 handler http.Handler 418 path string 419 protected bool 420 } 421 for i, test := range []Test{ 422 {s.Handler, "/nonswagger", true}, 423 {s.Handler, "/secret", true}, 424 } { 425 protected, called = false, false 426 427 var w io.Reader 428 req, err := http.NewRequest("GET", test.path, w) 429 if err != nil { 430 t.Errorf("%d: Unexpected http error: %v", i, err) 431 continue 432 } 433 434 test.handler.ServeHTTP(httptest.NewRecorder(), req) 435 436 if !called { 437 t.Errorf("%d: Expected handler to be called.", i) 438 } 439 if test.protected != protected { 440 t.Errorf("%d: Expected protected=%v, got protected=%v.", i, test.protected, protected) 441 } 442 } 443} 444 445// TestNotRestRoutesHaveAuth checks that special non-routes are behind authz/authn. 446func TestNotRestRoutesHaveAuth(t *testing.T) { 447 config, _ := setUp(t) 448 449 authz := mockAuthorizer{} 450 451 config.LegacyAPIGroupPrefixes = sets.NewString("/apiPrefix") 452 config.Authorization.Authorizer = &authz 453 454 config.EnableIndex = true 455 config.EnableProfiling = true 456 457 kubeVersion := fakeVersion() 458 config.Version = &kubeVersion 459 460 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 461 if err != nil { 462 t.Fatalf("Error in bringing up the server: %v", err) 463 } 464 465 for _, test := range []struct { 466 route string 467 }{ 468 {"/"}, 469 {"/debug/pprof/"}, 470 {"/debug/flags/"}, 471 {"/version"}, 472 } { 473 resp := httptest.NewRecorder() 474 req, _ := http.NewRequest("GET", test.route, nil) 475 s.Handler.ServeHTTP(resp, req) 476 if resp.Code != 200 { 477 t.Errorf("route %q expected to work: code %d", test.route, resp.Code) 478 continue 479 } 480 481 if authz.lastURI != test.route { 482 t.Errorf("route %q expected to go through authorization, last route did: %q", test.route, authz.lastURI) 483 } 484 } 485} 486 487type mockAuthorizer struct { 488 lastURI string 489} 490 491func (authz *mockAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { 492 authz.lastURI = a.GetPath() 493 return authorizer.DecisionAllow, "", nil 494} 495 496type testGetterStorage struct { 497 Version string 498} 499 500func (p *testGetterStorage) NamespaceScoped() bool { 501 return true 502} 503 504func (p *testGetterStorage) New() runtime.Object { 505 return &metav1.APIGroup{ 506 TypeMeta: metav1.TypeMeta{ 507 Kind: "Getter", 508 APIVersion: p.Version, 509 }, 510 } 511} 512 513func (p *testGetterStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { 514 return nil, nil 515} 516 517type testNoVerbsStorage struct { 518 Version string 519} 520 521func (p *testNoVerbsStorage) NamespaceScoped() bool { 522 return true 523} 524 525func (p *testNoVerbsStorage) New() runtime.Object { 526 return &metav1.APIGroup{ 527 TypeMeta: metav1.TypeMeta{ 528 Kind: "NoVerbs", 529 APIVersion: p.Version, 530 }, 531 } 532} 533 534func fakeVersion() version.Info { 535 return version.Info{ 536 Major: "42", 537 Minor: "42", 538 GitVersion: "42", 539 GitCommit: "34973274ccef6ab4dfaaf86599792fa9c3fe4689", 540 GitTreeState: "Dirty", 541 BuildDate: time.Now().String(), 542 GoVersion: goruntime.Version(), 543 Compiler: goruntime.Compiler, 544 Platform: fmt.Sprintf("%s/%s", goruntime.GOOS, goruntime.GOARCH), 545 } 546} 547 548// TestGracefulShutdown verifies server shutdown after request handler finish. 549func TestGracefulShutdown(t *testing.T) { 550 config, _ := setUp(t) 551 552 var graceShutdown bool 553 wg := sync.WaitGroup{} 554 wg.Add(1) 555 556 config.BuildHandlerChainFunc = func(apiHandler http.Handler, c *Config) http.Handler { 557 handler := genericfilters.WithWaitGroup(apiHandler, c.LongRunningFunc, c.HandlerChainWaitGroup) 558 handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) 559 return handler 560 } 561 562 s, err := config.Complete(nil).New("test", NewEmptyDelegate()) 563 if err != nil { 564 t.Fatalf("Error in bringing up the server: %v", err) 565 } 566 567 twoSecondHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 568 wg.Done() 569 time.Sleep(2 * time.Second) 570 w.WriteHeader(http.StatusOK) 571 graceShutdown = true 572 }) 573 okHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 574 w.WriteHeader(http.StatusOK) 575 }) 576 577 s.Handler.NonGoRestfulMux.Handle("/test", twoSecondHandler) 578 s.Handler.NonGoRestfulMux.Handle("/200", okHandler) 579 580 insecureServer := &http.Server{ 581 Addr: "0.0.0.0:0", 582 Handler: s.Handler, 583 } 584 stopCh := make(chan struct{}) 585 586 ln, err := net.Listen("tcp", insecureServer.Addr) 587 if err != nil { 588 t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err) 589 } 590 591 // get port 592 serverPort := ln.Addr().(*net.TCPAddr).Port 593 stoppedCh, _, err := RunServer(insecureServer, ln, 10*time.Second, stopCh) 594 if err != nil { 595 t.Fatalf("RunServer err: %v", err) 596 } 597 598 graceCh := make(chan struct{}) 599 // mock a client request 600 go func() { 601 resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/test") 602 if err != nil { 603 t.Errorf("Unexpected http error: %v", err) 604 } 605 if resp.StatusCode != http.StatusOK { 606 t.Errorf("Unexpected http status code: %v", resp.StatusCode) 607 } 608 close(graceCh) 609 }() 610 611 // close stopCh after request sent to server to guarantee request handler is running. 612 wg.Wait() 613 close(stopCh) 614 615 time.Sleep(500 * time.Millisecond) 616 if _, err := http.Get("http://127.0.0.1:" + strconv.Itoa(serverPort) + "/200"); err == nil { 617 t.Errorf("Unexpected http success after stopCh was closed") 618 } 619 620 // wait for wait group handler finish 621 s.HandlerChainWaitGroup.Wait() 622 <-stoppedCh 623 624 // check server all handlers finished. 625 if !graceShutdown { 626 t.Errorf("server shutdown not gracefully.") 627 } 628 // check client to make sure receive response. 629 select { 630 case <-graceCh: 631 t.Logf("server shutdown gracefully.") 632 case <-time.After(30 * time.Second): 633 t.Errorf("Timed out waiting for response.") 634 } 635} 636