1/* 2Copyright 2015 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package discovery 18 19import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "net/url" 24 "sort" 25 "strings" 26 "sync" 27 "time" 28 29 //lint:ignore SA1019 Keep using module since it's still being maintained and the api of google.golang.org/protobuf/proto differs 30 "github.com/golang/protobuf/proto" 31 openapi_v2 "github.com/googleapis/gnostic/openapiv2" 32 33 "k8s.io/apimachinery/pkg/api/errors" 34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 "k8s.io/apimachinery/pkg/runtime" 36 "k8s.io/apimachinery/pkg/runtime/schema" 37 "k8s.io/apimachinery/pkg/runtime/serializer" 38 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 39 "k8s.io/apimachinery/pkg/version" 40 "k8s.io/client-go/kubernetes/scheme" 41 restclient "k8s.io/client-go/rest" 42) 43 44const ( 45 // defaultRetries is the number of times a resource discovery is repeated if an api group disappears on the fly (e.g. CustomResourceDefinitions). 46 defaultRetries = 2 47 // protobuf mime type 48 mimePb = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf" 49 // defaultTimeout is the maximum amount of time per request when no timeout has been set on a RESTClient. 50 // Defaults to 32s in order to have a distinguishable length of time, relative to other timeouts that exist. 51 defaultTimeout = 32 * time.Second 52) 53 54// DiscoveryInterface holds the methods that discover server-supported API groups, 55// versions and resources. 56type DiscoveryInterface interface { 57 RESTClient() restclient.Interface 58 ServerGroupsInterface 59 ServerResourcesInterface 60 ServerVersionInterface 61 OpenAPISchemaInterface 62} 63 64// CachedDiscoveryInterface is a DiscoveryInterface with cache invalidation and freshness. 65// Note that If the ServerResourcesForGroupVersion method returns a cache miss 66// error, the user needs to explicitly call Invalidate to clear the cache, 67// otherwise the same cache miss error will be returned next time. 68type CachedDiscoveryInterface interface { 69 DiscoveryInterface 70 // Fresh is supposed to tell the caller whether or not to retry if the cache 71 // fails to find something (false = retry, true = no need to retry). 72 // 73 // TODO: this needs to be revisited, this interface can't be locked properly 74 // and doesn't make a lot of sense. 75 Fresh() bool 76 // Invalidate enforces that no cached data that is older than the current time 77 // is used. 78 Invalidate() 79} 80 81// ServerGroupsInterface has methods for obtaining supported groups on the API server 82type ServerGroupsInterface interface { 83 // ServerGroups returns the supported groups, with information like supported versions and the 84 // preferred version. 85 ServerGroups() (*metav1.APIGroupList, error) 86} 87 88// ServerResourcesInterface has methods for obtaining supported resources on the API server 89type ServerResourcesInterface interface { 90 // ServerResourcesForGroupVersion returns the supported resources for a group and version. 91 ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) 92 // ServerResources returns the supported resources for all groups and versions. 93 // 94 // The returned resource list might be non-nil with partial results even in the case of 95 // non-nil error. 96 // 97 // Deprecated: use ServerGroupsAndResources instead. 98 ServerResources() ([]*metav1.APIResourceList, error) 99 // ServerGroupsAndResources returns the supported groups and resources for all groups and versions. 100 // 101 // The returned group and resource lists might be non-nil with partial results even in the 102 // case of non-nil error. 103 ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) 104 // ServerPreferredResources returns the supported resources with the version preferred by the 105 // server. 106 // 107 // The returned group and resource lists might be non-nil with partial results even in the 108 // case of non-nil error. 109 ServerPreferredResources() ([]*metav1.APIResourceList, error) 110 // ServerPreferredNamespacedResources returns the supported namespaced resources with the 111 // version preferred by the server. 112 // 113 // The returned resource list might be non-nil with partial results even in the case of 114 // non-nil error. 115 ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) 116} 117 118// ServerVersionInterface has a method for retrieving the server's version. 119type ServerVersionInterface interface { 120 // ServerVersion retrieves and parses the server's version (git version). 121 ServerVersion() (*version.Info, error) 122} 123 124// OpenAPISchemaInterface has a method to retrieve the open API schema. 125type OpenAPISchemaInterface interface { 126 // OpenAPISchema retrieves and parses the swagger API schema the server supports. 127 OpenAPISchema() (*openapi_v2.Document, error) 128} 129 130// DiscoveryClient implements the functions that discover server-supported API groups, 131// versions and resources. 132type DiscoveryClient struct { 133 restClient restclient.Interface 134 135 LegacyPrefix string 136} 137 138// Convert metav1.APIVersions to metav1.APIGroup. APIVersions is used by legacy v1, so 139// group would be "". 140func apiVersionsToAPIGroup(apiVersions *metav1.APIVersions) (apiGroup metav1.APIGroup) { 141 groupVersions := []metav1.GroupVersionForDiscovery{} 142 for _, version := range apiVersions.Versions { 143 groupVersion := metav1.GroupVersionForDiscovery{ 144 GroupVersion: version, 145 Version: version, 146 } 147 groupVersions = append(groupVersions, groupVersion) 148 } 149 apiGroup.Versions = groupVersions 150 // There should be only one groupVersion returned at /api 151 apiGroup.PreferredVersion = groupVersions[0] 152 return 153} 154 155// ServerGroups returns the supported groups, with information like supported versions and the 156// preferred version. 157func (d *DiscoveryClient) ServerGroups() (apiGroupList *metav1.APIGroupList, err error) { 158 // Get the groupVersions exposed at /api 159 v := &metav1.APIVersions{} 160 err = d.restClient.Get().AbsPath(d.LegacyPrefix).Do(context.TODO()).Into(v) 161 apiGroup := metav1.APIGroup{} 162 if err == nil && len(v.Versions) != 0 { 163 apiGroup = apiVersionsToAPIGroup(v) 164 } 165 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 166 return nil, err 167 } 168 169 // Get the groupVersions exposed at /apis 170 apiGroupList = &metav1.APIGroupList{} 171 err = d.restClient.Get().AbsPath("/apis").Do(context.TODO()).Into(apiGroupList) 172 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 173 return nil, err 174 } 175 // to be compatible with a v1.0 server, if it's a 403 or 404, ignore and return whatever we got from /api 176 if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 177 apiGroupList = &metav1.APIGroupList{} 178 } 179 180 // prepend the group retrieved from /api to the list if not empty 181 if len(v.Versions) != 0 { 182 apiGroupList.Groups = append([]metav1.APIGroup{apiGroup}, apiGroupList.Groups...) 183 } 184 return apiGroupList, nil 185} 186 187// ServerResourcesForGroupVersion returns the supported resources for a group and version. 188func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *metav1.APIResourceList, err error) { 189 url := url.URL{} 190 if len(groupVersion) == 0 { 191 return nil, fmt.Errorf("groupVersion shouldn't be empty") 192 } 193 if len(d.LegacyPrefix) > 0 && groupVersion == "v1" { 194 url.Path = d.LegacyPrefix + "/" + groupVersion 195 } else { 196 url.Path = "/apis/" + groupVersion 197 } 198 resources = &metav1.APIResourceList{ 199 GroupVersion: groupVersion, 200 } 201 err = d.restClient.Get().AbsPath(url.String()).Do(context.TODO()).Into(resources) 202 if err != nil { 203 // ignore 403 or 404 error to be compatible with an v1.0 server. 204 if groupVersion == "v1" && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 205 return resources, nil 206 } 207 return nil, err 208 } 209 return resources, nil 210} 211 212// ServerResources returns the supported resources for all groups and versions. 213// Deprecated: use ServerGroupsAndResources instead. 214func (d *DiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { 215 _, rs, err := d.ServerGroupsAndResources() 216 return rs, err 217} 218 219// ServerGroupsAndResources returns the supported resources for all groups and versions. 220func (d *DiscoveryClient) ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 221 return withRetries(defaultRetries, func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 222 return ServerGroupsAndResources(d) 223 }) 224} 225 226// ErrGroupDiscoveryFailed is returned if one or more API groups fail to load. 227type ErrGroupDiscoveryFailed struct { 228 // Groups is a list of the groups that failed to load and the error cause 229 Groups map[schema.GroupVersion]error 230} 231 232// Error implements the error interface 233func (e *ErrGroupDiscoveryFailed) Error() string { 234 var groups []string 235 for k, v := range e.Groups { 236 groups = append(groups, fmt.Sprintf("%s: %v", k, v)) 237 } 238 sort.Strings(groups) 239 return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(groups, ", ")) 240} 241 242// IsGroupDiscoveryFailedError returns true if the provided error indicates the server was unable to discover 243// a complete list of APIs for the client to use. 244func IsGroupDiscoveryFailedError(err error) bool { 245 _, ok := err.(*ErrGroupDiscoveryFailed) 246 return err != nil && ok 247} 248 249// ServerResources uses the provided discovery interface to look up supported resources for all groups and versions. 250// Deprecated: use ServerGroupsAndResources instead. 251func ServerResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 252 _, rs, err := ServerGroupsAndResources(d) 253 return rs, err 254} 255 256func ServerGroupsAndResources(d DiscoveryInterface) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 257 sgs, err := d.ServerGroups() 258 if sgs == nil { 259 return nil, nil, err 260 } 261 resultGroups := []*metav1.APIGroup{} 262 for i := range sgs.Groups { 263 resultGroups = append(resultGroups, &sgs.Groups[i]) 264 } 265 266 groupVersionResources, failedGroups := fetchGroupVersionResources(d, sgs) 267 268 // order results by group/version discovery order 269 result := []*metav1.APIResourceList{} 270 for _, apiGroup := range sgs.Groups { 271 for _, version := range apiGroup.Versions { 272 gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 273 if resources, ok := groupVersionResources[gv]; ok { 274 result = append(result, resources) 275 } 276 } 277 } 278 279 if len(failedGroups) == 0 { 280 return resultGroups, result, nil 281 } 282 283 return resultGroups, result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 284} 285 286// ServerPreferredResources uses the provided discovery interface to look up preferred resources 287func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 288 serverGroupList, err := d.ServerGroups() 289 if err != nil { 290 return nil, err 291 } 292 293 groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList) 294 295 result := []*metav1.APIResourceList{} 296 grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource 297 grAPIResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource 298 gvAPIResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping 299 300 for _, apiGroup := range serverGroupList.Groups { 301 for _, version := range apiGroup.Versions { 302 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 303 304 apiResourceList, ok := groupVersionResources[groupVersion] 305 if !ok { 306 continue 307 } 308 309 // create empty list which is filled later in another loop 310 emptyAPIResourceList := metav1.APIResourceList{ 311 GroupVersion: version.GroupVersion, 312 } 313 gvAPIResourceLists[groupVersion] = &emptyAPIResourceList 314 result = append(result, &emptyAPIResourceList) 315 316 for i := range apiResourceList.APIResources { 317 apiResource := &apiResourceList.APIResources[i] 318 if strings.Contains(apiResource.Name, "/") { 319 continue 320 } 321 gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name} 322 if _, ok := grAPIResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version { 323 // only override with preferred version 324 continue 325 } 326 grVersions[gv] = version.Version 327 grAPIResources[gv] = apiResource 328 } 329 } 330 } 331 332 // group selected APIResources according to GroupVersion into APIResourceLists 333 for groupResource, apiResource := range grAPIResources { 334 version := grVersions[groupResource] 335 groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version} 336 apiResourceList := gvAPIResourceLists[groupVersion] 337 apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource) 338 } 339 340 if len(failedGroups) == 0 { 341 return result, nil 342 } 343 344 return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 345} 346 347// fetchServerResourcesForGroupVersions uses the discovery client to fetch the resources for the specified groups in parallel. 348func fetchGroupVersionResources(d DiscoveryInterface, apiGroups *metav1.APIGroupList) (map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error) { 349 groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) 350 failedGroups := make(map[schema.GroupVersion]error) 351 352 wg := &sync.WaitGroup{} 353 resultLock := &sync.Mutex{} 354 for _, apiGroup := range apiGroups.Groups { 355 for _, version := range apiGroup.Versions { 356 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 357 wg.Add(1) 358 go func() { 359 defer wg.Done() 360 defer utilruntime.HandleCrash() 361 362 apiResourceList, err := d.ServerResourcesForGroupVersion(groupVersion.String()) 363 364 // lock to record results 365 resultLock.Lock() 366 defer resultLock.Unlock() 367 368 if err != nil { 369 // TODO: maybe restrict this to NotFound errors 370 failedGroups[groupVersion] = err 371 } 372 if apiResourceList != nil { 373 // even in case of error, some fallback might have been returned 374 groupVersionResources[groupVersion] = apiResourceList 375 } 376 }() 377 } 378 } 379 wg.Wait() 380 381 return groupVersionResources, failedGroups 382} 383 384// ServerPreferredResources returns the supported resources with the version preferred by the 385// server. 386func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { 387 _, rs, err := withRetries(defaultRetries, func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 388 rs, err := ServerPreferredResources(d) 389 return nil, rs, err 390 }) 391 return rs, err 392} 393 394// ServerPreferredNamespacedResources returns the supported namespaced resources with the 395// version preferred by the server. 396func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { 397 return ServerPreferredNamespacedResources(d) 398} 399 400// ServerPreferredNamespacedResources uses the provided discovery interface to look up preferred namespaced resources 401func ServerPreferredNamespacedResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 402 all, err := ServerPreferredResources(d) 403 return FilteredBy(ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool { 404 return r.Namespaced 405 }), all), err 406} 407 408// ServerVersion retrieves and parses the server's version (git version). 409func (d *DiscoveryClient) ServerVersion() (*version.Info, error) { 410 body, err := d.restClient.Get().AbsPath("/version").Do(context.TODO()).Raw() 411 if err != nil { 412 return nil, err 413 } 414 var info version.Info 415 err = json.Unmarshal(body, &info) 416 if err != nil { 417 return nil, fmt.Errorf("unable to parse the server version: %v", err) 418 } 419 return &info, nil 420} 421 422// OpenAPISchema fetches the open api schema using a rest client and parses the proto. 423func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { 424 data, err := d.restClient.Get().AbsPath("/openapi/v2").SetHeader("Accept", mimePb).Do(context.TODO()).Raw() 425 if err != nil { 426 if errors.IsForbidden(err) || errors.IsNotFound(err) || errors.IsNotAcceptable(err) { 427 // single endpoint not found/registered in old server, try to fetch old endpoint 428 // TODO: remove this when kubectl/client-go don't work with 1.9 server 429 data, err = d.restClient.Get().AbsPath("/swagger-2.0.0.pb-v1").Do(context.TODO()).Raw() 430 if err != nil { 431 return nil, err 432 } 433 } else { 434 return nil, err 435 } 436 } 437 document := &openapi_v2.Document{} 438 err = proto.Unmarshal(data, document) 439 if err != nil { 440 return nil, err 441 } 442 return document, nil 443} 444 445// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns. 446func withRetries(maxRetries int, f func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error)) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 447 var result []*metav1.APIResourceList 448 var resultGroups []*metav1.APIGroup 449 var err error 450 for i := 0; i < maxRetries; i++ { 451 resultGroups, result, err = f() 452 if err == nil { 453 return resultGroups, result, nil 454 } 455 if _, ok := err.(*ErrGroupDiscoveryFailed); !ok { 456 return nil, nil, err 457 } 458 } 459 return resultGroups, result, err 460} 461 462func setDiscoveryDefaults(config *restclient.Config) error { 463 config.APIPath = "" 464 config.GroupVersion = nil 465 if config.Timeout == 0 { 466 config.Timeout = defaultTimeout 467 } 468 if config.Burst == 0 && config.QPS < 100 { 469 // discovery is expected to be bursty, increase the default burst 470 // to accommodate looking up resource info for many API groups. 471 // matches burst set by ConfigFlags#ToDiscoveryClient(). 472 // see https://issue.k8s.io/86149 473 config.Burst = 100 474 } 475 codec := runtime.NoopEncoder{Decoder: scheme.Codecs.UniversalDecoder()} 476 config.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec}) 477 if len(config.UserAgent) == 0 { 478 config.UserAgent = restclient.DefaultKubernetesUserAgent() 479 } 480 return nil 481} 482 483// NewDiscoveryClientForConfig creates a new DiscoveryClient for the given config. This client 484// can be used to discover supported resources in the API server. 485func NewDiscoveryClientForConfig(c *restclient.Config) (*DiscoveryClient, error) { 486 config := *c 487 if err := setDiscoveryDefaults(&config); err != nil { 488 return nil, err 489 } 490 client, err := restclient.UnversionedRESTClientFor(&config) 491 return &DiscoveryClient{restClient: client, LegacyPrefix: "/api"}, err 492} 493 494// NewDiscoveryClientForConfigOrDie creates a new DiscoveryClient for the given config. If 495// there is an error, it panics. 496func NewDiscoveryClientForConfigOrDie(c *restclient.Config) *DiscoveryClient { 497 client, err := NewDiscoveryClientForConfig(c) 498 if err != nil { 499 panic(err) 500 } 501 return client 502 503} 504 505// NewDiscoveryClient returns a new DiscoveryClient for the given RESTClient. 506func NewDiscoveryClient(c restclient.Interface) *DiscoveryClient { 507 return &DiscoveryClient{restClient: c, LegacyPrefix: "/api"} 508} 509 510// RESTClient returns a RESTClient that is used to communicate 511// with API server by this client implementation. 512func (d *DiscoveryClient) RESTClient() restclient.Interface { 513 if d == nil { 514 return nil 515 } 516 return d.restClient 517} 518