1/* 2Copyright 2015 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package discovery 18 19import ( 20 "encoding/json" 21 "fmt" 22 "net/url" 23 "sort" 24 "strings" 25 "sync" 26 "time" 27 28 "github.com/golang/protobuf/proto" 29 openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" 30 31 "k8s.io/apimachinery/pkg/api/errors" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/runtime" 34 "k8s.io/apimachinery/pkg/runtime/schema" 35 "k8s.io/apimachinery/pkg/runtime/serializer" 36 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 37 "k8s.io/apimachinery/pkg/version" 38 "k8s.io/client-go/kubernetes/scheme" 39 restclient "k8s.io/client-go/rest" 40) 41 42const ( 43 // defaultRetries is the number of times a resource discovery is repeated if an api group disappears on the fly (e.g. ThirdPartyResources). 44 defaultRetries = 2 45 // protobuf mime type 46 mimePb = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf" 47 // defaultTimeout is the maximum amount of time per request when no timeout has been set on a RESTClient. 48 // Defaults to 32s in order to have a distinguishable length of time, relative to other timeouts that exist. 49 defaultTimeout = 32 * time.Second 50) 51 52// DiscoveryInterface holds the methods that discover server-supported API groups, 53// versions and resources. 54type DiscoveryInterface interface { 55 RESTClient() restclient.Interface 56 ServerGroupsInterface 57 ServerResourcesInterface 58 ServerVersionInterface 59 OpenAPISchemaInterface 60} 61 62// CachedDiscoveryInterface is a DiscoveryInterface with cache invalidation and freshness. 63// Note that If the ServerResourcesForGroupVersion method returns a cache miss 64// error, the user needs to explicitly call Invalidate to clear the cache, 65// otherwise the same cache miss error will be returned next time. 66type CachedDiscoveryInterface interface { 67 DiscoveryInterface 68 // Fresh is supposed to tell the caller whether or not to retry if the cache 69 // fails to find something (false = retry, true = no need to retry). 70 // 71 // TODO: this needs to be revisited, this interface can't be locked properly 72 // and doesn't make a lot of sense. 73 Fresh() bool 74 // Invalidate enforces that no cached data that is older than the current time 75 // is used. 76 Invalidate() 77} 78 79// ServerGroupsInterface has methods for obtaining supported groups on the API server 80type ServerGroupsInterface interface { 81 // ServerGroups returns the supported groups, with information like supported versions and the 82 // preferred version. 83 ServerGroups() (*metav1.APIGroupList, error) 84} 85 86// ServerResourcesInterface has methods for obtaining supported resources on the API server 87type ServerResourcesInterface interface { 88 // ServerResourcesForGroupVersion returns the supported resources for a group and version. 89 ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) 90 // ServerResources returns the supported resources for all groups and versions. 91 // 92 // The returned resource list might be non-nil with partial results even in the case of 93 // non-nil error. 94 // 95 // Deprecated: use ServerGroupsAndResources instead. 96 ServerResources() ([]*metav1.APIResourceList, error) 97 // ServerResources returns the supported groups and resources for all groups and versions. 98 // 99 // The returned group and resource lists might be non-nil with partial results even in the 100 // case of non-nil error. 101 ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) 102 // ServerPreferredResources returns the supported resources with the version preferred by the 103 // server. 104 // 105 // The returned group and resource lists might be non-nil with partial results even in the 106 // case of non-nil error. 107 ServerPreferredResources() ([]*metav1.APIResourceList, error) 108 // ServerPreferredNamespacedResources returns the supported namespaced resources with the 109 // version preferred by the server. 110 // 111 // The returned resource list might be non-nil with partial results even in the case of 112 // non-nil error. 113 ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) 114} 115 116// ServerVersionInterface has a method for retrieving the server's version. 117type ServerVersionInterface interface { 118 // ServerVersion retrieves and parses the server's version (git version). 119 ServerVersion() (*version.Info, error) 120} 121 122// OpenAPISchemaInterface has a method to retrieve the open API schema. 123type OpenAPISchemaInterface interface { 124 // OpenAPISchema retrieves and parses the swagger API schema the server supports. 125 OpenAPISchema() (*openapi_v2.Document, error) 126} 127 128// DiscoveryClient implements the functions that discover server-supported API groups, 129// versions and resources. 130type DiscoveryClient struct { 131 restClient restclient.Interface 132 133 LegacyPrefix string 134} 135 136// Convert metav1.APIVersions to metav1.APIGroup. APIVersions is used by legacy v1, so 137// group would be "". 138func apiVersionsToAPIGroup(apiVersions *metav1.APIVersions) (apiGroup metav1.APIGroup) { 139 groupVersions := []metav1.GroupVersionForDiscovery{} 140 for _, version := range apiVersions.Versions { 141 groupVersion := metav1.GroupVersionForDiscovery{ 142 GroupVersion: version, 143 Version: version, 144 } 145 groupVersions = append(groupVersions, groupVersion) 146 } 147 apiGroup.Versions = groupVersions 148 // There should be only one groupVersion returned at /api 149 apiGroup.PreferredVersion = groupVersions[0] 150 return 151} 152 153// ServerGroups returns the supported groups, with information like supported versions and the 154// preferred version. 155func (d *DiscoveryClient) ServerGroups() (apiGroupList *metav1.APIGroupList, err error) { 156 // Get the groupVersions exposed at /api 157 v := &metav1.APIVersions{} 158 err = d.restClient.Get().AbsPath(d.LegacyPrefix).Do().Into(v) 159 apiGroup := metav1.APIGroup{} 160 if err == nil && len(v.Versions) != 0 { 161 apiGroup = apiVersionsToAPIGroup(v) 162 } 163 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 164 return nil, err 165 } 166 167 // Get the groupVersions exposed at /apis 168 apiGroupList = &metav1.APIGroupList{} 169 err = d.restClient.Get().AbsPath("/apis").Do().Into(apiGroupList) 170 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 171 return nil, err 172 } 173 // to be compatible with a v1.0 server, if it's a 403 or 404, ignore and return whatever we got from /api 174 if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 175 apiGroupList = &metav1.APIGroupList{} 176 } 177 178 // prepend the group retrieved from /api to the list if not empty 179 if len(v.Versions) != 0 { 180 apiGroupList.Groups = append([]metav1.APIGroup{apiGroup}, apiGroupList.Groups...) 181 } 182 return apiGroupList, nil 183} 184 185// ServerResourcesForGroupVersion returns the supported resources for a group and version. 186func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *metav1.APIResourceList, err error) { 187 url := url.URL{} 188 if len(groupVersion) == 0 { 189 return nil, fmt.Errorf("groupVersion shouldn't be empty") 190 } 191 if len(d.LegacyPrefix) > 0 && groupVersion == "v1" { 192 url.Path = d.LegacyPrefix + "/" + groupVersion 193 } else { 194 url.Path = "/apis/" + groupVersion 195 } 196 resources = &metav1.APIResourceList{ 197 GroupVersion: groupVersion, 198 } 199 err = d.restClient.Get().AbsPath(url.String()).Do().Into(resources) 200 if err != nil { 201 // ignore 403 or 404 error to be compatible with an v1.0 server. 202 if groupVersion == "v1" && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 203 return resources, nil 204 } 205 return nil, err 206 } 207 return resources, nil 208} 209 210// ServerResources returns the supported resources for all groups and versions. 211// Deprecated: use ServerGroupsAndResources instead. 212func (d *DiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { 213 _, rs, err := d.ServerGroupsAndResources() 214 return rs, err 215} 216 217// ServerGroupsAndResources returns the supported resources for all groups and versions. 218func (d *DiscoveryClient) ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 219 return withRetries(defaultRetries, func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 220 return ServerGroupsAndResources(d) 221 }) 222} 223 224// ErrGroupDiscoveryFailed is returned if one or more API groups fail to load. 225type ErrGroupDiscoveryFailed struct { 226 // Groups is a list of the groups that failed to load and the error cause 227 Groups map[schema.GroupVersion]error 228} 229 230// Error implements the error interface 231func (e *ErrGroupDiscoveryFailed) Error() string { 232 var groups []string 233 for k, v := range e.Groups { 234 groups = append(groups, fmt.Sprintf("%s: %v", k, v)) 235 } 236 sort.Strings(groups) 237 return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(groups, ", ")) 238} 239 240// IsGroupDiscoveryFailedError returns true if the provided error indicates the server was unable to discover 241// a complete list of APIs for the client to use. 242func IsGroupDiscoveryFailedError(err error) bool { 243 _, ok := err.(*ErrGroupDiscoveryFailed) 244 return err != nil && ok 245} 246 247// ServerResources uses the provided discovery interface to look up supported resources for all groups and versions. 248// Deprecated: use ServerGroupsAndResources instead. 249func ServerResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 250 _, rs, err := ServerGroupsAndResources(d) 251 return rs, err 252} 253 254func ServerGroupsAndResources(d DiscoveryInterface) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 255 sgs, err := d.ServerGroups() 256 if sgs == nil { 257 return nil, nil, err 258 } 259 resultGroups := []*metav1.APIGroup{} 260 for i := range sgs.Groups { 261 resultGroups = append(resultGroups, &sgs.Groups[i]) 262 } 263 264 groupVersionResources, failedGroups := fetchGroupVersionResources(d, sgs) 265 266 // order results by group/version discovery order 267 result := []*metav1.APIResourceList{} 268 for _, apiGroup := range sgs.Groups { 269 for _, version := range apiGroup.Versions { 270 gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 271 if resources, ok := groupVersionResources[gv]; ok { 272 result = append(result, resources) 273 } 274 } 275 } 276 277 if len(failedGroups) == 0 { 278 return resultGroups, result, nil 279 } 280 281 return resultGroups, result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 282} 283 284// ServerPreferredResources uses the provided discovery interface to look up preferred resources 285func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 286 serverGroupList, err := d.ServerGroups() 287 if err != nil { 288 return nil, err 289 } 290 291 groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList) 292 293 result := []*metav1.APIResourceList{} 294 grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource 295 grAPIResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource 296 gvAPIResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping 297 298 for _, apiGroup := range serverGroupList.Groups { 299 for _, version := range apiGroup.Versions { 300 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 301 302 apiResourceList, ok := groupVersionResources[groupVersion] 303 if !ok { 304 continue 305 } 306 307 // create empty list which is filled later in another loop 308 emptyAPIResourceList := metav1.APIResourceList{ 309 GroupVersion: version.GroupVersion, 310 } 311 gvAPIResourceLists[groupVersion] = &emptyAPIResourceList 312 result = append(result, &emptyAPIResourceList) 313 314 for i := range apiResourceList.APIResources { 315 apiResource := &apiResourceList.APIResources[i] 316 if strings.Contains(apiResource.Name, "/") { 317 continue 318 } 319 gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name} 320 if _, ok := grAPIResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version { 321 // only override with preferred version 322 continue 323 } 324 grVersions[gv] = version.Version 325 grAPIResources[gv] = apiResource 326 } 327 } 328 } 329 330 // group selected APIResources according to GroupVersion into APIResourceLists 331 for groupResource, apiResource := range grAPIResources { 332 version := grVersions[groupResource] 333 groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version} 334 apiResourceList := gvAPIResourceLists[groupVersion] 335 apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource) 336 } 337 338 if len(failedGroups) == 0 { 339 return result, nil 340 } 341 342 return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 343} 344 345// fetchServerResourcesForGroupVersions uses the discovery client to fetch the resources for the specified groups in parallel. 346func fetchGroupVersionResources(d DiscoveryInterface, apiGroups *metav1.APIGroupList) (map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error) { 347 groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) 348 failedGroups := make(map[schema.GroupVersion]error) 349 350 wg := &sync.WaitGroup{} 351 resultLock := &sync.Mutex{} 352 for _, apiGroup := range apiGroups.Groups { 353 for _, version := range apiGroup.Versions { 354 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 355 wg.Add(1) 356 go func() { 357 defer wg.Done() 358 defer utilruntime.HandleCrash() 359 360 apiResourceList, err := d.ServerResourcesForGroupVersion(groupVersion.String()) 361 362 // lock to record results 363 resultLock.Lock() 364 defer resultLock.Unlock() 365 366 if err != nil { 367 // TODO: maybe restrict this to NotFound errors 368 failedGroups[groupVersion] = err 369 } 370 if apiResourceList != nil { 371 // even in case of error, some fallback might have been returned 372 groupVersionResources[groupVersion] = apiResourceList 373 } 374 }() 375 } 376 } 377 wg.Wait() 378 379 return groupVersionResources, failedGroups 380} 381 382// ServerPreferredResources returns the supported resources with the version preferred by the 383// server. 384func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { 385 _, rs, err := withRetries(defaultRetries, func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 386 rs, err := ServerPreferredResources(d) 387 return nil, rs, err 388 }) 389 return rs, err 390} 391 392// ServerPreferredNamespacedResources returns the supported namespaced resources with the 393// version preferred by the server. 394func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { 395 return ServerPreferredNamespacedResources(d) 396} 397 398// ServerPreferredNamespacedResources uses the provided discovery interface to look up preferred namespaced resources 399func ServerPreferredNamespacedResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 400 all, err := ServerPreferredResources(d) 401 return FilteredBy(ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool { 402 return r.Namespaced 403 }), all), err 404} 405 406// ServerVersion retrieves and parses the server's version (git version). 407func (d *DiscoveryClient) ServerVersion() (*version.Info, error) { 408 body, err := d.restClient.Get().AbsPath("/version").Do().Raw() 409 if err != nil { 410 return nil, err 411 } 412 var info version.Info 413 err = json.Unmarshal(body, &info) 414 if err != nil { 415 return nil, fmt.Errorf("unable to parse the server version: %v", err) 416 } 417 return &info, nil 418} 419 420// OpenAPISchema fetches the open api schema using a rest client and parses the proto. 421func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { 422 data, err := d.restClient.Get().AbsPath("/openapi/v2").SetHeader("Accept", mimePb).Do().Raw() 423 if err != nil { 424 if errors.IsForbidden(err) || errors.IsNotFound(err) || errors.IsNotAcceptable(err) { 425 // single endpoint not found/registered in old server, try to fetch old endpoint 426 // TODO: remove this when kubectl/client-go don't work with 1.9 server 427 data, err = d.restClient.Get().AbsPath("/swagger-2.0.0.pb-v1").Do().Raw() 428 if err != nil { 429 return nil, err 430 } 431 } else { 432 return nil, err 433 } 434 } 435 document := &openapi_v2.Document{} 436 err = proto.Unmarshal(data, document) 437 if err != nil { 438 return nil, err 439 } 440 return document, nil 441} 442 443// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns. 444func withRetries(maxRetries int, f func() ([]*metav1.APIGroup, []*metav1.APIResourceList, error)) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 445 var result []*metav1.APIResourceList 446 var resultGroups []*metav1.APIGroup 447 var err error 448 for i := 0; i < maxRetries; i++ { 449 resultGroups, result, err = f() 450 if err == nil { 451 return resultGroups, result, nil 452 } 453 if _, ok := err.(*ErrGroupDiscoveryFailed); !ok { 454 return nil, nil, err 455 } 456 } 457 return resultGroups, result, err 458} 459 460func setDiscoveryDefaults(config *restclient.Config) error { 461 config.APIPath = "" 462 config.GroupVersion = nil 463 if config.Timeout == 0 { 464 config.Timeout = defaultTimeout 465 } 466 codec := runtime.NoopEncoder{Decoder: scheme.Codecs.UniversalDecoder()} 467 config.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec}) 468 if len(config.UserAgent) == 0 { 469 config.UserAgent = restclient.DefaultKubernetesUserAgent() 470 } 471 return nil 472} 473 474// NewDiscoveryClientForConfig creates a new DiscoveryClient for the given config. This client 475// can be used to discover supported resources in the API server. 476func NewDiscoveryClientForConfig(c *restclient.Config) (*DiscoveryClient, error) { 477 config := *c 478 if err := setDiscoveryDefaults(&config); err != nil { 479 return nil, err 480 } 481 client, err := restclient.UnversionedRESTClientFor(&config) 482 return &DiscoveryClient{restClient: client, LegacyPrefix: "/api"}, err 483} 484 485// NewDiscoveryClientForConfigOrDie creates a new DiscoveryClient for the given config. If 486// there is an error, it panics. 487func NewDiscoveryClientForConfigOrDie(c *restclient.Config) *DiscoveryClient { 488 client, err := NewDiscoveryClientForConfig(c) 489 if err != nil { 490 panic(err) 491 } 492 return client 493 494} 495 496// NewDiscoveryClient returns a new DiscoveryClient for the given RESTClient. 497func NewDiscoveryClient(c restclient.Interface) *DiscoveryClient { 498 return &DiscoveryClient{restClient: c, LegacyPrefix: "/api"} 499} 500 501// RESTClient returns a RESTClient that is used to communicate 502// with API server by this client implementation. 503func (d *DiscoveryClient) RESTClient() restclient.Interface { 504 if d == nil { 505 return nil 506 } 507 return d.restClient 508} 509