1/* 2Copyright 2015 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package discovery 18 19import ( 20 "encoding/json" 21 "fmt" 22 "net/url" 23 "sort" 24 "strings" 25 "sync" 26 "time" 27 28 "github.com/golang/protobuf/proto" 29 "github.com/googleapis/gnostic/OpenAPIv2" 30 31 "k8s.io/apimachinery/pkg/api/errors" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/runtime" 34 "k8s.io/apimachinery/pkg/runtime/schema" 35 "k8s.io/apimachinery/pkg/runtime/serializer" 36 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 37 "k8s.io/apimachinery/pkg/version" 38 "k8s.io/client-go/kubernetes/scheme" 39 restclient "k8s.io/client-go/rest" 40) 41 42const ( 43 // defaultRetries is the number of times a resource discovery is repeated if an api group disappears on the fly (e.g. ThirdPartyResources). 44 defaultRetries = 2 45 // protobuf mime type 46 mimePb = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf" 47 // defaultTimeout is the maximum amount of time per request when no timeout has been set on a RESTClient. 48 // Defaults to 32s in order to have a distinguishable length of time, relative to other timeouts that exist. 49 defaultTimeout = 32 * time.Second 50) 51 52// DiscoveryInterface holds the methods that discover server-supported API groups, 53// versions and resources. 54type DiscoveryInterface interface { 55 RESTClient() restclient.Interface 56 ServerGroupsInterface 57 ServerResourcesInterface 58 ServerVersionInterface 59 OpenAPISchemaInterface 60} 61 62// CachedDiscoveryInterface is a DiscoveryInterface with cache invalidation and freshness. 63type CachedDiscoveryInterface interface { 64 DiscoveryInterface 65 // Fresh is supposed to tell the caller whether or not to retry if the cache 66 // fails to find something (false = retry, true = no need to retry). 67 // 68 // TODO: this needs to be revisited, this interface can't be locked properly 69 // and doesn't make a lot of sense. 70 Fresh() bool 71 // Invalidate enforces that no cached data is used in the future that is older than the current time. 72 Invalidate() 73} 74 75// ServerGroupsInterface has methods for obtaining supported groups on the API server 76type ServerGroupsInterface interface { 77 // ServerGroups returns the supported groups, with information like supported versions and the 78 // preferred version. 79 ServerGroups() (*metav1.APIGroupList, error) 80} 81 82// ServerResourcesInterface has methods for obtaining supported resources on the API server 83type ServerResourcesInterface interface { 84 // ServerResourcesForGroupVersion returns the supported resources for a group and version. 85 ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) 86 // ServerResources returns the supported resources for all groups and versions. 87 ServerResources() ([]*metav1.APIResourceList, error) 88 // ServerPreferredResources returns the supported resources with the version preferred by the 89 // server. 90 ServerPreferredResources() ([]*metav1.APIResourceList, error) 91 // ServerPreferredNamespacedResources returns the supported namespaced resources with the 92 // version preferred by the server. 93 ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) 94} 95 96// ServerVersionInterface has a method for retrieving the server's version. 97type ServerVersionInterface interface { 98 // ServerVersion retrieves and parses the server's version (git version). 99 ServerVersion() (*version.Info, error) 100} 101 102// OpenAPISchemaInterface has a method to retrieve the open API schema. 103type OpenAPISchemaInterface interface { 104 // OpenAPISchema retrieves and parses the swagger API schema the server supports. 105 OpenAPISchema() (*openapi_v2.Document, error) 106} 107 108// DiscoveryClient implements the functions that discover server-supported API groups, 109// versions and resources. 110type DiscoveryClient struct { 111 restClient restclient.Interface 112 113 LegacyPrefix string 114} 115 116// Convert metav1.APIVersions to metav1.APIGroup. APIVersions is used by legacy v1, so 117// group would be "". 118func apiVersionsToAPIGroup(apiVersions *metav1.APIVersions) (apiGroup metav1.APIGroup) { 119 groupVersions := []metav1.GroupVersionForDiscovery{} 120 for _, version := range apiVersions.Versions { 121 groupVersion := metav1.GroupVersionForDiscovery{ 122 GroupVersion: version, 123 Version: version, 124 } 125 groupVersions = append(groupVersions, groupVersion) 126 } 127 apiGroup.Versions = groupVersions 128 // There should be only one groupVersion returned at /api 129 apiGroup.PreferredVersion = groupVersions[0] 130 return 131} 132 133// ServerGroups returns the supported groups, with information like supported versions and the 134// preferred version. 135func (d *DiscoveryClient) ServerGroups() (apiGroupList *metav1.APIGroupList, err error) { 136 // Get the groupVersions exposed at /api 137 v := &metav1.APIVersions{} 138 err = d.restClient.Get().AbsPath(d.LegacyPrefix).Do().Into(v) 139 apiGroup := metav1.APIGroup{} 140 if err == nil && len(v.Versions) != 0 { 141 apiGroup = apiVersionsToAPIGroup(v) 142 } 143 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 144 return nil, err 145 } 146 147 // Get the groupVersions exposed at /apis 148 apiGroupList = &metav1.APIGroupList{} 149 err = d.restClient.Get().AbsPath("/apis").Do().Into(apiGroupList) 150 if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) { 151 return nil, err 152 } 153 // to be compatible with a v1.0 server, if it's a 403 or 404, ignore and return whatever we got from /api 154 if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 155 apiGroupList = &metav1.APIGroupList{} 156 } 157 158 // prepend the group retrieved from /api to the list if not empty 159 if len(v.Versions) != 0 { 160 apiGroupList.Groups = append([]metav1.APIGroup{apiGroup}, apiGroupList.Groups...) 161 } 162 return apiGroupList, nil 163} 164 165// ServerResourcesForGroupVersion returns the supported resources for a group and version. 166func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *metav1.APIResourceList, err error) { 167 url := url.URL{} 168 if len(groupVersion) == 0 { 169 return nil, fmt.Errorf("groupVersion shouldn't be empty") 170 } 171 if len(d.LegacyPrefix) > 0 && groupVersion == "v1" { 172 url.Path = d.LegacyPrefix + "/" + groupVersion 173 } else { 174 url.Path = "/apis/" + groupVersion 175 } 176 resources = &metav1.APIResourceList{ 177 GroupVersion: groupVersion, 178 } 179 err = d.restClient.Get().AbsPath(url.String()).Do().Into(resources) 180 if err != nil { 181 // ignore 403 or 404 error to be compatible with an v1.0 server. 182 if groupVersion == "v1" && (errors.IsNotFound(err) || errors.IsForbidden(err)) { 183 return resources, nil 184 } 185 return nil, err 186 } 187 return resources, nil 188} 189 190// serverResources returns the supported resources for all groups and versions. 191func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) { 192 return ServerResources(d) 193} 194 195// ServerResources returns the supported resources for all groups and versions. 196func (d *DiscoveryClient) ServerResources() ([]*metav1.APIResourceList, error) { 197 return withRetries(defaultRetries, d.serverResources) 198} 199 200// ErrGroupDiscoveryFailed is returned if one or more API groups fail to load. 201type ErrGroupDiscoveryFailed struct { 202 // Groups is a list of the groups that failed to load and the error cause 203 Groups map[schema.GroupVersion]error 204} 205 206// Error implements the error interface 207func (e *ErrGroupDiscoveryFailed) Error() string { 208 var groups []string 209 for k, v := range e.Groups { 210 groups = append(groups, fmt.Sprintf("%s: %v", k, v)) 211 } 212 sort.Strings(groups) 213 return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(groups, ", ")) 214} 215 216// IsGroupDiscoveryFailedError returns true if the provided error indicates the server was unable to discover 217// a complete list of APIs for the client to use. 218func IsGroupDiscoveryFailedError(err error) bool { 219 _, ok := err.(*ErrGroupDiscoveryFailed) 220 return err != nil && ok 221} 222 223// serverPreferredResources returns the supported resources with the version preferred by the server. 224func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList, error) { 225 return ServerPreferredResources(d) 226} 227 228// ServerResources uses the provided discovery interface to look up supported resources for all groups and versions. 229func ServerResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 230 apiGroups, err := d.ServerGroups() 231 if err != nil { 232 return nil, err 233 } 234 235 groupVersionResources, failedGroups := fetchGroupVersionResources(d, apiGroups) 236 237 // order results by group/version discovery order 238 result := []*metav1.APIResourceList{} 239 for _, apiGroup := range apiGroups.Groups { 240 for _, version := range apiGroup.Versions { 241 gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 242 if resources, ok := groupVersionResources[gv]; ok { 243 result = append(result, resources) 244 } 245 } 246 } 247 248 if len(failedGroups) == 0 { 249 return result, nil 250 } 251 252 return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 253} 254 255// ServerPreferredResources uses the provided discovery interface to look up preferred resources 256func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 257 serverGroupList, err := d.ServerGroups() 258 if err != nil { 259 return nil, err 260 } 261 262 groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList) 263 264 result := []*metav1.APIResourceList{} 265 grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource 266 grApiResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource 267 gvApiResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping 268 269 for _, apiGroup := range serverGroupList.Groups { 270 for _, version := range apiGroup.Versions { 271 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 272 273 apiResourceList, ok := groupVersionResources[groupVersion] 274 if !ok { 275 continue 276 } 277 278 // create empty list which is filled later in another loop 279 emptyApiResourceList := metav1.APIResourceList{ 280 GroupVersion: version.GroupVersion, 281 } 282 gvApiResourceLists[groupVersion] = &emptyApiResourceList 283 result = append(result, &emptyApiResourceList) 284 285 for i := range apiResourceList.APIResources { 286 apiResource := &apiResourceList.APIResources[i] 287 if strings.Contains(apiResource.Name, "/") { 288 continue 289 } 290 gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name} 291 if _, ok := grApiResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version { 292 // only override with preferred version 293 continue 294 } 295 grVersions[gv] = version.Version 296 grApiResources[gv] = apiResource 297 } 298 } 299 } 300 301 // group selected APIResources according to GroupVersion into APIResourceLists 302 for groupResource, apiResource := range grApiResources { 303 version := grVersions[groupResource] 304 groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version} 305 apiResourceList := gvApiResourceLists[groupVersion] 306 apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource) 307 } 308 309 if len(failedGroups) == 0 { 310 return result, nil 311 } 312 313 return result, &ErrGroupDiscoveryFailed{Groups: failedGroups} 314} 315 316// fetchServerResourcesForGroupVersions uses the discovery client to fetch the resources for the specified groups in parallel 317func fetchGroupVersionResources(d DiscoveryInterface, apiGroups *metav1.APIGroupList) (map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error) { 318 groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) 319 failedGroups := make(map[schema.GroupVersion]error) 320 321 wg := &sync.WaitGroup{} 322 resultLock := &sync.Mutex{} 323 for _, apiGroup := range apiGroups.Groups { 324 for _, version := range apiGroup.Versions { 325 groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version} 326 wg.Add(1) 327 go func() { 328 defer wg.Done() 329 defer utilruntime.HandleCrash() 330 331 apiResourceList, err := d.ServerResourcesForGroupVersion(groupVersion.String()) 332 333 // lock to record results 334 resultLock.Lock() 335 defer resultLock.Unlock() 336 337 if err != nil { 338 // TODO: maybe restrict this to NotFound errors 339 failedGroups[groupVersion] = err 340 } else { 341 groupVersionResources[groupVersion] = apiResourceList 342 } 343 }() 344 } 345 } 346 wg.Wait() 347 348 return groupVersionResources, failedGroups 349} 350 351// ServerPreferredResources returns the supported resources with the version preferred by the 352// server. 353func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) { 354 return withRetries(defaultRetries, d.serverPreferredResources) 355} 356 357// ServerPreferredNamespacedResources returns the supported namespaced resources with the 358// version preferred by the server. 359func (d *DiscoveryClient) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { 360 return ServerPreferredNamespacedResources(d) 361} 362 363// ServerPreferredNamespacedResources uses the provided discovery interface to look up preferred namespaced resources 364func ServerPreferredNamespacedResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) { 365 all, err := ServerPreferredResources(d) 366 return FilteredBy(ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool { 367 return r.Namespaced 368 }), all), err 369} 370 371// ServerVersion retrieves and parses the server's version (git version). 372func (d *DiscoveryClient) ServerVersion() (*version.Info, error) { 373 body, err := d.restClient.Get().AbsPath("/version").Do().Raw() 374 if err != nil { 375 return nil, err 376 } 377 var info version.Info 378 err = json.Unmarshal(body, &info) 379 if err != nil { 380 return nil, fmt.Errorf("got '%s': %v", string(body), err) 381 } 382 return &info, nil 383} 384 385// OpenAPISchema fetches the open api schema using a rest client and parses the proto. 386func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { 387 data, err := d.restClient.Get().AbsPath("/openapi/v2").SetHeader("Accept", mimePb).Do().Raw() 388 if err != nil { 389 if errors.IsForbidden(err) || errors.IsNotFound(err) || errors.IsNotAcceptable(err) { 390 // single endpoint not found/registered in old server, try to fetch old endpoint 391 // TODO(roycaihw): remove this in 1.11 392 data, err = d.restClient.Get().AbsPath("/swagger-2.0.0.pb-v1").Do().Raw() 393 if err != nil { 394 return nil, err 395 } 396 } else { 397 return nil, err 398 } 399 } 400 document := &openapi_v2.Document{} 401 err = proto.Unmarshal(data, document) 402 if err != nil { 403 return nil, err 404 } 405 return document, nil 406} 407 408// withRetries retries the given recovery function in case the groups supported by the server change after ServerGroup() returns. 409func withRetries(maxRetries int, f func() ([]*metav1.APIResourceList, error)) ([]*metav1.APIResourceList, error) { 410 var result []*metav1.APIResourceList 411 var err error 412 for i := 0; i < maxRetries; i++ { 413 result, err = f() 414 if err == nil { 415 return result, nil 416 } 417 if _, ok := err.(*ErrGroupDiscoveryFailed); !ok { 418 return nil, err 419 } 420 } 421 return result, err 422} 423 424func setDiscoveryDefaults(config *restclient.Config) error { 425 config.APIPath = "" 426 config.GroupVersion = nil 427 if config.Timeout == 0 { 428 config.Timeout = defaultTimeout 429 } 430 codec := runtime.NoopEncoder{Decoder: scheme.Codecs.UniversalDecoder()} 431 config.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec}) 432 if len(config.UserAgent) == 0 { 433 config.UserAgent = restclient.DefaultKubernetesUserAgent() 434 } 435 return nil 436} 437 438// NewDiscoveryClientForConfig creates a new DiscoveryClient for the given config. This client 439// can be used to discover supported resources in the API server. 440func NewDiscoveryClientForConfig(c *restclient.Config) (*DiscoveryClient, error) { 441 config := *c 442 if err := setDiscoveryDefaults(&config); err != nil { 443 return nil, err 444 } 445 client, err := restclient.UnversionedRESTClientFor(&config) 446 return &DiscoveryClient{restClient: client, LegacyPrefix: "/api"}, err 447} 448 449// NewDiscoveryClientForConfigOrDie creates a new DiscoveryClient for the given config. If 450// there is an error, it panics. 451func NewDiscoveryClientForConfigOrDie(c *restclient.Config) *DiscoveryClient { 452 client, err := NewDiscoveryClientForConfig(c) 453 if err != nil { 454 panic(err) 455 } 456 return client 457 458} 459 460// NewDiscoveryClient returns a new DiscoveryClient for the given RESTClient. 461func NewDiscoveryClient(c restclient.Interface) *DiscoveryClient { 462 return &DiscoveryClient{restClient: c, LegacyPrefix: "/api"} 463} 464 465// RESTClient returns a RESTClient that is used to communicate 466// with API server by this client implementation. 467func (c *DiscoveryClient) RESTClient() restclient.Interface { 468 if c == nil { 469 return nil 470 } 471 return c.restClient 472} 473