1/* 2Copyright 2014 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package volume 18 19import ( 20 "fmt" 21 "net" 22 "strings" 23 "sync" 24 25 "k8s.io/apimachinery/pkg/util/sets" 26 "k8s.io/klog/v2" 27 "k8s.io/mount-utils" 28 "k8s.io/utils/exec" 29 30 authenticationv1 "k8s.io/api/authentication/v1" 31 v1 "k8s.io/api/core/v1" 32 "k8s.io/apimachinery/pkg/api/resource" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/types" 35 utilerrors "k8s.io/apimachinery/pkg/util/errors" 36 "k8s.io/apimachinery/pkg/util/validation" 37 "k8s.io/client-go/informers" 38 clientset "k8s.io/client-go/kubernetes" 39 storagelistersv1 "k8s.io/client-go/listers/storage/v1" 40 "k8s.io/client-go/tools/cache" 41 "k8s.io/client-go/tools/record" 42 cloudprovider "k8s.io/cloud-provider" 43 proxyutil "k8s.io/kubernetes/pkg/proxy/util" 44 "k8s.io/kubernetes/pkg/volume/util/hostutil" 45 "k8s.io/kubernetes/pkg/volume/util/recyclerclient" 46 "k8s.io/kubernetes/pkg/volume/util/subpath" 47) 48 49type ProbeOperation uint32 50type ProbeEvent struct { 51 Plugin VolumePlugin // VolumePlugin that was added/updated/removed. if ProbeEvent.Op is 'ProbeRemove', Plugin should be nil 52 PluginName string 53 Op ProbeOperation // The operation to the plugin 54} 55 56// CSIVolumePhaseType stores information about CSI volume path. 57type CSIVolumePhaseType string 58 59const ( 60 // Common parameter which can be specified in StorageClass to specify the desired FSType 61 // Provisioners SHOULD implement support for this if they are block device based 62 // Must be a filesystem type supported by the host operating system. 63 // Ex. "ext4", "xfs", "ntfs". Default value depends on the provisioner 64 VolumeParameterFSType = "fstype" 65 66 ProbeAddOrUpdate ProbeOperation = 1 << iota 67 ProbeRemove 68 CSIVolumeStaged CSIVolumePhaseType = "staged" 69 CSIVolumePublished CSIVolumePhaseType = "published" 70) 71 72var ( 73 deprecatedVolumeProviders = map[string]string{ 74 "kubernetes.io/cinder": "The Cinder volume provider is deprecated and will be removed in a future release", 75 "kubernetes.io/storageos": "The StorageOS volume provider is deprecated and will be removed in a future release", 76 "kubernetes.io/quobyte": "The Quobyte volume provider is deprecated and will be removed in a future release", 77 "kubernetes.io/flocker": "The Flocker volume provider is deprecated and will be removed in a future release", 78 } 79) 80 81// VolumeOptions contains option information about a volume. 82type VolumeOptions struct { 83 // The attributes below are required by volume.Provisioner 84 // TODO: refactor all of this out of volumes when an admin can configure 85 // many kinds of provisioners. 86 87 // Reclamation policy for a persistent volume 88 PersistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy 89 // Mount options for a persistent volume 90 MountOptions []string 91 // Suggested PV.Name of the PersistentVolume to provision. 92 // This is a generated name guaranteed to be unique in Kubernetes cluster. 93 // If you choose not to use it as volume name, ensure uniqueness by either 94 // combining it with your value or create unique values of your own. 95 PVName string 96 // PVC is reference to the claim that lead to provisioning of a new PV. 97 // Provisioners *must* create a PV that would be matched by this PVC, 98 // i.e. with required capacity, accessMode, labels matching PVC.Selector and 99 // so on. 100 PVC *v1.PersistentVolumeClaim 101 // Unique name of Kubernetes cluster. 102 ClusterName string 103 // Tags to attach to the real volume in the cloud provider - e.g. AWS EBS 104 CloudTags *map[string]string 105 // Volume provisioning parameters from StorageClass 106 Parameters map[string]string 107} 108 109// NodeResizeOptions contain options to be passed for node expansion. 110type NodeResizeOptions struct { 111 VolumeSpec *Spec 112 113 // DevicePath - location of actual device on the node. In case of CSI 114 // this just could be volumeID 115 DevicePath string 116 117 // DeviceMountPath location where device is mounted on the node. If volume type 118 // is attachable - this would be global mount path otherwise 119 // it would be location where volume was mounted for the pod 120 DeviceMountPath string 121 122 // DeviceStagingPath stores location where the volume is staged 123 DeviceStagePath string 124 125 NewSize resource.Quantity 126 OldSize resource.Quantity 127 128 // CSIVolumePhase contains volume phase on the node 129 CSIVolumePhase CSIVolumePhaseType 130} 131 132type DynamicPluginProber interface { 133 Init() error 134 135 // If an error occurs, events are undefined. 136 Probe() (events []ProbeEvent, err error) 137} 138 139// VolumePlugin is an interface to volume plugins that can be used on a 140// kubernetes node (e.g. by kubelet) to instantiate and manage volumes. 141type VolumePlugin interface { 142 // Init initializes the plugin. This will be called exactly once 143 // before any New* calls are made - implementations of plugins may 144 // depend on this. 145 Init(host VolumeHost) error 146 147 // Name returns the plugin's name. Plugins must use namespaced names 148 // such as "example.com/volume" and contain exactly one '/' character. 149 // The "kubernetes.io" namespace is reserved for plugins which are 150 // bundled with kubernetes. 151 GetPluginName() string 152 153 // GetVolumeName returns the name/ID to uniquely identifying the actual 154 // backing device, directory, path, etc. referenced by the specified volume 155 // spec. 156 // For Attachable volumes, this value must be able to be passed back to 157 // volume Detach methods to identify the device to act on. 158 // If the plugin does not support the given spec, this returns an error. 159 GetVolumeName(spec *Spec) (string, error) 160 161 // CanSupport tests whether the plugin supports a given volume 162 // specification from the API. The spec pointer should be considered 163 // const. 164 CanSupport(spec *Spec) bool 165 166 // RequiresRemount returns true if this plugin requires mount calls to be 167 // reexecuted. Atomically updating volumes, like Downward API, depend on 168 // this to update the contents of the volume. 169 RequiresRemount(spec *Spec) bool 170 171 // NewMounter creates a new volume.Mounter from an API specification. 172 // Ownership of the spec pointer in *not* transferred. 173 // - spec: The v1.Volume spec 174 // - pod: The enclosing pod 175 NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) 176 177 // NewUnmounter creates a new volume.Unmounter from recoverable state. 178 // - name: The volume name, as per the v1.Volume spec. 179 // - podUID: The UID of the enclosing pod 180 NewUnmounter(name string, podUID types.UID) (Unmounter, error) 181 182 // ConstructVolumeSpec constructs a volume spec based on the given volume name 183 // and volumePath. The spec may have incomplete information due to limited 184 // information from input. This function is used by volume manager to reconstruct 185 // volume spec by reading the volume directories from disk 186 ConstructVolumeSpec(volumeName, volumePath string) (*Spec, error) 187 188 // SupportsMountOption returns true if volume plugins supports Mount options 189 // Specifying mount options in a volume plugin that doesn't support 190 // user specified mount options will result in error creating persistent volumes 191 SupportsMountOption() bool 192 193 // SupportsBulkVolumeVerification checks if volume plugin type is capable 194 // of enabling bulk polling of all nodes. This can speed up verification of 195 // attached volumes by quite a bit, but underlying pluging must support it. 196 SupportsBulkVolumeVerification() bool 197} 198 199// PersistentVolumePlugin is an extended interface of VolumePlugin and is used 200// by volumes that want to provide long term persistence of data 201type PersistentVolumePlugin interface { 202 VolumePlugin 203 // GetAccessModes describes the ways a given volume can be accessed/mounted. 204 GetAccessModes() []v1.PersistentVolumeAccessMode 205} 206 207// RecyclableVolumePlugin is an extended interface of VolumePlugin and is used 208// by persistent volumes that want to be recycled before being made available 209// again to new claims 210type RecyclableVolumePlugin interface { 211 VolumePlugin 212 213 // Recycle knows how to reclaim this 214 // resource after the volume's release from a PersistentVolumeClaim. 215 // Recycle will use the provided recorder to write any events that might be 216 // interesting to user. It's expected that caller will pass these events to 217 // the PV being recycled. 218 Recycle(pvName string, spec *Spec, eventRecorder recyclerclient.RecycleEventRecorder) error 219} 220 221// DeletableVolumePlugin is an extended interface of VolumePlugin and is used 222// by persistent volumes that want to be deleted from the cluster after their 223// release from a PersistentVolumeClaim. 224type DeletableVolumePlugin interface { 225 VolumePlugin 226 // NewDeleter creates a new volume.Deleter which knows how to delete this 227 // resource in accordance with the underlying storage provider after the 228 // volume's release from a claim 229 NewDeleter(spec *Spec) (Deleter, error) 230} 231 232// ProvisionableVolumePlugin is an extended interface of VolumePlugin and is 233// used to create volumes for the cluster. 234type ProvisionableVolumePlugin interface { 235 VolumePlugin 236 // NewProvisioner creates a new volume.Provisioner which knows how to 237 // create PersistentVolumes in accordance with the plugin's underlying 238 // storage provider 239 NewProvisioner(options VolumeOptions) (Provisioner, error) 240} 241 242// AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment 243// to a node before mounting. 244type AttachableVolumePlugin interface { 245 DeviceMountableVolumePlugin 246 NewAttacher() (Attacher, error) 247 NewDetacher() (Detacher, error) 248 // CanAttach tests if provided volume spec is attachable 249 CanAttach(spec *Spec) (bool, error) 250} 251 252// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used 253// for volumes that requires mount device to a node before binding to volume to pod. 254type DeviceMountableVolumePlugin interface { 255 VolumePlugin 256 NewDeviceMounter() (DeviceMounter, error) 257 NewDeviceUnmounter() (DeviceUnmounter, error) 258 GetDeviceMountRefs(deviceMountPath string) ([]string, error) 259 // CanDeviceMount determines if device in volume.Spec is mountable 260 CanDeviceMount(spec *Spec) (bool, error) 261} 262 263// ExpandableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that can be 264// expanded via control-plane ExpandVolumeDevice call. 265type ExpandableVolumePlugin interface { 266 VolumePlugin 267 ExpandVolumeDevice(spec *Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) 268 RequiresFSResize() bool 269} 270 271// NodeExpandableVolumePlugin is an expanded interface of VolumePlugin and is used for volumes that 272// require expansion on the node via NodeExpand call. 273type NodeExpandableVolumePlugin interface { 274 VolumePlugin 275 RequiresFSResize() bool 276 // NodeExpand expands volume on given deviceMountPath and returns true if resize is successful. 277 NodeExpand(resizeOptions NodeResizeOptions) (bool, error) 278} 279 280// VolumePluginWithAttachLimits is an extended interface of VolumePlugin that restricts number of 281// volumes that can be attached to a node. 282type VolumePluginWithAttachLimits interface { 283 VolumePlugin 284 // Return maximum number of volumes that can be attached to a node for this plugin. 285 // The key must be same as string returned by VolumeLimitKey function. The returned 286 // map may look like: 287 // - { "storage-limits-aws-ebs": 39 } 288 // - { "storage-limits-gce-pd": 10 } 289 // A volume plugin may return error from this function - if it can not be used on a given node or not 290 // applicable in given environment (where environment could be cloudprovider or any other dependency) 291 // For example - calling this function for EBS volume plugin on a GCE node should 292 // result in error. 293 // The returned values are stored in node allocatable property and will be used 294 // by scheduler to determine how many pods with volumes can be scheduled on given node. 295 GetVolumeLimits() (map[string]int64, error) 296 // Return volume limit key string to be used in node capacity constraints 297 // The key must start with prefix storage-limits-. For example: 298 // - storage-limits-aws-ebs 299 // - storage-limits-csi-cinder 300 // The key should respect character limit of ResourceName type 301 // This function may be called by kubelet or scheduler to identify node allocatable property 302 // which stores volumes limits. 303 VolumeLimitKey(spec *Spec) string 304} 305 306// BlockVolumePlugin is an extend interface of VolumePlugin and is used for block volumes support. 307type BlockVolumePlugin interface { 308 VolumePlugin 309 // NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification. 310 // Ownership of the spec pointer in *not* transferred. 311 // - spec: The v1.Volume spec 312 // - pod: The enclosing pod 313 NewBlockVolumeMapper(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error) 314 // NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state. 315 // - name: The volume name, as per the v1.Volume spec. 316 // - podUID: The UID of the enclosing pod 317 NewBlockVolumeUnmapper(name string, podUID types.UID) (BlockVolumeUnmapper, error) 318 // ConstructBlockVolumeSpec constructs a volume spec based on the given 319 // podUID, volume name and a pod device map path. 320 // The spec may have incomplete information due to limited information 321 // from input. This function is used by volume manager to reconstruct 322 // volume spec by reading the volume directories from disk. 323 ConstructBlockVolumeSpec(podUID types.UID, volumeName, volumePath string) (*Spec, error) 324} 325 326// TODO(#14217) 327// As part of the Volume Host refactor we are starting to create Volume Hosts 328// for specific hosts. New methods for each specific host can be added here. 329// Currently consumers will do type assertions to get the specific type of Volume 330// Host; however, the end result should be that specific Volume Hosts are passed 331// to the specific functions they are needed in (instead of using a catch-all 332// VolumeHost interface) 333 334// KubeletVolumeHost is a Kubelet specific interface that plugins can use to access the kubelet. 335type KubeletVolumeHost interface { 336 // SetKubeletError lets plugins set an error on the Kubelet runtime status 337 // that will cause the Kubelet to post NotReady status with the error message provided 338 SetKubeletError(err error) 339 340 // GetInformerFactory returns the informer factory for CSIDriverLister 341 GetInformerFactory() informers.SharedInformerFactory 342 // CSIDriverLister returns the informer lister for the CSIDriver API Object 343 CSIDriverLister() storagelistersv1.CSIDriverLister 344 // CSIDriverSynced returns the informer synced for the CSIDriver API Object 345 CSIDriversSynced() cache.InformerSynced 346 // WaitForCacheSync is a helper function that waits for cache sync for CSIDriverLister 347 WaitForCacheSync() error 348 // Returns hostutil.HostUtils 349 GetHostUtil() hostutil.HostUtils 350} 351 352// AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use 353// to access methods on the Attach Detach Controller. 354type AttachDetachVolumeHost interface { 355 // CSINodeLister returns the informer lister for the CSINode API Object 356 CSINodeLister() storagelistersv1.CSINodeLister 357 358 // CSIDriverLister returns the informer lister for the CSIDriver API Object 359 CSIDriverLister() storagelistersv1.CSIDriverLister 360 361 // VolumeAttachmentLister returns the informer lister for the VolumeAttachment API Object 362 VolumeAttachmentLister() storagelistersv1.VolumeAttachmentLister 363 // IsAttachDetachController is an interface marker to strictly tie AttachDetachVolumeHost 364 // to the attachDetachController 365 IsAttachDetachController() bool 366} 367 368// VolumeHost is an interface that plugins can use to access the kubelet. 369type VolumeHost interface { 370 // GetPluginDir returns the absolute path to a directory under which 371 // a given plugin may store data. This directory might not actually 372 // exist on disk yet. For plugin data that is per-pod, see 373 // GetPodPluginDir(). 374 GetPluginDir(pluginName string) string 375 376 // GetVolumeDevicePluginDir returns the absolute path to a directory 377 // under which a given plugin may store data. 378 // ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/ 379 GetVolumeDevicePluginDir(pluginName string) string 380 381 // GetPodsDir returns the absolute path to a directory where all the pods 382 // information is stored 383 GetPodsDir() string 384 385 // GetPodVolumeDir returns the absolute path a directory which 386 // represents the named volume under the named plugin for the given 387 // pod. If the specified pod does not exist, the result of this call 388 // might not exist. 389 GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string 390 391 // GetPodPluginDir returns the absolute path to a directory under which 392 // a given plugin may store data for a given pod. If the specified pod 393 // does not exist, the result of this call might not exist. This 394 // directory might not actually exist on disk yet. 395 GetPodPluginDir(podUID types.UID, pluginName string) string 396 397 // GetPodVolumeDeviceDir returns the absolute path a directory which 398 // represents the named plugin for the given pod. 399 // If the specified pod does not exist, the result of this call 400 // might not exist. 401 // ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/ 402 GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string 403 404 // GetKubeClient returns a client interface 405 GetKubeClient() clientset.Interface 406 407 // NewWrapperMounter finds an appropriate plugin with which to handle 408 // the provided spec. This is used to implement volume plugins which 409 // "wrap" other plugins. For example, the "secret" volume is 410 // implemented in terms of the "emptyDir" volume. 411 NewWrapperMounter(volName string, spec Spec, pod *v1.Pod, opts VolumeOptions) (Mounter, error) 412 413 // NewWrapperUnmounter finds an appropriate plugin with which to handle 414 // the provided spec. See comments on NewWrapperMounter for more 415 // context. 416 NewWrapperUnmounter(volName string, spec Spec, podUID types.UID) (Unmounter, error) 417 418 // Get cloud provider from kubelet. 419 GetCloudProvider() cloudprovider.Interface 420 421 // Get mounter interface. 422 GetMounter(pluginName string) mount.Interface 423 424 // Returns the hostname of the host kubelet is running on 425 GetHostName() string 426 427 // Returns host IP or nil in the case of error. 428 GetHostIP() (net.IP, error) 429 430 // Returns node allocatable. 431 GetNodeAllocatable() (v1.ResourceList, error) 432 433 // Returns a function that returns a secret. 434 GetSecretFunc() func(namespace, name string) (*v1.Secret, error) 435 436 // Returns a function that returns a configmap. 437 GetConfigMapFunc() func(namespace, name string) (*v1.ConfigMap, error) 438 439 GetServiceAccountTokenFunc() func(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) 440 441 DeleteServiceAccountTokenFunc() func(podUID types.UID) 442 443 // Returns an interface that should be used to execute any utilities in volume plugins 444 GetExec(pluginName string) exec.Interface 445 446 // Returns the labels on the node 447 GetNodeLabels() (map[string]string, error) 448 449 // Returns the name of the node 450 GetNodeName() types.NodeName 451 452 // Returns the event recorder of kubelet. 453 GetEventRecorder() record.EventRecorder 454 455 // Returns an interface that should be used to execute subpath operations 456 GetSubpather() subpath.Interface 457 458 // Returns options to pass for proxyutil filtered dialers. 459 GetFilteredDialOptions() *proxyutil.FilteredDialOptions 460} 461 462// VolumePluginMgr tracks registered plugins. 463type VolumePluginMgr struct { 464 mutex sync.RWMutex 465 plugins map[string]VolumePlugin 466 prober DynamicPluginProber 467 probedPlugins map[string]VolumePlugin 468 loggedDeprecationWarnings sets.String 469 Host VolumeHost 470} 471 472// Spec is an internal representation of a volume. All API volume types translate to Spec. 473type Spec struct { 474 Volume *v1.Volume 475 PersistentVolume *v1.PersistentVolume 476 ReadOnly bool 477 InlineVolumeSpecForCSIMigration bool 478 Migrated bool 479} 480 481// Name returns the name of either Volume or PersistentVolume, one of which must not be nil. 482func (spec *Spec) Name() string { 483 switch { 484 case spec.Volume != nil: 485 return spec.Volume.Name 486 case spec.PersistentVolume != nil: 487 return spec.PersistentVolume.Name 488 default: 489 return "" 490 } 491} 492 493// IsKubeletExpandable returns true for volume types that can be expanded only by the node 494// and not the controller. Currently Flex volume is the only one in this category since 495// it is typically not installed on the controller 496func (spec *Spec) IsKubeletExpandable() bool { 497 switch { 498 case spec.Volume != nil: 499 return spec.Volume.FlexVolume != nil 500 case spec.PersistentVolume != nil: 501 return spec.PersistentVolume.Spec.FlexVolume != nil 502 default: 503 return false 504 505 } 506} 507 508// KubeletExpandablePluginName creates and returns a name for the plugin 509// this is used in context on the controller where the plugin lookup fails 510// as volume expansion on controller isn't supported, but a plugin name is 511// required 512func (spec *Spec) KubeletExpandablePluginName() string { 513 switch { 514 case spec.Volume != nil && spec.Volume.FlexVolume != nil: 515 return spec.Volume.FlexVolume.Driver 516 case spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil: 517 return spec.PersistentVolume.Spec.FlexVolume.Driver 518 default: 519 return "" 520 } 521} 522 523// VolumeConfig is how volume plugins receive configuration. An instance 524// specific to the plugin will be passed to the plugin's 525// ProbeVolumePlugins(config) func. Reasonable defaults will be provided by 526// the binary hosting the plugins while allowing override of those default 527// values. Those config values are then set to an instance of VolumeConfig 528// and passed to the plugin. 529// 530// Values in VolumeConfig are intended to be relevant to several plugins, but 531// not necessarily all plugins. The preference is to leverage strong typing 532// in this struct. All config items must have a descriptive but non-specific 533// name (i.e, RecyclerMinimumTimeout is OK but RecyclerMinimumTimeoutForNFS is 534// !OK). An instance of config will be given directly to the plugin, so 535// config names specific to plugins are unneeded and wrongly expose plugins in 536// this VolumeConfig struct. 537// 538// OtherAttributes is a map of string values intended for one-off 539// configuration of a plugin or config that is only relevant to a single 540// plugin. All values are passed by string and require interpretation by the 541// plugin. Passing config as strings is the least desirable option but can be 542// used for truly one-off configuration. The binary should still use strong 543// typing for this value when binding CLI values before they are passed as 544// strings in OtherAttributes. 545type VolumeConfig struct { 546 // RecyclerPodTemplate is pod template that understands how to scrub clean 547 // a persistent volume after its release. The template is used by plugins 548 // which override specific properties of the pod in accordance with that 549 // plugin. See NewPersistentVolumeRecyclerPodTemplate for the properties 550 // that are expected to be overridden. 551 RecyclerPodTemplate *v1.Pod 552 553 // RecyclerMinimumTimeout is the minimum amount of time in seconds for the 554 // recycler pod's ActiveDeadlineSeconds attribute. Added to the minimum 555 // timeout is the increment per Gi of capacity. 556 RecyclerMinimumTimeout int 557 558 // RecyclerTimeoutIncrement is the number of seconds added to the recycler 559 // pod's ActiveDeadlineSeconds for each Gi of capacity in the persistent 560 // volume. Example: 5Gi volume x 30s increment = 150s + 30s minimum = 180s 561 // ActiveDeadlineSeconds for recycler pod 562 RecyclerTimeoutIncrement int 563 564 // PVName is name of the PersistentVolume instance that is being recycled. 565 // It is used to generate unique recycler pod name. 566 PVName string 567 568 // OtherAttributes stores config as strings. These strings are opaque to 569 // the system and only understood by the binary hosting the plugin and the 570 // plugin itself. 571 OtherAttributes map[string]string 572 573 // ProvisioningEnabled configures whether provisioning of this plugin is 574 // enabled or not. Currently used only in host_path plugin. 575 ProvisioningEnabled bool 576} 577 578// NewSpecFromVolume creates an Spec from an v1.Volume 579func NewSpecFromVolume(vs *v1.Volume) *Spec { 580 return &Spec{ 581 Volume: vs, 582 } 583} 584 585// NewSpecFromPersistentVolume creates an Spec from an v1.PersistentVolume 586func NewSpecFromPersistentVolume(pv *v1.PersistentVolume, readOnly bool) *Spec { 587 return &Spec{ 588 PersistentVolume: pv, 589 ReadOnly: readOnly, 590 } 591} 592 593// InitPlugins initializes each plugin. All plugins must have unique names. 594// This must be called exactly once before any New* methods are called on any 595// plugins. 596func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error { 597 pm.mutex.Lock() 598 defer pm.mutex.Unlock() 599 600 pm.Host = host 601 pm.loggedDeprecationWarnings = sets.NewString() 602 603 if prober == nil { 604 // Use a dummy prober to prevent nil deference. 605 pm.prober = &dummyPluginProber{} 606 } else { 607 pm.prober = prober 608 } 609 if err := pm.prober.Init(); err != nil { 610 // Prober init failure should not affect the initialization of other plugins. 611 klog.ErrorS(err, "Error initializing dynamic plugin prober") 612 pm.prober = &dummyPluginProber{} 613 } 614 615 if pm.plugins == nil { 616 pm.plugins = map[string]VolumePlugin{} 617 } 618 if pm.probedPlugins == nil { 619 pm.probedPlugins = map[string]VolumePlugin{} 620 } 621 622 allErrs := []error{} 623 for _, plugin := range plugins { 624 name := plugin.GetPluginName() 625 if errs := validation.IsQualifiedName(name); len(errs) != 0 { 626 allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";"))) 627 continue 628 } 629 630 if _, found := pm.plugins[name]; found { 631 allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name)) 632 continue 633 } 634 err := plugin.Init(host) 635 if err != nil { 636 klog.ErrorS(err, "Failed to load volume plugin", "pluginName", name) 637 allErrs = append(allErrs, err) 638 continue 639 } 640 pm.plugins[name] = plugin 641 klog.V(1).InfoS("Loaded volume plugin", "pluginName", name) 642 } 643 return utilerrors.NewAggregate(allErrs) 644} 645 646func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error { 647 name := probedPlugin.GetPluginName() 648 if errs := validation.IsQualifiedName(name); len(errs) != 0 { 649 return fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")) 650 } 651 652 err := probedPlugin.Init(pm.Host) 653 if err != nil { 654 return fmt.Errorf("failed to load volume plugin %s, error: %s", name, err.Error()) 655 } 656 657 klog.V(1).InfoS("Loaded volume plugin", "pluginName", name) 658 return nil 659} 660 661// FindPluginBySpec looks for a plugin that can support a given volume 662// specification. If no plugins can support or more than one plugin can 663// support it, return error. 664func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) { 665 pm.mutex.RLock() 666 defer pm.mutex.RUnlock() 667 668 if spec == nil { 669 return nil, fmt.Errorf("could not find plugin because volume spec is nil") 670 } 671 672 matches := []VolumePlugin{} 673 for _, v := range pm.plugins { 674 if v.CanSupport(spec) { 675 matches = append(matches, v) 676 } 677 } 678 679 pm.refreshProbedPlugins() 680 for _, plugin := range pm.probedPlugins { 681 if plugin.CanSupport(spec) { 682 matches = append(matches, plugin) 683 } 684 } 685 686 if len(matches) == 0 { 687 return nil, fmt.Errorf("no volume plugin matched") 688 } 689 if len(matches) > 1 { 690 matchedPluginNames := []string{} 691 for _, plugin := range matches { 692 matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) 693 } 694 return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) 695 } 696 697 // Issue warning if the matched provider is deprecated 698 pm.logDeprecation(matches[0].GetPluginName()) 699 return matches[0], nil 700} 701 702// FindPluginByName fetches a plugin by name or by legacy name. If no plugin 703// is found, returns error. 704func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) { 705 pm.mutex.RLock() 706 defer pm.mutex.RUnlock() 707 708 // Once we can get rid of legacy names we can reduce this to a map lookup. 709 matches := []VolumePlugin{} 710 if v, found := pm.plugins[name]; found { 711 matches = append(matches, v) 712 } 713 714 pm.refreshProbedPlugins() 715 if plugin, found := pm.probedPlugins[name]; found { 716 matches = append(matches, plugin) 717 } 718 719 if len(matches) == 0 { 720 return nil, fmt.Errorf("no volume plugin matched name: %s", name) 721 } 722 if len(matches) > 1 { 723 matchedPluginNames := []string{} 724 for _, plugin := range matches { 725 matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) 726 } 727 return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) 728 } 729 730 // Issue warning if the matched provider is deprecated 731 pm.logDeprecation(matches[0].GetPluginName()) 732 return matches[0], nil 733} 734 735// logDeprecation logs warning when a deprecated plugin is used. 736func (pm *VolumePluginMgr) logDeprecation(plugin string) { 737 if detail, ok := deprecatedVolumeProviders[plugin]; ok && !pm.loggedDeprecationWarnings.Has(plugin) { 738 klog.Warningf("WARNING: %s built-in volume provider is now deprecated. %s", plugin, detail) 739 // Make sure the message is logged only once. It has Warning severity 740 // and we don't want to spam the log too much. 741 pm.loggedDeprecationWarnings.Insert(plugin) 742 } 743} 744 745// Check if probedPlugin cache update is required. 746// If it is, initialize all probed plugins and replace the cache with them. 747func (pm *VolumePluginMgr) refreshProbedPlugins() { 748 events, err := pm.prober.Probe() 749 if err != nil { 750 klog.ErrorS(err, "Error dynamically probing plugins") 751 return // Use cached plugins upon failure. 752 } 753 754 for _, event := range events { 755 if event.Op == ProbeAddOrUpdate { 756 if err := pm.initProbedPlugin(event.Plugin); err != nil { 757 klog.ErrorS(err, "Error initializing dynamically probed plugin", 758 "pluginName", event.Plugin.GetPluginName()) 759 continue 760 } 761 pm.probedPlugins[event.Plugin.GetPluginName()] = event.Plugin 762 } else if event.Op == ProbeRemove { 763 // Plugin is not available on ProbeRemove event, only PluginName 764 delete(pm.probedPlugins, event.PluginName) 765 } else { 766 klog.ErrorS(nil, "Unknown Operation on PluginName.", 767 "pluginName", event.Plugin.GetPluginName()) 768 } 769 } 770} 771 772// ListVolumePluginWithLimits returns plugins that have volume limits on nodes 773func (pm *VolumePluginMgr) ListVolumePluginWithLimits() []VolumePluginWithAttachLimits { 774 pm.mutex.RLock() 775 defer pm.mutex.RUnlock() 776 777 matchedPlugins := []VolumePluginWithAttachLimits{} 778 for _, v := range pm.plugins { 779 if plugin, ok := v.(VolumePluginWithAttachLimits); ok { 780 matchedPlugins = append(matchedPlugins, plugin) 781 } 782 } 783 return matchedPlugins 784} 785 786// FindPersistentPluginBySpec looks for a persistent volume plugin that can 787// support a given volume specification. If no plugin is found, return an 788// error 789func (pm *VolumePluginMgr) FindPersistentPluginBySpec(spec *Spec) (PersistentVolumePlugin, error) { 790 volumePlugin, err := pm.FindPluginBySpec(spec) 791 if err != nil { 792 return nil, fmt.Errorf("could not find volume plugin for spec: %#v", spec) 793 } 794 if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok { 795 return persistentVolumePlugin, nil 796 } 797 return nil, fmt.Errorf("no persistent volume plugin matched") 798} 799 800// FindVolumePluginWithLimitsBySpec returns volume plugin that has a limit on how many 801// of them can be attached to a node 802func (pm *VolumePluginMgr) FindVolumePluginWithLimitsBySpec(spec *Spec) (VolumePluginWithAttachLimits, error) { 803 volumePlugin, err := pm.FindPluginBySpec(spec) 804 if err != nil { 805 return nil, fmt.Errorf("could not find volume plugin for spec : %#v", spec) 806 } 807 808 if limitedPlugin, ok := volumePlugin.(VolumePluginWithAttachLimits); ok { 809 return limitedPlugin, nil 810 } 811 return nil, fmt.Errorf("no plugin with limits found") 812} 813 814// FindPersistentPluginByName fetches a persistent volume plugin by name. If 815// no plugin is found, returns error. 816func (pm *VolumePluginMgr) FindPersistentPluginByName(name string) (PersistentVolumePlugin, error) { 817 volumePlugin, err := pm.FindPluginByName(name) 818 if err != nil { 819 return nil, err 820 } 821 if persistentVolumePlugin, ok := volumePlugin.(PersistentVolumePlugin); ok { 822 return persistentVolumePlugin, nil 823 } 824 return nil, fmt.Errorf("no persistent volume plugin matched") 825} 826 827// FindRecyclablePluginByName fetches a persistent volume plugin by name. If 828// no plugin is found, returns error. 829func (pm *VolumePluginMgr) FindRecyclablePluginBySpec(spec *Spec) (RecyclableVolumePlugin, error) { 830 volumePlugin, err := pm.FindPluginBySpec(spec) 831 if err != nil { 832 return nil, err 833 } 834 if recyclableVolumePlugin, ok := volumePlugin.(RecyclableVolumePlugin); ok { 835 return recyclableVolumePlugin, nil 836 } 837 return nil, fmt.Errorf("no recyclable volume plugin matched") 838} 839 840// FindProvisionablePluginByName fetches a persistent volume plugin by name. If 841// no plugin is found, returns error. 842func (pm *VolumePluginMgr) FindProvisionablePluginByName(name string) (ProvisionableVolumePlugin, error) { 843 volumePlugin, err := pm.FindPluginByName(name) 844 if err != nil { 845 return nil, err 846 } 847 if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok { 848 return provisionableVolumePlugin, nil 849 } 850 return nil, fmt.Errorf("no provisionable volume plugin matched") 851} 852 853// FindDeletablePluginBySpec fetches a persistent volume plugin by spec. If 854// no plugin is found, returns error. 855func (pm *VolumePluginMgr) FindDeletablePluginBySpec(spec *Spec) (DeletableVolumePlugin, error) { 856 volumePlugin, err := pm.FindPluginBySpec(spec) 857 if err != nil { 858 return nil, err 859 } 860 if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok { 861 return deletableVolumePlugin, nil 862 } 863 return nil, fmt.Errorf("no deletable volume plugin matched") 864} 865 866// FindDeletablePluginByName fetches a persistent volume plugin by name. If 867// no plugin is found, returns error. 868func (pm *VolumePluginMgr) FindDeletablePluginByName(name string) (DeletableVolumePlugin, error) { 869 volumePlugin, err := pm.FindPluginByName(name) 870 if err != nil { 871 return nil, err 872 } 873 if deletableVolumePlugin, ok := volumePlugin.(DeletableVolumePlugin); ok { 874 return deletableVolumePlugin, nil 875 } 876 return nil, fmt.Errorf("no deletable volume plugin matched") 877} 878 879// FindCreatablePluginBySpec fetches a persistent volume plugin by name. If 880// no plugin is found, returns error. 881func (pm *VolumePluginMgr) FindCreatablePluginBySpec(spec *Spec) (ProvisionableVolumePlugin, error) { 882 volumePlugin, err := pm.FindPluginBySpec(spec) 883 if err != nil { 884 return nil, err 885 } 886 if provisionableVolumePlugin, ok := volumePlugin.(ProvisionableVolumePlugin); ok { 887 return provisionableVolumePlugin, nil 888 } 889 return nil, fmt.Errorf("no creatable volume plugin matched") 890} 891 892// FindAttachablePluginBySpec fetches a persistent volume plugin by spec. 893// Unlike the other "FindPlugin" methods, this does not return error if no 894// plugin is found. All volumes require a mounter and unmounter, but not 895// every volume will have an attacher/detacher. 896func (pm *VolumePluginMgr) FindAttachablePluginBySpec(spec *Spec) (AttachableVolumePlugin, error) { 897 volumePlugin, err := pm.FindPluginBySpec(spec) 898 if err != nil { 899 return nil, err 900 } 901 if attachableVolumePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok { 902 if canAttach, err := attachableVolumePlugin.CanAttach(spec); err != nil { 903 return nil, err 904 } else if canAttach { 905 return attachableVolumePlugin, nil 906 } 907 } 908 return nil, nil 909} 910 911// FindAttachablePluginByName fetches an attachable volume plugin by name. 912// Unlike the other "FindPlugin" methods, this does not return error if no 913// plugin is found. All volumes require a mounter and unmounter, but not 914// every volume will have an attacher/detacher. 915func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVolumePlugin, error) { 916 volumePlugin, err := pm.FindPluginByName(name) 917 if err != nil { 918 return nil, err 919 } 920 if attachablePlugin, ok := volumePlugin.(AttachableVolumePlugin); ok { 921 return attachablePlugin, nil 922 } 923 return nil, nil 924} 925 926// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec. 927func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) { 928 volumePlugin, err := pm.FindPluginBySpec(spec) 929 if err != nil { 930 return nil, err 931 } 932 if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { 933 if canMount, err := deviceMountableVolumePlugin.CanDeviceMount(spec); err != nil { 934 return nil, err 935 } else if canMount { 936 return deviceMountableVolumePlugin, nil 937 } 938 } 939 return nil, nil 940} 941 942// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name. 943func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) { 944 volumePlugin, err := pm.FindPluginByName(name) 945 if err != nil { 946 return nil, err 947 } 948 if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { 949 return deviceMountableVolumePlugin, nil 950 } 951 return nil, nil 952} 953 954// FindExpandablePluginBySpec fetches a persistent volume plugin by spec. 955func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) { 956 volumePlugin, err := pm.FindPluginBySpec(spec) 957 if err != nil { 958 if spec.IsKubeletExpandable() { 959 // for kubelet expandable volumes, return a noop plugin that 960 // returns success for expand on the controller 961 klog.V(4).InfoS("FindExpandablePluginBySpec -> returning noopExpandableVolumePluginInstance", "specName", spec.Name()) 962 return &noopExpandableVolumePluginInstance{spec}, nil 963 } 964 klog.V(4).InfoS("FindExpandablePluginBySpec -> err", "specName", spec.Name(), "err", err) 965 return nil, err 966 } 967 968 if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok { 969 return expandableVolumePlugin, nil 970 } 971 return nil, nil 972} 973 974// FindExpandablePluginBySpec fetches a persistent volume plugin by name. 975func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVolumePlugin, error) { 976 volumePlugin, err := pm.FindPluginByName(name) 977 if err != nil { 978 return nil, err 979 } 980 981 if expandableVolumePlugin, ok := volumePlugin.(ExpandableVolumePlugin); ok { 982 return expandableVolumePlugin, nil 983 } 984 return nil, nil 985} 986 987// FindMapperPluginBySpec fetches a block volume plugin by spec. 988func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) { 989 volumePlugin, err := pm.FindPluginBySpec(spec) 990 if err != nil { 991 return nil, err 992 } 993 994 if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { 995 return blockVolumePlugin, nil 996 } 997 return nil, nil 998} 999 1000// FindMapperPluginByName fetches a block volume plugin by name. 1001func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) { 1002 volumePlugin, err := pm.FindPluginByName(name) 1003 if err != nil { 1004 return nil, err 1005 } 1006 1007 if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok { 1008 return blockVolumePlugin, nil 1009 } 1010 return nil, nil 1011} 1012 1013// FindNodeExpandablePluginBySpec fetches a persistent volume plugin by spec 1014func (pm *VolumePluginMgr) FindNodeExpandablePluginBySpec(spec *Spec) (NodeExpandableVolumePlugin, error) { 1015 volumePlugin, err := pm.FindPluginBySpec(spec) 1016 if err != nil { 1017 return nil, err 1018 } 1019 if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { 1020 return fsResizablePlugin, nil 1021 } 1022 return nil, nil 1023} 1024 1025// FindNodeExpandablePluginByName fetches a persistent volume plugin by name 1026func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpandableVolumePlugin, error) { 1027 volumePlugin, err := pm.FindPluginByName(name) 1028 if err != nil { 1029 return nil, err 1030 } 1031 1032 if fsResizablePlugin, ok := volumePlugin.(NodeExpandableVolumePlugin); ok { 1033 return fsResizablePlugin, nil 1034 } 1035 1036 return nil, nil 1037} 1038 1039func (pm *VolumePluginMgr) Run(stopCh <-chan struct{}) { 1040 kletHost, ok := pm.Host.(KubeletVolumeHost) 1041 if ok { 1042 // start informer for CSIDriver 1043 informerFactory := kletHost.GetInformerFactory() 1044 informerFactory.Start(stopCh) 1045 informerFactory.WaitForCacheSync(stopCh) 1046 } 1047} 1048 1049// NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler 1050// pod. By default, a recycler pod simply runs "rm -rf" on a volume and tests 1051// for emptiness. Most attributes of the template will be correct for most 1052// plugin implementations. The following attributes can be overridden per 1053// plugin via configuration: 1054// 1055// 1. pod.Spec.Volumes[0].VolumeSource must be overridden. Recycler 1056// implementations without a valid VolumeSource will fail. 1057// 2. pod.GenerateName helps distinguish recycler pods by name. Recommended. 1058// Default is "pv-recycler-". 1059// 3. pod.Spec.ActiveDeadlineSeconds gives the recycler pod a maximum timeout 1060// before failing. Recommended. Default is 60 seconds. 1061// 1062// See HostPath and NFS for working recycler examples 1063func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod { 1064 timeout := int64(60) 1065 pod := &v1.Pod{ 1066 ObjectMeta: metav1.ObjectMeta{ 1067 GenerateName: "pv-recycler-", 1068 Namespace: metav1.NamespaceDefault, 1069 }, 1070 Spec: v1.PodSpec{ 1071 ActiveDeadlineSeconds: &timeout, 1072 RestartPolicy: v1.RestartPolicyNever, 1073 Volumes: []v1.Volume{ 1074 { 1075 Name: "vol", 1076 // IMPORTANT! All plugins using this template MUST 1077 // override pod.Spec.Volumes[0].VolumeSource Recycler 1078 // implementations without a valid VolumeSource will fail. 1079 VolumeSource: v1.VolumeSource{}, 1080 }, 1081 }, 1082 Containers: []v1.Container{ 1083 { 1084 Name: "pv-recycler", 1085 Image: "busybox:1.27", 1086 Command: []string{"/bin/sh"}, 1087 Args: []string{"-c", "test -e /scrub && rm -rf /scrub/..?* /scrub/.[!.]* /scrub/* && test -z \"$(ls -A /scrub)\" || exit 1"}, 1088 VolumeMounts: []v1.VolumeMount{ 1089 { 1090 Name: "vol", 1091 MountPath: "/scrub", 1092 }, 1093 }, 1094 }, 1095 }, 1096 }, 1097 } 1098 return pod 1099} 1100 1101// Check validity of recycle pod template 1102// List of checks: 1103// - at least one volume is defined in the recycle pod template 1104// If successful, returns nil 1105// if unsuccessful, returns an error. 1106func ValidateRecyclerPodTemplate(pod *v1.Pod) error { 1107 if len(pod.Spec.Volumes) < 1 { 1108 return fmt.Errorf("does not contain any volume(s)") 1109 } 1110 return nil 1111} 1112 1113type dummyPluginProber struct{} 1114 1115func (*dummyPluginProber) Init() error { return nil } 1116func (*dummyPluginProber) Probe() ([]ProbeEvent, error) { return nil, nil } 1117