1/* 2Copyright 2018 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package volumepathhandler 18 19import ( 20 "fmt" 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 25 "k8s.io/klog/v2" 26 "k8s.io/mount-utils" 27 utilexec "k8s.io/utils/exec" 28 29 "k8s.io/apimachinery/pkg/types" 30) 31 32const ( 33 losetupPath = "losetup" 34 ErrDeviceNotFound = "device not found" 35) 36 37// BlockVolumePathHandler defines a set of operations for handling block volume-related operations 38type BlockVolumePathHandler interface { 39 // MapDevice creates a symbolic link to block device under specified map path 40 MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error 41 // UnmapDevice removes a symbolic link to block device under specified map path 42 UnmapDevice(mapPath string, linkName string, bindMount bool) error 43 // RemovePath removes a file or directory on specified map path 44 RemoveMapPath(mapPath string) error 45 // IsSymlinkExist retruns true if specified symbolic link exists 46 IsSymlinkExist(mapPath string) (bool, error) 47 // IsDeviceBindMountExist retruns true if specified bind mount exists 48 IsDeviceBindMountExist(mapPath string) (bool, error) 49 // GetDeviceBindMountRefs searches bind mounts under global map path 50 GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error) 51 // FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath 52 // corresponding to map path symlink, and then return global map path with pod uuid. 53 FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) 54 // AttachFileDevice takes a path to a regular file and makes it available as an 55 // attached block device. 56 AttachFileDevice(path string) (string, error) 57 // DetachFileDevice takes a path to the attached block device and 58 // detach it from block device. 59 DetachFileDevice(path string) error 60 // GetLoopDevice returns the full path to the loop device associated with the given path. 61 GetLoopDevice(path string) (string, error) 62} 63 64// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler. 65func NewBlockVolumePathHandler() BlockVolumePathHandler { 66 var volumePathHandler VolumePathHandler 67 return volumePathHandler 68} 69 70// VolumePathHandler is path related operation handlers for block volume 71type VolumePathHandler struct { 72} 73 74// MapDevice creates a symbolic link to block device under specified map path 75func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error { 76 // Example of global map path: 77 // globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid} 78 // linkName: {podUid} 79 // 80 // Example of pod device map path: 81 // podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} 82 // linkName: {volumeName} 83 if len(devicePath) == 0 { 84 return fmt.Errorf("failed to map device to map path. devicePath is empty") 85 } 86 if len(mapPath) == 0 { 87 return fmt.Errorf("failed to map device to map path. mapPath is empty") 88 } 89 if !filepath.IsAbs(mapPath) { 90 return fmt.Errorf("the map path should be absolute: map path: %s", mapPath) 91 } 92 klog.V(5).Infof("MapDevice: devicePath %s", devicePath) 93 klog.V(5).Infof("MapDevice: mapPath %s", mapPath) 94 klog.V(5).Infof("MapDevice: linkName %s", linkName) 95 96 // Check and create mapPath 97 _, err := os.Stat(mapPath) 98 if err != nil && !os.IsNotExist(err) { 99 return fmt.Errorf("cannot validate map path: %s: %v", mapPath, err) 100 } 101 if err = os.MkdirAll(mapPath, 0750); err != nil { 102 return fmt.Errorf("failed to mkdir %s: %v", mapPath, err) 103 } 104 105 if bindMount { 106 return mapBindMountDevice(v, devicePath, mapPath, linkName) 107 } 108 return mapSymlinkDevice(v, devicePath, mapPath, linkName) 109} 110 111func mapBindMountDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error { 112 // Check bind mount exists 113 linkPath := filepath.Join(mapPath, string(linkName)) 114 115 file, err := os.Stat(linkPath) 116 if err != nil { 117 if !os.IsNotExist(err) { 118 return fmt.Errorf("failed to stat file %s: %v", linkPath, err) 119 } 120 121 // Create file 122 newFile, err := os.OpenFile(linkPath, os.O_CREATE|os.O_RDWR, 0750) 123 if err != nil { 124 return fmt.Errorf("failed to open file %s: %v", linkPath, err) 125 } 126 if err := newFile.Close(); err != nil { 127 return fmt.Errorf("failed to close file %s: %v", linkPath, err) 128 } 129 } else { 130 // Check if device file 131 // TODO: Need to check if this device file is actually the expected bind mount 132 if file.Mode()&os.ModeDevice == os.ModeDevice { 133 klog.Warningf("Warning: Map skipped because bind mount already exist on the path: %v", linkPath) 134 return nil 135 } 136 137 klog.Warningf("Warning: file %s is already exist but not mounted, skip creating file", linkPath) 138 } 139 140 // Bind mount file 141 mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()} 142 if err := mounter.MountSensitiveWithoutSystemd(devicePath, linkPath, "" /* fsType */, []string{"bind"}, nil); err != nil { 143 return fmt.Errorf("failed to bind mount devicePath: %s to linkPath %s: %v", devicePath, linkPath, err) 144 } 145 146 return nil 147} 148 149func mapSymlinkDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error { 150 // Remove old symbolic link(or file) then create new one. 151 // This should be done because current symbolic link is 152 // stale across node reboot. 153 linkPath := filepath.Join(mapPath, string(linkName)) 154 if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) { 155 return fmt.Errorf("failed to remove file %s: %v", linkPath, err) 156 } 157 return os.Symlink(devicePath, linkPath) 158} 159 160// UnmapDevice removes a symbolic link associated to block device under specified map path 161func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string, bindMount bool) error { 162 if len(mapPath) == 0 { 163 return fmt.Errorf("failed to unmap device from map path. mapPath is empty") 164 } 165 klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath) 166 klog.V(5).Infof("UnmapDevice: linkName %s", linkName) 167 168 if bindMount { 169 return unmapBindMountDevice(v, mapPath, linkName) 170 } 171 return unmapSymlinkDevice(v, mapPath, linkName) 172} 173 174func unmapBindMountDevice(v VolumePathHandler, mapPath string, linkName string) error { 175 // Check bind mount exists 176 linkPath := filepath.Join(mapPath, string(linkName)) 177 if isMountExist, checkErr := v.IsDeviceBindMountExist(linkPath); checkErr != nil { 178 return checkErr 179 } else if !isMountExist { 180 klog.Warningf("Warning: Unmap skipped because bind mount does not exist on the path: %v", linkPath) 181 182 // Check if linkPath still exists 183 if _, err := os.Stat(linkPath); err != nil { 184 if !os.IsNotExist(err) { 185 return fmt.Errorf("failed to check if path %s exists: %v", linkPath, err) 186 } 187 // linkPath has already been removed 188 return nil 189 } 190 // Remove file 191 if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) { 192 return fmt.Errorf("failed to remove file %s: %v", linkPath, err) 193 } 194 return nil 195 } 196 197 // Unmount file 198 mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()} 199 if err := mounter.Unmount(linkPath); err != nil { 200 return fmt.Errorf("failed to unmount linkPath %s: %v", linkPath, err) 201 } 202 203 // Remove file 204 if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) { 205 return fmt.Errorf("failed to remove file %s: %v", linkPath, err) 206 } 207 208 return nil 209} 210 211func unmapSymlinkDevice(v VolumePathHandler, mapPath string, linkName string) error { 212 // Check symbolic link exists 213 linkPath := filepath.Join(mapPath, string(linkName)) 214 if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil { 215 return checkErr 216 } else if !islinkExist { 217 klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath) 218 return nil 219 } 220 return os.Remove(linkPath) 221} 222 223// RemoveMapPath removes a file or directory on specified map path 224func (v VolumePathHandler) RemoveMapPath(mapPath string) error { 225 if len(mapPath) == 0 { 226 return fmt.Errorf("failed to remove map path. mapPath is empty") 227 } 228 klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath) 229 err := os.RemoveAll(mapPath) 230 if err != nil && !os.IsNotExist(err) { 231 return fmt.Errorf("failed to remove directory %s: %v", mapPath, err) 232 } 233 return nil 234} 235 236// IsSymlinkExist returns true if specified file exists and the type is symbolik link. 237// If file doesn't exist, or file exists but not symbolic link, return false with no error. 238// On other cases, return false with error from Lstat(). 239func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) { 240 fi, err := os.Lstat(mapPath) 241 if err != nil { 242 // If file doesn't exist, return false and no error 243 if os.IsNotExist(err) { 244 return false, nil 245 } 246 // Return error from Lstat() 247 return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err) 248 } 249 // If file exits and it's symbolic link, return true and no error 250 if fi.Mode()&os.ModeSymlink == os.ModeSymlink { 251 return true, nil 252 } 253 // If file exits but it's not symbolic link, return false and no error 254 return false, nil 255} 256 257// IsDeviceBindMountExist returns true if specified file exists and the type is device. 258// If file doesn't exist, or file exists but not device, return false with no error. 259// On other cases, return false with error from Lstat(). 260func (v VolumePathHandler) IsDeviceBindMountExist(mapPath string) (bool, error) { 261 fi, err := os.Lstat(mapPath) 262 if err != nil { 263 // If file doesn't exist, return false and no error 264 if os.IsNotExist(err) { 265 return false, nil 266 } 267 268 // Return error from Lstat() 269 return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err) 270 } 271 // If file exits and it's device, return true and no error 272 if fi.Mode()&os.ModeDevice == os.ModeDevice { 273 return true, nil 274 } 275 // If file exits but it's not device, return false and no error 276 return false, nil 277} 278 279// GetDeviceBindMountRefs searches bind mounts under global map path 280func (v VolumePathHandler) GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error) { 281 var refs []string 282 files, err := ioutil.ReadDir(mapPath) 283 if err != nil { 284 return nil, err 285 } 286 for _, file := range files { 287 if file.Mode()&os.ModeDevice != os.ModeDevice { 288 continue 289 } 290 filename := file.Name() 291 // TODO: Might need to check if the file is actually linked to devPath 292 refs = append(refs, filepath.Join(mapPath, filename)) 293 } 294 klog.V(5).Infof("GetDeviceBindMountRefs: refs %v", refs) 295 return refs, nil 296} 297