1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package storageos 18 19import ( 20 "errors" 21 "fmt" 22 "io/ioutil" 23 "os" 24 "path/filepath" 25 "strings" 26 27 storageosapi "github.com/storageos/go-api" 28 storageostypes "github.com/storageos/go-api/types" 29 "k8s.io/klog/v2" 30 proxyutil "k8s.io/kubernetes/pkg/proxy/util" 31 "k8s.io/kubernetes/pkg/volume" 32 utilexec "k8s.io/utils/exec" 33) 34 35const ( 36 losetupPath = "losetup" 37 38 modeBlock deviceType = iota 39 modeFile 40 modeUnsupported 41 42 //ErrDeviceNotFound defines "device not found" 43 ErrDeviceNotFound = "device not found" 44 //ErrDeviceNotSupported defines "device not supported" 45 ErrDeviceNotSupported = "device not supported" 46 //ErrNotAvailable defines "not available" 47 ErrNotAvailable = "not available" 48) 49 50type deviceType int 51 52// storageosVolume describes a provisioned volume 53type storageosVolume struct { 54 ID string 55 Name string 56 Namespace string 57 Description string 58 Pool string 59 SizeGB int 60 Labels map[string]string 61 FSType string 62} 63 64type storageosAPIConfig struct { 65 apiAddr string 66 apiUser string 67 apiPass string 68 apiVersion string 69} 70 71type apiImplementer interface { 72 Volume(namespace string, ref string) (*storageostypes.Volume, error) 73 VolumeCreate(opts storageostypes.VolumeCreateOptions) (*storageostypes.Volume, error) 74 VolumeMount(opts storageostypes.VolumeMountOptions) error 75 VolumeUnmount(opts storageostypes.VolumeUnmountOptions) error 76 VolumeDelete(opt storageostypes.DeleteOptions) error 77 Node(ref string) (*storageostypes.Node, error) 78} 79 80// storageosUtil is the utility structure to interact with the StorageOS API. 81type storageosUtil struct { 82 api apiImplementer 83 host volume.VolumeHost 84} 85 86func (u *storageosUtil) NewAPI(apiCfg *storageosAPIConfig) error { 87 if u.api != nil { 88 return nil 89 } 90 if u.host == nil { 91 return errors.New("host must not be nil") 92 } 93 if apiCfg == nil { 94 apiCfg = &storageosAPIConfig{ 95 apiAddr: defaultAPIAddress, 96 apiUser: defaultAPIUser, 97 apiPass: defaultAPIPassword, 98 apiVersion: defaultAPIVersion, 99 } 100 klog.V(4).Infof("using default StorageOS API settings: addr %s, version: %s", apiCfg.apiAddr, defaultAPIVersion) 101 } 102 103 api, err := storageosapi.NewVersionedClient(apiCfg.apiAddr, defaultAPIVersion) 104 if err != nil { 105 return err 106 } 107 api.SetAuth(apiCfg.apiUser, apiCfg.apiPass) 108 if err := api.SetDialContext(proxyutil.NewFilteredDialContext(api.GetDialContext(), nil, u.host.GetFilteredDialOptions())); err != nil { 109 return fmt.Errorf("failed to set DialContext in storageos client: %v", err) 110 } 111 u.api = api 112 return nil 113} 114 115// Creates a new StorageOS volume and makes it available as a device within 116// /var/lib/storageos/volumes. 117func (u *storageosUtil) CreateVolume(p *storageosProvisioner) (*storageosVolume, error) { 118 119 klog.V(4).Infof("creating StorageOS volume %q with namespace %q", p.volName, p.volNamespace) 120 121 if err := u.NewAPI(p.apiCfg); err != nil { 122 return nil, err 123 } 124 125 if p.labels == nil { 126 p.labels = make(map[string]string) 127 } 128 opts := storageostypes.VolumeCreateOptions{ 129 Name: p.volName, 130 Size: p.sizeGB, 131 Description: p.description, 132 Pool: p.pool, 133 FSType: p.fsType, 134 Namespace: p.volNamespace, 135 Labels: p.labels, 136 } 137 138 vol, err := u.api.VolumeCreate(opts) 139 if err != nil { 140 // don't log error details from client calls in events 141 klog.V(4).Infof("volume create failed for volume %q (%v)", opts.Name, err) 142 return nil, errors.New("volume create failed: see kube-controller-manager.log for details") 143 } 144 return &storageosVolume{ 145 ID: vol.ID, 146 Name: vol.Name, 147 Namespace: vol.Namespace, 148 Description: vol.Description, 149 Pool: vol.Pool, 150 FSType: vol.FSType, 151 SizeGB: int(vol.Size), 152 Labels: vol.Labels, 153 }, nil 154} 155 156// Attach exposes a volume on the host as a block device. StorageOS uses a 157// global namespace, so if the volume exists, it should already be available as 158// a device within `/var/lib/storageos/volumes/<id>`. 159// 160// Depending on the host capabilities, the device may be either a block device 161// or a file device. Block devices can be used directly, but file devices must 162// be made accessible as a block device before using. 163func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { 164 165 klog.V(4).Infof("attaching StorageOS volume %q with namespace %q", b.volName, b.volNamespace) 166 167 if err := u.NewAPI(b.apiCfg); err != nil { 168 return "", err 169 } 170 171 // Get the node's device path from the API, falling back to the default if 172 // not set on the node. 173 if b.deviceDir == "" { 174 b.deviceDir = u.DeviceDir(b) 175 } 176 177 vol, err := u.api.Volume(b.volNamespace, b.volName) 178 if err != nil { 179 klog.Warningf("volume retrieve failed for volume %q with namespace %q (%v)", b.volName, b.volNamespace, err) 180 return "", err 181 } 182 183 srcPath := filepath.Join(b.deviceDir, vol.ID) 184 dt, err := pathDeviceType(srcPath) 185 if err != nil { 186 klog.Warningf("volume source path %q for volume %q not ready (%v)", srcPath, b.volName, err) 187 return "", err 188 } 189 190 switch dt { 191 case modeBlock: 192 return srcPath, nil 193 case modeFile: 194 return attachFileDevice(srcPath, b.exec) 195 default: 196 return "", fmt.Errorf(ErrDeviceNotSupported) 197 } 198} 199 200// Detach detaches a volume from the host. This is only needed when NBD is not 201// enabled and loop devices are used to simulate a block device. 202func (u *storageosUtil) DetachVolume(b *storageosUnmounter, devicePath string) error { 203 204 klog.V(4).Infof("detaching StorageOS volume %q with namespace %q", b.volName, b.volNamespace) 205 206 if !isLoopDevice(devicePath) { 207 return nil 208 } 209 if _, err := os.Stat(devicePath); os.IsNotExist(err) { 210 return nil 211 } 212 return removeLoopDevice(devicePath, b.exec) 213} 214 215// AttachDevice attaches the volume device to the host at a given mount path. 216func (u *storageosUtil) AttachDevice(b *storageosMounter, deviceMountPath string) error { 217 218 klog.V(4).Infof("attaching StorageOS device for volume %q with namespace %q", b.volName, b.volNamespace) 219 220 if err := u.NewAPI(b.apiCfg); err != nil { 221 return err 222 } 223 224 opts := storageostypes.VolumeMountOptions{ 225 Name: b.volName, 226 Namespace: b.volNamespace, 227 FsType: b.fsType, 228 Mountpoint: deviceMountPath, 229 Client: b.plugin.host.GetHostName(), 230 } 231 if err := u.api.VolumeMount(opts); err != nil { 232 return err 233 } 234 return nil 235} 236 237// Mount mounts the volume on the host. 238func (u *storageosUtil) MountVolume(b *storageosMounter, mntDevice, deviceMountPath string) error { 239 240 klog.V(4).Infof("mounting StorageOS volume %q with namespace %q", b.volName, b.volNamespace) 241 242 notMnt, err := b.mounter.IsLikelyNotMountPoint(deviceMountPath) 243 if err != nil { 244 if os.IsNotExist(err) { 245 if err = os.MkdirAll(deviceMountPath, 0750); err != nil { 246 return err 247 } 248 notMnt = true 249 } else { 250 return err 251 } 252 } 253 if err = os.MkdirAll(deviceMountPath, 0750); err != nil { 254 klog.Errorf("mkdir failed on disk %s (%v)", deviceMountPath, err) 255 return err 256 } 257 options := []string{} 258 if b.readOnly { 259 options = append(options, "ro") 260 } 261 if notMnt { 262 err = b.diskMounter.FormatAndMount(mntDevice, deviceMountPath, b.fsType, options) 263 if err != nil { 264 os.Remove(deviceMountPath) 265 return err 266 } 267 } 268 return err 269} 270 271// Unmount removes the mount reference from the volume allowing it to be 272// re-mounted elsewhere. 273func (u *storageosUtil) UnmountVolume(b *storageosUnmounter) error { 274 275 klog.V(4).Infof("clearing StorageOS mount reference for volume %q with namespace %q", b.volName, b.volNamespace) 276 277 if err := u.NewAPI(b.apiCfg); err != nil { 278 // We can't always get the config we need, so allow the unmount to 279 // succeed even if we can't remove the mount reference from the API. 280 klog.Warningf("could not remove mount reference in the StorageOS API as no credentials available to the unmount operation") 281 return nil 282 } 283 284 opts := storageostypes.VolumeUnmountOptions{ 285 Name: b.volName, 286 Namespace: b.volNamespace, 287 Client: b.plugin.host.GetHostName(), 288 } 289 return u.api.VolumeUnmount(opts) 290} 291 292// Deletes a StorageOS volume. Assumes it has already been unmounted and detached. 293func (u *storageosUtil) DeleteVolume(d *storageosDeleter) error { 294 if err := u.NewAPI(d.apiCfg); err != nil { 295 return err 296 } 297 298 // Deletes must be forced as the StorageOS API will not normally delete 299 // volumes that it thinks are mounted. We can't be sure the unmount was 300 // registered via the API so we trust k8s to only delete volumes it knows 301 // are unmounted. 302 opts := storageostypes.DeleteOptions{ 303 Name: d.volName, 304 Namespace: d.volNamespace, 305 Force: true, 306 } 307 if err := u.api.VolumeDelete(opts); err != nil { 308 // don't log error details from client calls in events 309 klog.V(4).Infof("volume deleted failed for volume %q in namespace %q: %v", d.volName, d.volNamespace, err) 310 return errors.New("volume delete failed: see kube-controller-manager.log for details") 311 } 312 return nil 313} 314 315// Get the node's device path from the API, falling back to the default if not 316// specified. 317func (u *storageosUtil) DeviceDir(b *storageosMounter) string { 318 319 ctrl, err := u.api.Node(b.plugin.host.GetHostName()) 320 if err != nil { 321 klog.Warningf("node device path lookup failed: %v", err) 322 return defaultDeviceDir 323 } 324 if ctrl == nil || ctrl.DeviceDir == "" { 325 klog.Warningf("node device path not set, using default: %s", defaultDeviceDir) 326 return defaultDeviceDir 327 } 328 return ctrl.DeviceDir 329} 330 331// pathMode returns the FileMode for a path. 332func pathDeviceType(path string) (deviceType, error) { 333 fi, err := os.Stat(path) 334 if err != nil { 335 return modeUnsupported, err 336 } 337 switch mode := fi.Mode(); { 338 case mode&os.ModeDevice != 0: 339 return modeBlock, nil 340 case mode.IsRegular(): 341 return modeFile, nil 342 default: 343 return modeUnsupported, nil 344 } 345} 346 347// attachFileDevice takes a path to a regular file and makes it available as an 348// attached block device. 349func attachFileDevice(path string, exec utilexec.Interface) (string, error) { 350 blockDevicePath, err := getLoopDevice(path) 351 if err != nil && err.Error() != ErrDeviceNotFound { 352 return "", err 353 } 354 355 // If no existing loop device for the path, create one 356 if blockDevicePath == "" { 357 klog.V(4).Infof("Creating device for path: %s", path) 358 blockDevicePath, err = makeLoopDevice(path, exec) 359 if err != nil { 360 return "", err 361 } 362 } 363 return blockDevicePath, nil 364} 365 366// Returns the full path to the loop device associated with the given path. 367func getLoopDevice(path string) (string, error) { 368 _, err := os.Stat(path) 369 if os.IsNotExist(err) { 370 return "", errors.New(ErrNotAvailable) 371 } 372 if err != nil { 373 return "", fmt.Errorf("not attachable: %v", err) 374 } 375 376 return getLoopDeviceFromSysfs(path) 377} 378 379func makeLoopDevice(path string, exec utilexec.Interface) (string, error) { 380 args := []string{"-f", "-P", path} 381 out, err := exec.Command(losetupPath, args...).CombinedOutput() 382 if err != nil { 383 klog.V(2).Infof("Failed device create command for path %s: %v %s", path, err, out) 384 return "", err 385 } 386 387 return getLoopDeviceFromSysfs(path) 388} 389 390func removeLoopDevice(device string, exec utilexec.Interface) error { 391 args := []string{"-d", device} 392 out, err := exec.Command(losetupPath, args...).CombinedOutput() 393 if err != nil { 394 if !strings.Contains(string(out), "No such device or address") { 395 return err 396 } 397 } 398 return nil 399} 400 401func isLoopDevice(device string) bool { 402 return strings.HasPrefix(device, "/dev/loop") 403} 404 405// getLoopDeviceFromSysfs finds the backing file for a loop 406// device from sysfs via "/sys/block/loop*/loop/backing_file". 407func getLoopDeviceFromSysfs(path string) (string, error) { 408 // If the file is a symlink. 409 realPath, err := filepath.EvalSymlinks(path) 410 if err != nil { 411 return "", errors.New(ErrDeviceNotFound) 412 } 413 414 devices, err := filepath.Glob("/sys/block/loop*") 415 if err != nil { 416 return "", errors.New(ErrDeviceNotFound) 417 } 418 419 for _, device := range devices { 420 backingFile := fmt.Sprintf("%s/loop/backing_file", device) 421 422 // The contents of this file is the absolute path of "path". 423 data, err := ioutil.ReadFile(backingFile) 424 if err != nil { 425 continue 426 } 427 428 // Return the first match. 429 backingFilePath := strings.TrimSpace(string(data)) 430 if backingFilePath == path || backingFilePath == realPath { 431 return fmt.Sprintf("/dev/%s", filepath.Base(device)), nil 432 } 433 } 434 435 return "", errors.New(ErrDeviceNotFound) 436} 437