1// +build !providerless 2 3/* 4Copyright 2014 The Kubernetes Authors. 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19package gcepd 20 21import ( 22 "fmt" 23 "path/filepath" 24 "regexp" 25 "strings" 26 "time" 27 28 "k8s.io/klog/v2" 29 "k8s.io/mount-utils" 30 "k8s.io/utils/exec" 31 utilpath "k8s.io/utils/path" 32 33 v1 "k8s.io/api/core/v1" 34 "k8s.io/apimachinery/pkg/util/sets" 35 cloudprovider "k8s.io/cloud-provider" 36 cloudvolume "k8s.io/cloud-provider/volume" 37 volumehelpers "k8s.io/cloud-provider/volume/helpers" 38 "k8s.io/kubernetes/pkg/volume" 39 volumeutil "k8s.io/kubernetes/pkg/volume/util" 40 gcecloud "k8s.io/legacy-cloud-providers/gce" 41) 42 43const ( 44 diskByIDPath = "/dev/disk/by-id/" 45 diskGooglePrefix = "google-" 46 diskScsiGooglePrefix = "scsi-0Google_PersistentDisk_" 47 diskPartitionSuffix = "-part" 48 diskSDPath = "/dev/sd" 49 diskSDPattern = "/dev/sd*" 50 maxRetries = 10 51 checkSleepDuration = time.Second 52 maxRegionalPDZones = 2 53 54 // Replication type constants must be lower case. 55 replicationTypeNone = "none" 56 replicationTypeRegionalPD = "regional-pd" 57 58 // scsi_id output should be in the form of: 59 // 0Google PersistentDisk <disk name> 60 scsiPattern = `^0Google\s+PersistentDisk\s+([\S]+)\s*$` 61) 62 63var ( 64 // errorSleepDuration is modified only in unit tests and should be constant 65 // otherwise. 66 errorSleepDuration = 5 * time.Second 67 68 // regex to parse scsi_id output and extract the serial 69 scsiRegex = regexp.MustCompile(scsiPattern) 70) 71 72// GCEDiskUtil provides operation for GCE PD 73type GCEDiskUtil struct{} 74 75// DeleteVolume deletes a GCE PD 76// Returns: error 77func (util *GCEDiskUtil) DeleteVolume(d *gcePersistentDiskDeleter) error { 78 cloud, err := getCloudProvider(d.gcePersistentDisk.plugin.host.GetCloudProvider()) 79 if err != nil { 80 return err 81 } 82 83 if err = cloud.DeleteDisk(d.pdName); err != nil { 84 klog.V(2).Infof("Error deleting GCE PD volume %s: %v", d.pdName, err) 85 // GCE cloud provider returns volume.deletedVolumeInUseError when 86 // necessary, no handling needed here. 87 return err 88 } 89 klog.V(2).Infof("Successfully deleted GCE PD volume %s", d.pdName) 90 return nil 91} 92 93// CreateVolume creates a GCE PD. 94// Returns: gcePDName, volumeSizeGB, labels, fsType, error 95func (util *GCEDiskUtil) CreateVolume(c *gcePersistentDiskProvisioner, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm) (string, int, map[string]string, string, error) { 96 cloud, err := getCloudProvider(c.gcePersistentDisk.plugin.host.GetCloudProvider()) 97 if err != nil { 98 return "", 0, nil, "", err 99 } 100 101 name := volumeutil.GenerateVolumeName(c.options.ClusterName, c.options.PVName, 63) // GCE PD name can have up to 63 characters 102 capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] 103 // GCE PDs are allocated in chunks of GiBs 104 requestGB, err := volumehelpers.RoundUpToGiB(capacity) 105 if err != nil { 106 return "", 0, nil, "", err 107 } 108 109 // Apply Parameters. 110 // Values for parameter "replication-type" are canonicalized to lower case. 111 // Values for other parameters are case-insensitive, and we leave validation of these values 112 // to the cloud provider. 113 diskType := "" 114 configuredZone := "" 115 var configuredZones sets.String 116 zonePresent := false 117 zonesPresent := false 118 replicationType := replicationTypeNone 119 fstype := "" 120 for k, v := range c.options.Parameters { 121 switch strings.ToLower(k) { 122 case "type": 123 diskType = v 124 case "zone": 125 zonePresent = true 126 configuredZone = v 127 case "zones": 128 zonesPresent = true 129 configuredZones, err = volumehelpers.ZonesToSet(v) 130 if err != nil { 131 return "", 0, nil, "", err 132 } 133 case "replication-type": 134 replicationType = strings.ToLower(v) 135 case volume.VolumeParameterFSType: 136 fstype = v 137 default: 138 return "", 0, nil, "", fmt.Errorf("invalid option %q for volume plugin %s", k, c.plugin.GetPluginName()) 139 } 140 } 141 142 // TODO: implement PVC.Selector parsing 143 if c.options.PVC.Spec.Selector != nil { 144 return "", 0, nil, "", fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on GCE") 145 } 146 147 var activezones sets.String 148 activezones, err = cloud.GetAllCurrentZones() 149 if err != nil { 150 return "", 0, nil, "", err 151 } 152 153 var disk *gcecloud.Disk 154 switch replicationType { 155 case replicationTypeRegionalPD: 156 selectedZones, err := volumehelpers.SelectZonesForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name, maxRegionalPDZones) 157 if err != nil { 158 klog.V(2).Infof("Error selecting zones for regional GCE PD volume: %v", err) 159 return "", 0, nil, "", err 160 } 161 disk, err = cloud.CreateRegionalDisk( 162 name, 163 diskType, 164 selectedZones, 165 requestGB, 166 *c.options.CloudTags) 167 if err != nil { 168 klog.V(2).Infof("Error creating regional GCE PD volume: %v", err) 169 return "", 0, nil, "", err 170 } 171 klog.V(2).Infof("Successfully created Regional GCE PD volume %s", name) 172 173 case replicationTypeNone: 174 selectedZone, err := volumehelpers.SelectZoneForVolume(zonePresent, zonesPresent, configuredZone, configuredZones, activezones, node, allowedTopologies, c.options.PVC.Name) 175 if err != nil { 176 return "", 0, nil, "", err 177 } 178 disk, err = cloud.CreateDisk( 179 name, 180 diskType, 181 selectedZone, 182 requestGB, 183 *c.options.CloudTags) 184 if err != nil { 185 klog.V(2).Infof("Error creating single-zone GCE PD volume: %v", err) 186 return "", 0, nil, "", err 187 } 188 klog.V(2).Infof("Successfully created single-zone GCE PD volume %s", name) 189 190 default: 191 return "", 0, nil, "", fmt.Errorf("replication-type of '%s' is not supported", replicationType) 192 } 193 194 labels, err := cloud.GetAutoLabelsForPD(disk) 195 if err != nil { 196 // We don't really want to leak the volume here... 197 klog.Errorf("error getting labels for volume %q: %v", name, err) 198 } 199 200 return name, int(requestGB), labels, fstype, nil 201} 202 203// Returns the first path that exists, or empty string if none exist. 204func verifyDevicePath(devicePaths []string, sdBeforeSet sets.String, diskName string) (string, error) { 205 if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil { 206 // It's possible udevadm was called on other disks so it should not block this 207 // call. If it did fail on this disk, then the devicePath will either 208 // not exist or be wrong. If it's wrong, then the scsi_id check below will fail. 209 klog.Errorf("udevadmChangeToNewDrives failed with: %v", err) 210 } 211 212 for _, path := range devicePaths { 213 if pathExists, err := mount.PathExists(path); err != nil { 214 return "", fmt.Errorf("error checking if path exists: %v", err) 215 } else if pathExists { 216 // validate that the path actually resolves to the correct disk 217 serial, err := getScsiSerial(path, diskName) 218 if err != nil { 219 return "", fmt.Errorf("failed to get scsi serial %v", err) 220 } 221 if serial != diskName { 222 // The device link is not pointing to the correct device 223 // Trigger udev on this device to try to fix the link 224 if udevErr := udevadmChangeToDrive(path); udevErr != nil { 225 klog.Errorf("udevadmChangeToDrive %q failed with: %v", path, err) 226 } 227 228 // Return error to retry WaitForAttach and verifyDevicePath 229 return "", fmt.Errorf("scsi_id serial %q for device %q doesn't match disk %q", serial, path, diskName) 230 } 231 // The device link is correct 232 return path, nil 233 } 234 } 235 236 return "", nil 237} 238 239// Calls scsi_id on the given devicePath to get the serial number reported by that device. 240func getScsiSerial(devicePath, diskName string) (string, error) { 241 exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, "/lib/udev/scsi_id") 242 if err != nil { 243 return "", fmt.Errorf("failed to check scsi_id existence: %v", err) 244 } 245 246 if !exists { 247 klog.V(6).Infof("scsi_id doesn't exist; skipping check for %v", devicePath) 248 return diskName, nil 249 } 250 251 out, err := exec.New().Command( 252 "/lib/udev/scsi_id", 253 "--page=0x83", 254 "--whitelisted", 255 fmt.Sprintf("--device=%v", devicePath)).CombinedOutput() 256 if err != nil { 257 return "", fmt.Errorf("scsi_id failed for device %q with %v", devicePath, err) 258 } 259 260 return parseScsiSerial(string(out)) 261} 262 263// Parse the output returned by scsi_id and extract the serial number 264func parseScsiSerial(output string) (string, error) { 265 substrings := scsiRegex.FindStringSubmatch(output) 266 if substrings == nil { 267 return "", fmt.Errorf("scsi_id output cannot be parsed: %q", output) 268 } 269 270 return substrings[1], nil 271} 272 273// Returns list of all /dev/disk/by-id/* paths for given PD. 274func getDiskByIDPaths(pdName string, partition string) []string { 275 devicePaths := []string{ 276 filepath.Join(diskByIDPath, diskGooglePrefix+pdName), 277 filepath.Join(diskByIDPath, diskScsiGooglePrefix+pdName), 278 } 279 280 if partition != "" { 281 for i, path := range devicePaths { 282 devicePaths[i] = path + diskPartitionSuffix + partition 283 } 284 } 285 286 return devicePaths 287} 288 289// Return cloud provider 290func getCloudProvider(cloudProvider cloudprovider.Interface) (*gcecloud.Cloud, error) { 291 var err error 292 for numRetries := 0; numRetries < maxRetries; numRetries++ { 293 gceCloudProvider, ok := cloudProvider.(*gcecloud.Cloud) 294 if !ok || gceCloudProvider == nil { 295 // Retry on error. See issue #11321 296 klog.Errorf("Failed to get GCE Cloud Provider. plugin.host.GetCloudProvider returned %v instead", cloudProvider) 297 time.Sleep(errorSleepDuration) 298 continue 299 } 300 301 return gceCloudProvider, nil 302 } 303 304 return nil, fmt.Errorf("failed to get GCE GCECloudProvider with error %v", err) 305} 306 307// Triggers the application of udev rules by calling "udevadm trigger 308// --action=change" for newly created "/dev/sd*" drives (exist only in 309// after set). This is workaround for Issue #7972. Once the underlying 310// issue has been resolved, this may be removed. 311func udevadmChangeToNewDrives(sdBeforeSet sets.String) error { 312 sdAfter, err := filepath.Glob(diskSDPattern) 313 if err != nil { 314 return fmt.Errorf("error filepath.Glob(\"%s\"): %v\r", diskSDPattern, err) 315 } 316 317 for _, sd := range sdAfter { 318 if !sdBeforeSet.Has(sd) { 319 return udevadmChangeToDrive(sd) 320 } 321 } 322 323 return nil 324} 325 326// Calls "udevadm trigger --action=change" on the specified drive. 327// drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it. 328// This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. 329func udevadmChangeToDrive(drivePath string) error { 330 klog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath) 331 332 // Evaluate symlink, if any 333 drive, err := filepath.EvalSymlinks(drivePath) 334 if err != nil { 335 return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v", drivePath, err) 336 } 337 klog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive) 338 339 // Check to make sure input is "/dev/sd*" 340 if !strings.Contains(drive, diskSDPath) { 341 return fmt.Errorf("udevadmChangeToDrive: expected input in the form \"%s\" but drive is %q", diskSDPattern, drive) 342 } 343 344 // Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..." 345 _, err = exec.New().Command( 346 "udevadm", 347 "trigger", 348 "--action=change", 349 fmt.Sprintf("--property-match=DEVNAME=%s", drive)).CombinedOutput() 350 if err != nil { 351 return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with %v", drive, err) 352 } 353 return nil 354} 355 356// Checks whether the given GCE PD volume spec is associated with a regional PD. 357func isRegionalPD(spec *volume.Spec) bool { 358 if spec.PersistentVolume != nil { 359 zonesLabel := spec.PersistentVolume.Labels[v1.LabelTopologyZone] 360 if zonesLabel == "" { 361 zonesLabel = spec.PersistentVolume.Labels[v1.LabelFailureDomainBetaZone] 362 } 363 zones := strings.Split(zonesLabel, cloudvolume.LabelMultiZoneDelimiter) 364 return len(zones) > 1 365 } 366 return false 367} 368