1// +build windows 2 3package lcow // import "github.com/docker/docker/daemon/graphdriver/lcow" 4 5import ( 6 "bytes" 7 "fmt" 8 "io" 9 "strings" 10 "sync" 11 "time" 12 13 "github.com/Microsoft/hcsshim" 14 "github.com/Microsoft/opengcs/client" 15 "github.com/pkg/errors" 16 "github.com/sirupsen/logrus" 17) 18 19// Code for all the service VM management for the LCOW graphdriver 20 21var errVMisTerminating = errors.New("service VM is shutting down") 22var errVMUnknown = errors.New("service vm id is unknown") 23var errVMStillHasReference = errors.New("Attemping to delete a VM that is still being used") 24 25// serviceVMMap is the struct representing the id -> service VM mapping. 26type serviceVMMap struct { 27 sync.Mutex 28 svms map[string]*serviceVMMapItem 29} 30 31// serviceVMMapItem is our internal structure representing an item in our 32// map of service VMs we are maintaining. 33type serviceVMMapItem struct { 34 svm *serviceVM // actual service vm object 35 refCount int // refcount for VM 36} 37 38// attachedVHD is for reference counting SCSI disks attached to a service VM, 39// and for a counter used to generate a short path name for the container path. 40type attachedVHD struct { 41 refCount int 42 attachCounter uint64 43} 44 45type serviceVM struct { 46 sync.Mutex // Serialises operations being performed in this service VM. 47 scratchAttached bool // Has a scratch been attached? 48 config *client.Config // Represents the service VM item. 49 50 // Indicates that the vm is started 51 startStatus chan interface{} 52 startError error 53 54 // Indicates that the vm is stopped 55 stopStatus chan interface{} 56 stopError error 57 58 attachCounter uint64 // Increasing counter for each add 59 attachedVHDs map[string]*attachedVHD // Map ref counting all the VHDS we've hot-added/hot-removed. 60 unionMounts map[string]int // Map ref counting all the union filesystems we mounted. 61} 62 63// add will add an id to the service vm map. There are three cases: 64// - entry doesn't exist: 65// - add id to map and return a new vm that the caller can manually configure+start 66// - entry does exist 67// - return vm in map and increment ref count 68// - entry does exist but the ref count is 0 69// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop 70func (svmMap *serviceVMMap) add(id string) (svm *serviceVM, alreadyExists bool, err error) { 71 svmMap.Lock() 72 defer svmMap.Unlock() 73 if svm, ok := svmMap.svms[id]; ok { 74 if svm.refCount == 0 { 75 return svm.svm, true, errVMisTerminating 76 } 77 svm.refCount++ 78 return svm.svm, true, nil 79 } 80 81 // Doesn't exist, so create an empty svm to put into map and return 82 newSVM := &serviceVM{ 83 startStatus: make(chan interface{}), 84 stopStatus: make(chan interface{}), 85 attachedVHDs: make(map[string]*attachedVHD), 86 unionMounts: make(map[string]int), 87 config: &client.Config{}, 88 } 89 svmMap.svms[id] = &serviceVMMapItem{ 90 svm: newSVM, 91 refCount: 1, 92 } 93 return newSVM, false, nil 94} 95 96// get will get the service vm from the map. There are three cases: 97// - entry doesn't exist: 98// - return errVMUnknown 99// - entry does exist 100// - return vm with no error 101// - entry does exist but the ref count is 0 102// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop 103func (svmMap *serviceVMMap) get(id string) (*serviceVM, error) { 104 svmMap.Lock() 105 defer svmMap.Unlock() 106 svm, ok := svmMap.svms[id] 107 if !ok { 108 return nil, errVMUnknown 109 } 110 if svm.refCount == 0 { 111 return svm.svm, errVMisTerminating 112 } 113 return svm.svm, nil 114} 115 116// decrementRefCount decrements the ref count of the given ID from the map. There are four cases: 117// - entry doesn't exist: 118// - return errVMUnknown 119// - entry does exist but the ref count is 0 120// - return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop 121// - entry does exist but ref count is 1 122// - return vm and set lastRef to true. The caller can then stop the vm, delete the id from this map 123// - and execute svm.signalStopFinished to signal the threads that the svm has been terminated. 124// - entry does exist and ref count > 1 125// - just reduce ref count and return svm 126func (svmMap *serviceVMMap) decrementRefCount(id string) (_ *serviceVM, lastRef bool, _ error) { 127 svmMap.Lock() 128 defer svmMap.Unlock() 129 130 svm, ok := svmMap.svms[id] 131 if !ok { 132 return nil, false, errVMUnknown 133 } 134 if svm.refCount == 0 { 135 return svm.svm, false, errVMisTerminating 136 } 137 svm.refCount-- 138 return svm.svm, svm.refCount == 0, nil 139} 140 141// setRefCountZero works the same way as decrementRefCount, but sets ref count to 0 instead of decrementing it. 142func (svmMap *serviceVMMap) setRefCountZero(id string) (*serviceVM, error) { 143 svmMap.Lock() 144 defer svmMap.Unlock() 145 146 svm, ok := svmMap.svms[id] 147 if !ok { 148 return nil, errVMUnknown 149 } 150 if svm.refCount == 0 { 151 return svm.svm, errVMisTerminating 152 } 153 svm.refCount = 0 154 return svm.svm, nil 155} 156 157// deleteID deletes the given ID from the map. If the refcount is not 0 or the 158// VM does not exist, then this function returns an error. 159func (svmMap *serviceVMMap) deleteID(id string) error { 160 svmMap.Lock() 161 defer svmMap.Unlock() 162 svm, ok := svmMap.svms[id] 163 if !ok { 164 return errVMUnknown 165 } 166 if svm.refCount != 0 { 167 return errVMStillHasReference 168 } 169 delete(svmMap.svms, id) 170 return nil 171} 172 173func (svm *serviceVM) signalStartFinished(err error) { 174 svm.Lock() 175 svm.startError = err 176 svm.Unlock() 177 close(svm.startStatus) 178} 179 180func (svm *serviceVM) getStartError() error { 181 <-svm.startStatus 182 svm.Lock() 183 defer svm.Unlock() 184 return svm.startError 185} 186 187func (svm *serviceVM) signalStopFinished(err error) { 188 svm.Lock() 189 svm.stopError = err 190 svm.Unlock() 191 close(svm.stopStatus) 192} 193 194func (svm *serviceVM) getStopError() error { 195 <-svm.stopStatus 196 svm.Lock() 197 defer svm.Unlock() 198 return svm.stopError 199} 200 201// hotAddVHDs waits for the service vm to start and then attaches the vhds. 202func (svm *serviceVM) hotAddVHDs(mvds ...hcsshim.MappedVirtualDisk) error { 203 if err := svm.getStartError(); err != nil { 204 return err 205 } 206 return svm.hotAddVHDsAtStart(mvds...) 207} 208 209// hotAddVHDsAtStart works the same way as hotAddVHDs but does not wait for the VM to start. 210func (svm *serviceVM) hotAddVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error { 211 svm.Lock() 212 defer svm.Unlock() 213 for i, mvd := range mvds { 214 if _, ok := svm.attachedVHDs[mvd.HostPath]; ok { 215 svm.attachedVHDs[mvd.HostPath].refCount++ 216 logrus.Debugf("lcowdriver: UVM %s: %s already present, refCount now %d", svm.config.Name, mvd.HostPath, svm.attachedVHDs[mvd.HostPath].refCount) 217 continue 218 } 219 220 svm.attachCounter++ 221 shortContainerPath := remapLongToShortContainerPath(mvd.ContainerPath, svm.attachCounter, svm.config.Name) 222 if err := svm.config.HotAddVhd(mvd.HostPath, shortContainerPath, mvd.ReadOnly, !mvd.AttachOnly); err != nil { 223 svm.hotRemoveVHDsNoLock(mvds[:i]...) 224 return err 225 } 226 svm.attachedVHDs[mvd.HostPath] = &attachedVHD{refCount: 1, attachCounter: svm.attachCounter} 227 } 228 return nil 229} 230 231// hotRemoveVHDs waits for the service vm to start and then removes the vhds. 232// The service VM must not be locked when calling this function. 233func (svm *serviceVM) hotRemoveVHDs(mvds ...hcsshim.MappedVirtualDisk) error { 234 if err := svm.getStartError(); err != nil { 235 return err 236 } 237 svm.Lock() 238 defer svm.Unlock() 239 return svm.hotRemoveVHDsNoLock(mvds...) 240} 241 242// hotRemoveVHDsNoLock removes VHDs from a service VM. When calling this function, 243// the contract is the service VM lock must be held. 244func (svm *serviceVM) hotRemoveVHDsNoLock(mvds ...hcsshim.MappedVirtualDisk) error { 245 var retErr error 246 for _, mvd := range mvds { 247 if _, ok := svm.attachedVHDs[mvd.HostPath]; !ok { 248 // We continue instead of returning an error if we try to hot remove a non-existent VHD. 249 // This is because one of the callers of the function is graphdriver.Put(). Since graphdriver.Get() 250 // defers the VM start to the first operation, it's possible that nothing have been hot-added 251 // when Put() is called. To avoid Put returning an error in that case, we simply continue if we 252 // don't find the vhd attached. 253 logrus.Debugf("lcowdriver: UVM %s: %s is not attached, not doing anything", svm.config.Name, mvd.HostPath) 254 continue 255 } 256 257 if svm.attachedVHDs[mvd.HostPath].refCount > 1 { 258 svm.attachedVHDs[mvd.HostPath].refCount-- 259 logrus.Debugf("lcowdriver: UVM %s: %s refCount dropped to %d. not removing from UVM", svm.config.Name, mvd.HostPath, svm.attachedVHDs[mvd.HostPath].refCount) 260 continue 261 } 262 263 // last reference to VHD, so remove from VM and map 264 if err := svm.config.HotRemoveVhd(mvd.HostPath); err == nil { 265 delete(svm.attachedVHDs, mvd.HostPath) 266 } else { 267 // Take note of the error, but still continue to remove the other VHDs 268 logrus.Warnf("Failed to hot remove %s: %s", mvd.HostPath, err) 269 if retErr == nil { 270 retErr = err 271 } 272 } 273 } 274 return retErr 275} 276 277func (svm *serviceVM) createExt4VHDX(destFile string, sizeGB uint32, cacheFile string) error { 278 if err := svm.getStartError(); err != nil { 279 return err 280 } 281 282 svm.Lock() 283 defer svm.Unlock() 284 return svm.config.CreateExt4Vhdx(destFile, sizeGB, cacheFile) 285} 286 287// getShortContainerPath looks up where a SCSI disk was actually mounted 288// in a service VM when we remapped a long path name to a short name. 289func (svm *serviceVM) getShortContainerPath(mvd *hcsshim.MappedVirtualDisk) string { 290 if mvd.ContainerPath == "" { 291 return "" 292 } 293 avhd, ok := svm.attachedVHDs[mvd.HostPath] 294 if !ok { 295 return "" 296 } 297 return fmt.Sprintf("/tmp/d%d", avhd.attachCounter) 298} 299 300func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedVirtualDisk) (err error) { 301 if len(mvds) == 0 { 302 return fmt.Errorf("createUnionMount: error must have at least 1 layer") 303 } 304 305 if err = svm.getStartError(); err != nil { 306 return err 307 } 308 309 svm.Lock() 310 defer svm.Unlock() 311 if _, ok := svm.unionMounts[mountName]; ok { 312 svm.unionMounts[mountName]++ 313 return nil 314 } 315 316 var lowerLayers []string 317 if mvds[0].ReadOnly { 318 lowerLayers = append(lowerLayers, svm.getShortContainerPath(&mvds[0])) 319 } 320 321 for i := 1; i < len(mvds); i++ { 322 lowerLayers = append(lowerLayers, svm.getShortContainerPath(&mvds[i])) 323 } 324 325 logrus.Debugf("Doing the overlay mount with union directory=%s", mountName) 326 errOut := &bytes.Buffer{} 327 if err = svm.runProcess(fmt.Sprintf("mkdir -p %s", mountName), nil, nil, errOut); err != nil { 328 return errors.Wrapf(err, "mkdir -p %s failed (%s)", mountName, errOut.String()) 329 } 330 331 var cmd string 332 if len(mvds) == 1 { 333 // `FROM SCRATCH` case and the only layer. No overlay required. 334 cmd = fmt.Sprintf("mount %s %s", svm.getShortContainerPath(&mvds[0]), mountName) 335 } else if mvds[0].ReadOnly { 336 // Readonly overlay 337 cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s %s", 338 strings.Join(lowerLayers, ","), 339 mountName) 340 } else { 341 upper := fmt.Sprintf("%s/upper", svm.getShortContainerPath(&mvds[0])) 342 work := fmt.Sprintf("%s/work", svm.getShortContainerPath(&mvds[0])) 343 344 errOut := &bytes.Buffer{} 345 if err = svm.runProcess(fmt.Sprintf("mkdir -p %s %s", upper, work), nil, nil, errOut); err != nil { 346 return errors.Wrapf(err, "mkdir -p %s failed (%s)", mountName, errOut.String()) 347 } 348 349 cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s,upperdir=%s,workdir=%s %s", 350 strings.Join(lowerLayers, ":"), 351 upper, 352 work, 353 mountName) 354 } 355 356 logrus.Debugf("createUnionMount: Executing mount=%s", cmd) 357 errOut = &bytes.Buffer{} 358 if err = svm.runProcess(cmd, nil, nil, errOut); err != nil { 359 return errors.Wrapf(err, "%s failed (%s)", cmd, errOut.String()) 360 } 361 362 svm.unionMounts[mountName] = 1 363 return nil 364} 365 366func (svm *serviceVM) deleteUnionMount(mountName string, disks ...hcsshim.MappedVirtualDisk) error { 367 if err := svm.getStartError(); err != nil { 368 return err 369 } 370 371 svm.Lock() 372 defer svm.Unlock() 373 if _, ok := svm.unionMounts[mountName]; !ok { 374 return nil 375 } 376 377 if svm.unionMounts[mountName] > 1 { 378 svm.unionMounts[mountName]-- 379 return nil 380 } 381 382 logrus.Debugf("Removing union mount %s", mountName) 383 if err := svm.runProcess(fmt.Sprintf("umount %s", mountName), nil, nil, nil); err != nil { 384 return err 385 } 386 387 delete(svm.unionMounts, mountName) 388 return nil 389} 390 391func (svm *serviceVM) runProcess(command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { 392 var process hcsshim.Process 393 var err error 394 errOut := &bytes.Buffer{} 395 396 if stderr != nil { 397 process, err = svm.config.RunProcess(command, stdin, stdout, stderr) 398 } else { 399 process, err = svm.config.RunProcess(command, stdin, stdout, errOut) 400 } 401 if err != nil { 402 return err 403 } 404 defer process.Close() 405 406 process.WaitTimeout(time.Duration(int(time.Second) * svm.config.UvmTimeoutSeconds)) 407 exitCode, err := process.ExitCode() 408 if err != nil { 409 return err 410 } 411 412 if exitCode != 0 { 413 // If the caller isn't explicitly capturing stderr output, then capture it here instead. 414 e := fmt.Sprintf("svm.runProcess: command %s failed with exit code %d", command, exitCode) 415 if stderr == nil { 416 e = fmt.Sprintf("%s. (%s)", e, errOut.String()) 417 } 418 return fmt.Errorf(e) 419 } 420 return nil 421} 422