1// Copyright 2016-2018 VMware, Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package management 16 17import ( 18 "context" 19 "crypto/tls" 20 "crypto/x509" 21 "encoding/json" 22 "fmt" 23 "io/ioutil" 24 "net" 25 "net/http" 26 "net/url" 27 "os" 28 "path" 29 "strconv" 30 "strings" 31 "syscall" 32 "time" 33 34 dockertypes "github.com/docker/docker/api/types" 35 "github.com/docker/docker/opts" 36 37 "github.com/vmware/govmomi/object" 38 "github.com/vmware/govmomi/vim25/soap" 39 "github.com/vmware/govmomi/vim25/types" 40 "github.com/vmware/vic/lib/config" 41 "github.com/vmware/vic/lib/config/executor" 42 "github.com/vmware/vic/lib/constants" 43 "github.com/vmware/vic/lib/install/data" 44 "github.com/vmware/vic/lib/spec" 45 "github.com/vmware/vic/lib/tether/shared" 46 "github.com/vmware/vic/pkg/errors" 47 "github.com/vmware/vic/pkg/ip" 48 "github.com/vmware/vic/pkg/retry" 49 "github.com/vmware/vic/pkg/trace" 50 "github.com/vmware/vic/pkg/vsphere/compute" 51 "github.com/vmware/vic/pkg/vsphere/diag" 52 "github.com/vmware/vic/pkg/vsphere/extraconfig" 53 "github.com/vmware/vic/pkg/vsphere/extraconfig/vmomi" 54 "github.com/vmware/vic/pkg/vsphere/tasks" 55 "github.com/vmware/vic/pkg/vsphere/vm" 56) 57 58const ( 59 portLayerPort = constants.SerialOverLANPort 60 61 // this is generated in the crypto/tls.alert code 62 badTLSCertificate = "tls: bad certificate" 63 64 // This is a constant also used in the lib/apiservers/engine/backends/system.go to assign custom info the docker types.info struct 65 volumeStoresID = "VolumeStores" 66) 67 68var ( 69 lastSeenProgressMessage string 70 unitNumber int32 71) 72 73func (d *Dispatcher) isVCH(vm *vm.VirtualMachine) (bool, error) { 74 if vm == nil { 75 return false, errors.New("nil parameter") 76 } 77 defer trace.End(trace.Begin(vm.InventoryPath)) 78 79 info, err := vm.FetchExtraConfig(d.op) 80 if err != nil { 81 err = errors.Errorf("Failed to fetch guest info of appliance vm: %s", err) 82 return false, err 83 } 84 85 var remoteConf config.VirtualContainerHostConfigSpec 86 extraconfig.Decode(extraconfig.MapSource(info), &remoteConf) 87 88 // if the moref of the target matches where we expect to find it for a VCH, run with it 89 if remoteConf.ExecutorConfig.ID == vm.Reference().String() || remoteConf.IsCreating() { 90 return true, nil 91 } 92 93 return false, nil 94} 95 96func (d *Dispatcher) isContainerVM(vm *vm.VirtualMachine) (bool, error) { 97 if vm == nil { 98 return false, errors.New("nil parameter") 99 } 100 defer trace.End(trace.Begin(vm.InventoryPath)) 101 var cspec executor.ExecutorConfig 102 info, err := vm.FetchExtraConfig(d.op) 103 if err != nil { 104 err = errors.Errorf("Failed to fetch guest info of appliance vm: %s", err) 105 return false, err 106 } 107 extraconfig.Decode(extraconfig.MapSource(info), &cspec) 108 if cspec.Version == nil { 109 return false, nil 110 } 111 return true, nil 112} 113 114func (d *Dispatcher) checkExistence(conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) error { 115 defer trace.End(trace.Begin("")) 116 var err error 117 var orp *object.ResourcePool 118 if orp, err = d.findResourcePool(d.vchPoolPath); err != nil { 119 return err 120 } 121 if orp == nil { 122 return nil 123 } 124 125 rp := compute.NewResourcePool(d.op, d.session, orp.Reference()) 126 vm, err := rp.GetChildVM(d.op, conf.Name) 127 if err != nil { 128 return err 129 } 130 if vm == nil { 131 return nil 132 } 133 134 d.op.Debug("Appliance is found") 135 if ok, verr := d.isVCH(vm); !ok { 136 verr = errors.Errorf("Found virtual machine %q, but it is not a VCH. Please choose a different virtual app.", conf.Name) 137 return verr 138 } 139 err = errors.Errorf("A VCH with the name %q already exists. Please choose a different name before attempting another install", conf.Name) 140 return err 141} 142 143func (d *Dispatcher) powerOffVM(vm *vm.VirtualMachine) error { 144 var err error 145 power, err := vm.PowerState(d.op) 146 if err == nil && power == types.VirtualMachinePowerStatePoweredOff { 147 d.op.Debugf("VM is already powered off: %s", vm.Reference()) 148 return nil 149 } 150 151 if err != nil { 152 d.op.Warnf("Failed to get vm power status %q: %s", vm.Reference(), err) 153 } 154 155 // try guest shutdown first 156 tools, err := vm.IsToolsRunning(d.op) 157 if tools { 158 d.op.Debugf("Performing guest shutdown for %s", vm.Reference()) 159 err = vm.ShutdownGuest(d.op) 160 if err == nil { 161 // just enough time for the endpoint to shutdown cleanly even having timed out internally 162 timeout, cancel := trace.WithTimeout(&d.op, shared.GuestShutdownTimeout+(5*time.Second), "Shut down endpointVM") 163 err = vm.WaitForPowerState(timeout, types.VirtualMachinePowerStatePoweredOff) 164 cancel() 165 if err == nil { 166 return nil 167 } 168 169 d.op.Warnf("Guest shutdown timed out, resorting to power off - sessions may be left open: %s", err) 170 } 171 172 // this may well be because of delay in reporting the VM powering off so the actual detail is recorded as debug 173 // if this is the case then we will catch it via the IsAlreadyPoweredOff call below 174 d.op.Debugf("Guest shutdown failed: %s", err) 175 } else { 176 d.op.Warnf("Guest tools unavailable, resorting to power off - sessions will be left open") 177 } 178 179 _, err = vm.WaitForResult(d.op, func(ctx context.Context) (tasks.Task, error) { 180 return vm.PowerOff(ctx) 181 }) 182 183 if vm.IsAlreadyPoweredOffError(err) { 184 err = nil 185 } 186 187 return err 188} 189 190func (d *Dispatcher) deleteVM(vm *vm.VirtualMachine, force bool) error { 191 defer trace.End(trace.Begin(fmt.Sprintf("vm %q, force %t", vm.String(), force))) 192 193 var err error 194 power, err := vm.PowerState(d.op) 195 if err != nil || power != types.VirtualMachinePowerStatePoweredOff { 196 if err != nil { 197 d.op.Warnf("Failed to get vm power status %q: %s", vm.Reference(), err) 198 } 199 if !force { 200 if err != nil { 201 return err 202 } 203 name, _ := vm.ObjectName(d.op) 204 if name != "" { 205 err = errors.Errorf("VM %q is powered on", name) 206 } else { 207 err = errors.Errorf("VM %q is powered on", vm.Reference()) 208 } 209 return err 210 } 211 212 err = d.powerOffVM(vm) 213 if err != nil { 214 d.op.Debugf("Failed to power off existing appliance for %s, try to remove anyway", err) 215 } 216 } 217 218 // get the actual folder name before we delete it 219 folder, err := vm.DatastoreFolderName(d.op) 220 if err != nil { 221 // failed to get folder name, might not be able to remove files for this VM 222 name, _ := vm.ObjectName(d.op) 223 if name == "" { 224 d.op.Errorf("Unable to automatically remove all files in datastore for VM %q", vm.Reference()) 225 } else { 226 // try to use the vm name in place of folder 227 d.op.Infof("Delete will attempt to remove datastore files for VM %q", name) 228 folder = name 229 } 230 } 231 232 // Power off the VM if necessary 233 retryErrHandler := func(err error) bool { 234 if tasks.IsInvalidPowerStateError(err) { 235 _, terr := vm.WaitForResult(d.op, func(ctx context.Context) (tasks.Task, error) { 236 return vm.PowerOff(ctx) 237 }) 238 239 if terr == nil || tasks.IsTransientError(d.op, terr) || tasks.IsInvalidPowerStateError(terr) { 240 return true 241 } 242 } 243 244 return tasks.IsTransientError(d.op, err) || tasks.IsConcurrentAccessError(err) 245 } 246 247 // Only retry VM destroy on ConcurrentAccess error 248 err = retry.Do(d.op, func() error { 249 _, err := vm.DeleteExceptDisks(d.op) 250 return err 251 }, retryErrHandler) 252 253 if err != nil { 254 d.op.Warnf("Destroy VM %s failed with %s, unregister the VM instead", vm.Reference(), err) 255 256 err = retry.Do(d.op, func() error { 257 return vm.Unregister(d.op) 258 }, retryErrHandler) 259 260 if err != nil { 261 d.op.Errorf("Unregister the VM failed: %s", err) 262 return err 263 } 264 } 265 266 if _, err = d.deleteDatastoreFiles(d.session.Datastore, folder, true); err != nil { 267 d.op.Warnf("Failed to remove datastore files for VM %s with folder path %s: %s", vm.Reference(), folder, err) 268 } 269 270 return nil 271} 272 273func (d *Dispatcher) addNetworkDevices(conf *config.VirtualContainerHostConfigSpec, cspec *spec.VirtualMachineConfigSpec, devices object.VirtualDeviceList) (object.VirtualDeviceList, error) { 274 defer trace.End(trace.Begin("")) 275 276 // network name:alias, to avoid create multiple devices for same network 277 slots := make(map[int32]bool) 278 nets := make(map[string]*executor.NetworkEndpoint) 279 280 for name, endpoint := range conf.ExecutorConfig.Networks { 281 if pnic, ok := nets[endpoint.Network.Common.ID]; ok { 282 // there's already a NIC on this network 283 endpoint.Common.ID = pnic.Common.ID 284 d.op.Infof("Network role %q is sharing NIC with %q", name, pnic.Network.Common.Name) 285 continue 286 } 287 288 moref := new(types.ManagedObjectReference) 289 if ok := moref.FromString(endpoint.Network.ID); !ok { 290 return nil, fmt.Errorf("serialized managed object reference in unexpected format: %q", endpoint.Network.ID) 291 } 292 obj, err := d.session.Finder.ObjectReference(d.op, *moref) 293 if err != nil { 294 return nil, fmt.Errorf("unable to reacquire reference for network %q from serialized form: %q", endpoint.Network.Name, endpoint.Network.ID) 295 } 296 network, ok := obj.(object.NetworkReference) 297 if !ok { 298 return nil, fmt.Errorf("reacquired reference for network %q, from serialized form %q, was not a network: %T", endpoint.Network.Name, endpoint.Network.ID, obj) 299 } 300 301 backing, err := network.EthernetCardBackingInfo(d.op) 302 if err != nil { 303 err = errors.Errorf("Failed to get network backing info for %q: %s", network, err) 304 return nil, err 305 } 306 307 nic, err := devices.CreateEthernetCard("vmxnet3", backing) 308 if err != nil { 309 err = errors.Errorf("Failed to create Ethernet Card spec for %s", err) 310 return nil, err 311 } 312 313 slot := cspec.AssignSlotNumber(nic, slots) 314 if slot == constants.NilSlot { 315 err = errors.Errorf("Failed to assign stable PCI slot for %q network card", name) 316 } 317 318 endpoint.Common.ID = strconv.Itoa(int(slot)) 319 slots[slot] = true 320 d.op.Debugf("Setting %q to slot %d", name, slot) 321 322 devices = append(devices, nic) 323 324 nets[endpoint.Network.Common.ID] = endpoint 325 } 326 return devices, nil 327} 328 329func (d *Dispatcher) addIDEController(devices object.VirtualDeviceList) (object.VirtualDeviceList, error) { 330 defer trace.End(trace.Begin("")) 331 332 // IDE controller 333 scsi, err := devices.CreateIDEController() 334 if err != nil { 335 return nil, err 336 } 337 devices = append(devices, scsi) 338 return devices, nil 339} 340 341func (d *Dispatcher) addParaVirtualSCSIController(devices object.VirtualDeviceList) (object.VirtualDeviceList, error) { 342 defer trace.End(trace.Begin("")) 343 344 // para virtual SCSI controller 345 scsi, err := devices.CreateSCSIController("pvscsi") 346 if err != nil { 347 return nil, err 348 } 349 devices = append(devices, scsi) 350 return devices, nil 351} 352 353func (d *Dispatcher) createApplianceSpec(conf *config.VirtualContainerHostConfigSpec, vConf *data.InstallerData) (*types.VirtualMachineConfigSpec, error) { 354 defer trace.End(trace.Begin("")) 355 356 var devices object.VirtualDeviceList 357 var err error 358 var cpus int32 // appliance number of CPUs 359 var memory int64 // appliance memory in MB 360 361 // set to creating VCH 362 conf.SetIsCreating(true) 363 364 cfg, err := d.encodeConfig(conf) 365 if err != nil { 366 return nil, err 367 } 368 369 if vConf.ApplianceSize.CPU.Limit != nil { 370 cpus = int32(*vConf.ApplianceSize.CPU.Limit) 371 } 372 if vConf.ApplianceSize.Memory.Limit != nil { 373 memory = *vConf.ApplianceSize.Memory.Limit 374 } 375 376 spec := &spec.VirtualMachineConfigSpec{ 377 VirtualMachineConfigSpec: &types.VirtualMachineConfigSpec{ 378 Name: conf.Name, 379 GuestId: string(types.VirtualMachineGuestOsIdentifierOtherGuest64), 380 AlternateGuestName: constants.DefaultAltVCHGuestName(), 381 Files: &types.VirtualMachineFileInfo{VmPathName: fmt.Sprintf("[%s]", conf.ImageStores[0].Host)}, 382 NumCPUs: cpus, 383 MemoryMB: memory, 384 // Encode the config both here and after the VMs created so that it can be identified as a VCH appliance as soon as 385 // creation is complete. 386 ExtraConfig: append(vmomi.OptionValueFromMap(cfg, true), &types.OptionValue{Key: "answer.msg.serial.file.open", Value: "Append"}), 387 }, 388 } 389 390 if devices, err = d.addIDEController(devices); err != nil { 391 return nil, err 392 } 393 394 if devices, err = d.addParaVirtualSCSIController(devices); err != nil { 395 return nil, err 396 } 397 398 if devices, err = d.addNetworkDevices(conf, spec, devices); err != nil { 399 return nil, err 400 } 401 402 deviceChange, err := devices.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd) 403 if err != nil { 404 return nil, err 405 } 406 407 spec.DeviceChange = deviceChange 408 return spec.VirtualMachineConfigSpec, nil 409} 410 411func isManagedObjectNotFoundError(err error) bool { 412 if soap.IsSoapFault(err) { 413 _, ok := soap.ToSoapFault(err).VimFault().(types.ManagedObjectNotFound) 414 return ok 415 } 416 417 return false 418} 419 420func (d *Dispatcher) findApplianceByID(conf *config.VirtualContainerHostConfigSpec) (*vm.VirtualMachine, error) { 421 defer trace.End(trace.Begin("")) 422 423 var err error 424 var vmm *vm.VirtualMachine 425 426 moref := new(types.ManagedObjectReference) 427 if ok := moref.FromString(conf.ID); !ok { 428 message := "Failed to get appliance VM mob reference" 429 d.op.Error(message) 430 return nil, errors.New(message) 431 } 432 ref, err := d.session.Finder.ObjectReference(d.op, *moref) 433 if err != nil { 434 if !isManagedObjectNotFoundError(err) { 435 err = errors.Errorf("Failed to query appliance (%q): %s", moref, err) 436 return nil, err 437 } 438 d.op.Debug("Appliance is not found") 439 return nil, nil 440 441 } 442 ovm, ok := ref.(*object.VirtualMachine) 443 if !ok { 444 d.op.Errorf("Failed to find VM %q: %s", moref, err) 445 return nil, err 446 } 447 vmm = vm.NewVirtualMachine(d.op, d.session, ovm.Reference()) 448 449 element, err := d.session.Finder.Element(d.op, vmm.Reference()) 450 if err != nil { 451 return nil, err 452 } 453 vmm.SetInventoryPath(element.Path) 454 return vmm, nil 455} 456 457func (d *Dispatcher) configIso(conf *config.VirtualContainerHostConfigSpec, vm *vm.VirtualMachine, settings *data.InstallerData) (object.VirtualDeviceList, error) { 458 defer trace.End(trace.Begin("")) 459 460 var devices object.VirtualDeviceList 461 var err error 462 463 vmDevices, err := vm.Device(d.op) 464 if err != nil { 465 d.op.Errorf("Failed to get vm devices for appliance: %s", err) 466 return nil, err 467 } 468 ide, err := vmDevices.FindIDEController("") 469 if err != nil { 470 d.op.Errorf("Failed to find IDE controller for appliance: %s", err) 471 return nil, err 472 } 473 cdrom, err := devices.CreateCdrom(ide) 474 if err != nil { 475 d.op.Errorf("Failed to create Cdrom device for appliance: %s", err) 476 return nil, err 477 } 478 cdrom = devices.InsertIso(cdrom, fmt.Sprintf("[%s] %s/%s", conf.ImageStores[0].Host, d.vmPathName, settings.ApplianceISO)) 479 devices = append(devices, cdrom) 480 return devices, nil 481} 482 483func (d *Dispatcher) configLogging(conf *config.VirtualContainerHostConfigSpec, vm *vm.VirtualMachine, settings *data.InstallerData) (object.VirtualDeviceList, error) { 484 defer trace.End(trace.Begin("")) 485 486 devices, err := vm.Device(d.op) 487 if err != nil { 488 d.op.Errorf("Failed to get vm devices for appliance: %s", err) 489 return nil, err 490 } 491 492 p, err := devices.CreateSerialPort() 493 if err != nil { 494 return nil, err 495 } 496 497 err = vm.AddDevice(d.op, p) 498 if err != nil { 499 return nil, err 500 } 501 502 devices, err = vm.Device(d.op) 503 if err != nil { 504 d.op.Errorf("Failed to get vm devices for appliance: %s", err) 505 return nil, err 506 } 507 508 serial, err := devices.FindSerialPort("") 509 if err != nil { 510 d.op.Errorf("Failed to locate serial port for persistent log configuration: %s", err) 511 return nil, err 512 } 513 514 // TODO: we need to add an accessor for generating paths within the VM directory 515 vmx, err := vm.VMPathName(d.op) 516 if err != nil { 517 d.op.Errorf("Unable to determine path of appliance VM: %s", err) 518 return nil, err 519 } 520 521 // TODO: move this construction into the spec package and update portlayer/logging to use it as well 522 serial.Backing = &types.VirtualSerialPortFileBackingInfo{ 523 VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ 524 // name consistency with containerVM 525 FileName: fmt.Sprintf("%s/tether.debug", path.Dir(vmx)), 526 }, 527 } 528 529 return []types.BaseVirtualDevice{serial}, nil 530} 531 532func (d *Dispatcher) setDockerPort(conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) { 533 if conf.HostCertificate != nil { 534 d.DockerPort = fmt.Sprintf("%d", opts.DefaultTLSHTTPPort) 535 } else { 536 d.DockerPort = fmt.Sprintf("%d", opts.DefaultHTTPPort) 537 } 538} 539 540func (d *Dispatcher) createAppliance(conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) error { 541 defer trace.End(trace.Begin("")) 542 d.op.Info("Creating appliance on target") 543 544 spec, err := d.createApplianceSpec(conf, settings) 545 if err != nil { 546 d.op.Errorf("Unable to create appliance spec: %s", err) 547 return err 548 } 549 550 // Create the VCH inventory folder 551 if d.isVC { 552 d.op.Info("Creating the VCH folder") 553 // update the session pointer with the VCH Folder 554 d.session.VCHFolder, err = d.session.VMFolder.CreateFolder(d.op, spec.Name) 555 if err != nil { 556 if soap.IsSoapFault(err) { 557 switch soap.ToSoapFault(err).VimFault().(type) { 558 case types.DuplicateName: 559 return fmt.Errorf("The specified VCH name (%s) is already in use", conf.Name) 560 } 561 } 562 return fmt.Errorf("Unable to create the VCH Folder(%s): %s", conf.Name, err) 563 } 564 } 565 566 d.op.Info("Creating the VCH VM") 567 info, err := tasks.WaitForResult(d.op, func(ctx context.Context) (tasks.Task, error) { 568 return d.session.VCHFolder.CreateVM(ctx, *spec, d.vchPool, d.session.Host) 569 }) 570 571 if err != nil { 572 d.op.Errorf("Unable to create the appliance VM: %s", err) 573 return err 574 } 575 if info.Error != nil || info.State != types.TaskInfoStateSuccess { 576 d.op.Errorf("Create appliance reported: %s", info.Error.LocalizedMessage) 577 } 578 579 if err = d.createVolumeStores(conf); err != nil { 580 return errors.Errorf("Exiting because we could not create volume stores due to error: %s", err) 581 } 582 583 // get VM reference and save it 584 moref := info.Result.(types.ManagedObjectReference) 585 conf.SetMoref(&moref) 586 obj, err := d.session.Finder.ObjectReference(d.op, moref) 587 if err != nil { 588 d.op.Errorf("Failed to reacquire reference to appliance VM after creation: %s", err) 589 return err 590 } 591 gvm, ok := obj.(*object.VirtualMachine) 592 if !ok { 593 return fmt.Errorf("Required reference after appliance creation was not for a VM: %T", obj) 594 } 595 vm2 := vm.NewVirtualMachineFromVM(d.op, d.session, gvm) 596 597 vm2.DisableDestroy(d.op) 598 599 // update the displayname to the actual folder name used 600 if d.vmPathName, err = vm2.DatastoreFolderName(d.op); err != nil { 601 d.op.Errorf("Failed to get canonical name for appliance: %s", err) 602 return err 603 } 604 d.op.Debugf("vm folder name: %q", d.vmPathName) 605 d.op.Debugf("vm inventory path: %q", vm2.InventoryPath) 606 607 vicadmin := executor.Cmd{ 608 Path: "/sbin/vicadmin", 609 Args: []string{ 610 "/sbin/vicadmin", 611 "--dc=" + settings.DatacenterName, 612 "--pool=" + settings.ResourcePoolPath, 613 "--cluster=" + settings.ClusterPath, 614 }, 615 Env: []string{ 616 "PATH=/sbin:/bin", 617 "GOTRACEBACK=all", 618 }, 619 Dir: "/home/vicadmin", 620 } 621 if settings.HTTPProxy != nil { 622 vicadmin.Env = append(vicadmin.Env, fmt.Sprintf("%s=%s", config.VICAdminHTTPProxy, settings.HTTPProxy.String())) 623 } 624 if settings.HTTPSProxy != nil { 625 vicadmin.Env = append(vicadmin.Env, fmt.Sprintf("%s=%s", config.VICAdminHTTPSProxy, settings.HTTPSProxy.String())) 626 } 627 if settings.NoProxy != nil { 628 vicadmin.Env = append(vicadmin.Env, fmt.Sprintf("%s=%s", config.VICAdminNoProxy, *settings.NoProxy)) 629 } 630 631 conf.AddComponent(config.VicAdminService, &executor.SessionConfig{ 632 User: "vicadmin", 633 Group: "vicadmin", 634 Cmd: vicadmin, 635 Restart: true, 636 Active: true, 637 }, 638 ) 639 640 d.setDockerPort(conf, settings) 641 642 personality := executor.Cmd{ 643 Path: "/sbin/docker-engine-server", 644 Args: []string{ 645 "/sbin/docker-engine-server", 646 //FIXME: hack during config migration 647 "-port=" + d.DockerPort, 648 fmt.Sprintf("-port-layer-port=%d", portLayerPort), 649 }, 650 Env: []string{ 651 "PATH=/sbin", 652 "GOTRACEBACK=all", 653 }, 654 } 655 if settings.HTTPProxy != nil { 656 personality.Env = append(personality.Env, fmt.Sprintf("%s=%s", config.GeneralHTTPProxy, settings.HTTPProxy.String())) 657 } 658 if settings.HTTPSProxy != nil { 659 personality.Env = append(personality.Env, fmt.Sprintf("%s=%s", config.GeneralHTTPSProxy, settings.HTTPSProxy.String())) 660 } 661 if settings.NoProxy != nil { 662 personality.Env = append(personality.Env, fmt.Sprintf("%s=%s", config.GeneralNoProxy, *settings.NoProxy)) 663 } 664 665 conf.AddComponent(config.PersonaService, &executor.SessionConfig{ 666 // currently needed for iptables interaction 667 // User: "nobody", 668 // Group: "nobody", 669 Cmd: personality, 670 Restart: true, 671 Active: true, 672 }, 673 ) 674 675 cfg := &executor.SessionConfig{ 676 Cmd: executor.Cmd{ 677 Path: "/sbin/port-layer-server", 678 Args: []string{ 679 "/sbin/port-layer-server", 680 "--host=localhost", 681 fmt.Sprintf("--port=%d", portLayerPort), 682 }, 683 Env: []string{ 684 //FIXME: hack during config migration 685 "VC_URL=" + conf.Target, 686 "DC_PATH=" + settings.DatacenterName, 687 "CS_PATH=" + settings.ClusterPath, 688 "POOL_PATH=" + settings.ResourcePoolPath, 689 "DS_PATH=" + conf.ImageStores[0].Host, 690 }, 691 }, 692 Restart: true, 693 Active: true, 694 } 695 696 conf.AddComponent(config.PortLayerService, cfg) 697 698 // fix up those parts of the config that depend on the final applianceVM folder name 699 conf.BootstrapImagePath = fmt.Sprintf("[%s] %s/%s", conf.ImageStores[0].Host, d.vmPathName, settings.BootstrapISO) 700 701 if len(conf.ImageStores[0].Path) == 0 { 702 conf.ImageStores[0].Path = d.vmPathName 703 } 704 705 // apply the fixed-up configuration 706 spec, err = d.reconfigureApplianceSpec(vm2, conf, settings) 707 if err != nil { 708 d.op.Errorf("Error while getting appliance reconfig spec: %s", err) 709 return err 710 } 711 712 // reconfig 713 info, err = vm2.WaitForResult(d.op, func(ctx context.Context) (tasks.Task, error) { 714 return vm2.Reconfigure(ctx, *spec) 715 }) 716 717 if err != nil { 718 d.op.Errorf("Error while setting component parameters to appliance: %s", err) 719 return err 720 } 721 if info.State != types.TaskInfoStateSuccess { 722 d.op.Errorf("Setting parameters to appliance reported: %s", info.Error.LocalizedMessage) 723 return err 724 } 725 726 d.appliance = vm2 727 return nil 728} 729 730func (d *Dispatcher) encodeConfig(conf *config.VirtualContainerHostConfigSpec) (map[string]string, error) { 731 d.op.Debug("generating new config secret key") 732 733 s, err := extraconfig.NewSecretKey() 734 if err != nil { 735 return nil, err 736 } 737 d.secret = s 738 739 cfg := make(map[string]string) 740 extraconfig.Encode(d.secret.Sink(extraconfig.MapSink(cfg)), conf) 741 return cfg, nil 742} 743 744func (d *Dispatcher) decryptVCHConfig(vm *vm.VirtualMachine, cfg map[string]string) (*config.VirtualContainerHostConfigSpec, error) { 745 defer trace.End(trace.Begin("")) 746 747 if d.secret == nil { 748 name, err := vm.ObjectName(d.op) 749 if err != nil { 750 err = errors.Errorf("Failed to get vm name %q: %s", vm.Reference(), err) 751 return nil, err 752 } 753 // set session datastore to where the VM is running 754 ds, err := d.getImageDatastore(vm, nil, true) 755 if err != nil { 756 err = errors.Errorf("Failure finding image store from VCH VM %q: %s", name, err) 757 return nil, err 758 } 759 path, err := vm.DatastoreFolderName(d.op) 760 if err != nil { 761 err = errors.Errorf("Failed to get VM %q datastore path: %s", name, err) 762 return nil, err 763 } 764 s, err := d.GuestInfoSecret(name, path, ds) 765 if err != nil { 766 return nil, err 767 } 768 d.secret = s 769 } 770 771 conf := &config.VirtualContainerHostConfigSpec{} 772 extraconfig.Decode(d.secret.Source(extraconfig.MapSource(cfg)), conf) 773 return conf, nil 774} 775 776func (d *Dispatcher) reconfigureApplianceSpec(vm *vm.VirtualMachine, conf *config.VirtualContainerHostConfigSpec, settings *data.InstallerData) (*types.VirtualMachineConfigSpec, error) { 777 defer trace.End(trace.Begin("")) 778 779 var devices object.VirtualDeviceList 780 var err error 781 782 spec := &types.VirtualMachineConfigSpec{} 783 784 // create new devices 785 if devices, err = d.configIso(conf, vm, settings); err != nil { 786 return nil, err 787 } 788 789 newDevices, err := devices.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd) 790 if err != nil { 791 d.op.Errorf("Failed to create config spec for appliance: %s", err) 792 return nil, err 793 } 794 795 spec.DeviceChange = newDevices 796 797 // update existing devices 798 if devices, err = d.configLogging(conf, vm, settings); err != nil { 799 return nil, err 800 } 801 802 updateDevices, err := devices.ConfigSpec(types.VirtualDeviceConfigSpecOperationEdit) 803 if err != nil { 804 d.op.Errorf("Failed to create config spec for logging update: %s", err) 805 return nil, err 806 } 807 808 spec.DeviceChange = append(spec.DeviceChange, updateDevices...) 809 810 cfg, err := d.encodeConfig(conf) 811 if err != nil { 812 return nil, err 813 } 814 815 spec.ExtraConfig = append(spec.ExtraConfig, vmomi.OptionValueFromMap(cfg, true)...) 816 return spec, nil 817} 818 819// applianceConfiguration updates the configuration passed in with the latest from the appliance VM. 820// there's no guarantee of consistency within the configuration at this time 821func (d *Dispatcher) applianceConfiguration(conf *config.VirtualContainerHostConfigSpec) error { 822 defer trace.End(trace.Begin("")) 823 824 extraConfig, err := d.appliance.FetchExtraConfig(d.op) 825 if err != nil { 826 return err 827 } 828 829 extraconfig.Decode(extraconfig.MapSource(extraConfig), conf) 830 return nil 831} 832 833// waitForKey squashes the return values and simpy blocks until the key is updated or there is an error 834func (d *Dispatcher) waitForKey(key string) { 835 defer trace.End(trace.Begin(key)) 836 837 d.appliance.WaitForKeyInExtraConfig(d.op, key) 838 return 839} 840 841// isPortLayerRunning decodes the `docker info` response to check if the portlayer is running 842func isPortLayerRunning(op trace.Operation, res *http.Response, conf *config.VirtualContainerHostConfigSpec) bool { 843 defer res.Body.Close() 844 resBody, err := ioutil.ReadAll(res.Body) 845 if err != nil { 846 op.Debugf("error while reading res body: %s", err.Error()) 847 return false 848 } 849 850 var sysInfo dockertypes.Info 851 if err = json.Unmarshal(resBody, &sysInfo); err != nil { 852 op.Debugf("error while unmarshalling res body: %s", err.Error()) 853 return false 854 } 855 // At this point the portlayer is up successfully. However, we need to report the Volume Stores that were not created successfully. 856 volumeStoresLine := "" 857 858 for _, value := range sysInfo.SystemStatus { 859 if value[0] == volumeStoresID { 860 op.Debugf("Portlayer has established volume stores (%s)", value[1]) 861 volumeStoresLine = value[1] 862 break 863 } 864 } 865 866 allVolumeStoresPresent := confirmVolumeStores(op, conf, volumeStoresLine) 867 if !allVolumeStoresPresent { 868 op.Error("Not all configured volume stores are online - check port layer log via vicadmin") 869 } 870 871 for _, status := range sysInfo.SystemStatus { 872 if status[0] == sysInfo.Driver { 873 return status[1] == "RUNNING" 874 } 875 } 876 877 return false 878} 879 880// confirmVolumeStores is a helper function that will log and warn the vic-machine user if some of their volumestores did not present in the portlayer 881func confirmVolumeStores(op trace.Operation, conf *config.VirtualContainerHostConfigSpec, rawVolumeStores string) bool { 882 establishedVolumeStores := make(map[string]struct{}) 883 884 splitStores := strings.Split(rawVolumeStores, " ") 885 for _, v := range splitStores { 886 establishedVolumeStores[v] = struct{}{} 887 } 888 889 result := true 890 for k := range conf.VolumeLocations { 891 if _, ok := establishedVolumeStores[k]; !ok { 892 op.Errorf("VolumeStore (%s) cannot be brought online - check network, nfs server, and --volume-store configurations", k) 893 result = false 894 } 895 } 896 return result 897} 898 899// CheckDockerAPI checks if the appliance components are initialized by issuing 900// `docker info` to the appliance 901func (d *Dispatcher) CheckDockerAPI(conf *config.VirtualContainerHostConfigSpec, clientCert *tls.Certificate) error { 902 defer trace.End(trace.Begin("")) 903 904 var ( 905 proto string 906 client *http.Client 907 res *http.Response 908 err error 909 req *http.Request 910 tlsErrExpected bool 911 ) 912 913 if conf.HostCertificate.IsNil() { 914 // TLS disabled 915 proto = "http" 916 client = &http.Client{} 917 } else { 918 // TLS enabled 919 proto = "https" 920 921 // #nosec: TLS InsecureSkipVerify set true 922 tr := &http.Transport{ 923 TLSClientConfig: &tls.Config{ 924 InsecureSkipVerify: true, 925 }, 926 } 927 928 // appliance is configured for tlsverify, but we don't have a client certificate 929 if len(conf.CertificateAuthorities) > 0 { 930 // if tlsverify was configured at all then we must verify the remote 931 tr.TLSClientConfig.InsecureSkipVerify = false 932 933 func() { 934 d.op.Debug("Loading CAs for client auth") 935 pool, err := x509.SystemCertPool() 936 if err != nil { 937 d.op.Warn("Unable to load system root certificates - continuing with only the provided CA") 938 pool = x509.NewCertPool() 939 } 940 941 if !pool.AppendCertsFromPEM(conf.CertificateAuthorities) { 942 d.op.Warn("Unable add CAs from config to validation pool") 943 } 944 945 // tr.TLSClientConfig.ClientCAs = pool 946 tr.TLSClientConfig.RootCAs = pool 947 948 if clientCert == nil { 949 // we know this will fail, but we can try to distinguish the expected error vs 950 // unresponsive endpoint 951 tlsErrExpected = true 952 d.op.Debug("CA configured on appliance but no client certificate available") 953 return 954 } 955 956 cert, err := conf.HostCertificate.X509Certificate() 957 if err != nil { 958 d.op.Debug("Unable to extract host certificate: %s", err) 959 tlsErrExpected = true 960 return 961 } 962 963 cip := net.ParseIP(d.HostIP) 964 if err != nil { 965 d.op.Debug("Unable to process Docker API host address from %q: %s", d.HostIP, err) 966 tlsErrExpected = true 967 return 968 } 969 970 // find the name to use and override the IP if found 971 addr, err := addrToUse(d.op, []net.IP{cip}, cert, conf.CertificateAuthorities) 972 if err != nil { 973 d.op.Debug("Unable to determine address to use with remote certificate, checking SANs") 974 // #nosec: Errors unhandled . 975 addr, _ = viableHostAddress(d.op, []net.IP{cip}, cert, conf.CertificateAuthorities) 976 d.op.Debugf("Using host address: %s", addr) 977 } 978 if addr != "" { 979 d.HostIP = addr 980 } else { 981 d.op.Debug("Failed to find a viable address for Docker API from certificates") 982 // Server certificate won't validate since we don't have a hostname 983 tlsErrExpected = true 984 } 985 d.op.Debugf("Host address set to: %q", d.HostIP) 986 }() 987 } 988 989 if clientCert != nil { 990 d.op.Debug("Assigning certificates for client auth") 991 tr.TLSClientConfig.Certificates = []tls.Certificate{*clientCert} 992 } 993 994 client = &http.Client{Transport: tr} 995 } 996 997 dockerInfoURL := fmt.Sprintf("%s://%s:%s/info", proto, d.HostIP, d.DockerPort) 998 d.op.Debugf("Docker API endpoint: %s", dockerInfoURL) 999 req, err = http.NewRequest("GET", dockerInfoURL, nil) 1000 if err != nil { 1001 return errors.New("invalid HTTP request for docker info") 1002 } 1003 req = req.WithContext(d.op) 1004 1005 ticker := time.NewTicker(time.Second) 1006 defer ticker.Stop() 1007 for { 1008 res, err = client.Do(req) 1009 if err == nil { 1010 if res.StatusCode == http.StatusOK { 1011 if isPortLayerRunning(d.op, res, conf) { 1012 d.op.Debug("Confirmed port layer is operational") 1013 break 1014 } 1015 } 1016 1017 d.op.Debugf("Received HTTP status %d: %s", res.StatusCode, res.Status) 1018 } else { 1019 // DEBU[2016-10-11T22:22:38Z] Error received from endpoint: Get https://192.168.78.127:2376/info: dial tcp 192.168.78.127:2376: getsockopt: connection refused &{%!t(string=Get) %!t(string=https://192.168.78.127:2376/info) %!t(*net.OpError=&{dial tcp <nil> 0xc4204505a0 0xc4203a5e00})} 1020 // DEBU[2016-10-11T22:22:39Z] Components not yet initialized, retrying 1021 // ERR=&url.Error{ 1022 // Op: "Get", 1023 // URL: "https://192.168.78.127:2376/info", 1024 // Err: &net.OpError{ 1025 // Op: "dial", 1026 // Net: "tcp", 1027 // Source: nil, 1028 // Addr: &net.TCPAddr{ 1029 // IP: {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xc0, 0xa8, 0x4e, 0x7f}, 1030 // Port: 2376, 1031 // Zone: "", 1032 // }, 1033 // Err: &os.SyscallError{ 1034 // Syscall: "getsockopt", 1035 // Err: syscall.Errno(0x6f), 1036 // }, 1037 // }, 1038 // } 1039 // DEBU[2016-10-11T22:22:41Z] Error received from endpoint: Get https://192.168.78.127:2376/info: remote error: tls: bad certificate &{%!t(string=Get) %!t(string=https://192.168.78.127:2376/info) %!t(*net.OpError=&{remote error <nil> <nil> 42})} 1040 // DEBU[2016-10-11T22:22:42Z] Components not yet initialized, retrying 1041 // ERR=&url.Error{ 1042 // Op: "Get", 1043 // URL: "https://192.168.78.127:2376/info", 1044 // Err: &net.OpError{ 1045 // Op: "remote error", 1046 // Net: "", 1047 // Source: nil, 1048 // Addr: nil, 1049 // Err: tls.alert(0x2a), 1050 // }, 1051 // } 1052 1053 // ECONNREFUSED: 111, 0x6f 1054 1055 uerr, ok := err.(*url.Error) 1056 if ok { 1057 switch neterr := uerr.Err.(type) { 1058 case *net.OpError: 1059 switch root := neterr.Err.(type) { 1060 case *os.SyscallError: 1061 if root.Err == syscall.Errno(syscall.ECONNREFUSED) { 1062 // waiting for API server to start 1063 d.op.Debug("connection refused") 1064 } else { 1065 d.op.Debugf("Error was expected to be ECONNREFUSED: %#v", root.Err) 1066 } 1067 default: 1068 errmsg := root.Error() 1069 1070 if tlsErrExpected { 1071 d.op.Warnf("Expected TLS error without access to client certificate, received error: %s", errmsg) 1072 return nil 1073 } 1074 1075 // the TLS package doesn't expose the raw reason codes 1076 // but we're actually looking for alertBadCertificate (42) 1077 if errmsg == badTLSCertificate { 1078 // TODO: programmatic check for clock skew on host 1079 d.op.Error("Connection failed with TLS error \"bad certificate\" - check for clock skew on the host") 1080 } else { 1081 d.op.Errorf("Connection failed with error: %s", root) 1082 } 1083 1084 return fmt.Errorf("failed to connect to %s: %s", dockerInfoURL, root) 1085 } 1086 1087 case x509.UnknownAuthorityError: 1088 // This will occur if the server certificate was signed by a CA that is not the one used for client authentication 1089 // and does not have a trusted root registered on the system running vic-machine 1090 msg := fmt.Sprintf("Unable to validate server certificate with configured CAs (unknown CA): %s", neterr.Error()) 1091 if tlsErrExpected { 1092 // Legitimate deployment so no error, but definitely requires a warning. 1093 d.op.Warn(msg) 1094 return nil 1095 } 1096 // TLS error not expected, the validation failure is a problem 1097 d.op.Error(msg) 1098 return neterr 1099 1100 case x509.HostnameError: 1101 // e.g. "doesn't contain any IP SANs" 1102 msg := fmt.Sprintf("Server certificate hostname doesn't match: %s", neterr.Error()) 1103 if tlsErrExpected { 1104 d.op.Warn(msg) 1105 return nil 1106 } 1107 d.op.Error(msg) 1108 return neterr 1109 1110 default: 1111 d.op.Debugf("Unhandled net error type: %#v", neterr) 1112 return neterr 1113 } 1114 } else { 1115 d.op.Debugf("Error type was expected to be url.Error: %#v", err) 1116 } 1117 } 1118 1119 select { 1120 case <-ticker.C: 1121 case <-d.op.Done(): 1122 return d.op.Err() 1123 } 1124 1125 d.op.Debug("Components not yet initialized, retrying") 1126 } 1127 1128 return nil 1129} 1130 1131// ensureApplianceInitializes checks if the appliance component processes are launched correctly 1132func (d *Dispatcher) ensureApplianceInitializes(conf *config.VirtualContainerHostConfigSpec) error { 1133 defer trace.End(trace.Begin("")) 1134 1135 if d.appliance == nil { 1136 return errors.New("cannot validate appliance due to missing VM reference") 1137 } 1138 1139 d.op.Info("Waiting for IP information") 1140 d.waitForKey(extraconfig.CalculateKey(conf, "ExecutorConfig.Networks.client.Assigned.IP", "")) 1141 ctxerr := d.op.Err() 1142 1143 if ctxerr == nil { 1144 d.op.Info("Waiting for major appliance components to launch") 1145 for _, k := range extraconfig.CalculateKeys(conf, "ExecutorConfig.Sessions.*.Started", "") { 1146 d.waitForKey(k) 1147 } 1148 } 1149 1150 // at this point either everything has succeeded or we're going into diagnostics, ignore error 1151 // as we're only using it for IP in the success case 1152 updateErr := d.applianceConfiguration(conf) 1153 1154 // confirm components launched correctly 1155 d.op.Debug(" State of components:") 1156 for name, session := range conf.ExecutorConfig.Sessions { 1157 status := "waiting to launch" 1158 if session.Started == "true" { 1159 status = "started successfully" 1160 } else if session.Started != "" { 1161 status = session.Started 1162 d.op.Errorf(" Component did not launch successfully - %s: %s", name, status) 1163 } 1164 1165 d.op.Debugf(" %q: %q", name, status) 1166 } 1167 1168 // TODO: we should call to the general vic-machine inspect implementation here for more detail 1169 // but instead... 1170 if !ip.IsUnspecifiedIP(conf.ExecutorConfig.Networks["client"].Assigned.IP) { 1171 d.HostIP = conf.ExecutorConfig.Networks["client"].Assigned.IP.String() 1172 d.op.Infof("Obtained IP address for client interface: %q", d.HostIP) 1173 return nil 1174 } 1175 1176 // it's possible we timed out... get updated info having adjusted context to allow it 1177 // keeping it short 1178 ctxerr = d.op.Err() 1179 1180 baseOp := trace.NewOperationWithLoggerFrom(context.Background(), d.op, "ensureApplianceInitializes") 1181 op, cancel := trace.WithTimeout(&baseOp, 10*time.Second, "ensureApplianceInitializes timeout") 1182 defer cancel() 1183 d.op = op 1184 err := d.applianceConfiguration(conf) 1185 if err != nil { 1186 return fmt.Errorf("unable to retrieve updated configuration from appliance for diagnostics: %s", err) 1187 } 1188 1189 if ctxerr == context.DeadlineExceeded { 1190 d.op.Info("") 1191 d.op.Error("Failed to obtain IP address for client interface") 1192 d.op.Info("Use vic-machine inspect to see if VCH has received an IP address at a later time") 1193 d.op.Info(" State of all interfaces:") 1194 1195 // if we timed out, then report status - if cancelled this doesn't need reporting 1196 for name, net := range conf.ExecutorConfig.Networks { 1197 addr := net.Assigned.String() 1198 if ip.IsUnspecifiedIP(net.Assigned.IP) { 1199 addr = "waiting for IP" 1200 } 1201 d.op.Infof(" %q IP: %q", name, addr) 1202 } 1203 1204 // if we timed out, then report status - if cancelled this doesn't need reporting 1205 d.op.Info(" State of components:") 1206 for name, session := range conf.ExecutorConfig.Sessions { 1207 status := "waiting to launch" 1208 if session.Started == "true" { 1209 status = "started successfully" 1210 } else if session.Started != "" { 1211 status = session.Started 1212 } 1213 d.op.Infof(" %q: %q", name, status) 1214 } 1215 1216 return errors.New("Failed to obtain IP address for client interface (timed out)") 1217 } 1218 1219 return fmt.Errorf("Failed to get IP address information from appliance: %s", updateErr) 1220} 1221 1222// CheckServiceReady checks if service is launched correctly, including ip address, service initialization, VC connection and Docker API 1223// Should expand this method for any more VCH service checking 1224func (d *Dispatcher) CheckServiceReady(ctx context.Context, conf *config.VirtualContainerHostConfigSpec, clientCert *tls.Certificate) error { 1225 defer func(oldOp trace.Operation) { d.op = oldOp }(d.op) 1226 d.op = trace.FromContext(ctx, "CheckServiceReady") 1227 1228 if err := d.ensureApplianceInitializes(conf); err != nil { 1229 return err 1230 } 1231 1232 // vic-init will try to reach out to the vSphere target. 1233 d.op.Info("Checking VCH connectivity with vSphere target") 1234 // Checking access to vSphere API 1235 if cd, err := d.CheckAccessToVCAPI(d.appliance, conf.Target); err == nil { 1236 code := int(cd) 1237 if code > 0 { 1238 d.op.Warnf("vSphere API Test: %s %s", conf.Target, diag.UserReadableVCAPITestDescription(code)) 1239 } else { 1240 d.op.Infof("vSphere API Test: %s %s", conf.Target, diag.UserReadableVCAPITestDescription(code)) 1241 } 1242 } else { 1243 d.op.Warnf("Could not run VCH vSphere API target check due to %v but the VCH may still function normally", err) 1244 } 1245 1246 if err := d.CheckDockerAPI(conf, clientCert); err != nil { 1247 err = errors.Errorf("Docker API endpoint check failed: %s", err) 1248 // log with info because this might not be an error 1249 d.op.Info(err) 1250 return err 1251 } 1252 return nil 1253} 1254 1255// deleteFolder deletes the supplied folder if it is empty. During a VCH Delete there is a slight possibility vic 1256// could delete a folder it didn't create. The only time a VCH would be in a folder vic didn't create 1257// would be outside of normal vic operations. There is no risk of loss of data as it will this will only 1258// delete an empty folder. 1259func (d *Dispatcher) deleteFolder(folder *object.Folder) { 1260 // only continue if VC and the target Folder is NOT the datacenter wide VM Folder 1261 if d.isVC && folder != nil && folder.Reference() != d.session.VMFolder.Reference() { 1262 children, err := folder.Children(d.op) 1263 if err != nil { 1264 d.op.Errorf("Unable to retrieve Folder(%s) contents: %s", folder.InventoryPath, err) 1265 return 1266 } 1267 if len(children) > 0 { 1268 d.op.Warnf("Folder(%s) is not empty and will not be removed", folder.InventoryPath) 1269 return 1270 } 1271 d.op.Debugf("Destroying folder %s", folder.Name()) 1272 _, err = tasks.WaitForResult(d.op, func(ctx context.Context) (tasks.Task, error) { 1273 return folder.Destroy(d.op) 1274 }) 1275 if err != nil { 1276 d.op.Errorf("Failed to remove Folder(%s) - manual removal may be needed: %s", folder.InventoryPath, err) 1277 } 1278 } 1279} 1280