1// Copyright 2016-2017 VMware, Inc. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package management 16 17import ( 18 "context" 19 "crypto/x509" 20 "errors" 21 "fmt" 22 "math" 23 "net" 24 "os" 25 "strings" 26 "time" 27 28 "github.com/vmware/govmomi/guest" 29 "github.com/vmware/govmomi/object" 30 "github.com/vmware/govmomi/vim25/types" 31 "github.com/vmware/vic/lib/config" 32 "github.com/vmware/vic/pkg/trace" 33 "github.com/vmware/vic/pkg/vsphere/compute" 34 "github.com/vmware/vic/pkg/vsphere/diagnostic" 35 "github.com/vmware/vic/pkg/vsphere/extraconfig" 36 "github.com/vmware/vic/pkg/vsphere/session" 37 "github.com/vmware/vic/pkg/vsphere/vm" 38) 39 40// Action is the current action being performed 41type Action int 42 43// Action definitions 44const ( 45 ActionConfigure Action = iota 46 ActionCreate 47 ActionDebug 48 ActionDelete 49 ActionInspect 50 ActionInspectCertificates 51 ActionInspectLogs 52 ActionList 53 ActionRollback 54 ActionUpdate 55 ActionUpgrade 56) 57 58// stringer for action 59func (a Action) String() string { 60 var act string 61 switch a { 62 case ActionConfigure: 63 act = "configure" 64 case ActionCreate: 65 act = "create" 66 case ActionDebug: 67 act = "debug" 68 case ActionDelete: 69 act = "delete" 70 case ActionInspect, ActionInspectCertificates, ActionInspectLogs: 71 act = "inspect" 72 case ActionList: 73 act = "list" 74 case ActionRollback: 75 act = "rollback" 76 case ActionUpdate: 77 act = "update" 78 case ActionUpgrade: 79 act = "upgrade" 80 } 81 return act 82} 83 84type Dispatcher struct { 85 Action 86 87 session *session.Session 88 op trace.Operation 89 force bool 90 secret *extraconfig.SecretKey 91 92 isVC bool 93 vchPoolPath string 94 vmPathName string 95 dockertlsargs string 96 97 DockerPort string 98 HostIP string 99 100 vchPool *object.ResourcePool 101 vchVapp *object.VirtualApp 102 appliance *vm.VirtualMachine 103 104 oldApplianceISO string 105 oldVCHResources *config.Resources 106 107 sshEnabled bool 108 parentResourcepool *compute.ResourcePool 109} 110 111type diagnosticLog struct { 112 key string 113 name string 114 start int32 115 host *object.HostSystem 116 collect bool 117} 118 119var diagnosticLogs = make(map[string]*diagnosticLog) 120 121// NewDispatcher creates a dispatcher that can act upon VIC management operations. 122// clientCert is an optional client certificate to allow interaction with the Docker API for verification 123// force will ignore some errors 124func NewDispatcher(ctx context.Context, s *session.Session, action Action, force bool) *Dispatcher { 125 defer trace.End(trace.Begin("")) 126 isVC := s.IsVC() 127 e := &Dispatcher{ 128 Action: action, 129 session: s, 130 op: trace.FromContext(ctx, "Dispatcher"), 131 isVC: isVC, 132 force: force, 133 } 134 return e 135} 136 137// Get the current log header LineEnd of the hostd/vpxd logs based on VCH configuration 138// With this we avoid collecting log file data that existed prior to install. 139func (d *Dispatcher) InitDiagnosticLogsFromConf(conf *config.VirtualContainerHostConfigSpec) { 140 defer trace.End(trace.Begin("")) 141 142 if d.isVC { 143 diagnosticLogs[d.session.ServiceContent.About.InstanceUuid] = 144 &diagnosticLog{"vpxd:vpxd.log", "vpxd.log", 0, nil, true} 145 } 146 147 var err error 148 // try best to get datastore and cluster, but do not return for any error. The least is to collect VC log only 149 if d.session.Datastore == nil { 150 if len(conf.ImageStores) > 0 { 151 if d.session.Datastore, err = d.session.Finder.DatastoreOrDefault(d.op, conf.ImageStores[0].Host); err != nil { 152 d.op.Debugf("Failure finding image store from VCH config (%s): %s", conf.ImageStores[0].Host, err.Error()) 153 } else { 154 d.op.Debugf("Found ds: %s", conf.ImageStores[0].Host) 155 } 156 } else { 157 d.op.Debug("Image datastore is empty") 158 } 159 } 160 161 // find the host(s) attached to given storage 162 if d.session.Cluster == nil { 163 if len(conf.ComputeResources) > 0 { 164 rp := compute.NewResourcePool(d.op, d.session, conf.ComputeResources[0]) 165 if d.session.Cluster, err = rp.GetCluster(d.op); err != nil { 166 d.op.Debugf("Unable to get cluster for given resource pool %s: %s", conf.ComputeResources[0], err) 167 } 168 } else { 169 d.op.Debug("Compute resource is empty") 170 } 171 } 172 173 var hosts []*object.HostSystem 174 if d.session.Datastore != nil && d.session.Cluster != nil { 175 hosts, err = d.session.Datastore.AttachedClusterHosts(d.op, d.session.Cluster) 176 if err != nil { 177 d.op.Debugf("Unable to get the list of hosts attached to given storage: %s", err) 178 } 179 } 180 181 if d.session.Host == nil { 182 // vCenter w/ auto DRS. 183 // Set collect=false here as we do not want to collect all hosts logs, 184 // just the hostd log where the VM is placed. 185 for _, host := range hosts { 186 diagnosticLogs[host.Reference().Value] = 187 &diagnosticLog{"hostd", "hostd.log", 0, host, false} 188 } 189 } else { 190 // vCenter w/ manual DRS or standalone ESXi 191 var host *object.HostSystem 192 if d.isVC { 193 host = d.session.Host 194 } 195 196 diagnosticLogs[d.session.Host.Reference().Value] = 197 &diagnosticLog{"hostd", "hostd.log", 0, host, true} 198 } 199 200 m := diagnostic.NewDiagnosticManager(d.session) 201 202 for k, l := range diagnosticLogs { 203 if l == nil { 204 continue 205 } 206 // get LineEnd without any LineText 207 h, err := m.BrowseLog(d.op, l.host, l.key, math.MaxInt32, 0) 208 if err != nil { 209 d.op.Warnf("Disabling %s %s collection (%s)", k, l.name, err) 210 diagnosticLogs[k] = nil 211 continue 212 } 213 214 l.start = h.LineEnd 215 } 216} 217 218// Get the current log header LineEnd of the hostd/vpxd logs based on vch VM hardwares, cause VCH configuration might not be available at this time 219// With this we avoid collecting log file data that existed prior to install. 220func (d *Dispatcher) InitDiagnosticLogsFromVCH(vch *vm.VirtualMachine) { 221 defer trace.End(trace.Begin("")) 222 223 if d.isVC { 224 diagnosticLogs[d.session.ServiceContent.About.InstanceUuid] = 225 &diagnosticLog{"vpxd:vpxd.log", "vpxd.log", 0, nil, true} 226 } 227 228 var err error 229 // where the VM is running 230 ds, err := d.getImageDatastore(vch, nil, true) 231 if err != nil { 232 d.op.Debugf("Failure finding image store from VCH VM %s: %s", vch.Reference(), err.Error()) 233 } 234 235 var hosts []*object.HostSystem 236 if ds != nil && d.session.Cluster != nil { 237 hosts, err = ds.AttachedClusterHosts(d.op, d.session.Cluster) 238 if err != nil { 239 d.op.Debugf("Unable to get the list of hosts attached to given storage: %s", err) 240 } 241 } 242 243 for _, host := range hosts { 244 diagnosticLogs[host.Reference().Value] = 245 &diagnosticLog{"hostd", "hostd.log", 0, host, false} 246 } 247 248 m := diagnostic.NewDiagnosticManager(d.session) 249 250 for k, l := range diagnosticLogs { 251 if l == nil { 252 continue 253 } 254 // get LineEnd without any LineText 255 h, err := m.BrowseLog(d.op, l.host, l.key, math.MaxInt32, 0) 256 257 if err != nil { 258 d.op.Warnf("Disabling %s %s collection (%s)", k, l.name, err) 259 diagnosticLogs[k] = nil 260 continue 261 } 262 263 l.start = h.LineEnd 264 } 265} 266 267func (d *Dispatcher) CollectDiagnosticLogs() { 268 defer trace.End(trace.Begin("")) 269 270 m := diagnostic.NewDiagnosticManager(d.session) 271 272 for k, l := range diagnosticLogs { 273 if l == nil || !l.collect { 274 continue 275 } 276 277 d.op.Infof("Collecting %s %s", k, l.name) 278 279 var lines []string 280 start := l.start 281 282 for i := 0; i < 2; i++ { 283 h, err := m.BrowseLog(d.op, l.host, l.key, start, 0) 284 if err != nil { 285 d.op.Errorf("Failed to collect %s %s: %s", k, l.name, err) 286 break 287 } 288 289 lines = h.LineText 290 if len(lines) != 0 { 291 break // l.start was still valid, log was not rolled over 292 } 293 294 // log rolled over, start at the beginning. 295 // TODO: If this actually happens we will have missed some log data, 296 // it is possible to get data from the previous log too. 297 start = 0 298 d.op.Infof("%s %s rolled over", k, l.name) 299 } 300 301 if len(lines) == 0 { 302 d.op.Warnf("No log data for %s %s", k, l.name) 303 continue 304 } 305 306 f, err := os.Create(l.name) 307 if err != nil { 308 d.op.Errorf("Failed to create local %s: %s", l.name, err) 309 continue 310 } 311 defer f.Close() 312 313 for _, line := range lines { 314 fmt.Fprintln(f, line) 315 } 316 } 317} 318 319func (d *Dispatcher) opManager(vch *vm.VirtualMachine) (*guest.ProcessManager, error) { 320 state, err := vch.PowerState(d.op) 321 if err != nil { 322 return nil, fmt.Errorf("Failed to get appliance power state, service might not be available at this moment.") 323 } 324 if state != types.VirtualMachinePowerStatePoweredOn { 325 return nil, fmt.Errorf("VCH appliance is not powered on, state %s", state) 326 } 327 328 running, err := vch.IsToolsRunning(d.op) 329 if err != nil || !running { 330 return nil, errors.New("Tools are not running in the appliance, unable to continue") 331 } 332 333 manager := guest.NewOperationsManager(d.session.Client.Client, vch.Reference()) 334 processManager, err := manager.ProcessManager(d.op) 335 if err != nil { 336 return nil, fmt.Errorf("Unable to manage processes in appliance VM: %s", err) 337 } 338 return processManager, nil 339} 340 341// opManagerWait polls for state of the process with the given pid, waiting until the process has completed. 342// The pid param must be one returned by ProcessManager.StartProgram. 343func (d *Dispatcher) opManagerWait(op trace.Operation, pm *guest.ProcessManager, auth types.BaseGuestAuthentication, pid int64) (*types.GuestProcessInfo, error) { 344 pids := []int64{pid} 345 346 for { 347 select { 348 case <-time.After(time.Millisecond * 250): 349 case <-op.Done(): 350 return nil, fmt.Errorf("opManagerWait(%d): %s", pid, op.Err()) 351 } 352 353 procs, err := pm.ListProcesses(op, auth, pids) 354 if err != nil { 355 return nil, err 356 } 357 358 if len(procs) == 1 && procs[0].EndTime != nil { 359 return &procs[0], nil 360 } 361 } 362} 363 364func (d *Dispatcher) CheckAccessToVCAPI(vch *vm.VirtualMachine, target string) (int64, error) { 365 pm, err := d.opManager(vch) 366 if err != nil { 367 return -1, err 368 } 369 auth := types.NamePasswordAuthentication{} 370 spec := types.GuestProgramSpec{ 371 ProgramPath: "test-vc-api", 372 Arguments: target, 373 } 374 pid, err := pm.StartProgram(d.op, &auth, &spec) 375 if err != nil { 376 return -1, err 377 } 378 379 info, err := d.opManagerWait(d.op, pm, &auth, pid) 380 if err != nil { 381 return -1, err 382 } 383 384 return int64(info.ExitCode), nil 385} 386 387// addrToUse given candidateIPs, determines an address in cert that resolves to 388// a candidateIP - this address can be used as the remote address to connect to with 389// cert to ensure that certificate validation is successful 390// if none can be found, return empty string and an err 391func addrToUse(op trace.Operation, candidateIPs []net.IP, cert *x509.Certificate, cas []byte) (string, error) { 392 if cert == nil { 393 return "", errors.New("unable to determine suitable address with nil certificate") 394 } 395 396 pool, err := x509.SystemCertPool() 397 if err != nil { 398 op.Warnf("Failed to load system cert pool: %s. Using empty pool.", err) 399 pool = x509.NewCertPool() 400 } 401 pool.AppendCertsFromPEM(cas) 402 403 // update target to use FQDN 404 for _, ip := range candidateIPs { 405 names, err := net.LookupAddr(ip.String()) 406 if err != nil { 407 op.Debugf("Unable to perform reverse lookup of IP address %s: %s", ip, err) 408 } 409 410 // check all the returned names, and lastly the raw IP 411 for _, n := range append(names, ip.String()) { 412 opts := x509.VerifyOptions{ 413 Roots: pool, 414 DNSName: n, 415 } 416 417 _, err := cert.Verify(opts) 418 if err == nil { 419 // this identifier will work 420 op.Debugf("Matched %q for use against host certificate", n) 421 // trim '.' fqdn suffix if fqdn 422 return strings.TrimSuffix(n, "."), nil 423 } 424 425 op.Debugf("Checked %q, no match for host certificate", n) 426 } 427 } 428 429 // no viable address 430 return "", errors.New("unable to determine viable address") 431} 432 433/// viableHostAddresses attempts to determine which possibles addresses in the certificate 434// are viable from the current location. 435// This will return all IP addresses - it attempts to validate DNS names via resolution. 436// This does NOT check connectivity 437func viableHostAddress(op trace.Operation, candidateIPs []net.IP, cert *x509.Certificate, cas []byte) (string, error) { 438 if cert == nil { 439 return "", fmt.Errorf("unable to determine suitable address with nil certificate") 440 } 441 442 op.Debug("Loading CAs for client auth") 443 pool := x509.NewCertPool() 444 pool.AppendCertsFromPEM(cas) 445 446 dnsnames := cert.DNSNames 447 448 // assemble the common name and alt names 449 ip := net.ParseIP(cert.Subject.CommonName) 450 if ip != nil { 451 candidateIPs = append(candidateIPs, ip) 452 } else { 453 // assume it's dns 454 dnsnames = append([]string{cert.Subject.CommonName}, dnsnames...) 455 } 456 457 // turn the DNS names into IPs 458 for _, n := range dnsnames { 459 // see which resolve from here 460 ips, err := net.LookupIP(n) 461 if err != nil { 462 op.Debugf("Unable to perform IP lookup of %q: %s", n, err) 463 } 464 // Allow wildcard names for later validation 465 if len(ips) == 0 && !strings.HasPrefix(n, "*") { 466 op.Debugf("Discarding name from viable set: %s", n) 467 continue 468 } 469 470 candidateIPs = append(candidateIPs, ips...) 471 } 472 473 // always add all the altname IPs - we're not checking for connectivity 474 candidateIPs = append(candidateIPs, cert.IPAddresses...) 475 476 return addrToUse(op, candidateIPs, cert, cas) 477} 478