1""" VM commands module. """ 2 3import math 4import os 5import socket 6import time 7import errno 8 9from gandi.cli.core.base import GandiModule 10from gandi.cli.core.utils import randomstring 11from gandi.cli.modules.datacenter import Datacenter 12from gandi.cli.modules.sshkey import SshkeyHelper 13from gandi.cli.core.utils import MigrationNotFinalized 14 15 16class Iaas(GandiModule, SshkeyHelper): 17 18 """ Module to handle CLI commands. 19 20 $ gandi vm console 21 $ gandi vm create 22 $ gandi vm delete 23 $ gandi vm images 24 $ gandi vm info 25 $ gandi vm kernels 26 $ gandi vm list 27 $ gandi vm reboot 28 $ gandi vm ssh 29 $ gandi vm start 30 $ gandi vm stop 31 $ gandi vm update 32 33 """ 34 35 @classmethod 36 def list(cls, options=None): 37 """List virtual machines.""" 38 if not options: 39 options = {} 40 41 return cls.call('hosting.vm.list', options) 42 43 @classmethod 44 def resource_list(cls): 45 """ Get the possible list of resources (hostname, id). """ 46 items = cls.list({'items_per_page': 500}) 47 ret = [vm['hostname'] for vm in items] 48 ret.extend([str(vm['id']) for vm in items]) 49 return ret 50 51 @classmethod 52 def info(cls, id): 53 """Display information about a virtual machine.""" 54 return cls.call('hosting.vm.info', cls.usable_id(id)) 55 56 @classmethod 57 def stop(cls, resources, background=False): 58 """Stop a virtual machine.""" 59 if not isinstance(resources, (list, tuple)): 60 resources = [resources] 61 62 opers = [] 63 for item in resources: 64 oper = cls.call('hosting.vm.stop', cls.usable_id(item)) 65 if isinstance(oper, list): 66 opers.extend(oper) 67 else: 68 opers.append(oper) 69 70 if background: 71 return opers 72 73 # interactive mode, run a progress bar 74 instance_info = "'%s'" % ', '.join(resources) 75 cls.echo('Stopping your Virtual Machine(s) %s.' % instance_info) 76 cls.display_progress(opers) 77 78 @classmethod 79 def start(cls, resources, background=False): 80 """Start a virtual machine.""" 81 if not isinstance(resources, (list, tuple)): 82 resources = [resources] 83 84 opers = [] 85 for item in resources: 86 oper = cls.call('hosting.vm.start', cls.usable_id(item)) 87 if isinstance(oper, list): 88 opers.extend(oper) 89 else: 90 opers.append(oper) 91 92 if background: 93 return opers 94 95 # interactive mode, run a progress bar 96 instance_info = "'%s'" % ', '.join(resources) 97 cls.echo('Starting your Virtual Machine(s) %s.' % instance_info) 98 cls.display_progress(opers) 99 100 @classmethod 101 def reboot(cls, resources, background=False): 102 """Reboot a virtual machine.""" 103 if not isinstance(resources, (list, tuple)): 104 resources = [resources] 105 106 opers = [] 107 for item in resources: 108 oper = cls.call('hosting.vm.reboot', cls.usable_id(item)) 109 if isinstance(oper, list): 110 opers.extend(oper) 111 else: 112 opers.append(oper) 113 114 if background: 115 return opers 116 117 # interactive mode, run a progress bar 118 instance_info = "'%s'" % ', '.join(resources) 119 cls.echo('Rebooting your Virtual Machine(s) %s.' % instance_info) 120 cls.display_progress(opers) 121 122 @classmethod 123 def delete(cls, resources, background=False): 124 """Delete a virtual machine.""" 125 if not isinstance(resources, (list, tuple)): 126 resources = [resources] 127 128 opers = [] 129 for item in resources: 130 oper = cls.call('hosting.vm.delete', cls.usable_id(item)) 131 if not oper: 132 continue 133 134 if isinstance(oper, list): 135 opers.extend(oper) 136 else: 137 opers.append(oper) 138 139 if background: 140 return opers 141 142 # interactive mode, run a progress bar 143 instance_info = "'%s'" % ', '.join(resources) 144 cls.echo('Deleting your Virtual Machine(s) %s.' % instance_info) 145 if opers: 146 cls.display_progress(opers) 147 148 @classmethod 149 def required_max_memory(cls, id, memory): 150 """ 151 Recommend a max_memory setting for this vm given memory. If the 152 VM already has a nice setting, return None. The max_memory 153 param cannot be fixed too high, because page table allocation 154 would cost too much for small memory profile. Use a range as below. 155 """ 156 best = int(max(2 ** math.ceil(math.log(memory, 2)), 2048)) 157 158 actual_vm = cls.info(id) 159 160 if (actual_vm['state'] == 'running' 161 and actual_vm['vm_max_memory'] != best): 162 return best 163 164 @classmethod 165 def update(cls, id, memory, cores, console, password, background, 166 max_memory): 167 """Update a virtual machine.""" 168 if not background and not cls.intty(): 169 background = True 170 171 vm_params = {} 172 173 if memory: 174 vm_params['memory'] = memory 175 176 if cores: 177 vm_params['cores'] = cores 178 179 if console: 180 vm_params['console'] = console 181 182 if password: 183 vm_params['password'] = password 184 185 if max_memory: 186 vm_params['vm_max_memory'] = max_memory 187 188 result = cls.call('hosting.vm.update', cls.usable_id(id), vm_params) 189 if background: 190 return result 191 192 # interactive mode, run a progress bar 193 cls.echo('Updating your Virtual Machine %s.' % id) 194 cls.display_progress(result) 195 196 @classmethod 197 def create(cls, datacenter, memory, cores, ip_version, bandwidth, 198 login, password, hostname, image, run, background, sshkey, 199 size, vlan, ip, script, script_args, ssh): 200 """Create a new virtual machine.""" 201 from gandi.cli.modules.network import Ip, Iface 202 if not background and not cls.intty(): 203 background = True 204 205 datacenter_id_ = int(Datacenter.usable_id(datacenter)) 206 207 if not hostname: 208 hostname = randomstring('vm') 209 disk_name = 'sys_%s' % hostname[2:] 210 else: 211 disk_name = 'sys_%s' % hostname.replace('.', '') 212 213 vm_params = { 214 'hostname': hostname, 215 'datacenter_id': datacenter_id_, 216 'memory': memory, 217 'cores': cores, 218 } 219 220 if login: 221 vm_params['login'] = login 222 223 if run: 224 vm_params['run'] = run 225 226 if password: 227 vm_params['password'] = password 228 229 if ip_version: 230 vm_params['ip_version'] = ip_version 231 vm_params['bandwidth'] = bandwidth 232 233 if script: 234 with open(script) as fd: 235 vm_params['script'] = fd.read() 236 if script_args: 237 vm_params['script_args'] = script_args 238 239 vm_params.update(cls.convert_sshkey(sshkey)) 240 241 # XXX: name of disk is limited to 15 chars in ext2fs, ext3fs 242 # but api allow 255, so we limit to 15 for now 243 disk_params = {'datacenter_id': vm_params['datacenter_id'], 244 'name': disk_name[:15]} 245 246 if size: 247 if isinstance(size, tuple): 248 prefix, size = size 249 disk_params['size'] = size 250 251 sys_disk_id_ = int(Image.usable_id(image, datacenter_id_)) 252 253 ip_summary = [] 254 if ip_version == 4: 255 ip_summary = ['v4', 'v6'] 256 elif ip_version == 6: 257 ip_summary = ['v6'] 258 259 if vlan: 260 ip_ = None 261 ip_summary.append('private') 262 if ip: 263 try: 264 ip_ = Ip.info(ip) 265 except Exception: 266 pass 267 else: 268 if not Ip._check_and_detach(ip_, None): 269 return 270 if ip_: 271 iface_id = ip_['iface_id'] 272 else: 273 ip_create = Ip.create(4, 274 vm_params['datacenter_id'], 275 bandwidth, 276 None, 277 vlan, 278 ip) 279 280 iface_id = ip_create['iface_id'] 281 282 # if there is a public ip, will attach this one later, else give 283 # the iface to vm.create 284 if not ip_version: 285 vm_params['iface_id'] = iface_id 286 287 result = cls.call('hosting.vm.create_from', vm_params, disk_params, 288 sys_disk_id_) 289 290 cls.echo('* Configuration used: %d cores, %dMb memory, ip %s, ' 291 'image %s, hostname: %s, datacenter: %s' % 292 (cores, memory, '+'.join(ip_summary), image, hostname, 293 datacenter)) 294 295 # background mode, bail out now (skip interactive part) 296 if background and (not vlan or not ip_version): 297 return result 298 299 # interactive mode, run a progress bar 300 cls.echo('Creating your Virtual Machine %s.' % hostname) 301 cls.display_progress(result) 302 cls.echo('Your Virtual Machine %s has been created.' % hostname) 303 304 vm_id = None 305 for oper in result: 306 if oper.get('vm_id'): 307 vm_id = oper.get('vm_id') 308 break 309 310 if vlan and ip_version: 311 attach = Iface._attach(iface_id, vm_id) 312 if background: 313 return attach 314 315 if 'ssh_key' not in vm_params and 'keys' not in vm_params: 316 return 317 318 if vm_id and ip_version: 319 cls.wait_for_sshd(vm_id) 320 if ssh: 321 cls.ssh_keyscan(vm_id) 322 cls.ssh(vm_id, 'root', None) 323 324 @classmethod 325 def need_finalize(cls, resource): 326 """Check if vm migration need to be finalized.""" 327 vm_id = cls.usable_id(resource) 328 params = {'type': 'hosting_migration_vm', 329 'step': 'RUN', 330 'vm_id': vm_id} 331 result = cls.call('operation.list', params) 332 if not result or len(result) > 1: 333 raise MigrationNotFinalized('Cannot find VM %s ' 334 'migration operation.' % resource) 335 336 need_finalize = result[0]['params']['inner_step'] == 'wait_finalize' 337 if not need_finalize: 338 raise MigrationNotFinalized('VM %s migration does not need ' 339 'finalization.' % resource) 340 341 @classmethod 342 def check_can_migrate(cls, resource): 343 """Check if virtual machine can be migrated to another datacenter.""" 344 vm_id = cls.usable_id(resource) 345 result = cls.call('hosting.vm.can_migrate', vm_id) 346 347 if not result['can_migrate']: 348 if result['matched']: 349 matched = result['matched'][0] 350 cls.echo('Your VM %s cannot be migrated yet. Migration will ' 351 'be available when datacenter %s is opened.' 352 % (resource, matched)) 353 else: 354 cls.echo('Your VM %s cannot be migrated.' % resource) 355 return False 356 357 return True 358 359 @classmethod 360 def migrate(cls, resource, background=False, finalize=False): 361 """ Migrate a virtual machine to another datacenter. """ 362 vm_id = cls.usable_id(resource) 363 if finalize: 364 verb = 'Finalizing' 365 result = cls.call('hosting.vm.migrate', vm_id, True) 366 else: 367 verb = 'Starting' 368 result = cls.call('hosting.vm.migrate', vm_id, False) 369 370 dcs = {} 371 for dc in Datacenter.list(): 372 dcs[dc['id']] = dc['dc_code'] 373 374 oper = cls.call('operation.info', result['id']) 375 dc_from = dcs[oper['params']['from_dc_id']] 376 dc_to = dcs[oper['params']['to_dc_id']] 377 migration_msg = ('* %s the migration of VM %s ' 378 'from datacenter %s to %s' 379 % (verb, resource, dc_from, dc_to)) 380 cls.echo(migration_msg) 381 382 if background: 383 return result 384 385 cls.echo('VM migration in progress.') 386 cls.display_progress(result) 387 cls.echo('Your VM %s has been migrated.' % resource) 388 return result 389 390 @classmethod 391 def from_hostname(cls, hostname): 392 """Retrieve virtual machine id associated to a hostname.""" 393 result = cls.list({'hostname': str(hostname)}) 394 if result: 395 return result[0]['id'] 396 397 @classmethod 398 def usable_id(cls, id): 399 """ Retrieve id from input which can be hostname or id.""" 400 try: 401 # id is maybe a hostname 402 qry_id = cls.from_hostname(id) 403 if not qry_id: 404 qry_id = int(id) 405 except Exception: 406 qry_id = None 407 408 if not qry_id: 409 msg = 'unknown identifier %s' % id 410 cls.error(msg) 411 412 return qry_id 413 414 @classmethod 415 def vm_ip(cls, vm_id): 416 """Return the first usable ip address for this vm. 417 Returns a (version, ip) tuple.""" 418 vm_info = cls.info(vm_id) 419 420 for iface in vm_info['ifaces']: 421 if iface['type'] == 'private': 422 continue 423 for ip in iface['ips']: 424 return ip['version'], ip['ip'] 425 426 @classmethod 427 def wait_for_sshd(cls, vm_id): 428 """Insist on having the vm booted and sshd 429 listening""" 430 cls.echo('Waiting for the vm to come online') 431 version, ip_addr = cls.vm_ip(vm_id) 432 give_up = time.time() + 300 433 last_error = None 434 while time.time() < give_up: 435 try: 436 inet = socket.AF_INET 437 if version == 6: 438 inet = socket.AF_INET6 439 sd = socket.socket(inet, socket.SOCK_STREAM, 440 socket.IPPROTO_TCP) 441 sd.settimeout(5) 442 sd.connect((ip_addr, 22)) 443 sd.recv(1024) 444 return 445 except socket.error as err: 446 if err.errno == errno.EHOSTUNREACH and version == 6: 447 cls.error('%s is not reachable, you may be missing ' 448 'IPv6 connectivity' % ip_addr) 449 last_error = err 450 time.sleep(1) 451 except Exception as err: 452 last_error = err 453 time.sleep(1) 454 cls.error('VM did not spin up (last error: %s)' % last_error) 455 456 @classmethod 457 def ssh_keyscan(cls, vm_id): 458 """Wipe this old key and learn the new one from a freshly 459 created vm. This is a security risk for this VM, however 460 we dont have another way to learn the key yet, so do this 461 for the user.""" 462 cls.echo('Wiping old key and learning the new one') 463 _version, ip_addr = cls.vm_ip(vm_id) 464 cls.execute('ssh-keygen -R "%s"' % ip_addr) 465 466 for _ in range(5): 467 output = cls.exec_output('ssh-keyscan "%s"' % ip_addr) 468 if output: 469 with open(os.path.expanduser('~/.ssh/known_hosts'), 'a') as f: 470 f.write(output) 471 return True 472 time.sleep(.5) 473 474 @classmethod 475 def scp(cls, vm_id, login, identity, local_file, remote_file): 476 """Copy file to remote VM.""" 477 cmd = ['scp'] 478 if identity: 479 cmd.extend(('-i', identity,)) 480 481 version, ip_addr = cls.vm_ip(vm_id) 482 if version == 6: 483 ip_addr = '[%s]' % ip_addr 484 485 cmd.extend((local_file, '%s@%s:%s' % 486 (login, ip_addr, remote_file),)) 487 cls.echo('Running %s' % ' '.join(cmd)) 488 for _ in range(5): 489 ret = cls.execute(cmd, False) 490 if ret: 491 break 492 time.sleep(.5) 493 return ret 494 495 @classmethod 496 def ssh(cls, vm_id, login, identity, args=None): 497 """Spawn an ssh session to virtual machine.""" 498 cmd = ['ssh'] 499 if identity: 500 cmd.extend(('-i', identity,)) 501 502 version, ip_addr = cls.vm_ip(vm_id) 503 if version == 6: 504 cmd.append('-6') 505 506 if not ip_addr: 507 cls.echo('No IP address found for vm %s, aborting.' % vm_id) 508 return 509 510 cmd.append('%s@%s' % (login, ip_addr,)) 511 512 if args: 513 cmd.extend(args) 514 515 cls.echo('Requesting access using: %s ...' % ' '.join(cmd)) 516 return cls.execute(cmd, False) 517 518 @classmethod 519 def console(cls, id): 520 """Open a console to virtual machine.""" 521 vm_info = cls.info(id) 522 if not vm_info['console']: 523 # first activate console 524 cls.update(id, memory=None, cores=None, console=True, 525 password=None, background=False, max_memory=None) 526 # now we can connect 527 # retrieve ip of vm 528 vm_info = cls.info(id) 529 version, ip_addr = cls.vm_ip(id) 530 531 console_url = vm_info.get('console_url', 'console.gandi.net') 532 access = 'ssh %s@%s' % (ip_addr, console_url) 533 cls.execute(access) 534 535 536class Image(GandiModule): 537 538 """ Module to handle CLI commands. 539 540 $ gandi vm images 541 542 """ 543 544 @classmethod 545 def list(cls, datacenter=None, label=None): 546 """List available images for vm creation.""" 547 options = {} 548 if datacenter: 549 datacenter_id = int(Datacenter.usable_id(datacenter)) 550 options['datacenter_id'] = datacenter_id 551 552 # implement a filter by label as API doesn't handle it 553 images = cls.safe_call('hosting.image.list', options) 554 if not label: 555 return images 556 return [img for img in images 557 if label.lower() in img['label'].lower()] 558 559 @classmethod 560 def is_deprecated(cls, label, datacenter=None): 561 """Check if image if flagged as deprecated.""" 562 images = cls.list(datacenter, label) 563 images_visibility = dict([(image['label'], image['visibility']) 564 for image in images]) 565 return images_visibility.get(label, 'all') == 'deprecated' 566 567 @classmethod 568 def from_label(cls, label, datacenter=None): 569 """Retrieve disk image id associated to a label.""" 570 result = cls.list(datacenter=datacenter) 571 image_labels = dict([(image['label'], image['disk_id']) 572 for image in result]) 573 574 return image_labels.get(label) 575 576 @classmethod 577 def from_sysdisk(cls, label): 578 """Retrieve disk id from available system disks""" 579 disks = cls.safe_call('hosting.disk.list', {'name': label}) 580 if len(disks): 581 return disks[0]['id'] 582 583 @classmethod 584 def usable_id(cls, id, datacenter=None): 585 """ Retrieve id from input which can be label or id.""" 586 try: 587 qry_id = int(id) 588 except Exception: 589 # if id is a string, prefer a system disk then a label 590 qry_id = cls.from_sysdisk(id) or cls.from_label(id, datacenter) 591 592 if not qry_id: 593 msg = 'unknown identifier %s' % id 594 cls.error(msg) 595 596 return qry_id 597 598 599class Kernel(GandiModule): 600 601 """ Module to handle Gandi Kernels. """ 602 603 @classmethod 604 def list(cls, datacenter=None, flavor=None, match='', exact_match=False): 605 """ List available kernels for datacenter.""" 606 if not datacenter: 607 dc_ids = [dc['id'] for dc in Datacenter.filtered_list()] 608 kmap = {} 609 for dc_id in dc_ids: 610 vals = cls.safe_call('hosting.disk.list_kernels', dc_id) 611 for key in vals: 612 kmap.setdefault(key, []).extend(vals.get(key, [])) 613 # remove duplicates 614 for key in kmap: 615 kmap[key] = list(set(kmap[key])) 616 else: 617 dc_id = Datacenter.usable_id(datacenter) 618 kmap = cls.safe_call('hosting.disk.list_kernels', dc_id) 619 620 if match: 621 for flav in kmap: 622 if exact_match: 623 kmap[flav] = [x for x in kmap[flav] if match == x] 624 else: 625 kmap[flav] = [x for x in kmap[flav] if match in x] 626 if flavor: 627 if flavor not in kmap: 628 cls.error('flavor %s not supported here' % flavor) 629 return dict([(flavor, kmap[flavor])]) 630 631 return kmap 632 633 @classmethod 634 def is_available(cls, disk, kernel): 635 """ Check if kernel is available for disk.""" 636 kmap = cls.list(disk['datacenter_id'], None, kernel, True) 637 for flavor in kmap: 638 if kernel in kmap[flavor]: 639 return True 640 return False 641