1#!/usr/bin/env python 2 3# Bareos python class for Ovirt related backup and restore 4 5import bareosfd 6from bareos_fd_consts import ( 7 bJobMessageType, 8 bFileType, 9 bRCs, 10 bIOPS, 11 bEventType, 12 bVariable, 13 bCFs, 14) 15 16import os 17import io 18import time 19import uuid 20import json 21import ConfigParser as configparser 22 23import lxml.etree 24 25import logging 26import ovirtsdk4 as sdk 27import ovirtsdk4.types as types 28 29import ssl 30 31from httplib import HTTPSConnection 32 33try: 34 from urllib.parse import urlparse 35except ImportError: 36 from urlparse import urlparse 37 38import BareosFdPluginBaseclass 39 40# The name of the application, to be used as the 'origin' of events 41# sent to the audit log: 42APPLICATION_NAME = "Bareos oVirt plugin" 43 44 45class BareosFdPluginOvirt(BareosFdPluginBaseclass.BareosFdPluginBaseclass): 46 """ 47 Plugin for oVirt backup and restore 48 """ 49 50 def __init__(self, context, plugindef): 51 bareosfd.DebugMessage( 52 context, 53 100, 54 "Constructor called in module %s with plugindef=%s\n" 55 % (__name__, plugindef), 56 ) 57 super(BareosFdPluginOvirt, self).__init__(context, plugindef) 58 59 if self.mandatory_options is None: 60 self.mandatory_options = [] 61 62 self.config = None 63 64 self.ovirt = BareosOvirtWrapper() 65 66 def parse_plugin_definition(self, context, plugindef): 67 """ 68 Parses the plugin arguments 69 """ 70 super(BareosFdPluginOvirt, self).parse_plugin_definition(context, plugindef) 71 72 bareosfd.DebugMessage( 73 context, 74 100, 75 "BareosFdPluginOvirt:parse_plugin_definition() called with options '%s' \n" 76 % str(self.options), 77 ) 78 79 # if the option config_file is present, parse the given file 80 config_file = self.options.get("config_file") 81 if config_file: 82 if not self.parse_config_file(context): 83 return bRCs["bRC_Error"] 84 85 self.ovirt.set_options(self.options) 86 return bRCs["bRC_OK"] 87 88 def check_options(self, context, mandatory_options=None): 89 """ 90 Check Plugin options 91 Note: this is called by parent class parse_plugin_definition(). 92 This plugin does not yet implement any checks. 93 If it is required to know if it is a backup or a restore, it 94 may make more sense to invoke the options checking from 95 start_backup_job() and start_restore_job() 96 """ 97 return bRCs["bRC_OK"] 98 99 def start_backup_job(self, context): 100 """ 101 Start of Backup Job. Called just before backup job really start. 102 Overload this to arrange whatever you have to do at this time. 103 """ 104 bareosfd.DebugMessage( 105 context, 100, "BareosFdPluginOvirt:start_backup_job() called\n" 106 ) 107 108 if chr(self.level) != "F": 109 bareosfd.JobMessage( 110 context, 111 bJobMessageType["M_FATAL"], 112 "BareosFdPluginOvirt can only perform level F (Full) backups, but level is %s\n" 113 % (chr(self.level)), 114 ) 115 return bRCs["bRC_Error"] 116 117 if not self.ovirt.connect_api(context): 118 return bRCs["bRC_Error"] 119 120 return self.ovirt.prepare_vm_backup(context) 121 122 def start_backup_file(self, context, savepkt): 123 """ 124 Defines the file to backup and creates the savepkt. 125 """ 126 bareosfd.DebugMessage( 127 context, 100, "BareosFdPluginOvirt:start_backup_file() called\n" 128 ) 129 130 if not self.ovirt.backup_objects: 131 bareosfd.JobMessage( 132 context, bJobMessageType["M_ERROR"], "Nothing to backup.\n" 133 ) 134 self.backup_obj = None 135 return bRCs["bRC_Skip"] 136 137 self.backup_obj = self.ovirt.backup_objects.pop(0) 138 139 # create a stat packet for a restore object 140 statp = bareosfd.StatPacket() 141 savepkt.statp = statp 142 143 if "file" in self.backup_obj: 144 145 # regular file 146 vmfile = self.backup_obj["file"] 147 148 bareosfd.DebugMessage( 149 context, 150 100, 151 "BareosFdPluginOvirt:start_backup_file() backup regular file '%s' of VM '%s'\n" 152 % (vmfile["filename"], self.backup_obj["vmname"]), 153 ) 154 155 savepkt.type = bFileType["FT_REG"] 156 savepkt.fname = "/VMS/%s-%s/%s" % ( 157 self.backup_obj["vmname"], 158 self.backup_obj["vmid"], 159 vmfile["filename"], 160 ) 161 self.backup_obj["file"]["fh"] = io.BytesIO(vmfile["data"]) 162 163 elif "disk" in self.backup_obj: 164 165 # disk file 166 disk = self.backup_obj["disk"] 167 # snapshot 168 snapshot = self.backup_obj["snapshot"] 169 170 bareosfd.DebugMessage( 171 context, 172 100, 173 "BareosFdPluginOvirt:start_backup_file() backup disk file '%s'('%s'/'%s') of VM '%s'\n" 174 % (disk.alias, disk.id, snapshot.id, self.backup_obj["vmname"]), 175 ) 176 177 savepkt.type = bFileType["FT_REG"] 178 savepkt.fname = "/VMS/%s-%s/%s-%s/%s" % ( 179 self.backup_obj["vmname"], 180 self.backup_obj["vmid"], 181 disk.alias, 182 disk.id, 183 snapshot.id, 184 ) 185 186 try: 187 self.ovirt.start_download(context, snapshot, disk) 188 except Exception as e: 189 bareosfd.JobMessage( 190 context, 191 bJobMessageType["M_ERROR"], 192 "BareosFdPluginOvirt:start_backup_file() Error: %s" % str(e), 193 ) 194 self.ovirt.end_transfer(context) 195 return bRCs["bRC_Error"] 196 197 elif "disk_metadata" in self.backup_obj: 198 # save disk metadata as restoreobject 199 200 disk_alias = self.backup_obj["disk_metadata"]["alias"] 201 disk_id = self.backup_obj["disk_metadata"]["id"] 202 snapshot_id = self.backup_obj["snapshot_id"] 203 disk_metadata_json = json.dumps( 204 {"disk_metadata": self.backup_obj["disk_metadata"]} 205 ) 206 207 bareosfd.DebugMessage( 208 context, 209 100, 210 "BareosFdPluginOvirt:start_backup_file() backup disk metadata '%s'('%s'/'%s') of VM '%s': %s\n" 211 % ( 212 disk_alias, 213 disk_id, 214 snapshot_id, 215 self.backup_obj["vmname"], 216 disk_metadata_json, 217 ), 218 ) 219 220 savepkt.type = bFileType["FT_RESTORE_FIRST"] 221 savepkt.fname = "/VMS/%s-%s/%s-%s/%s.metadata" % ( 222 self.backup_obj["vmname"], 223 self.backup_obj["vmid"], 224 disk_alias, 225 disk_id, 226 snapshot_id, 227 ) 228 savepkt.object_name = savepkt.fname 229 savepkt.object = bytearray(disk_metadata_json) 230 savepkt.object_len = len(savepkt.object) 231 savepkt.object_index = int(time.time()) 232 233 else: 234 bareosfd.JobMessage( 235 context, 236 bJobMessageType["M_FATAL"], 237 "BareosFdPluginOvirt:start_backup_file(): Invalid data in backup_obj, keys: %s\n" 238 % (self.backup_obj.keys()), 239 ) 240 return bRCs["bRC_Error"] 241 242 bareosfd.JobMessage( 243 context, 244 bJobMessageType["M_INFO"], 245 "Starting backup of %s\n" % savepkt.fname, 246 ) 247 248 return bRCs["bRC_OK"] 249 250 def create_file(self, context, restorepkt): 251 """ 252 Creates the file to be restored and directory structure, if needed. 253 Adapt this in your derived class, if you need modifications for 254 virtual files or similar 255 """ 256 bareosfd.DebugMessage( 257 context, 258 100, 259 "create_file() entry point in Python called with %s\n" % (restorepkt), 260 ) 261 262 # Process includes/excludes for restore to oVirt. Note that it is more 263 # efficient to mark only the disks to restore, as skipping here can not 264 # prevent from receiving the data from bareos-sd which is useless for 265 # excluded disks. 266 # 267 # When restoring locally, all disks will be restored without filtering. 268 if ( 269 not restorepkt.ofname.endswith(".ovf") 270 and not self.options.get("local") == "yes" 271 ): 272 disk_alias = self.ovirt.get_ovf_disk_alias_by_basename( 273 context, restorepkt.ofname 274 ) 275 276 if not self.ovirt.is_disk_alias_included(context, disk_alias): 277 restorepkt.create_status = bCFs["CF_SKIP"] 278 return bRCs["bRC_OK"] 279 280 if self.ovirt.is_disk_alias_excluded(context, disk_alias): 281 restorepkt.create_status = bCFs["CF_SKIP"] 282 return bRCs["bRC_OK"] 283 284 if self.options.get("local") == "yes": 285 FNAME = restorepkt.ofname 286 dirname = os.path.dirname(FNAME) 287 if not os.path.exists(dirname): 288 bareosfd.DebugMessage( 289 context, 290 200, 291 "Directory %s does not exist, creating it now\n" % dirname, 292 ) 293 os.makedirs(dirname) 294 # open creates the file, if not yet existing, we close it again right 295 # always it will be opened again in plugin_io. 296 # But: only do this for regular files, prevent from 297 # IOError: (21, 'Is a directory', '/tmp/bareos-restores/my/dir/') 298 # if it's a directory 299 if restorepkt.type == bFileType["FT_REG"]: 300 open(FNAME, "wb").close() 301 else: 302 if not restorepkt.ofname.endswith(".ovf"): 303 FNAME = restorepkt.ofname 304 FNAME = FNAME.decode("utf-8") 305 306 disk = self.ovirt.get_vm_disk_by_basename(context, FNAME) 307 if disk is None: 308 bareosfd.JobMessage( 309 context, 310 bJobMessageType["M_ERROR"], 311 "BareosFdPluginOvirt:create_file() Unable to restore disk %s.\n" 312 % (FNAME), 313 ) 314 return bRCs["bRC_Error"] 315 else: 316 self.ovirt.start_upload(context, disk) 317 318 if restorepkt.type == bFileType["FT_REG"]: 319 restorepkt.create_status = bCFs["CF_EXTRACT"] 320 return bRCs["bRC_OK"] 321 322 def start_restore_job(self, context): 323 """ 324 Start of Restore Job. Called , if you have Restore objects. 325 Overload this to handle restore objects, if applicable 326 """ 327 bareosfd.DebugMessage( 328 context, 100, "BareosFdPluginOvirt:start_restore_job() called\n" 329 ) 330 if self.options.get("local") == "yes": 331 bareosfd.DebugMessage( 332 context, 333 100, 334 "BareosFdPluginOvirt:start_restore_job(): restore to local file, skipping checks\n", 335 ) 336 return bRCs["bRC_OK"] 337 else: 338 # restore to VM to OVirt 339 if not self.ovirt.connect_api(context): 340 return bRCs["bRC_Error"] 341 342 return bRCs["bRC_OK"] 343 344 def start_restore_file(self, context, cmd): 345 bareosfd.DebugMessage( 346 context, 347 100, 348 "BareosFdPluginOvirt:start_restore_file() called with %s\n" % (cmd), 349 ) 350 return bRCs["bRC_OK"] 351 352 def plugin_io(self, context, IOP): 353 """ 354 Called for io operations. 355 """ 356 bareosfd.DebugMessage( 357 context, 100, "plugin_io called with function %s\n" % (IOP.func) 358 ) 359 bareosfd.DebugMessage(context, 100, "FNAME is set to %s\n" % (self.FNAME)) 360 self.jobType = chr(bareosfd.GetValue(context, bVariable["bVarType"])) 361 bareosfd.DebugMessage( 362 context, 363 100, 364 "BareosFdPluginOvirt::plugin_io() jobType: %s\n" % (self.jobType), 365 ) 366 367 if IOP.func == bIOPS["IO_OPEN"]: 368 369 self.FNAME = IOP.fname 370 bareosfd.DebugMessage( 371 context, 100, "self.FNAME was set to %s from IOP.fname\n" % (self.FNAME) 372 ) 373 374 if self.options.get("local") == "yes": 375 try: 376 if IOP.flags & (os.O_CREAT | os.O_WRONLY): 377 bareosfd.DebugMessage( 378 context, 379 100, 380 "Open file %s for writing with %s\n" % (self.FNAME, IOP), 381 ) 382 383 dirname = os.path.dirname(self.FNAME) 384 if not os.path.exists(dirname): 385 bareosfd.DebugMessage( 386 context, 387 100, 388 "Directory %s does not exist, creating it now\n" 389 % (dirname), 390 ) 391 os.makedirs(dirname) 392 self.file = open(self.FNAME, "wb") 393 else: 394 IOP.status = -1 395 bareosfd.JobMessage( 396 context, 397 bJobMessageType["M_FATAL"], 398 "plugin_io: option local=yes can only be used on restore\n", 399 ) 400 return bRCs["bRC_Error"] 401 except (OSError, IOError) as io_open_error: 402 IOP.status = -1 403 bareosfd.DebugMessage( 404 context, 405 100, 406 "plugin_io: failed to open %s: %s\n" 407 % (self.FNAME, io_open_error.strerror), 408 ) 409 bareosfd.JobMessage( 410 context, 411 bJobMessageType["M_FATAL"], 412 "plugin_io: failed to open %s: %s\n" 413 % (self.FNAME, io_open_error.strerror), 414 ) 415 return bRCs["bRC_Error"] 416 417 return bRCs["bRC_OK"] 418 419 elif IOP.func == bIOPS["IO_CLOSE"]: 420 if self.file is not None: 421 bareosfd.DebugMessage(context, 100, "Closing file " + "\n") 422 self.file.close() 423 elif self.jobType == "B": 424 if "file" in self.backup_obj: 425 self.backup_obj["file"]["fh"].close() 426 elif "disk" in self.backup_obj: 427 bareosfd.DebugMessage( 428 context, 100, "plugin_io: calling end_transfer()\n" 429 ) 430 # Backup Job 431 self.ovirt.end_transfer(context) 432 elif self.jobType == "R": 433 if self.FNAME.endswith(".ovf"): 434 return self.ovirt.prepare_vm_restore(context) 435 else: 436 self.ovirt.end_transfer(context) 437 return bRCs["bRC_OK"] 438 elif IOP.func == bIOPS["IO_SEEK"]: 439 return bRCs["bRC_OK"] 440 elif IOP.func == bIOPS["IO_READ"]: 441 if "file" in self.backup_obj: 442 IOP.buf = bytearray(IOP.count) 443 IOP.status = self.backup_obj["file"]["fh"].readinto(IOP.buf) 444 IOP.io_errno = 0 445 bareosfd.DebugMessage( 446 context, 447 100, 448 "plugin_io: read from file %s\n" 449 % (self.backup_obj["file"]["filename"]), 450 ) 451 elif "disk" in self.backup_obj: 452 bareosfd.DebugMessage( 453 context, 454 100, 455 "plugin_io: read from disk %s\n" % (self.backup_obj["disk"].alias), 456 ) 457 try: 458 IOP.buf = bytearray(IOP.count) 459 chunk = self.ovirt.process_download(context, IOP.count) 460 IOP.buf[:] = chunk 461 IOP.status = len(IOP.buf) 462 IOP.io_errno = 0 463 except Exception as e: 464 bareosfd.JobMessage( 465 context, 466 bJobMessageType["M_ERROR"], 467 "BareosFdPluginOvirt:plugin_io() Error: %s" % str(e), 468 ) 469 self.ovirt.end_transfer(context) 470 return bRCs["bRC_Error"] 471 else: 472 bareosfd.JobMessage( 473 context, 474 bJobMessageType["M_ERROR"], 475 "BareosFdPluginOvirt:plugin_io() Unable to read data to backup.", 476 ) 477 return bRCs["bRC_Error"] 478 return bRCs["bRC_OK"] 479 elif IOP.func == bIOPS["IO_WRITE"]: 480 if self.file is not None: 481 try: 482 bareosfd.DebugMessage( 483 context, 200, "Writing buffer to file %s\n" % (self.FNAME) 484 ) 485 self.file.write(IOP.buf) 486 IOP.status = IOP.count 487 IOP.io_errno = 0 488 except IOError as msg: 489 IOP.io_errno = -1 490 bareosfd.DebugMessage( 491 context, 100, "Error writing data: " + msg + "\n" 492 ) 493 return bRCs["bRC_Error"] 494 elif self.FNAME.endswith(".ovf"): 495 self.ovirt.process_ovf(context, IOP.buf) 496 IOP.status = IOP.count 497 IOP.io_errno = 0 498 else: 499 self.ovirt.process_upload(context, IOP.buf) 500 IOP.status = IOP.count 501 IOP.io_errno = 0 502 return bRCs["bRC_OK"] 503 504 def handle_plugin_event(self, context, event): 505 506 if event == bEventType["bEventEndBackupJob"]: 507 bareosfd.DebugMessage( 508 context, 509 100, 510 "BareosFdPluginOvirt::handle_plugin_event() called with bEventEndBackupJob\n", 511 ) 512 bareosfd.DebugMessage(context, 100, "removing Snapshot\n") 513 self.ovirt.end_vm_backup(context) 514 515 elif event == bEventType["bEventEndFileSet"]: 516 bareosfd.DebugMessage( 517 context, 518 100, 519 "BareosFdPluginOvirt::handle_plugin_event() called with bEventEndFileSet\n", 520 ) 521 522 elif event == bEventType["bEventStartBackupJob"]: 523 bareosfd.DebugMessage( 524 context, 525 100, 526 "BareosFdPluginOvirt::handle_plugin_event() called with bEventStartBackupJob\n", 527 ) 528 529 return self.start_backup_job(context) 530 531 elif event == bEventType["bEventStartRestoreJob"]: 532 bareosfd.DebugMessage( 533 context, 534 100, 535 "BareosFdPluginOvirt::handle_plugin_event() called with bEventStartRestoreJob\n", 536 ) 537 538 return self.start_restore_job(context) 539 540 elif event == bEventType["bEventEndRestoreJob"]: 541 bareosfd.DebugMessage( 542 context, 543 100, 544 "BareosFdPluginOvirt::handle_plugin_event() called with bEventEndRestoreJob\n", 545 ) 546 bareosfd.DebugMessage(context, 100, "removing Snapshot\n") 547 self.ovirt.end_vm_restore(context) 548 else: 549 bareosfd.DebugMessage( 550 context, 551 100, 552 "BareosFdPluginOvirt::handle_plugin_event() called with event %s\n" 553 % (event), 554 ) 555 556 return bRCs["bRC_OK"] 557 558 def end_backup_file(self, context): 559 bareosfd.DebugMessage( 560 context, 561 100, 562 "BareosFdPluginOvirt::end_backup_file() entry point in Python called\n", 563 ) 564 if self.ovirt.transfer_start_time: 565 elapsed_seconds = round(time.time() - self.ovirt.transfer_start_time, 2) 566 download_rate = round( 567 self.ovirt.init_bytes_to_transf / 1000.0 / elapsed_seconds, 1 568 ) 569 bareosfd.JobMessage( 570 context, 571 bJobMessageType["M_INFO"], 572 " Transfer time: %s s bytes: %s rate: %s KB/s\n" 573 % (elapsed_seconds, self.ovirt.init_bytes_to_transf, download_rate), 574 ) 575 self.ovirt.transfer_start_time = None 576 577 if self.ovirt.backup_objects: 578 return bRCs["bRC_More"] 579 else: 580 return bRCs["bRC_OK"] 581 582 def end_restore_file(self, context): 583 bareosfd.DebugMessage( 584 context, 585 100, 586 "BareosFdPluginOvirt::end_restore_file() entry point in Python called\n", 587 ) 588 if self.ovirt.transfer_start_time: 589 elapsed_seconds = round(time.time() - self.ovirt.transfer_start_time, 2) 590 download_rate = round( 591 self.ovirt.init_bytes_to_transf / 1000.0 / elapsed_seconds, 1 592 ) 593 bareosfd.JobMessage( 594 context, 595 bJobMessageType["M_INFO"], 596 " Upload time: %s s bytes: %s rate: %s KB/s\n" 597 % (elapsed_seconds, self.ovirt.init_bytes_to_transf, download_rate), 598 ) 599 self.ovirt.transfer_start_time = None 600 return bRCs["bRC_OK"] 601 602 def restore_object_data(self, context, ROP): 603 """ 604 Note: 605 This is called in two cases: 606 - on diff/inc backup (should be called only once) 607 - on restore (for every job id being restored) 608 But at the point in time called, it is not possible 609 to distinguish which of them it is, because job type 610 is "I" until the bEventStartBackupJob event 611 """ 612 bareosfd.DebugMessage( 613 context, 614 100, 615 "BareosFdPluginOvirt:restore_object_data() called with ROP:%s\n" % (ROP), 616 ) 617 bareosfd.DebugMessage( 618 context, 619 100, 620 "ROP.object_name(%s): %s\n" % (type(ROP.object_name), ROP.object_name), 621 ) 622 bareosfd.DebugMessage( 623 context, 624 100, 625 "ROP.plugin_name(%s): %s\n" % (type(ROP.plugin_name), ROP.plugin_name), 626 ) 627 bareosfd.DebugMessage( 628 context, 629 100, 630 "ROP.object_len(%s): %s\n" % (type(ROP.object_len), ROP.object_len), 631 ) 632 bareosfd.DebugMessage( 633 context, 634 100, 635 "ROP.object_full_len(%s): %s\n" 636 % (type(ROP.object_full_len), ROP.object_full_len), 637 ) 638 bareosfd.DebugMessage( 639 context, 100, "ROP.object(%s): %s\n" % (type(ROP.object), ROP.object) 640 ) 641 ro_data = json.loads(str(ROP.object)) 642 self.ovirt.disk_metadata_by_id[ro_data["disk_metadata"]["id"]] = ro_data[ 643 "disk_metadata" 644 ] 645 646 return bRCs["bRC_OK"] 647 648 def parse_config_file(self, context): 649 """ 650 Parse the config file given in the config_file plugin option 651 """ 652 bareosfd.DebugMessage( 653 context, 654 100, 655 "BareosFdPluginOvirt: parse_config_file(): parse %s\n" 656 % (self.options["config_file"]), 657 ) 658 659 self.config = configparser.ConfigParser() 660 661 try: 662 self.config.readfp(open(self.options["config_file"])) 663 except IOError as err: 664 bareosfd.JobMessage( 665 context, 666 bJobMessageType["M_FATAL"], 667 "BareosFdPluginOvirt: Error reading config file %s: %s\n" 668 % (self.options["config_file"], err.strerror), 669 ) 670 return False 671 672 return self.check_config(context) 673 674 def check_config(self, context): 675 """ 676 Check the configuration and set or override options if necessary, 677 considering mandatory: username and password in the [credentials] section 678 """ 679 bareosfd.DebugMessage(context, 100, "BareosFdPluginOvirt: check_config()\n") 680 mandatory_sections = ["credentials"] 681 mandatory_options = {} 682 mandatory_options["credentials"] = ["username", "password"] 683 684 for section in mandatory_sections: 685 if not self.config.has_section(section): 686 bareosfd.JobMessage( 687 context, 688 bJobMessageType["M_FATAL"], 689 "BareosFdPluginOvirt: Section [%s] missing in config file %s\n" 690 % (section, self.options["config_file"]), 691 ) 692 return False 693 694 for option in mandatory_options[section]: 695 if not self.config.has_option(section, option): 696 bareosfd.JobMessage( 697 context, 698 bJobMessageType["M_FATAL"], 699 "BareosFdPluginOvirt: Options %s missing in Section [%s] in config file %s\n" 700 % (option, section, self.options["config_file"]), 701 ) 702 return False 703 704 plugin_option = self.options.get(option) 705 if plugin_option: 706 bareosfd.JobMessage( 707 context, 708 bJobMessageType["M_WARNING"], 709 "BareosFdPluginOvirt: Overriding plugin option %s from config file %s\n" 710 % (option, self.options["config_file"]), 711 ) 712 713 self.options[option] = self.config.get(section, option) 714 715 return True 716 717 718class BareosOvirtWrapper(object): 719 """ 720 Ovirt wrapper class 721 """ 722 723 def __init__(self): 724 725 self.options = None 726 self.ca = None 727 728 self.connection = None 729 self.system_service = None 730 self.vms_service = None 731 self.storage_domains_service = None 732 self.events_service = None 733 self.network_profiles = None 734 735 self.vm = None 736 self.ovf_data = None 737 self.ovf = None 738 self.vm_service = None 739 self.snap_service = None 740 self.transfer_service = None 741 self.event_id = None 742 743 self.proxy_connection = None 744 self.bytes_to_transf = None 745 self.init_bytes_to_transf = None 746 self.transfer_start_time = None 747 748 self.backup_objects = None 749 self.restore_objects = None 750 self.ovf_disks_by_alias_and_fileref = {} 751 self.disk_metadata_by_id = {} 752 self.all_disks_excluded = False 753 754 self.old_new_ids = {} 755 756 self.snapshot_remove_timeout = 300 757 758 def set_options(self, options): 759 # make the plugin options also available in this class 760 self.options = options 761 762 def connect_api(self, context): 763 764 # ca certificate 765 self.ca = self.options["ca"] 766 767 # set server url 768 server_url = "https://%s/ovirt-engine/api" % str(self.options["server"]) 769 770 ovirt_sdk_debug = False 771 if self.options.get("ovirt_sdk_debug_log") is not None: 772 logging.basicConfig( 773 level=logging.DEBUG, filename=self.options["ovirt_sdk_debug_log"] 774 ) 775 ovirt_sdk_debug = True 776 777 # Create a connection to the server: 778 self.connection = sdk.Connection( 779 url=server_url, 780 username=self.options["username"], 781 password=self.options["password"], 782 ca_file=self.ca, 783 debug=ovirt_sdk_debug, 784 log=logging.getLogger(), 785 ) 786 787 if not self.connection: 788 bareosfd.JobMessage( 789 context, 790 bJobMessageType["M_FATAL"], 791 "Cannot connect to host %s with user %s and ca file %s\n" 792 % (self.options["server"], self.options["username"], self.ca), 793 ) 794 return False 795 796 # Get a reference to the root service: 797 self.system_service = self.connection.system_service() 798 799 # Get the reference to the "vms" service: 800 self.vms_service = self.system_service.vms_service() 801 802 if not self.vms_service: 803 return False 804 805 # get network profiles 806 profiles_service = self.system_service.vnic_profiles_service() 807 self.network_profiles = { 808 profile.name: profile.id for profile in profiles_service.list() 809 } 810 811 # Get the reference to the service that we will use to send events to 812 # the audit log: 813 self.events_service = self.system_service.events_service() 814 815 # In order to send events we need to also send unique integer ids. 816 self.event_id = int(time.time()) 817 818 bareosfd.DebugMessage( 819 context, 820 100, 821 ( 822 "Successfully connected to Ovirt Engine API on '%s' with" 823 " user %s and ca file %s\n" 824 ) 825 % (self.options["server"], self.options["username"], self.ca), 826 ) 827 828 return True 829 830 def prepare_vm_backup(self, context): 831 """ 832 prepare VM backup: 833 - get vm details 834 - take snapshot 835 - get disk devices 836 """ 837 bareosfd.DebugMessage( 838 context, 839 100, 840 "BareosOvirtWrapper::prepare_vm_backup: prepare VM to backup\n", 841 ) 842 843 if not self.get_vm(context): 844 bareosfd.DebugMessage(context, 100, "Error getting details for VM\n") 845 846 return bRCs["bRC_Error"] 847 else: 848 # Locate the service that manages the virtual machine: 849 self.vm_service = self.vms_service.vm_service(self.vm.id) 850 851 # check if vm have snapshosts 852 snaps_service = self.vm_service.snapshots_service() 853 if len(snaps_service.list()) > 1: 854 bareosfd.JobMessage( 855 context, 856 bJobMessageType["M_FATAL"], 857 "Error '%s' already have snapshosts. This is not supported\n" 858 % (self.vm.name), 859 ) 860 return bRCs["bRC_Error"] 861 862 bareosfd.DebugMessage( 863 context, 100, "Start the backup of VM %s\n" % (self.vm.name) 864 ) 865 866 # Save the OVF to a file, so that we can use to restore the virtual 867 # machine later. The name of the file is the name of the virtual 868 # machine, followed by a dash and the identifier of the virtual machine, 869 # to make it unique: 870 ovf_data = self.vm.initialization.configuration.data 871 ovf_file = "%s-%s.ovf" % (self.vm.name, self.vm.id) 872 if self.backup_objects is None: 873 self.backup_objects = [] 874 875 self.backup_objects.append( 876 { 877 "vmname": self.vm.name, 878 "vmid": self.vm.id, 879 "file": {"data": ovf_data, "filename": ovf_file}, 880 } 881 ) 882 883 # create vm snapshots 884 self.create_vm_snapshot(context) 885 886 # get vm backup disks from snapshot 887 if not self.all_disks_excluded and not self.get_vm_backup_disks(context): 888 bareosfd.JobMessage( 889 context, 890 bJobMessageType["M_FATAL"], 891 "Error getting Backup Disks VM %s from snapshot\n" % (self.vm.name), 892 ) 893 return bRCs["bRC_Error"] 894 895 return bRCs["bRC_OK"] 896 897 def get_vm(self, context): 898 search = None 899 if "uuid" in self.options: 900 search = "id=%s" % str(self.options["uuid"]) 901 elif "vm_name" in self.options: 902 search = "name=%s" % str(self.options["vm_name"]) 903 904 if search is not None: 905 bareosfd.DebugMessage(context, 100, "get_vm search vm by '%s'\n" % (search)) 906 res = self.vms_service.list(search=search, all_content=True) 907 if len(res) > 0: 908 self.vm = res[0] 909 return True 910 return False 911 912 def create_vm_snapshot(self, context): 913 """ 914 Creates a snapshot 915 """ 916 917 # Create an unique description for the snapshot, so that it is easier 918 # for the administrator to identify this snapshot as a temporary one 919 # created just for backup purposes: 920 snap_description = "%s-backup-%s" % (self.vm.name, uuid.uuid4()) 921 922 # Send an external event to indicate to the administrator that the 923 # backup of the virtual machine is starting. Note that the description 924 # of the event contains the name of the virtual machine and the name of 925 # the temporary snapshot, this way, if something fails, the administrator 926 # will know what snapshot was used and remove it manually. 927 self.events_service.add( 928 event=types.Event( 929 vm=types.Vm(id=self.vm.id), 930 origin=APPLICATION_NAME, 931 severity=types.LogSeverity.NORMAL, 932 custom_id=self.event_id, 933 description=( 934 "Backup of virtual machine '%s' using snapshot '%s' is " 935 "starting." % (self.vm.name, snap_description) 936 ), 937 ), 938 ) 939 self.event_id += 1 940 941 # Send the request to create the snapshot. Note that this will return 942 # before the snapshot is completely created, so we will later need to 943 # wait till the snapshot is completely created. 944 # The snapshot will not include memory. Change to True the parameter 945 # persist_memorystate to get it (in that case the VM will be paused for a while). 946 snaps_service = self.vm_service.snapshots_service() 947 snap = snaps_service.add( 948 snapshot=types.Snapshot( 949 description=snap_description, 950 persist_memorystate=False, 951 disk_attachments=self.get_vm_disks_for_snapshot(context), 952 ), 953 ) 954 bareosfd.JobMessage( 955 context, 956 bJobMessageType["M_INFO"], 957 "Sent request to create snapshot '%s', the id is '%s'.\n" 958 % (snap.description, snap.id), 959 ) 960 961 # Poll and wait till the status of the snapshot is 'ok', which means 962 # that it is completely created: 963 self.snap_service = snaps_service.snapshot_service(snap.id) 964 while snap.snapshot_status != types.SnapshotStatus.OK: 965 bareosfd.DebugMessage( 966 context, 967 100, 968 "Waiting till the snapshot is created, the status is now '%s'.\n" 969 % snap.snapshot_status, 970 ) 971 time.sleep(1) 972 snap = self.snap_service.get() 973 time.sleep(1) 974 975 # wait some time until snapshot real complete 976 time.sleep(10) 977 978 bareosfd.JobMessage( 979 context, bJobMessageType["M_INFO"], "' The snapshot is now complete.\n" 980 ) 981 982 def get_vm_disks_for_snapshot(self, context): 983 # return list of disks for snapshot, process include/exclude if given 984 disk_attachments_service = self.vm_service.disk_attachments_service() 985 disk_attachments = disk_attachments_service.list() 986 included_disk_ids = [] 987 included_disks = None 988 for disk_attachment in disk_attachments: 989 disk = self.connection.follow_link(disk_attachment.disk) 990 991 if not self.is_disk_alias_included(context, disk.alias): 992 continue 993 if self.is_disk_alias_excluded(context, disk.alias): 994 continue 995 996 included_disk_ids.append(disk.id) 997 998 if len(included_disk_ids) == 0: 999 # No disks to backup, snapshot will only contain VM config 1000 # Note: The comma must not be omitted here: 1001 # included_disks = (types.DiskAttachment(),) 1002 self.all_disks_excluded = True 1003 bareosfd.JobMessage( 1004 context, 1005 bJobMessageType["M_INFO"], 1006 "All disks excluded, only backing up VM configuration.\n", 1007 ) 1008 included_disks = [types.DiskAttachment()] 1009 1010 else: 1011 included_disks = [ 1012 types.DiskAttachment(disk=types.Disk(id=disk_id)) 1013 for disk_id in included_disk_ids 1014 ] 1015 1016 return included_disks 1017 1018 def get_vm_backup_disks(self, context): 1019 1020 has_disks = False 1021 1022 if self.backup_objects is None: 1023 self.backup_objects = [] 1024 1025 # get snapshot 1026 self.snap_service.get() 1027 1028 # Get a reference to the storage domains service: 1029 storage_domains_service = self.system_service.storage_domains_service() 1030 1031 # Retrieve the descriptions of the disks of the snapshot: 1032 snap_disks_service = self.snap_service.disks_service() 1033 snap_disks = snap_disks_service.list() 1034 1035 # download disk snaphost 1036 for snap_disk in snap_disks: 1037 disk_id = snap_disk.id 1038 disk_alias = snap_disk.alias 1039 bareosfd.DebugMessage( 1040 context, 1041 200, 1042 "get_vm_backup_disks(): disk_id: '%s' disk_alias '%s'\n" 1043 % (disk_id, disk_alias), 1044 ) 1045 1046 sd_id = snap_disk.storage_domains[0].id 1047 1048 # Get a reference to the storage domain service in which the disk snapshots reside: 1049 storage_domain_service = storage_domains_service.storage_domain_service( 1050 sd_id 1051 ) 1052 1053 # Get a reference to the disk snapshots service: 1054 disk_snapshot_service = storage_domain_service.disk_snapshots_service() 1055 1056 # Get a list of disk snapshots by a disk ID 1057 all_disk_snapshots = disk_snapshot_service.list() 1058 1059 # Filter disk snapshots list by disk id 1060 disk_snapshots = [s for s in all_disk_snapshots if s.disk.id == disk_id] 1061 1062 # Download disk snapshot 1063 if len(disk_snapshots) > 0: 1064 for disk_snapshot in disk_snapshots: 1065 self.backup_objects.append( 1066 { 1067 "vmname": self.vm.name, 1068 "vmid": self.vm.id, 1069 "snapshot": disk_snapshot, 1070 "disk": snap_disk, 1071 } 1072 ) 1073 has_disks = True 1074 return has_disks 1075 1076 def is_disk_alias_included(self, context, disk_alias): 1077 if self.options.get("include_disk_aliases") is None: 1078 return True 1079 1080 include_disk_aliases = self.options["include_disk_aliases"].split(",") 1081 if disk_alias in include_disk_aliases: 1082 bareosfd.JobMessage( 1083 context, 1084 bJobMessageType["M_INFO"], 1085 "Including disk with alias %s\n" % (disk_alias), 1086 ) 1087 return True 1088 1089 return False 1090 1091 def is_disk_alias_excluded(self, context, disk_alias): 1092 if self.options.get("exclude_disk_aliases") is None: 1093 return False 1094 1095 exclude_disk_aliases = self.options["exclude_disk_aliases"].split(",") 1096 1097 if "*" in exclude_disk_aliases or disk_alias in exclude_disk_aliases: 1098 bareosfd.JobMessage( 1099 context, 1100 bJobMessageType["M_INFO"], 1101 "Excluding disk with alias %s\n" % (disk_alias), 1102 ) 1103 return True 1104 1105 return False 1106 1107 def get_transfer_service(self, image_transfer): 1108 # Get a reference to the service that manages the image transfer: 1109 transfers_service = self.system_service.image_transfers_service() 1110 1111 # Add a new image transfer: 1112 transfer = transfers_service.add(image_transfer) 1113 1114 # Get reference to the created transfer service: 1115 transfer_service = transfers_service.image_transfer_service(transfer.id) 1116 1117 while transfer.phase == types.ImageTransferPhase.INITIALIZING: 1118 time.sleep(1) 1119 transfer = transfer_service.get() 1120 1121 return transfer_service 1122 1123 def get_proxy_connection(self, proxy_url): 1124 # At this stage, the SDK granted the permission to start transferring the disk, and the 1125 # user should choose its preferred tool for doing it - regardless of the SDK. 1126 # In this example, we will use Python's httplib.HTTPSConnection for transferring the data. 1127 context = ssl.create_default_context() 1128 1129 # Note that ovirt-imageio-proxy by default checks the certificates, so if you don't have 1130 # your CA certificate of the engine in the system, you need to pass it to HTTPSConnection. 1131 context.load_verify_locations(cafile=self.ca) 1132 1133 return HTTPSConnection(proxy_url.hostname, proxy_url.port, context=context) 1134 1135 def start_download(self, context, snapshot, disk): 1136 1137 bareosfd.JobMessage( 1138 context, 1139 bJobMessageType["M_INFO"], 1140 "Downloading snapshot '%s' of disk '%s'('%s')\n" 1141 % (snapshot.id, disk.alias, disk.id), 1142 ) 1143 1144 self.transfer_service = self.get_transfer_service( 1145 types.ImageTransfer( 1146 snapshot=types.DiskSnapshot(id=snapshot.id), 1147 direction=types.ImageTransferDirection.DOWNLOAD, 1148 ) 1149 ) 1150 transfer = self.transfer_service.get() 1151 proxy_url = urlparse(transfer.proxy_url) 1152 self.proxy_connection = self.get_proxy_connection(proxy_url) 1153 1154 # Set needed headers for downloading: 1155 transfer_headers = {"Authorization": transfer.signed_ticket} 1156 1157 # Perform the request. 1158 self.proxy_connection.request("GET", proxy_url.path, headers=transfer_headers) 1159 # Get response 1160 self.response = self.proxy_connection.getresponse() 1161 1162 # Check the response status: 1163 if self.response.status >= 300: 1164 bareosfd.JobMessage( 1165 context, bJobMessageType["M_ERROR"], "Error: %s" % self.response.read() 1166 ) 1167 1168 self.bytes_to_transf = int(self.response.getheader("Content-Length")) 1169 1170 self.backup_objects.insert( 1171 0, 1172 { 1173 "vmname": self.vm.name, 1174 "vmid": self.vm.id, 1175 "snapshot_id": snapshot.id, 1176 "disk_metadata": { 1177 "id": disk.id, 1178 "alias": disk.alias, 1179 "effective_size": self.bytes_to_transf, 1180 }, 1181 }, 1182 ) 1183 1184 bareosfd.JobMessage( 1185 context, 1186 bJobMessageType["M_INFO"], 1187 " Transfer disk snapshot of %s bytes\n" % (str(self.bytes_to_transf)), 1188 ) 1189 1190 self.init_bytes_to_transf = self.bytes_to_transf 1191 self.transfer_start_time = time.time() 1192 1193 def process_download(self, context, chunk_size): 1194 1195 chunk = "" 1196 1197 bareosfd.DebugMessage( 1198 context, 1199 100, 1200 "process_download(): transfer %s of %s (%s) \n" 1201 % ( 1202 self.bytes_to_transf, 1203 self.response.getheader("Content-Length"), 1204 chunk_size, 1205 ), 1206 ) 1207 1208 if self.bytes_to_transf > 0: 1209 1210 # Calculate next chunk to read 1211 to_read = min(self.bytes_to_transf, chunk_size) 1212 1213 # Read next info buffer 1214 chunk = self.response.read(to_read) 1215 1216 if chunk == "": 1217 bareosfd.DebugMessage( 1218 context, 100, "process_download(): Socket disconnected. \n" 1219 ) 1220 bareosfd.JobMessage( 1221 context, 1222 bJobMessageType["M_ERROR"], 1223 "process_download(): Socket disconnected.", 1224 ) 1225 1226 raise RuntimeError("Socket disconnected") 1227 1228 # Update bytes_to_transf 1229 self.bytes_to_transf -= len(chunk) 1230 1231 completed = 1 - ( 1232 self.bytes_to_transf / float(self.response.getheader("Content-Length")) 1233 ) 1234 1235 bareosfd.DebugMessage( 1236 context, 100, "process_download(): Completed {:.0%}\n".format(completed) 1237 ) 1238 1239 return chunk 1240 1241 def prepare_vm_restore(self, context): 1242 restore_existing_vm = False 1243 if self.connection is None: 1244 # if not connected yet 1245 if not self.connect_api(context): 1246 return bRCs["bRC_Error"] 1247 1248 if self.ovf_data is None: 1249 bareosfd.JobMessage( 1250 context, 1251 bJobMessageType["M_FATAL"], 1252 "Unable to restore VM. No OVF data. \n", 1253 ) 1254 return bRCs["bRC_Error"] 1255 else: 1256 if "storage_domain" not in self.options: 1257 bareosfd.JobMessage( 1258 context, 1259 bJobMessageType["M_FATAL"], 1260 "No storage domain specified.\n", 1261 ) 1262 return bRCs["bRC_Error"] 1263 1264 storage_domain = self.options["storage_domain"] 1265 1266 # Parse the OVF as XML document: 1267 self.ovf = Ovf(self.ovf_data) 1268 1269 bareosfd.DebugMessage( 1270 context, 200, "prepare_vm_restore(): %s\n" % (self.ovf) 1271 ) 1272 1273 # Find the name of the virtual machine within the OVF: 1274 vm_name = None 1275 if "vm_name" in self.options: 1276 vm_name = self.options["vm_name"] 1277 1278 if vm_name is None: 1279 vm_name = self.ovf.get_element("vm_name") 1280 1281 # Find the cluster name of the virtual machine within the OVF: 1282 cluster_name = None 1283 if "cluster_name" in self.options: 1284 cluster_name = self.options["cluster_name"] 1285 1286 if cluster_name is None: 1287 # Find the cluster name of the virtual machine within the OVF: 1288 cluster_name = self.ovf.get_element("cluster_name") 1289 1290 # check if VM exists 1291 res = self.vms_service.list( 1292 search="name=%s and cluster=%s" % (str(vm_name), str(cluster_name)) 1293 ) 1294 if len(res) > 1: 1295 bareosfd.JobMessage( 1296 context, 1297 bJobMessageType["M_FATAL"], 1298 "Found %s VMs with name '%s'\n" % (len(res), str(vm_name)), 1299 ) 1300 return bRCs["bRC_Error"] 1301 1302 if len(res) == 1: 1303 if not self.options.get("overwrite") == "yes": 1304 bareosfd.JobMessage( 1305 context, 1306 bJobMessageType["M_FATAL"], 1307 "If you are sure you want to overwrite the existing VM '%s', please add the plugin option 'overwrite=yes'\n" 1308 % (str(vm_name)), 1309 ) 1310 return bRCs["bRC_Error"] 1311 1312 bareosfd.JobMessage( 1313 context, 1314 bJobMessageType["M_INFO"], 1315 "Restore to existing VM '%s'\n" % str(vm_name), 1316 ) 1317 self.vm = res[0] 1318 1319 if self.vm.status != types.VmStatus.DOWN: 1320 bareosfd.JobMessage( 1321 context, 1322 bJobMessageType["M_FATAL"], 1323 "VM '%s' must be down for restore, but status is %s\n" 1324 % (str(vm_name), self.vm.status), 1325 ) 1326 return bRCs["bRC_Error"] 1327 1328 restore_existing_vm = True 1329 1330 else: 1331 self.create_vm(context, vm_name, cluster_name) 1332 self.add_nics_to_vm(context) 1333 1334 # Extract disk information from OVF 1335 disk_elements = self.ovf.get_elements("disk_elements") 1336 1337 if self.restore_objects is None: 1338 self.restore_objects = [] 1339 1340 for disk_element in disk_elements: 1341 # Get disk properties: 1342 props = {} 1343 for key, value in disk_element.items(): 1344 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["ovf"], "") 1345 props[key] = value 1346 1347 # set storage domain 1348 props["storage_domain"] = storage_domain 1349 self.restore_objects.append(props) 1350 1351 # used by get_ovf_disk_alias_by_basename(), needed to process 1352 # includes/excludes on restore 1353 self.ovf_disks_by_alias_and_fileref[ 1354 props["disk-alias"] + "-" + props["fileRef"] 1355 ] = props 1356 1357 if restore_existing_vm: 1358 # When restoring to existing VM, old and new disk ID are identical 1359 # Important: The diskId property in the OVF is not the oVirt 1360 # diskId, which can be found in the first part of the OVF fileRef 1361 old_disk_id = os.path.dirname(props["fileRef"]) 1362 self.old_new_ids[old_disk_id] = old_disk_id 1363 1364 bareosfd.DebugMessage( 1365 context, 1366 200, 1367 "end of prepare_vm_restore(): self.restore_objects: %s self.old_new_ids: %s\n" 1368 % (self.restore_objects, self.old_new_ids), 1369 ) 1370 1371 return bRCs["bRC_OK"] 1372 1373 def create_vm(self, context, vm_name, cluster_name): 1374 1375 vm_template = "Blank" 1376 if "vm_template" in self.options: 1377 vm_template = self.options["vm_template"] 1378 1379 # Add the virtual machine, the transferred disks will be 1380 # attached to this virtual machine: 1381 bareosfd.JobMessage( 1382 context, bJobMessageType["M_INFO"], "Adding virtual machine %s\n" % vm_name 1383 ) 1384 1385 # vm type (server/desktop) 1386 vm_type = "server" 1387 if "vm_type" in self.options: 1388 vm_type = self.options["vm_type"] 1389 1390 # vm memory and cpu 1391 vm_memory = None 1392 if "vm_memory" in self.options: 1393 vm_memory = int(self.options["vm_memory"]) * 2 ** 20 1394 1395 vm_cpu = None 1396 if "vm_cpu" in self.options: 1397 vm_cpu_arr = self.options["vm_cpu"].split(",") 1398 if len(vm_cpu_arr) > 0: 1399 if len(vm_cpu_arr) < 2: 1400 vm_cpu_arr.append(1) 1401 vm_cpu_arr.append(1) 1402 elif len(vm_cpu_arr) < 3: 1403 vm_cpu_arr.append(1) 1404 1405 vm_cpu = types.Cpu( 1406 topology=types.CpuTopology( 1407 cores=int(vm_cpu_arr[0]), 1408 sockets=int(vm_cpu_arr[1]), 1409 threads=int(vm_cpu_arr[2]), 1410 ) 1411 ) 1412 1413 if vm_memory is None or vm_cpu is None: 1414 # Find the resources section 1415 resources_elements = self.ovf.get_elements("resources_elements") 1416 for resource in resources_elements: 1417 # Get resource properties: 1418 props = {} 1419 for e in resource: 1420 key = e.tag 1421 value = e.text 1422 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["rasd"], "") 1423 props[key] = value 1424 1425 if vm_cpu is None: 1426 # for ResourceType = 3 (CPU) 1427 if int(props["ResourceType"]) == 3: 1428 vm_cpu = types.Cpu( 1429 topology=types.CpuTopology( 1430 cores=int(props["cpu_per_socket"]), 1431 sockets=int(props["num_of_sockets"]), 1432 threads=int(props["threads_per_cpu"]), 1433 ) 1434 ) 1435 if vm_memory is None: 1436 # for ResourceType = 4 (Memory) 1437 if int(props["ResourceType"]) == 4: 1438 vm_memory = int(props["VirtualQuantity"]) 1439 if props["AllocationUnits"] == "GigaBytes": 1440 vm_memory = vm_memory * 2 ** 30 1441 elif props["AllocationUnits"] == "MegaBytes": 1442 vm_memory = vm_memory * 2 ** 20 1443 elif props["AllocationUnits"] == "KiloBytes": 1444 vm_memory = vm_memory * 2 ** 10 1445 1446 vm_memory_policy = None 1447 if vm_memory is not None: 1448 vm_maxmemory = 4 * vm_memory # 4x memory 1449 vm_memory_policy = types.MemoryPolicy( 1450 guaranteed=vm_memory, max=vm_maxmemory 1451 ) 1452 1453 self.vm = self.vms_service.add( 1454 types.Vm( 1455 name=vm_name, 1456 type=types.VmType(vm_type), 1457 memory=vm_memory, 1458 memory_policy=vm_memory_policy, 1459 cpu=vm_cpu, 1460 template=types.Template(name=vm_template), 1461 cluster=types.Cluster(name=cluster_name), 1462 ), 1463 ) 1464 1465 def add_nics_to_vm(self, context): 1466 # Find the network section 1467 network_elements = self.ovf.get_elements("network_elements") 1468 bareosfd.DebugMessage( 1469 context, 1470 200, 1471 "add_nics_to_vm(): network_elements: %s\n" % (network_elements), 1472 ) 1473 for nic in network_elements: 1474 # Get nic properties: 1475 props = {} 1476 for e in nic: 1477 key = e.tag 1478 value = e.text 1479 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["rasd"], "") 1480 props[key] = value 1481 1482 bareosfd.DebugMessage( 1483 context, 200, "add_nics_to_vm(): props: %s\n" % (props) 1484 ) 1485 1486 network = props["Connection"] 1487 if network not in self.network_profiles: 1488 bareosfd.JobMessage( 1489 context, 1490 bJobMessageType["M_WARNING"], 1491 "No network profile found for '%s'\n" % (network), 1492 ) 1493 else: 1494 profile_id = self.network_profiles[network] 1495 1496 # Locate the service that manages the network interface cards of the 1497 # virtual machine: 1498 nics_service = self.vms_service.vm_service(self.vm.id).nics_service() 1499 1500 # Use the "add" method of the network interface cards service to add the 1501 # new network interface card: 1502 nics_service.add( 1503 types.Nic( 1504 name=props["Name"], 1505 description=props["Caption"], 1506 vnic_profile=types.VnicProfile(id=profile_id), 1507 ), 1508 ) 1509 bareosfd.DebugMessage( 1510 context, 1511 200, 1512 "add_nics_to_vm(): added NIC with props %s\n" % (props), 1513 ) 1514 1515 def get_ovf_disk_alias_by_basename(self, context, fname): 1516 """ 1517 Return the disk alias name from OVF data for given fname (full path), 1518 this is used on restore by create_file() to process includes/excludes 1519 """ 1520 1521 dirpath = os.path.dirname(fname) 1522 dirname = os.path.basename(dirpath) 1523 basename = os.path.basename(fname) 1524 relname = "%s/%s" % (dirname, basename) 1525 1526 return self.ovf_disks_by_alias_and_fileref[relname]["disk-alias"] 1527 1528 def get_vm_disk_by_basename(self, context, fname): 1529 1530 dirpath = os.path.dirname(fname) 1531 dirname = os.path.basename(dirpath) 1532 basename = os.path.basename(fname) 1533 relname = "%s/%s" % (dirname, basename) 1534 1535 found = None 1536 bareosfd.DebugMessage( 1537 context, 1538 200, 1539 "get_vm_disk_by_basename(): %s %s\n" % (basename, self.restore_objects), 1540 ) 1541 if self.restore_objects is not None: 1542 i = 0 1543 while i < len(self.restore_objects) and found is None: 1544 obj = self.restore_objects[i] 1545 key = "%s-%s" % (obj["disk-alias"], obj["fileRef"]) 1546 bareosfd.DebugMessage( 1547 context, 1548 200, 1549 "get_vm_disk_by_basename(): lookup %s == %s and %s == %s\n" 1550 % (repr(relname), repr(key), repr(basename), repr(obj["diskId"])), 1551 ) 1552 if relname == key and basename == obj["diskId"]: 1553 bareosfd.DebugMessage( 1554 context, 200, "get_vm_disk_by_basename(): lookup matched\n" 1555 ) 1556 old_disk_id = os.path.dirname(obj["fileRef"]) 1557 1558 new_disk_id = None 1559 if old_disk_id in self.old_new_ids: 1560 bareosfd.DebugMessage( 1561 context, 1562 200, 1563 "get_vm_disk_by_basename(): disk id %s found in old_new_ids: %s\n" 1564 % (old_disk_id, self.old_new_ids), 1565 ) 1566 new_disk_id = self.old_new_ids[old_disk_id] 1567 1568 # get base disks 1569 if not obj["parentRef"]: 1570 disk = self.get_or_add_vm_disk(context, obj, new_disk_id) 1571 1572 if disk is not None: 1573 new_disk_id = disk.id 1574 self.old_new_ids[old_disk_id] = new_disk_id 1575 found = disk 1576 else: 1577 bareosfd.JobMessage( 1578 context, 1579 bJobMessageType["M_WARNING"], 1580 "The backup have snapshots and only base will be restored\n", 1581 ) 1582 1583 i += 1 1584 bareosfd.DebugMessage( 1585 context, 200, "get_vm_disk_by_basename(): found disk %s\n" % (found) 1586 ) 1587 return found 1588 1589 def get_or_add_vm_disk(self, context, obj, disk_id=None): 1590 # Create the disks: 1591 disks_service = self.system_service.disks_service() 1592 1593 disk = None 1594 if "storage_domain" not in obj: 1595 bareosfd.JobMessage( 1596 context, bJobMessageType["M_FATAL"], "No storage domain specified.\n" 1597 ) 1598 else: 1599 storage_domain = obj["storage_domain"] 1600 1601 # Find the disk we want to download by the id: 1602 if disk_id: 1603 disk_service = disks_service.disk_service(disk_id) 1604 if disk_service: 1605 disk = disk_service.get() 1606 1607 if not disk: 1608 # Locate the service that manages the disk attachments of the virtual 1609 # machine: 1610 disk_attachments_service = self.vms_service.vm_service( 1611 self.vm.id 1612 ).disk_attachments_service() 1613 1614 # Determine the volume format 1615 disk_format = types.DiskFormat.RAW 1616 if "volume-format" in obj and obj["volume-format"] == "COW": 1617 disk_format = types.DiskFormat.COW 1618 1619 disk_alias = None 1620 if "disk-alias" in obj: 1621 disk_alias = obj["disk-alias"] 1622 1623 description = None 1624 if "description" in obj: 1625 description = obj["description"] 1626 1627 size = 0 1628 if "size" in obj: 1629 size = int(obj["size"]) * 2 ** 30 1630 1631 actual_size = 0 1632 if "actual_size" in obj: 1633 actual_size = int(obj["actual_size"]) * 2 ** 30 1634 1635 disk_interface = types.DiskInterface.VIRTIO 1636 if "disk-interface" in obj: 1637 if obj["disk-interface"] == "VirtIO_SCSI": 1638 disk_interface = types.DiskInterface.VIRTIO_SCSI 1639 elif obj["disk-interface"] == "IDE": 1640 disk_interface = types.DiskInterface.IDE 1641 1642 disk_bootable = False 1643 if "boot" in obj: 1644 if obj["boot"] == "true": 1645 disk_bootable = True 1646 1647 ##### 1648 # Use the "add" method of the disk attachments service to add the disk. 1649 # Note that the size of the disk, the `provisioned_size` attribute, is 1650 # specified in bytes, so to create a disk of 10 GiB the value should 1651 # be 10 * 2^30. 1652 disk_attachment = disk_attachments_service.add( 1653 types.DiskAttachment( 1654 disk=types.Disk( 1655 id=disk_id, 1656 name=disk_alias, 1657 description=description, 1658 format=disk_format, 1659 provisioned_size=size, 1660 initial_size=actual_size, 1661 sparse=disk_format == types.DiskFormat.COW, 1662 storage_domains=[types.StorageDomain(name=storage_domain)], 1663 ), 1664 interface=disk_interface, 1665 bootable=disk_bootable, 1666 active=True, 1667 ), 1668 ) 1669 1670 # 'Waiting for Disk creation to finish' 1671 disk_service = disks_service.disk_service(disk_attachment.disk.id) 1672 while True: 1673 time.sleep(5) 1674 disk = disk_service.get() 1675 if disk.status == types.DiskStatus.OK: 1676 time.sleep(5) 1677 break 1678 1679 return disk 1680 1681 def start_upload(self, context, disk): 1682 1683 bareosfd.JobMessage( 1684 context, 1685 bJobMessageType["M_INFO"], 1686 "Uploading disk '%s'('%s')\n" % (disk.alias, disk.id), 1687 ) 1688 bareosfd.DebugMessage( 1689 context, 100, "Uploading disk '%s'('%s')\n" % (disk.alias, disk.id) 1690 ) 1691 bareosfd.DebugMessage(context, 200, "old_new_ids: %s\n" % (self.old_new_ids)) 1692 bareosfd.DebugMessage( 1693 context, 200, "self.restore_objects: %s\n" % (self.restore_objects) 1694 ) 1695 1696 self.transfer_service = self.get_transfer_service( 1697 types.ImageTransfer( 1698 image=types.Image(id=disk.id), 1699 direction=types.ImageTransferDirection.UPLOAD, 1700 ) 1701 ) 1702 transfer = self.transfer_service.get() 1703 proxy_url = urlparse(transfer.proxy_url) 1704 self.proxy_connection = self.get_proxy_connection(proxy_url) 1705 1706 # Send the request head 1707 self.proxy_connection.putrequest("PUT", proxy_url.path) 1708 self.proxy_connection.putheader("Authorization", transfer.signed_ticket) 1709 1710 # To prevent from errors on transfer, the exact number of bytes that 1711 # will be sent must be used, we call it effective_size here. It was 1712 # saved at backup time in restoreobjects, see start_backup_file(), 1713 # and restoreobjects are restored before any file. 1714 new_old_ids = {v: k for k, v in self.old_new_ids.items()} 1715 old_id = new_old_ids[disk.id] 1716 effective_size = self.disk_metadata_by_id[old_id]["effective_size"] 1717 self.init_bytes_to_transf = self.bytes_to_transf = effective_size 1718 1719 content_range = "bytes %d-%d/%d" % ( 1720 0, 1721 self.bytes_to_transf - 1, 1722 self.bytes_to_transf, 1723 ) 1724 self.proxy_connection.putheader("Content-Range", content_range) 1725 self.proxy_connection.putheader("Content-Length", "%d" % (self.bytes_to_transf)) 1726 self.proxy_connection.endheaders() 1727 1728 bareosfd.JobMessage( 1729 context, 1730 bJobMessageType["M_INFO"], 1731 " Upload disk of %s bytes\n" % (str(self.bytes_to_transf)), 1732 ) 1733 self.transfer_start_time = time.time() 1734 1735 def process_ovf(self, context, chunk): 1736 if self.ovf_data is None: 1737 self.ovf_data = str(chunk) 1738 else: 1739 self.ovf_data = self.ovf_data.str(chunk) 1740 1741 def process_upload(self, context, chunk): 1742 1743 bareosfd.DebugMessage( 1744 context, 1745 100, 1746 "process_upload(): transfer %s of %s (%s) \n" 1747 % (self.bytes_to_transf, self.init_bytes_to_transf, len(chunk)), 1748 ) 1749 1750 self.proxy_connection.send(chunk) 1751 1752 self.bytes_to_transf -= len(chunk) 1753 1754 completed = 1 - (self.bytes_to_transf / float(self.init_bytes_to_transf)) 1755 1756 bareosfd.DebugMessage( 1757 context, 100, "process_upload(): Completed {:.0%}\n".format(completed) 1758 ) 1759 1760 def end_transfer(self, context): 1761 1762 bareosfd.DebugMessage(context, 100, "end_transfer()\n") 1763 1764 # Finalize the session. 1765 if self.transfer_service is not None: 1766 self.transfer_service.finalize() 1767 1768 def end_vm_backup(self, context): 1769 1770 if self.snap_service: 1771 t_start = int(time.time()) 1772 snap = self.snap_service.get() 1773 1774 # Remove the snapshot: 1775 snapshot_deleted_success = False 1776 1777 bareosfd.JobMessage( 1778 context, 1779 bJobMessageType["M_INFO"], 1780 "Sending request to remove snapshot '%s', the id is '%s'.\n" 1781 % (snap.description, snap.id), 1782 ) 1783 1784 while True: 1785 try: 1786 self.snap_service.remove() 1787 except sdk.Error as e: 1788 if e.code in [400, 409]: 1789 1790 # Fail after defined timeout 1791 elapsed = int(time.time()) - t_start 1792 if elapsed >= self.snapshot_remove_timeout: 1793 bareosfd.JobMessage( 1794 context, 1795 bJobMessageType["M_WARNING"], 1796 "Remove snapshot timed out after %s s, reason: %s! Please remove it manually.\n" 1797 % (elapsed, e.message), 1798 ) 1799 return bRCs["bRC_Error"] 1800 1801 bareosfd.DebugMessage( 1802 context, 1803 100, 1804 "Could not remove snapshot, reason: %s, retrying until timeout (%s seconds left).\n" 1805 % (e.message, self.snapshot_remove_timeout - elapsed), 1806 ) 1807 bareosfd.JobMessage( 1808 context, 1809 bJobMessageType["M_INFO"], 1810 "Still waiting for snapshot removal, retrying until timeout (%s seconds left).\n" 1811 % (self.snapshot_remove_timeout - elapsed), 1812 ) 1813 else: 1814 bareosfd.JobMessage( 1815 context, 1816 bJobMessageType["M_WARNING"], 1817 "Unexpected error removing snapshot: %s, Please remove it manually.\n" 1818 % e.message, 1819 ) 1820 return bRCs["bRC_Error"] 1821 1822 if self.wait_for_snapshot_removal(snap.id): 1823 snapshot_deleted_success = True 1824 break 1825 else: 1826 # the snapshot still exists 1827 continue 1828 1829 if snapshot_deleted_success: 1830 bareosfd.JobMessage( 1831 context, 1832 bJobMessageType["M_INFO"], 1833 "Removed the snapshot '%s'.\n" % snap.description, 1834 ) 1835 1836 # Send an external event to indicate to the administrator that the 1837 # backup of the virtual machine is completed: 1838 self.events_service.add( 1839 event=types.Event( 1840 vm=types.Vm(id=self.vm.id), 1841 origin=APPLICATION_NAME, 1842 severity=types.LogSeverity.NORMAL, 1843 custom_id=self.event_id, 1844 description=( 1845 "Backup of virtual machine '%s' using snapshot '%s' is " 1846 "completed." % (self.vm.name, snap.description) 1847 ), 1848 ), 1849 ) 1850 self.event_id += 1 1851 1852 if self.connection: 1853 # Close the connection to the server: 1854 self.connection.close() 1855 1856 def wait_for_snapshot_removal(self, snapshot_id, timeout=30, delay=10): 1857 t_start = int(time.time()) 1858 snaps_service = self.vm_service.snapshots_service() 1859 1860 while True: 1861 snaps_map = {snap.id: snap.description for snap in snaps_service.list()} 1862 1863 if snapshot_id not in snaps_map: 1864 return True 1865 1866 if int(time.time()) - t_start > timeout: 1867 return False 1868 1869 time.sleep(delay) 1870 1871 return 1872 1873 def end_vm_restore(self, context): 1874 1875 # Send an external event to indicate to the administrator that the 1876 # restore of the virtual machine is completed: 1877 self.events_service.add( 1878 event=types.Event( 1879 vm=types.Vm(id=self.vm.id), 1880 origin=APPLICATION_NAME, 1881 severity=types.LogSeverity.NORMAL, 1882 custom_id=self.event_id, 1883 description=( 1884 "Restore of virtual machine '%s' is " "completed." % (self.vm.name) 1885 ), 1886 ), 1887 ) 1888 self.event_id += 1 1889 1890 if self.connection: 1891 # Close the connection to the server: 1892 self.connection.close() 1893 1894 1895class Ovf(object): 1896 """ 1897 This class is used to encapsulate XPath expressions to extract data from OVF 1898 """ 1899 1900 # OVF Namespaces 1901 OVF_NAMESPACES = { 1902 "ovf": "http://schemas.dmtf.org/ovf/envelope/1/", 1903 "xsi": "http://www.w3.org/2001/XMLSchema-instance", 1904 "rasd": "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData", 1905 } 1906 1907 # XPath expressions 1908 OVF_XPATH_EXPRESSIONS = { 1909 "vm_name": '/ovf:Envelope/Content[@xsi:type="ovf:VirtualSystem_Type"]/Name', 1910 "cluster_name": '/ovf:Envelope/Content[@xsi:type="ovf:VirtualSystem_Type"]/ClusterName', 1911 "network_elements": '/ovf:Envelope/Content[@xsi:type="ovf:VirtualSystem_Type"]/Section[@xsi:type="ovf:VirtualHardwareSection_Type"]/Item[Type="interface"]', 1912 "disk_elements": '/ovf:Envelope/Section[@xsi:type="ovf:DiskSection_Type"]/Disk', 1913 "resources_elements": '/ovf:Envelope/Content[@xsi:type="ovf:VirtualSystem_Type"]/Section[@xsi:type="ovf:VirtualHardwareSection_Type"]/Item[rasd:ResourceType=3 or rasd:ResourceType=4]', 1914 } 1915 1916 def __init__(self, ovf_data=None): 1917 """ 1918 Creates a new OVF object 1919 1920 This method supports the following parameters: 1921 1922 `ovf_data`:: A string containing the OVF XML data 1923 """ 1924 1925 # Parse the OVF as XML document: 1926 self.ovf = lxml.etree.fromstring(ovf_data) 1927 1928 def get_element(self, element_name): 1929 """ 1930 Method to extract a single element from OVF data 1931 1932 This method supports the following parameters: 1933 `element_name':: A string with the element name to extract 1934 """ 1935 return self.ovf.xpath( 1936 self.OVF_XPATH_EXPRESSIONS[element_name], namespaces=self.OVF_NAMESPACES 1937 )[0].text 1938 1939 def get_elements(self, element_name): 1940 """ 1941 Method to extract a list of elements from OVF data 1942 1943 This method supports the following parameters: 1944 `element_name':: A string with the element name to extract 1945 """ 1946 return self.ovf.xpath( 1947 self.OVF_XPATH_EXPRESSIONS[element_name], namespaces=self.OVF_NAMESPACES 1948 ) 1949 1950 1951# vim: tabstop=4 shiftwidth=4 softtabstop=4 expandtab 1952