1#!/usr/bin/env python 2# BAREOS - Backup Archiving REcovery Open Sourced 3# 4# Copyright (C) 2018-2021 Bareos GmbH & Co. KG 5# 6# This program is Free Software; you can redistribute it and/or 7# modify it under the terms of version three of the GNU Affero General Public 8# License as published by the Free Software Foundation and included 9# in the file LICENSE. 10# 11# This program is distributed in the hope that it will be useful, but 12# WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14# Affero General Public License for more details. 15# 16# You should have received a copy of the GNU Affero General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19# 02110-1301, USA. 20 21 22# Bareos python class for Ovirt related backup and restore 23 24import bareosfd 25 26import os 27import io 28import time 29import uuid 30import json 31 32try: 33 from configparser import ConfigParser 34except ImportError: 35 from ConfigParser import SafeConfigParser as ConfigParser 36 37 38import xml.etree.ElementTree 39 40 41import ssl 42 43from sys import version_info 44 45try: 46 from httplib import HTTPSConnection 47except ImportError: 48 from http.client import HTTPSConnection 49 50try: 51 from urllib.parse import urlparse 52except ImportError: 53 from urlparse import urlparse 54 55import BareosFdPluginBaseclass 56 57import logging 58 59SDK_IMPORT_ERROR = False 60try: 61 import ovirtsdk4 as sdk 62 import ovirtsdk4.types as types 63except ImportError: 64 SDK_IMPORT_ERROR = True 65 66# The name of the application, to be used as the 'origin' of events 67# sent to the audit log: 68APPLICATION_NAME = "Bareos oVirt plugin" 69 70 71class BareosFdPluginOvirt(BareosFdPluginBaseclass.BareosFdPluginBaseclass): 72 """ 73 Plugin for oVirt backup and restore 74 """ 75 76 def __init__(self, plugindef): 77 bareosfd.DebugMessage( 78 100, 79 "Constructor called in module %s with plugindef=%s\n" 80 % (__name__, plugindef), 81 ) 82 bareosfd.DebugMessage( 83 100, 84 "Python Version: %s.%s.%s\n" 85 % (version_info.major, version_info.minor, version_info.micro), 86 ) 87 super(BareosFdPluginOvirt, self).__init__(plugindef) 88 89 if self.mandatory_options is None: 90 self.mandatory_options = [] 91 92 self.config = None 93 94 self.ovirt = BareosOvirtWrapper() 95 96 def parse_plugin_definition(self, plugindef): 97 """ 98 Parses the plugin arguments 99 """ 100 super(BareosFdPluginOvirt, self).parse_plugin_definition(plugindef) 101 102 bareosfd.DebugMessage( 103 100, 104 "BareosFdPluginOvirt:parse_plugin_definition() called with options '%s' \n" 105 % str(self.options), 106 ) 107 108 # if the option config_file is present, parse the given file 109 config_file = self.options.get("config_file") 110 if config_file: 111 if not self.parse_config_file(): 112 return bareosfd.bRC_Error 113 114 self.ovirt.set_options(self.options) 115 return bareosfd.bRC_OK 116 117 def check_options(self, mandatory_options=None): 118 """ 119 Check Plugin options 120 Note: this is called by parent class parse_plugin_definition(). 121 This plugin does not yet implement any checks for plugin options, 122 but it's used to report if the Python oVirt SDK is not installed. 123 If it is required to know if it is a backup or a restore, it 124 may make more sense to invoke the options checking from 125 start_backup_job() and start_restore_job() 126 """ 127 128 if SDK_IMPORT_ERROR: 129 bareosfd.JobMessage( 130 bareosfd.M_FATAL, 131 "Please install the Python SDK for oVirt Engine API.\n", 132 ) 133 return bareosfd.bRC_Error 134 135 return bareosfd.bRC_OK 136 137 def start_backup_job(self): 138 """ 139 Start of Backup Job. Called just before backup job really start. 140 Overload this to arrange whatever you have to do at this time. 141 """ 142 bareosfd.DebugMessage(100, "BareosFdPluginOvirt:start_backup_job() called\n") 143 144 bareosfd.JobMessage( 145 bareosfd.M_INFO, 146 "Using oVirt SDK Version %s\n" % sdk.version.VERSION, 147 ) 148 149 if chr(self.level) != "F": 150 bareosfd.JobMessage( 151 bareosfd.M_FATAL, 152 "BareosFdPluginOvirt can only perform level F (Full) backups, but level is %s\n" 153 % (chr(self.level)), 154 ) 155 return bareosfd.bRC_Error 156 157 if not self.ovirt.connect_api(): 158 return bareosfd.bRC_Error 159 160 return self.ovirt.prepare_vm_backup() 161 162 def start_backup_file(self, savepkt): 163 """ 164 Defines the file to backup and creates the savepkt. 165 """ 166 bareosfd.DebugMessage(100, "BareosFdPluginOvirt:start_backup_file() called\n") 167 168 if not self.ovirt.backup_objects: 169 bareosfd.JobMessage(bareosfd.M_ERROR, "Nothing to backup.\n") 170 self.backup_obj = None 171 return bareosfd.bRC_Skip 172 173 self.backup_obj = self.ovirt.backup_objects.pop(0) 174 175 # create a stat packet for a restore object 176 statp = bareosfd.StatPacket() 177 savepkt.statp = statp 178 179 if "file" in self.backup_obj: 180 181 # regular file 182 vmfile = self.backup_obj["file"] 183 184 bareosfd.DebugMessage( 185 100, 186 "BareosFdPluginOvirt:start_backup_file() backup regular file '%s' of VM '%s'\n" 187 % (vmfile["filename"], self.backup_obj["vmname"]), 188 ) 189 190 savepkt.type = bareosfd.FT_REG 191 savepkt.fname = "/VMS/%s-%s/%s" % ( 192 self.backup_obj["vmname"], 193 self.backup_obj["vmid"], 194 vmfile["filename"], 195 ) 196 self.backup_obj["file"]["fh"] = io.BytesIO( 197 StringCodec.encode_ovf_data(vmfile["data"]) 198 ) 199 200 elif "disk" in self.backup_obj: 201 202 # disk file 203 disk = self.backup_obj["disk"] 204 # snapshot 205 snapshot = self.backup_obj["snapshot"] 206 207 bareosfd.DebugMessage( 208 100, 209 "BareosFdPluginOvirt:start_backup_file() backup disk file '%s'('%s'/'%s') of VM '%s'\n" 210 % (disk.alias, disk.id, snapshot.id, self.backup_obj["vmname"]), 211 ) 212 213 savepkt.type = bareosfd.FT_REG 214 savepkt.fname = "/VMS/%s-%s/%s-%s/%s" % ( 215 self.backup_obj["vmname"], 216 self.backup_obj["vmid"], 217 disk.alias, 218 disk.id, 219 snapshot.id, 220 ) 221 222 try: 223 self.ovirt.start_download(snapshot, disk) 224 except Exception as e: 225 bareosfd.JobMessage( 226 bareosfd.M_ERROR, 227 "BareosFdPluginOvirt:start_backup_file() Error: %s\n" % str(e), 228 ) 229 self.ovirt.end_transfer() 230 return bareosfd.bRC_Error 231 232 elif "disk_metadata" in self.backup_obj: 233 # save disk metadata as restoreobject 234 235 disk_alias = self.backup_obj["disk_metadata"]["alias"] 236 disk_id = self.backup_obj["disk_metadata"]["id"] 237 snapshot_id = self.backup_obj["snapshot_id"] 238 disk_metadata_json = json.dumps( 239 {"disk_metadata": self.backup_obj["disk_metadata"]} 240 ) 241 242 bareosfd.DebugMessage( 243 100, 244 "BareosFdPluginOvirt:start_backup_file() backup disk metadata '%s'('%s'/'%s') of VM '%s': %s\n" 245 % ( 246 disk_alias, 247 disk_id, 248 snapshot_id, 249 self.backup_obj["vmname"], 250 disk_metadata_json, 251 ), 252 ) 253 254 savepkt.type = bareosfd.FT_RESTORE_FIRST 255 savepkt.fname = "/VMS/%s-%s/%s-%s/%s.metadata" % ( 256 self.backup_obj["vmname"], 257 self.backup_obj["vmid"], 258 disk_alias, 259 disk_id, 260 snapshot_id, 261 ) 262 savepkt.object_name = savepkt.fname 263 savepkt.object = bytearray(disk_metadata_json, "utf-8") 264 savepkt.object_len = len(savepkt.object) 265 savepkt.object_index = int(time.time()) 266 267 else: 268 bareosfd.JobMessage( 269 bareosfd.M_FATAL, 270 "BareosFdPluginOvirt:start_backup_file(): Invalid data in backup_obj, keys: %s\n" 271 % (self.backup_obj.keys()), 272 ) 273 return bareosfd.bRC_Error 274 275 bareosfd.JobMessage( 276 bareosfd.M_INFO, 277 "Starting backup of %s\n" % savepkt.fname, 278 ) 279 280 return bareosfd.bRC_OK 281 282 def create_file(self, restorepkt): 283 """ 284 Creates the file to be restored and directory structure, if needed. 285 Adapt this in your derived class, if you need modifications for 286 virtual files or similar 287 """ 288 bareosfd.DebugMessage( 289 100, 290 "create_file() entry point in Python called with %s\n" % (restorepkt), 291 ) 292 bareosfd.DebugMessage( 293 200, 294 "create_file() Type of restorepkt.ofname: %s\n" % type(restorepkt.ofname), 295 ) 296 297 # Process includes/excludes for restore to oVirt. Note that it is more 298 # efficient to mark only the disks to restore, as skipping here can not 299 # prevent from receiving the data from bareos-sd which is useless for 300 # excluded disks. 301 # 302 # When restoring locally, all disks will be restored without filtering. 303 if ( 304 not restorepkt.ofname.endswith(".ovf") 305 and not self.options.get("local") == "yes" 306 ): 307 disk_alias = self.ovirt.get_ovf_disk_alias_by_basename( 308 StringCodec.decode_fname(restorepkt.ofname) 309 ) 310 disk_alias = StringCodec.encode_disk_alias(disk_alias) 311 bareosfd.DebugMessage( 312 200, 313 "create_file() disk_alias: %s (%s)\n" 314 % (repr(disk_alias), type(disk_alias)), 315 ) 316 317 if not self.ovirt.is_disk_alias_included(disk_alias): 318 bareosfd.DebugMessage( 319 200, 320 "create_file() Skipping disk_alias: %s (is not included)\n" 321 % (repr(disk_alias)), 322 ) 323 restorepkt.create_status = bareosfd.CF_SKIP 324 return bareosfd.bRC_OK 325 326 if self.ovirt.is_disk_alias_excluded(disk_alias): 327 bareosfd.DebugMessage( 328 200, 329 "create_file() Skipping disk_alias: %s (is exluded)\n" 330 % (repr(disk_alias)), 331 ) 332 restorepkt.create_status = bareosfd.CF_SKIP 333 return bareosfd.bRC_OK 334 335 if self.options.get("local") == "yes": 336 FNAME = restorepkt.ofname 337 dirname = os.path.dirname(FNAME) 338 if not os.path.exists(dirname): 339 bareosfd.DebugMessage( 340 200, 341 "Directory %s does not exist, creating it now\n" % dirname, 342 ) 343 os.makedirs(dirname) 344 # open creates the file, if not yet existing, we close it again right 345 # always it will be opened again in plugin_io. 346 # But: only do this for regular files, prevent from 347 # IOError: (21, 'Is a directory', '/tmp/bareos-restores/my/dir/') 348 # if it's a directory 349 if restorepkt.type == bareosfd.FT_REG: 350 open(FNAME, "wb").close() 351 else: 352 if not restorepkt.ofname.endswith(".ovf"): 353 FNAME = StringCodec.decode_fname(restorepkt.ofname) 354 355 disk = self.ovirt.get_vm_disk_by_basename(FNAME) 356 if disk is None: 357 bareosfd.JobMessage( 358 bareosfd.M_ERROR, 359 "BareosFdPluginOvirt:create_file() Unable to restore disk %s.\n" 360 % (FNAME), 361 ) 362 return bareosfd.bRC_Error 363 else: 364 self.ovirt.start_upload(disk) 365 366 if restorepkt.type == bareosfd.FT_REG: 367 restorepkt.create_status = bareosfd.CF_EXTRACT 368 return bareosfd.bRC_OK 369 370 def start_restore_job(self): 371 """ 372 Start of Restore Job. Called , if you have Restore objects. 373 Overload this to handle restore objects, if applicable 374 """ 375 bareosfd.DebugMessage(100, "BareosFdPluginOvirt:start_restore_job() called\n") 376 377 bareosfd.JobMessage( 378 bareosfd.M_INFO, 379 "Using oVirt SDK Version %s\n" % sdk.version.VERSION, 380 ) 381 382 if self.options.get("local") == "yes": 383 bareosfd.DebugMessage( 384 100, 385 "BareosFdPluginOvirt:start_restore_job(): restore to local file, skipping checks\n", 386 ) 387 return bareosfd.bRC_OK 388 else: 389 # restore to VM to OVirt 390 if not self.ovirt.connect_api(): 391 return bareosfd.bRC_Error 392 393 return bareosfd.bRC_OK 394 395 def start_restore_file(self, cmd): 396 bareosfd.DebugMessage( 397 100, 398 "BareosFdPluginOvirt:start_restore_file() called with %s\n" % (cmd), 399 ) 400 return bareosfd.bRC_OK 401 402 def plugin_io(self, IOP): 403 """ 404 Called for io operations. 405 """ 406 bareosfd.DebugMessage(100, "plugin_io called with function %s\n" % (IOP.func)) 407 bareosfd.DebugMessage(100, "FNAME is set to %s\n" % (self.FNAME)) 408 self.jobType = chr(bareosfd.GetValue(bareosfd.bVarType)) 409 bareosfd.DebugMessage( 410 100, 411 "BareosFdPluginOvirt::plugin_io() jobType: %s\n" % (self.jobType), 412 ) 413 414 if IOP.func == bareosfd.IO_OPEN: 415 416 self.FNAME = IOP.fname 417 bareosfd.DebugMessage( 418 100, "self.FNAME was set to %s from IOP.fname\n" % (self.FNAME) 419 ) 420 bareosfd.DebugMessage(100, "type(self.FNAME): %s\n" % (type(self.FNAME))) 421 422 if self.options.get("local") == "yes": 423 try: 424 if IOP.flags & (os.O_CREAT | os.O_WRONLY): 425 bareosfd.DebugMessage( 426 100, 427 "Open file %s for writing with %s\n" % (self.FNAME, IOP), 428 ) 429 430 dirname = os.path.dirname(self.FNAME) 431 if not os.path.exists(dirname): 432 bareosfd.DebugMessage( 433 100, 434 "Directory %s does not exist, creating it now\n" 435 % (dirname), 436 ) 437 os.makedirs(dirname) 438 self.file = open(self.FNAME, "wb") 439 else: 440 IOP.status = -1 441 bareosfd.JobMessage( 442 bareosfd.M_FATAL, 443 "plugin_io: option local=yes can only be used on restore\n", 444 ) 445 return bareosfd.bRC_Error 446 except (OSError, IOError) as io_open_error: 447 IOP.status = -1 448 bareosfd.DebugMessage( 449 100, 450 "plugin_io: failed to open %s: %s\n" 451 % (self.FNAME, io_open_error.strerror), 452 ) 453 bareosfd.JobMessage( 454 bareosfd.M_FATAL, 455 "plugin_io: failed to open %s: %s\n" 456 % (self.FNAME, io_open_error.strerror), 457 ) 458 return bareosfd.bRC_Error 459 460 return bareosfd.bRC_OK 461 462 elif IOP.func == bareosfd.IO_CLOSE: 463 if self.file is not None: 464 bareosfd.DebugMessage(100, "Closing file " + "\n") 465 self.file.close() 466 elif self.jobType == "B": 467 if "file" in self.backup_obj: 468 self.backup_obj["file"]["fh"].close() 469 elif "disk" in self.backup_obj: 470 bareosfd.DebugMessage(100, "plugin_io: calling end_transfer()\n") 471 # Backup Job 472 self.ovirt.end_transfer() 473 elif self.jobType == "R": 474 if self.FNAME.endswith(".ovf"): 475 return self.ovirt.prepare_vm_restore() 476 else: 477 self.ovirt.end_transfer() 478 return bareosfd.bRC_OK 479 elif IOP.func == bareosfd.IO_SEEK: 480 return bareosfd.bRC_OK 481 elif IOP.func == bareosfd.IO_READ: 482 if "file" in self.backup_obj: 483 IOP.buf = bytearray(IOP.count) 484 IOP.status = self.backup_obj["file"]["fh"].readinto(IOP.buf) 485 IOP.io_errno = 0 486 bareosfd.DebugMessage( 487 100, 488 "plugin_io: read from file %s\n" 489 % (self.backup_obj["file"]["filename"]), 490 ) 491 elif "disk" in self.backup_obj: 492 bareosfd.DebugMessage( 493 100, 494 "plugin_io: read from disk %s\n" % (self.backup_obj["disk"].alias), 495 ) 496 try: 497 IOP.buf = bytearray(IOP.count) 498 chunk = self.ovirt.process_download(IOP.count) 499 IOP.buf[:] = chunk 500 IOP.status = len(IOP.buf) 501 IOP.io_errno = 0 502 except Exception as e: 503 bareosfd.JobMessage( 504 bareosfd.M_ERROR, 505 "BareosFdPluginOvirt:plugin_io() Error: %s\n" % str(e), 506 ) 507 self.ovirt.end_transfer() 508 return bareosfd.bRC_Error 509 else: 510 bareosfd.JobMessage( 511 bareosfd.M_ERROR, 512 "BareosFdPluginOvirt:plugin_io() Unable to read data to backup.", 513 ) 514 return bareosfd.bRC_Error 515 return bareosfd.bRC_OK 516 elif IOP.func == bareosfd.IO_WRITE: 517 if self.file is not None: 518 try: 519 bareosfd.DebugMessage( 520 200, "Writing buffer to file %s\n" % (self.FNAME) 521 ) 522 self.file.write(IOP.buf) 523 IOP.status = IOP.count 524 IOP.io_errno = 0 525 except IOError as msg: 526 IOP.io_errno = -1 527 bareosfd.DebugMessage(100, "Error writing data: " + msg + "\n") 528 return bareosfd.bRC_Error 529 elif self.FNAME.endswith(".ovf"): 530 self.ovirt.process_ovf(IOP.buf) 531 IOP.status = IOP.count 532 IOP.io_errno = 0 533 else: 534 self.ovirt.process_upload(IOP.buf) 535 IOP.status = IOP.count 536 IOP.io_errno = 0 537 return bareosfd.bRC_OK 538 539 def handle_plugin_event(self, event): 540 541 if event == bareosfd.bEventEndBackupJob: 542 bareosfd.DebugMessage( 543 100, 544 "BareosFdPluginOvirt::handle_plugin_event() called with bareosfd.bEventEndBackupJob\n", 545 ) 546 bareosfd.DebugMessage(100, "removing Snapshot\n") 547 self.ovirt.end_vm_backup() 548 549 elif event == bareosfd.bEventEndFileSet: 550 bareosfd.DebugMessage( 551 100, 552 "BareosFdPluginOvirt::handle_plugin_event() called with bareosfd.bEventEndFileSet\n", 553 ) 554 555 elif event == bareosfd.bEventStartBackupJob: 556 bareosfd.DebugMessage( 557 100, 558 "BareosFdPluginOvirt::handle_plugin_event() called with bareosfd.bEventStartBackupJob\n", 559 ) 560 561 return self.start_backup_job() 562 563 elif event == bareosfd.bEventStartRestoreJob: 564 bareosfd.DebugMessage( 565 100, 566 "BareosFdPluginOvirt::handle_plugin_event() called with bareosfd.bEventStartRestoreJob\n", 567 ) 568 569 return self.start_restore_job() 570 571 elif event == bareosfd.bEventEndRestoreJob: 572 bareosfd.DebugMessage( 573 100, 574 "BareosFdPluginOvirt::handle_plugin_event() called with bareosfd.bEventEndRestoreJob\n", 575 ) 576 bareosfd.DebugMessage(100, "removing Snapshot\n") 577 self.ovirt.end_vm_restore() 578 else: 579 bareosfd.DebugMessage( 580 100, 581 "BareosFdPluginOvirt::handle_plugin_event() called with event %s\n" 582 % (event), 583 ) 584 585 return bareosfd.bRC_OK 586 587 def end_backup_file(self): 588 bareosfd.DebugMessage( 589 100, 590 "BareosFdPluginOvirt::end_backup_file() entry point in Python called\n", 591 ) 592 if self.ovirt.transfer_start_time: 593 elapsed_seconds = round(time.time() - self.ovirt.transfer_start_time, 2) 594 download_rate = round( 595 self.ovirt.init_bytes_to_transf / 1000.0 / elapsed_seconds, 1 596 ) 597 bareosfd.JobMessage( 598 bareosfd.M_INFO, 599 " Transfer time: %s s bytes: %s rate: %s KB/s\n" 600 % (elapsed_seconds, self.ovirt.init_bytes_to_transf, download_rate), 601 ) 602 self.ovirt.transfer_start_time = None 603 604 if self.ovirt.backup_objects: 605 return bareosfd.bRC_More 606 else: 607 return bareosfd.bRC_OK 608 609 def end_restore_file(self): 610 bareosfd.DebugMessage( 611 100, 612 "BareosFdPluginOvirt::end_restore_file() entry point in Python called\n", 613 ) 614 if self.ovirt.transfer_start_time: 615 elapsed_seconds = round(time.time() - self.ovirt.transfer_start_time, 2) 616 download_rate = round( 617 self.ovirt.init_bytes_to_transf / 1000.0 / elapsed_seconds, 1 618 ) 619 bareosfd.JobMessage( 620 bareosfd.M_INFO, 621 " Upload time: %s s bytes: %s rate: %s KB/s\n" 622 % (elapsed_seconds, self.ovirt.init_bytes_to_transf, download_rate), 623 ) 624 self.ovirt.transfer_start_time = None 625 return bareosfd.bRC_OK 626 627 def restore_object_data(self, ROP): 628 """ 629 Note: 630 This is called in two cases: 631 - on diff/inc backup (should be called only once) 632 - on restore (for every job id being restored) 633 But at the point in time called, it is not possible 634 to distinguish which of them it is, because job type 635 is "I" until the bareosfd.bEventStartBackupJob event 636 """ 637 bareosfd.DebugMessage( 638 100, 639 "BareosFdPluginOvirt:restore_object_data() called with ROP:%s\n" % (ROP), 640 ) 641 bareosfd.DebugMessage( 642 100, 643 "ROP.object_name(%s): %s\n" % (type(ROP.object_name), ROP.object_name), 644 ) 645 bareosfd.DebugMessage( 646 100, 647 "ROP.plugin_name(%s): %s\n" % (type(ROP.plugin_name), ROP.plugin_name), 648 ) 649 bareosfd.DebugMessage( 650 100, 651 "ROP.object_len(%s): %s\n" % (type(ROP.object_len), ROP.object_len), 652 ) 653 bareosfd.DebugMessage( 654 100, 655 "ROP.object_full_len(%s): %s\n" 656 % (type(ROP.object_full_len), ROP.object_full_len), 657 ) 658 bareosfd.DebugMessage( 659 100, "ROP.object(%s): %s\n" % (type(ROP.object), ROP.object) 660 ) 661 ro_data = json.loads(str(StringCodec.decode_object_data(ROP.object))) 662 self.ovirt.disk_metadata_by_id[ro_data["disk_metadata"]["id"]] = ro_data[ 663 "disk_metadata" 664 ] 665 666 return bareosfd.bRC_OK 667 668 def parse_config_file(self): 669 """ 670 Parse the config file given in the config_file plugin option 671 """ 672 bareosfd.DebugMessage( 673 100, 674 "BareosFdPluginOvirt: parse_config_file(): parse %s\n" 675 % (self.options["config_file"]), 676 ) 677 678 self.config = ConfigParser() 679 680 try: 681 self.config.readfp(open(self.options["config_file"])) 682 except IOError as err: 683 bareosfd.JobMessage( 684 bareosfd.M_FATAL, 685 "BareosFdPluginOvirt: Error reading config file %s: %s\n" 686 % (self.options["config_file"], err.strerror), 687 ) 688 return False 689 690 return self.check_config() 691 692 def check_config(self): 693 """ 694 Check the configuration and set or override options if necessary, 695 considering mandatory: username and password in the [credentials] section 696 """ 697 bareosfd.DebugMessage(100, "BareosFdPluginOvirt: check_config()\n") 698 mandatory_sections = ["credentials"] 699 mandatory_options = {} 700 mandatory_options["credentials"] = ["username", "password"] 701 702 for section in mandatory_sections: 703 if not self.config.has_section(section): 704 bareosfd.JobMessage( 705 bareosfd.M_FATAL, 706 "BareosFdPluginOvirt: Section [%s] missing in config file %s\n" 707 % (section, self.options["config_file"]), 708 ) 709 return False 710 711 for option in mandatory_options[section]: 712 if not self.config.has_option(section, option): 713 bareosfd.JobMessage( 714 bareosfd.M_FATAL, 715 "BareosFdPluginOvirt: Options %s missing in Section [%s] in config file %s\n" 716 % (option, section, self.options["config_file"]), 717 ) 718 return False 719 720 plugin_option = self.options.get(option) 721 if plugin_option: 722 bareosfd.JobMessage( 723 bareosfd.M_WARNING, 724 "BareosFdPluginOvirt: Overriding plugin option %s from config file %s\n" 725 % (option, self.options["config_file"]), 726 ) 727 728 self.options[option] = self.config.get(section, option) 729 730 return True 731 732 733class BareosOvirtWrapper(object): 734 """ 735 Ovirt wrapper class 736 """ 737 738 def __init__(self): 739 740 self.options = None 741 self.ca = None 742 743 self.connection = None 744 self.system_service = None 745 self.vms_service = None 746 self.storage_domains_service = None 747 self.events_service = None 748 self.network_profiles = None 749 750 self.vm = None 751 self.ovf_data = None 752 self.ovf = None 753 self.vm_service = None 754 self.snap_service = None 755 self.transfer_service = None 756 self.event_id = None 757 758 self.proxy_connection = None 759 self.bytes_to_transf = None 760 self.init_bytes_to_transf = None 761 self.transfer_start_time = None 762 763 self.backup_objects = None 764 self.restore_objects = None 765 self.ovf_disks_by_alias_and_fileref = {} 766 self.disk_metadata_by_id = {} 767 self.all_disks_excluded = False 768 769 self.old_new_ids = {} 770 771 self.snapshot_remove_timeout = 300 772 773 def set_options(self, options): 774 # make the plugin options also available in this class 775 self.options = options 776 777 def connect_api(self): 778 779 # ca certificate 780 self.ca = self.options["ca"] 781 782 # set server url 783 server_url = "https://%s/ovirt-engine/api" % str(self.options["server"]) 784 785 ovirt_sdk_debug = False 786 if self.options.get("ovirt_sdk_debug_log") is not None: 787 logging.basicConfig( 788 level=logging.DEBUG, filename=self.options["ovirt_sdk_debug_log"] 789 ) 790 ovirt_sdk_debug = True 791 792 # Create a connection to the server: 793 self.connection = sdk.Connection( 794 url=server_url, 795 username=self.options["username"], 796 password=self.options["password"], 797 ca_file=self.ca, 798 debug=ovirt_sdk_debug, 799 log=logging.getLogger(), 800 ) 801 802 if not self.connection: 803 bareosfd.JobMessage( 804 bareosfd.M_FATAL, 805 "Cannot connect to host %s with user %s and ca file %s\n" 806 % (self.options["server"], self.options["username"], self.ca), 807 ) 808 return False 809 810 # Get a reference to the root service: 811 self.system_service = self.connection.system_service() 812 813 # Get the reference to the "vms" service: 814 self.vms_service = self.system_service.vms_service() 815 816 if not self.vms_service: 817 return False 818 819 # get network profiles 820 profiles_service = self.system_service.vnic_profiles_service() 821 self.network_profiles = { 822 profile.name: profile.id for profile in profiles_service.list() 823 } 824 825 # Get the reference to the service that we will use to send events to 826 # the audit log: 827 self.events_service = self.system_service.events_service() 828 829 # In order to send events we need to also send unique integer ids. 830 self.event_id = int(time.time()) 831 832 bareosfd.DebugMessage( 833 100, 834 ( 835 "Successfully connected to Ovirt Engine API on '%s' with" 836 " user %s and ca file %s\n" 837 ) 838 % (self.options["server"], self.options["username"], self.ca), 839 ) 840 841 return True 842 843 def prepare_vm_backup(self): 844 """ 845 prepare VM backup: 846 - get vm details 847 - take snapshot 848 - get disk devices 849 """ 850 bareosfd.DebugMessage( 851 100, 852 "BareosOvirtWrapper::prepare_vm_backup: prepare VM to backup\n", 853 ) 854 855 if not self.get_vm(): 856 bareosfd.DebugMessage(100, "Error getting details for VM\n") 857 858 return bareosfd.bRC_Error 859 else: 860 # Locate the service that manages the virtual machine: 861 self.vm_service = self.vms_service.vm_service(self.vm.id) 862 863 # check if vm have snapshots 864 snaps_service = self.vm_service.snapshots_service() 865 if len(snaps_service.list()) > 1: 866 bareosfd.JobMessage( 867 bareosfd.M_FATAL, 868 "Error '%s' already has %d snapshots. This is not supported\n" 869 % (self.vm.name, len(snaps_service.list()) - 1), 870 ) 871 return bareosfd.bRC_Error 872 873 bareosfd.DebugMessage(100, "Start the backup of VM %s\n" % (self.vm.name)) 874 875 # Save the OVF to a file, so that we can use to restore the virtual 876 # machine later. The name of the file is the name of the virtual 877 # machine, followed by a dash and the identifier of the virtual machine, 878 # to make it unique: 879 ovf_data = self.vm.initialization.configuration.data 880 ovf_file = "%s-%s.ovf" % (self.vm.name, self.vm.id) 881 if self.backup_objects is None: 882 self.backup_objects = [] 883 884 self.backup_objects.append( 885 { 886 "vmname": self.vm.name, 887 "vmid": self.vm.id, 888 "file": {"data": ovf_data, "filename": ovf_file}, 889 } 890 ) 891 892 # create vm snapshots 893 self.create_vm_snapshot() 894 895 # get vm backup disks from snapshot 896 if not self.all_disks_excluded and not self.get_vm_backup_disks(): 897 bareosfd.JobMessage( 898 bareosfd.M_FATAL, 899 "Error getting Backup Disks VM %s from snapshot\n" % (self.vm.name), 900 ) 901 return bareosfd.bRC_Error 902 903 return bareosfd.bRC_OK 904 905 def get_vm(self): 906 search = None 907 if "uuid" in self.options: 908 search = "id=%s" % str(self.options["uuid"]) 909 elif "vm_name" in self.options: 910 search = "name=%s" % str(self.options["vm_name"]) 911 912 if search is not None: 913 bareosfd.DebugMessage(100, "get_vm search vm by '%s'\n" % (search)) 914 res = self.vms_service.list(search=search, all_content=True) 915 if len(res) > 0: 916 self.vm = res[0] 917 return True 918 return False 919 920 def create_vm_snapshot(self): 921 """ 922 Creates a snapshot 923 """ 924 925 # Create an unique description for the snapshot, so that it is easier 926 # for the administrator to identify this snapshot as a temporary one 927 # created just for backup purposes: 928 snap_description = "%s-backup-%s" % (self.vm.name, uuid.uuid4()) 929 930 # Send an external event to indicate to the administrator that the 931 # backup of the virtual machine is starting. Note that the description 932 # of the event contains the name of the virtual machine and the name of 933 # the temporary snapshot, this way, if something fails, the administrator 934 # will know what snapshot was used and remove it manually. 935 self.events_service.add( 936 event=types.Event( 937 vm=types.Vm(id=self.vm.id), 938 origin=APPLICATION_NAME, 939 severity=types.LogSeverity.NORMAL, 940 custom_id=self.event_id, 941 description=( 942 "Backup of virtual machine '%s' using snapshot '%s' is " 943 "starting." % (self.vm.name, snap_description) 944 ), 945 ), 946 ) 947 self.event_id += 1 948 949 # Send the request to create the snapshot. Note that this will return 950 # before the snapshot is completely created, so we will later need to 951 # wait till the snapshot is completely created. 952 # The snapshot will not include memory. Change to True the parameter 953 # persist_memorystate to get it (in that case the VM will be paused for a while). 954 snaps_service = self.vm_service.snapshots_service() 955 snap = snaps_service.add( 956 snapshot=types.Snapshot( 957 description=snap_description, 958 persist_memorystate=False, 959 disk_attachments=self.get_vm_disks_for_snapshot(), 960 ), 961 ) 962 bareosfd.JobMessage( 963 bareosfd.M_INFO, 964 "Sent request to create snapshot '%s', the id is '%s'.\n" 965 % (snap.description, snap.id), 966 ) 967 968 # Poll and wait till the status of the snapshot is 'ok', which means 969 # that it is completely created: 970 self.snap_service = snaps_service.snapshot_service(snap.id) 971 while snap.snapshot_status != types.SnapshotStatus.OK: 972 bareosfd.DebugMessage( 973 100, 974 "Waiting till the snapshot is created, the status is now '%s'.\n" 975 % snap.snapshot_status, 976 ) 977 time.sleep(1) 978 snap = self.snap_service.get() 979 time.sleep(1) 980 981 # wait some time until snapshot real complete 982 time.sleep(10) 983 984 bareosfd.JobMessage(bareosfd.M_INFO, "' The snapshot is now complete.\n") 985 986 def get_vm_disks_for_snapshot(self): 987 # return list of disks for snapshot, process include/exclude if given 988 disk_attachments_service = self.vm_service.disk_attachments_service() 989 disk_attachments = disk_attachments_service.list() 990 included_disk_ids = [] 991 included_disks = None 992 for disk_attachment in disk_attachments: 993 disk = self.connection.follow_link(disk_attachment.disk) 994 995 if not self.is_disk_alias_included(disk.alias): 996 continue 997 if self.is_disk_alias_excluded(disk.alias): 998 continue 999 1000 included_disk_ids.append(disk.id) 1001 1002 if len(included_disk_ids) == 0: 1003 # No disks to backup, snapshot will only contain VM config 1004 # Note: The comma must not be omitted here: 1005 # included_disks = (types.DiskAttachment(),) 1006 self.all_disks_excluded = True 1007 bareosfd.JobMessage( 1008 bareosfd.M_INFO, 1009 "All disks excluded, only backing up VM configuration.\n", 1010 ) 1011 included_disks = [types.DiskAttachment()] 1012 1013 else: 1014 included_disks = [ 1015 types.DiskAttachment(disk=types.Disk(id=disk_id)) 1016 for disk_id in included_disk_ids 1017 ] 1018 1019 return included_disks 1020 1021 def get_vm_backup_disks(self): 1022 1023 has_disks = False 1024 1025 if self.backup_objects is None: 1026 self.backup_objects = [] 1027 1028 # get snapshot 1029 self.snap_service.get() 1030 1031 # Get a reference to the storage domains service: 1032 storage_domains_service = self.system_service.storage_domains_service() 1033 1034 # Retrieve the descriptions of the disks of the snapshot: 1035 snap_disks_service = self.snap_service.disks_service() 1036 snap_disks = snap_disks_service.list() 1037 1038 # download disk snapshot 1039 for snap_disk in snap_disks: 1040 disk_id = snap_disk.id 1041 disk_alias = snap_disk.alias 1042 bareosfd.DebugMessage( 1043 200, 1044 "get_vm_backup_disks(): disk_id: '%s' disk_alias '%s'\n" 1045 % (disk_id, disk_alias), 1046 ) 1047 1048 sd_id = snap_disk.storage_domains[0].id 1049 1050 # Get a reference to the storage domain service in which the disk snapshots reside: 1051 storage_domain_service = storage_domains_service.storage_domain_service( 1052 sd_id 1053 ) 1054 1055 # Get a reference to the disk snapshots service: 1056 disk_snapshot_service = storage_domain_service.disk_snapshots_service() 1057 1058 # Get a list of disk snapshots by a disk ID 1059 all_disk_snapshots = disk_snapshot_service.list() 1060 1061 # Filter disk snapshots list by disk id 1062 disk_snapshots = [s for s in all_disk_snapshots if s.disk.id == disk_id] 1063 1064 # Download disk snapshot 1065 if len(disk_snapshots) > 0: 1066 for disk_snapshot in disk_snapshots: 1067 self.backup_objects.append( 1068 { 1069 "vmname": self.vm.name, 1070 "vmid": self.vm.id, 1071 "snapshot": disk_snapshot, 1072 "disk": snap_disk, 1073 } 1074 ) 1075 has_disks = True 1076 return has_disks 1077 1078 def is_disk_alias_included(self, disk_alias): 1079 if self.options.get("include_disk_aliases") is None: 1080 return True 1081 1082 include_disk_aliases = self.options["include_disk_aliases"].split(",") 1083 bareosfd.DebugMessage( 1084 200, 1085 "is_disk_alias_included() disk_alias: %s include_disk_aliases: %s\n" 1086 % (repr(disk_alias), repr(include_disk_aliases)), 1087 ) 1088 if disk_alias in include_disk_aliases: 1089 bareosfd.JobMessage( 1090 bareosfd.M_INFO, 1091 "Including disk with alias %s\n" % (disk_alias), 1092 ) 1093 return True 1094 1095 return False 1096 1097 def is_disk_alias_excluded(self, disk_alias): 1098 if self.options.get("exclude_disk_aliases") is None: 1099 return False 1100 1101 exclude_disk_aliases = self.options["exclude_disk_aliases"].split(",") 1102 1103 if "*" in exclude_disk_aliases or disk_alias in exclude_disk_aliases: 1104 bareosfd.JobMessage( 1105 bareosfd.M_INFO, 1106 "Excluding disk with alias %s\n" % (disk_alias), 1107 ) 1108 return True 1109 1110 return False 1111 1112 def get_transfer_service(self, image_transfer): 1113 # Get a reference to the service that manages the image transfer: 1114 transfers_service = self.system_service.image_transfers_service() 1115 1116 # Add a new image transfer: 1117 transfer = transfers_service.add(image_transfer) 1118 1119 # Get reference to the created transfer service: 1120 transfer_service = transfers_service.image_transfer_service(transfer.id) 1121 1122 while transfer.phase == types.ImageTransferPhase.INITIALIZING: 1123 time.sleep(1) 1124 transfer = transfer_service.get() 1125 1126 return transfer_service 1127 1128 def get_proxy_connection(self, proxy_url): 1129 # At this stage, the SDK granted the permission to start transferring the disk, and the 1130 # user should choose its preferred tool for doing it - regardless of the SDK. 1131 # In this example, we will use Python's httplib.HTTPSConnection for transferring the data. 1132 sslcontext = ssl.create_default_context() 1133 1134 # Note that ovirt-imageio-proxy by default checks the certificates, so if you don't have 1135 # your CA certificate of the engine in the system, you need to pass it to HTTPSConnection. 1136 sslcontext.load_verify_locations(cafile=self.ca) 1137 1138 return HTTPSConnection(proxy_url.hostname, proxy_url.port, context=sslcontext) 1139 1140 def start_download(self, snapshot, disk): 1141 1142 bareosfd.JobMessage( 1143 bareosfd.M_INFO, 1144 "Downloading snapshot '%s' of disk '%s'('%s')\n" 1145 % (snapshot.id, disk.alias, disk.id), 1146 ) 1147 1148 self.transfer_service = self.get_transfer_service( 1149 types.ImageTransfer( 1150 snapshot=types.DiskSnapshot(id=snapshot.id), 1151 direction=types.ImageTransferDirection.DOWNLOAD, 1152 ) 1153 ) 1154 transfer = self.transfer_service.get() 1155 proxy_url = urlparse(transfer.proxy_url) 1156 self.proxy_connection = self.get_proxy_connection(proxy_url) 1157 1158 self.proxy_connection.request("GET", proxy_url.path) 1159 1160 # Get response 1161 self.response = self.proxy_connection.getresponse() 1162 1163 # Check the response status: 1164 if self.response.status >= 300: 1165 bareosfd.JobMessage(bareosfd.M_ERROR, "Error: %s" % self.response.read()) 1166 1167 self.bytes_to_transf = int(self.response.getheader("Content-Length")) 1168 1169 self.backup_objects.insert( 1170 0, 1171 { 1172 "vmname": self.vm.name, 1173 "vmid": self.vm.id, 1174 "snapshot_id": snapshot.id, 1175 "disk_metadata": { 1176 "id": disk.id, 1177 "alias": disk.alias, 1178 "effective_size": self.bytes_to_transf, 1179 }, 1180 }, 1181 ) 1182 1183 bareosfd.JobMessage( 1184 bareosfd.M_INFO, 1185 " Transfer disk snapshot of %s bytes\n" % (str(self.bytes_to_transf)), 1186 ) 1187 1188 self.init_bytes_to_transf = self.bytes_to_transf 1189 self.transfer_start_time = time.time() 1190 1191 def process_download(self, chunk_size): 1192 1193 chunk = b"" 1194 1195 bareosfd.DebugMessage( 1196 100, 1197 "process_download(): transfer %s of %s (%s) \n" 1198 % ( 1199 self.bytes_to_transf, 1200 self.response.getheader("Content-Length"), 1201 chunk_size, 1202 ), 1203 ) 1204 1205 if self.bytes_to_transf > 0: 1206 1207 # Calculate next chunk to read 1208 to_read = min(self.bytes_to_transf, chunk_size) 1209 1210 # Read next info buffer 1211 chunk = self.response.read(to_read) 1212 1213 if chunk == "": 1214 bareosfd.DebugMessage( 1215 100, "process_download(): Socket disconnected. \n" 1216 ) 1217 bareosfd.JobMessage( 1218 bareosfd.M_ERROR, 1219 "process_download(): Socket disconnected.", 1220 ) 1221 1222 raise RuntimeError("Socket disconnected") 1223 1224 # Update bytes_to_transf 1225 self.bytes_to_transf -= len(chunk) 1226 1227 completed = 1 - ( 1228 self.bytes_to_transf / float(self.response.getheader("Content-Length")) 1229 ) 1230 1231 bareosfd.DebugMessage( 1232 100, "process_download(): Completed {:.0%}\n".format(completed) 1233 ) 1234 1235 return chunk 1236 1237 def prepare_vm_restore(self): 1238 restore_existing_vm = False 1239 if self.connection is None: 1240 # if not connected yet 1241 if not self.connect_api(): 1242 return bareosfd.bRC_Error 1243 1244 if self.ovf_data is None: 1245 bareosfd.JobMessage( 1246 bareosfd.M_FATAL, 1247 "Unable to restore VM. No OVF data. \n", 1248 ) 1249 return bareosfd.bRC_Error 1250 else: 1251 if "storage_domain" not in self.options: 1252 bareosfd.JobMessage( 1253 bareosfd.M_FATAL, 1254 "No storage domain specified.\n", 1255 ) 1256 return bareosfd.bRC_Error 1257 1258 storage_domain = self.options["storage_domain"] 1259 1260 # Parse the OVF as XML document: 1261 self.ovf = Ovf(self.ovf_data) 1262 1263 bareosfd.DebugMessage(200, "prepare_vm_restore(): %s\n" % (self.ovf)) 1264 1265 # Find the name of the virtual machine within the OVF: 1266 vm_name = None 1267 if "vm_name" in self.options: 1268 vm_name = self.options["vm_name"] 1269 1270 if vm_name is None: 1271 vm_name = self.ovf.get_element("vm_name") 1272 1273 # Find the cluster name of the virtual machine within the OVF: 1274 cluster_name = None 1275 if "cluster_name" in self.options: 1276 cluster_name = self.options["cluster_name"] 1277 1278 if cluster_name is None: 1279 # Find the cluster name of the virtual machine within the OVF: 1280 cluster_name = self.ovf.get_element("cluster_name") 1281 1282 # check if VM exists 1283 res = self.vms_service.list( 1284 search="name=%s and cluster=%s" % (str(vm_name), str(cluster_name)) 1285 ) 1286 if len(res) > 1: 1287 bareosfd.JobMessage( 1288 bareosfd.M_FATAL, 1289 "Found %s VMs with name '%s'\n" % (len(res), str(vm_name)), 1290 ) 1291 return bareosfd.bRC_Error 1292 1293 if len(res) == 1: 1294 if not self.options.get("overwrite") == "yes": 1295 bareosfd.JobMessage( 1296 bareosfd.M_FATAL, 1297 "If you are sure you want to overwrite the existing VM '%s', please add the plugin option 'overwrite=yes'\n" 1298 % (str(vm_name)), 1299 ) 1300 return bareosfd.bRC_Error 1301 1302 bareosfd.JobMessage( 1303 bareosfd.M_INFO, 1304 "Restore to existing VM '%s'\n" % str(vm_name), 1305 ) 1306 self.vm = res[0] 1307 1308 if self.vm.status != types.VmStatus.DOWN: 1309 bareosfd.JobMessage( 1310 bareosfd.M_FATAL, 1311 "VM '%s' must be down for restore, but status is %s\n" 1312 % (str(vm_name), self.vm.status), 1313 ) 1314 return bareosfd.bRC_Error 1315 1316 restore_existing_vm = True 1317 1318 else: 1319 self.create_vm(vm_name, cluster_name) 1320 self.add_nics_to_vm() 1321 1322 # Extract disk information from OVF 1323 disk_elements = self.ovf.get_elements("disk_elements") 1324 1325 if self.restore_objects is None: 1326 self.restore_objects = [] 1327 1328 for disk_element in disk_elements: 1329 # Get disk properties: 1330 props = {} 1331 for key, value in disk_element.items(): 1332 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["ovf"], "") 1333 props[key] = value 1334 1335 # set storage domain 1336 props["storage_domain"] = storage_domain 1337 self.restore_objects.append(props) 1338 1339 # used by get_ovf_disk_alias_by_basename(), needed to process 1340 # includes/excludes on restore 1341 self.ovf_disks_by_alias_and_fileref[ 1342 props["disk-alias"] + "-" + props["fileRef"] 1343 ] = props 1344 1345 if restore_existing_vm: 1346 # When restoring to existing VM, old and new disk ID are identical 1347 # Important: The diskId property in the OVF is not the oVirt 1348 # diskId, which can be found in the first part of the OVF fileRef 1349 old_disk_id = os.path.dirname(props["fileRef"]) 1350 self.old_new_ids[old_disk_id] = old_disk_id 1351 1352 bareosfd.DebugMessage( 1353 200, 1354 "end of prepare_vm_restore(): self.restore_objects: %s self.old_new_ids: %s\n" 1355 % (self.restore_objects, self.old_new_ids), 1356 ) 1357 1358 return bareosfd.bRC_OK 1359 1360 def create_vm(self, vm_name, cluster_name): 1361 1362 vm_template = "Blank" 1363 if "vm_template" in self.options: 1364 vm_template = self.options["vm_template"] 1365 1366 # Add the virtual machine, the transferred disks will be 1367 # attached to this virtual machine: 1368 bareosfd.JobMessage(bareosfd.M_INFO, "Adding virtual machine %s\n" % vm_name) 1369 1370 # vm type (server/desktop) 1371 vm_type = "server" 1372 if "vm_type" in self.options: 1373 vm_type = self.options["vm_type"] 1374 1375 # vm memory and cpu 1376 vm_memory = None 1377 if "vm_memory" in self.options: 1378 vm_memory = int(self.options["vm_memory"]) * 2 ** 20 1379 1380 vm_cpu = None 1381 if "vm_cpu" in self.options: 1382 vm_cpu_arr = self.options["vm_cpu"].split(",") 1383 if len(vm_cpu_arr) > 0: 1384 if len(vm_cpu_arr) < 2: 1385 vm_cpu_arr.append(1) 1386 vm_cpu_arr.append(1) 1387 elif len(vm_cpu_arr) < 3: 1388 vm_cpu_arr.append(1) 1389 1390 vm_cpu = types.Cpu( 1391 topology=types.CpuTopology( 1392 cores=int(vm_cpu_arr[0]), 1393 sockets=int(vm_cpu_arr[1]), 1394 threads=int(vm_cpu_arr[2]), 1395 ) 1396 ) 1397 1398 if vm_memory is None or vm_cpu is None: 1399 # Find the resources section 1400 resources_elements = self.ovf.get_elements("resources_elements") 1401 for resource in resources_elements: 1402 # Get resource properties: 1403 props = {} 1404 for e in resource: 1405 key = e.tag 1406 value = e.text 1407 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["rasd"], "") 1408 props[key] = value 1409 1410 if vm_cpu is None: 1411 # for ResourceType = 3 (CPU) 1412 if int(props["ResourceType"]) == 3: 1413 vm_cpu = types.Cpu( 1414 topology=types.CpuTopology( 1415 cores=int(props["cpu_per_socket"]), 1416 sockets=int(props["num_of_sockets"]), 1417 threads=int(props["threads_per_cpu"]), 1418 ) 1419 ) 1420 if vm_memory is None: 1421 # for ResourceType = 4 (Memory) 1422 if int(props["ResourceType"]) == 4: 1423 vm_memory = int(props["VirtualQuantity"]) 1424 if props["AllocationUnits"] == "GigaBytes": 1425 vm_memory = vm_memory * 2 ** 30 1426 elif props["AllocationUnits"] == "MegaBytes": 1427 vm_memory = vm_memory * 2 ** 20 1428 elif props["AllocationUnits"] == "KiloBytes": 1429 vm_memory = vm_memory * 2 ** 10 1430 1431 vm_memory_policy = None 1432 if vm_memory is not None: 1433 vm_maxmemory = 4 * vm_memory # 4x memory 1434 vm_memory_policy = types.MemoryPolicy( 1435 guaranteed=vm_memory, max=vm_maxmemory 1436 ) 1437 1438 self.vm = self.vms_service.add( 1439 types.Vm( 1440 name=vm_name, 1441 type=types.VmType(vm_type), 1442 memory=vm_memory, 1443 memory_policy=vm_memory_policy, 1444 cpu=vm_cpu, 1445 template=types.Template(name=vm_template), 1446 cluster=types.Cluster(name=cluster_name), 1447 ), 1448 ) 1449 1450 def add_nics_to_vm(self): 1451 # Find the network section 1452 network_elements = self.ovf.get_elements("network_elements") 1453 bareosfd.DebugMessage( 1454 200, 1455 "add_nics_to_vm(): network_elements: %s\n" % (network_elements), 1456 ) 1457 for nic in network_elements: 1458 # Get nic properties: 1459 props = {} 1460 for e in nic: 1461 key = e.tag 1462 value = e.text 1463 key = key.replace("{%s}" % self.ovf.OVF_NAMESPACES["rasd"], "") 1464 props[key] = value 1465 1466 bareosfd.DebugMessage(200, "add_nics_to_vm(): props: %s\n" % (props)) 1467 1468 network = props["Connection"] 1469 if network not in self.network_profiles: 1470 bareosfd.JobMessage( 1471 bareosfd.M_WARNING, 1472 "No network profile found for '%s'\n" % (network), 1473 ) 1474 else: 1475 profile_id = self.network_profiles[network] 1476 1477 # Locate the service that manages the network interface cards of the 1478 # virtual machine: 1479 nics_service = self.vms_service.vm_service(self.vm.id).nics_service() 1480 1481 # Use the "add" method of the network interface cards service to add the 1482 # new network interface card: 1483 nics_service.add( 1484 types.Nic( 1485 name=props["Name"], 1486 description=props["Caption"], 1487 vnic_profile=types.VnicProfile(id=profile_id), 1488 ), 1489 ) 1490 bareosfd.DebugMessage( 1491 200, 1492 "add_nics_to_vm(): added NIC with props %s\n" % (props), 1493 ) 1494 1495 def get_ovf_disk_alias_by_basename(self, fname): 1496 """ 1497 Return the disk alias name from OVF data for given fname (full path), 1498 this is used on restore by create_file() to process includes/excludes 1499 """ 1500 1501 dirpath = os.path.dirname(fname) 1502 dirname = os.path.basename(dirpath) 1503 basename = os.path.basename(fname) 1504 bareosfd.DebugMessage( 1505 150, 1506 "get_ovf_disk_alias_by_basename(): self.ovf_disks_by_alias_and_fileref: %s\n" 1507 % (self.ovf_disks_by_alias_and_fileref), 1508 ) 1509 relname = "%s/%s" % (dirname, basename) 1510 1511 return self.ovf_disks_by_alias_and_fileref[relname]["disk-alias"] 1512 1513 def get_vm_disk_by_basename(self, fname): 1514 1515 dirpath = os.path.dirname(fname) 1516 dirname = os.path.basename(dirpath) 1517 basename = os.path.basename(fname) 1518 relname = "%s/%s" % (dirname, basename) 1519 1520 found = None 1521 bareosfd.DebugMessage( 1522 200, 1523 "get_vm_disk_by_basename(): %s %s\n" % (basename, self.restore_objects), 1524 ) 1525 if self.restore_objects is not None: 1526 i = 0 1527 while i < len(self.restore_objects) and found is None: 1528 obj = self.restore_objects[i] 1529 key = "%s-%s" % (obj["disk-alias"], obj["fileRef"]) 1530 bareosfd.DebugMessage( 1531 200, 1532 "get_vm_disk_by_basename(): lookup %s == %s and %s == %s\n" 1533 % (repr(relname), repr(key), repr(basename), repr(obj["diskId"])), 1534 ) 1535 if relname == key and basename == obj["diskId"]: 1536 bareosfd.DebugMessage( 1537 200, "get_vm_disk_by_basename(): lookup matched\n" 1538 ) 1539 old_disk_id = os.path.dirname(obj["fileRef"]) 1540 1541 new_disk_id = None 1542 if old_disk_id in self.old_new_ids: 1543 bareosfd.DebugMessage( 1544 200, 1545 "get_vm_disk_by_basename(): disk id %s found in old_new_ids: %s\n" 1546 % (old_disk_id, self.old_new_ids), 1547 ) 1548 new_disk_id = self.old_new_ids[old_disk_id] 1549 1550 # get base disks 1551 if not obj["parentRef"]: 1552 disk = self.get_or_add_vm_disk(obj, new_disk_id) 1553 1554 if disk is not None: 1555 new_disk_id = disk.id 1556 self.old_new_ids[old_disk_id] = new_disk_id 1557 found = disk 1558 else: 1559 bareosfd.JobMessage( 1560 bareosfd.M_WARNING, 1561 "The backup have snapshots and only base will be restored\n", 1562 ) 1563 1564 i += 1 1565 bareosfd.DebugMessage( 1566 200, "get_vm_disk_by_basename(): found disk %s\n" % (found) 1567 ) 1568 return found 1569 1570 def get_or_add_vm_disk(self, obj, disk_id=None): 1571 # Create the disks: 1572 disks_service = self.system_service.disks_service() 1573 1574 disk = None 1575 if "storage_domain" not in obj: 1576 bareosfd.JobMessage(bareosfd.M_FATAL, "No storage domain specified.\n") 1577 else: 1578 storage_domain = obj["storage_domain"] 1579 1580 # Find the disk we want to download by the id: 1581 if disk_id: 1582 disk_service = disks_service.disk_service(disk_id) 1583 if disk_service: 1584 disk = disk_service.get() 1585 1586 if not disk: 1587 # Locate the service that manages the disk attachments of the virtual 1588 # machine: 1589 disk_attachments_service = self.vms_service.vm_service( 1590 self.vm.id 1591 ).disk_attachments_service() 1592 1593 # Determine the volume format 1594 disk_format = types.DiskFormat.RAW 1595 if "volume-format" in obj and obj["volume-format"] == "COW": 1596 disk_format = types.DiskFormat.COW 1597 1598 disk_alias = None 1599 if "disk-alias" in obj: 1600 disk_alias = obj["disk-alias"] 1601 1602 description = None 1603 if "description" in obj: 1604 description = obj["description"] 1605 1606 size = 0 1607 if "size" in obj: 1608 size = int(obj["size"]) * 2 ** 30 1609 1610 actual_size = 0 1611 if "actual_size" in obj: 1612 actual_size = int(obj["actual_size"]) * 2 ** 30 1613 1614 disk_interface = types.DiskInterface.VIRTIO 1615 if "disk-interface" in obj: 1616 if obj["disk-interface"] == "VirtIO_SCSI": 1617 disk_interface = types.DiskInterface.VIRTIO_SCSI 1618 elif obj["disk-interface"] == "IDE": 1619 disk_interface = types.DiskInterface.IDE 1620 1621 disk_bootable = False 1622 if "boot" in obj: 1623 if obj["boot"] == "true": 1624 disk_bootable = True 1625 1626 ##### 1627 # Use the "add" method of the disk attachments service to add the disk. 1628 # Note that the size of the disk, the `provisioned_size` attribute, is 1629 # specified in bytes, so to create a disk of 10 GiB the value should 1630 # be 10 * 2^30. 1631 disk_attachment = disk_attachments_service.add( 1632 types.DiskAttachment( 1633 disk=types.Disk( 1634 id=disk_id, 1635 name=disk_alias, 1636 description=description, 1637 format=disk_format, 1638 provisioned_size=size, 1639 initial_size=actual_size, 1640 sparse=disk_format == types.DiskFormat.COW, 1641 storage_domains=[types.StorageDomain(name=storage_domain)], 1642 ), 1643 interface=disk_interface, 1644 bootable=disk_bootable, 1645 active=True, 1646 ), 1647 ) 1648 1649 # 'Waiting for Disk creation to finish' 1650 disk_service = disks_service.disk_service(disk_attachment.disk.id) 1651 while True: 1652 time.sleep(5) 1653 disk = disk_service.get() 1654 if disk.status == types.DiskStatus.OK: 1655 time.sleep(5) 1656 break 1657 1658 return disk 1659 1660 def start_upload(self, disk): 1661 1662 bareosfd.JobMessage( 1663 bareosfd.M_INFO, 1664 "Uploading disk '%s'('%s')\n" % (disk.alias, disk.id), 1665 ) 1666 bareosfd.DebugMessage( 1667 100, "Uploading disk '%s'('%s')\n" % (disk.alias, disk.id) 1668 ) 1669 bareosfd.DebugMessage(200, "old_new_ids: %s\n" % (self.old_new_ids)) 1670 bareosfd.DebugMessage( 1671 200, "self.restore_objects: %s\n" % (self.restore_objects) 1672 ) 1673 1674 self.transfer_service = self.get_transfer_service( 1675 types.ImageTransfer( 1676 image=types.Image(id=disk.id), 1677 direction=types.ImageTransferDirection.UPLOAD, 1678 ) 1679 ) 1680 transfer = self.transfer_service.get() 1681 proxy_url = urlparse(transfer.proxy_url) 1682 self.proxy_connection = self.get_proxy_connection(proxy_url) 1683 1684 # Send the request head 1685 self.proxy_connection.putrequest("PUT", proxy_url.path) 1686 1687 # To prevent from errors on transfer, the exact number of bytes that 1688 # will be sent must be used, we call it effective_size here. It was 1689 # saved at backup time in restoreobjects, see start_backup_file(), 1690 # and restoreobjects are restored before any file. 1691 new_old_ids = {v: k for k, v in self.old_new_ids.items()} 1692 old_id = new_old_ids[disk.id] 1693 effective_size = self.disk_metadata_by_id[old_id]["effective_size"] 1694 self.init_bytes_to_transf = self.bytes_to_transf = effective_size 1695 1696 content_range = "bytes %d-%d/%d" % ( 1697 0, 1698 self.bytes_to_transf - 1, 1699 self.bytes_to_transf, 1700 ) 1701 self.proxy_connection.putheader("Content-Range", content_range) 1702 self.proxy_connection.putheader("Content-Length", "%d" % (self.bytes_to_transf)) 1703 self.proxy_connection.endheaders() 1704 1705 bareosfd.JobMessage( 1706 bareosfd.M_INFO, 1707 " Upload disk of %s bytes\n" % (str(self.bytes_to_transf)), 1708 ) 1709 self.transfer_start_time = time.time() 1710 1711 def process_ovf(self, chunk): 1712 chunk = StringCodec.decode_ovf_chunk(chunk) 1713 if self.ovf_data is None: 1714 self.ovf_data = chunk 1715 else: 1716 self.ovf_data += chunk 1717 1718 def process_upload(self, chunk): 1719 1720 bareosfd.DebugMessage( 1721 100, 1722 "process_upload(): transfer %s of %s (%s) \n" 1723 % (self.bytes_to_transf, self.init_bytes_to_transf, len(chunk)), 1724 ) 1725 1726 self.proxy_connection.send(chunk) 1727 1728 self.bytes_to_transf -= len(chunk) 1729 1730 completed = 1 - (self.bytes_to_transf / float(self.init_bytes_to_transf)) 1731 1732 bareosfd.DebugMessage( 1733 100, "process_upload(): Completed {:.0%}\n".format(completed) 1734 ) 1735 1736 def end_transfer(self): 1737 1738 bareosfd.DebugMessage(100, "end_transfer()\n") 1739 1740 # Finalize the session. 1741 if self.transfer_service is not None: 1742 self.transfer_service.finalize() 1743 1744 def end_vm_backup(self): 1745 1746 if self.snap_service: 1747 t_start = int(time.time()) 1748 snap = self.snap_service.get() 1749 1750 # Remove the snapshot: 1751 snapshot_deleted_success = False 1752 1753 bareosfd.JobMessage( 1754 bareosfd.M_INFO, 1755 "Sending request to remove snapshot '%s', the id is '%s'.\n" 1756 % (snap.description, snap.id), 1757 ) 1758 1759 while True: 1760 try: 1761 self.snap_service.remove() 1762 except sdk.Error as e: 1763 if e.code in [400, 409]: 1764 1765 # Fail after defined timeout 1766 elapsed = int(time.time()) - t_start 1767 if elapsed >= self.snapshot_remove_timeout: 1768 bareosfd.JobMessage( 1769 bareosfd.M_WARNING, 1770 "Remove snapshot timed out after %s s, reason: %s! Please remove it manually.\n" 1771 % (elapsed, e), 1772 ) 1773 return bareosfd.bRC_Error 1774 1775 bareosfd.DebugMessage( 1776 100, 1777 "Could not remove snapshot, reason: %s, retrying until timeout (%s seconds left).\n" 1778 % (e, self.snapshot_remove_timeout - elapsed), 1779 ) 1780 bareosfd.JobMessage( 1781 bareosfd.M_INFO, 1782 "Still waiting for snapshot removal, retrying until timeout (%s seconds left).\n" 1783 % (self.snapshot_remove_timeout - elapsed), 1784 ) 1785 else: 1786 bareosfd.JobMessage( 1787 bareosfd.M_WARNING, 1788 "Unexpected error removing snapshot: %s, Please remove it manually.\n" 1789 % e, 1790 ) 1791 return bareosfd.bRC_Error 1792 1793 if self.wait_for_snapshot_removal(snap.id): 1794 snapshot_deleted_success = True 1795 break 1796 else: 1797 # the snapshot still exists 1798 continue 1799 1800 if snapshot_deleted_success: 1801 bareosfd.JobMessage( 1802 bareosfd.M_INFO, 1803 "Removed the snapshot '%s'.\n" % snap.description, 1804 ) 1805 1806 # Send an external event to indicate to the administrator that the 1807 # backup of the virtual machine is completed: 1808 self.events_service.add( 1809 event=types.Event( 1810 vm=types.Vm(id=self.vm.id), 1811 origin=APPLICATION_NAME, 1812 severity=types.LogSeverity.NORMAL, 1813 custom_id=self.event_id, 1814 description=( 1815 "Backup of virtual machine '%s' using snapshot '%s' is " 1816 "completed." % (self.vm.name, snap.description) 1817 ), 1818 ), 1819 ) 1820 self.event_id += 1 1821 1822 if self.connection: 1823 # Close the connection to the server: 1824 self.connection.close() 1825 1826 def wait_for_snapshot_removal(self, snapshot_id, timeout=30, delay=10): 1827 t_start = int(time.time()) 1828 snaps_service = self.vm_service.snapshots_service() 1829 1830 while True: 1831 snaps_map = {snap.id: snap.description for snap in snaps_service.list()} 1832 1833 if snapshot_id not in snaps_map: 1834 return True 1835 1836 if int(time.time()) - t_start > timeout: 1837 return False 1838 1839 time.sleep(delay) 1840 1841 return 1842 1843 def end_vm_restore(self): 1844 1845 # Send an external event to indicate to the administrator that the 1846 # restore of the virtual machine is completed: 1847 self.events_service.add( 1848 event=types.Event( 1849 vm=types.Vm(id=self.vm.id), 1850 origin=APPLICATION_NAME, 1851 severity=types.LogSeverity.NORMAL, 1852 custom_id=self.event_id, 1853 description=( 1854 "Restore of virtual machine '%s' is " "completed." % (self.vm.name) 1855 ), 1856 ), 1857 ) 1858 self.event_id += 1 1859 1860 if self.connection: 1861 # Close the connection to the server: 1862 self.connection.close() 1863 1864 1865class Ovf(object): 1866 """ 1867 This class is used to encapsulate XPath expressions to extract data from OVF 1868 """ 1869 1870 # OVF Namespaces 1871 OVF_NAMESPACES = { 1872 "ovf": "http://schemas.dmtf.org/ovf/envelope/1/", 1873 "xsi": "http://www.w3.org/2001/XMLSchema-instance", 1874 "rasd": "http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData", 1875 } 1876 1877 # XPath expressions 1878 OVF_XPATH_EXPRESSIONS = { 1879 "vm_name": './Content[@xsi:type="ovf:VirtualSystem_Type"]/Name', 1880 "cluster_name": './Content[@xsi:type="ovf:VirtualSystem_Type"]/ClusterName', 1881 "network_elements": [ 1882 './Content[@xsi:type="ovf:VirtualSystem_Type"]/Section[@xsi:type="ovf:VirtualHardwareSection_Type"]/Item[Type="interface"]' 1883 ], 1884 "disk_elements": ['./Section[@xsi:type="ovf:DiskSection_Type"]/Disk'], 1885 "resources_elements": [ 1886 './Content[@xsi:type="ovf:VirtualSystem_Type"]/Section[@xsi:type="ovf:VirtualHardwareSection_Type"]/Item[rasd:ResourceType="3"]', 1887 './Content[@xsi:type="ovf:VirtualSystem_Type"]/Section[@xsi:type="ovf:VirtualHardwareSection_Type"]/Item[rasd:ResourceType="4"]', 1888 ], 1889 } 1890 1891 def __init__(self, ovf_data=None): 1892 """ 1893 Creates a new OVF object 1894 1895 This method supports the following parameters: 1896 1897 `ovf_data`:: A string containing the OVF XML data 1898 """ 1899 1900 # Parse the OVF as XML document: 1901 self.ovf = xml.etree.ElementTree.fromstring(ovf_data) 1902 1903 # register namespaces 1904 for prefix, uri in self.OVF_NAMESPACES.items(): 1905 xml.etree.ElementTree.register_namespace(prefix, uri) 1906 1907 def get_element(self, element_name): 1908 """ 1909 Method to extract a single element from OVF data 1910 1911 This method supports the following parameters: 1912 `element_name':: A string with the element name to extract 1913 """ 1914 return self.ovf.findall( 1915 self.OVF_XPATH_EXPRESSIONS[element_name], self.OVF_NAMESPACES 1916 )[0].text 1917 1918 def get_elements(self, element_name): 1919 """ 1920 Method to extract a list of elements from OVF data 1921 1922 This method supports the following parameters: 1923 `element_name':: A string with the element name to extract 1924 """ 1925 results = [] 1926 for xpath_expression in self.OVF_XPATH_EXPRESSIONS[element_name]: 1927 results += self.ovf.findall(xpath_expression, self.OVF_NAMESPACES) 1928 return results 1929 1930 1931class StringCodec: 1932 @staticmethod 1933 def encode_ovf_data(var): 1934 if version_info.major < 3: 1935 return var 1936 else: 1937 return var.encode("utf-8") 1938 1939 @staticmethod 1940 def encode_disk_alias(var): 1941 if version_info.major < 3: 1942 return var.encode("utf-8") 1943 else: 1944 return var 1945 1946 @staticmethod 1947 def decode_fname(var): 1948 if version_info.major < 3: 1949 return var.decode("utf-8") 1950 else: 1951 return var 1952 1953 @staticmethod 1954 def decode_object_data(var): 1955 if version_info.major < 3: 1956 return var 1957 else: 1958 return var.decode("utf-8") 1959 1960 @staticmethod 1961 def decode_ovf_chunk(var): 1962 if version_info.major < 3: 1963 return str(var) 1964 else: 1965 return var.decode("utf-8") 1966 1967 1968# vim: tabstop=4 shiftwidth=4 softtabstop=4 expandtab 1969