1#
2# Storage lookup/creation helpers
3#
4# Copyright 2013 Red Hat, Inc.
5#
6# This work is licensed under the GNU GPLv2 or later.
7# See the COPYING file in the top-level directory.
8
9import os
10import re
11import stat
12import subprocess
13
14import libvirt
15
16from .logger import log
17from .storage import StoragePool, StorageVolume
18from . import xmlutil
19
20
21def _lookup_vol_by_path(conn, path):
22    """
23    Try to find a volume matching the full passed path. Call info() on
24    it to ensure the volume wasn't removed behind libvirt's back
25    """
26    try:
27        vol = conn.storageVolLookupByPath(path)
28        vol.info()
29        return vol, None
30    except libvirt.libvirtError as e:
31        # test_urls trigger empty errors here, because python
32        # garbage collection kicks in after the failure but before
33        # we read the error code, and libvirt virStoragePoolFree
34        # public entry point clears the cached error. So ignore
35        # an empty error code
36        if (e.get_error_code() and
37            e.get_error_code() != libvirt.VIR_ERR_NO_STORAGE_VOL):
38            raise  # pragma: no cover
39        return None, e
40
41
42def _lookup_vol_by_basename(pool, path):
43    """
44    Try to lookup a volume for 'path' in parent 'pool' by it's filename.
45    This sometimes works in cases where full volume path lookup doesn't,
46    since not all libvirt storage backends implement path lookup.
47    """
48    name = os.path.basename(path)
49    if name in pool.listVolumes():
50        return pool.storageVolLookupByName(name)
51
52
53def _get_block_size(path):  # pragma: no cover
54    try:
55        fd = os.open(path, os.O_RDONLY)
56        # os.SEEK_END is not present on all systems
57        size = os.lseek(fd, 0, 2)
58        os.close(fd)
59    except Exception:
60        size = 0
61    return size
62
63
64def _get_size(path):
65    if not os.path.exists(path):
66        return 0
67    if _stat_is_block(path):
68        return _get_block_size(path)  # pragma: no cover
69    return os.path.getsize(path)
70
71
72def _stat_is_block(path):
73    if not os.path.exists(path):
74        return False
75    return stat.S_ISBLK(os.stat(path)[stat.ST_MODE])
76
77
78def _check_if_path_managed(conn, path):
79    """
80    Try to lookup storage objects for the passed path.
81
82    Returns (volume, parent pool). Only one is returned at a time.
83    """
84    vol, ignore = _lookup_vol_by_path(conn, path)
85    if vol:
86        return vol, vol.storagePoolLookupByVolume()
87
88    pool = StoragePool.lookup_pool_by_path(conn, os.path.dirname(path))
89    if not pool:
90        return None, None
91
92    # We have the parent pool, but didn't find a volume on first lookup
93    # attempt. Refresh the pool and try again, in case we were just out
94    # of date or the pool was inactive.
95    try:
96        StoragePool.ensure_pool_is_running(pool, refresh=True)
97        vol, verr = _lookup_vol_by_path(conn, path)
98        if verr:
99            try:
100                vol = _lookup_vol_by_basename(pool, path)
101            except Exception:  # pragma: no cover
102                pass
103    except Exception as e:  # pragma: no cover
104        vol = None
105        pool = None
106        verr = str(e)
107
108    if not vol and not pool and verr:  # pragma: no cover
109        raise ValueError(_("Cannot use storage %(path)s: %(err)s") %
110            {'path': path, 'err': verr})
111
112    return vol, pool
113
114
115def _can_auto_manage(path):
116    path = path or ""
117    skip_prefixes = ["/dev", "/sys", "/proc"]
118
119    if path_is_url(path):
120        return False
121
122    for prefix in skip_prefixes:
123        if path.startswith(prefix + "/") or path == prefix:
124            return False
125    return True
126
127
128def _get_storage_search_path(path):
129    # If the passed path is one of our artificial rbd:// style
130    # URIs, parse out the path component, since that is what is needed
131    # for looking up storage volumes by target path
132    from .uri import URI
133    uriobj = URI(path)
134    if uriobj.scheme == "rbd":
135        return uriobj.path.strip("/")
136    return path
137
138
139def manage_path(conn, path):
140    """
141    If path is not managed, try to create a storage pool to probe the path
142    """
143    if not conn.support.conn_storage():
144        return None, None  # pragma: no cover
145    if not path:
146        return None, None
147
148    if not path_is_url(path) and not path_is_network_vol(conn, path):
149        path = os.path.abspath(path)
150
151    searchpath = _get_storage_search_path(path)
152    vol, pool = _check_if_path_managed(conn, searchpath)
153    if vol or pool or not _can_auto_manage(path):
154        return vol, pool
155
156    dirname = os.path.dirname(path)
157    poolname = os.path.basename(dirname).replace(" ", "_")
158    if not poolname:
159        poolname = "dirpool"
160    poolname = StoragePool.find_free_name(conn, poolname)
161    log.debug("Attempting to build pool=%s target=%s", poolname, dirname)
162
163    poolxml = StoragePool(conn)
164    poolxml.name = poolname
165    poolxml.type = poolxml.TYPE_DIR
166    poolxml.target_path = dirname
167    pool = poolxml.install(build=False, create=True, autostart=True)
168
169    vol = _lookup_vol_by_basename(pool, path)
170    return vol, pool
171
172
173def path_is_url(path):
174    """
175    Detect if path is a URL
176    """
177    return bool(re.match(r"[a-zA-Z]+(\+[a-zA-Z]+)?://.*", path or ""))
178
179
180def path_is_network_vol(conn, path):
181    """
182    Detect if path is a network volume such as rbd, gluster, etc
183    """
184    for volxml in conn.fetch_all_vols():
185        if path and volxml.target_path == path:
186            return volxml.type == "network"
187    return False
188
189
190def _get_dev_type(path, vol_xml, vol_object, pool_xml, remote):
191    """
192    Try to get device type for volume.
193    """
194    if vol_xml and vol_xml.type:
195        return vol_xml.type
196
197    if pool_xml:
198        t = pool_xml.get_disk_type()
199        if t == StorageVolume.TYPE_BLOCK:
200            return "block"
201        elif t == StorageVolume.TYPE_NETWORK:
202            return "network"
203
204    if vol_object:  # pragma: no cover
205        # This path is hard to test, because test suite XML always has
206        # the vol_xml.type set
207        t = vol_object.info()[0]
208        if t == StorageVolume.TYPE_FILE:
209            return "file"
210        elif t == StorageVolume.TYPE_BLOCK:
211            return "block"
212        elif t == StorageVolume.TYPE_NETWORK:
213            return "network"
214
215    if path:
216        if path_is_url(path):
217            return "network"
218
219        if remote:
220            if not _can_auto_manage(path):
221                # Just a heurisitic, if this path is one of the ones
222                # we don't try to auto-import, then consider it a
223                # block device, because managing those correctly is difficult
224                return "block"
225
226        else:
227            if os.path.isdir(path):
228                return "dir"
229            elif _stat_is_block(path):
230                return "block"  # pragma: no cover
231
232    return "file"
233
234
235def path_definitely_exists(conn, path):
236    """
237    Return True if the path certainly exists, False if we are unsure.
238    See DeviceDisk entry point for more details
239    """
240    if path is None:
241        return False
242
243    try:
244        (vol, pool) = _check_if_path_managed(conn, path)
245        ignore = pool
246        if vol:
247            return True
248
249        if not conn.is_remote():
250            return os.path.exists(path)
251    except Exception:  # pragma: no cover
252        pass
253
254    return False
255
256
257#########################
258# ACL/path perm helpers #
259#########################
260
261SETFACL = "setfacl"
262
263
264def _fix_perms_acl(dirname, username):
265    cmd = [SETFACL, "--modify", "user:%s:x" % username, dirname]
266    proc = subprocess.Popen(cmd,
267                            stdout=subprocess.PIPE,
268                            stderr=subprocess.PIPE)
269    out, err = proc.communicate()
270
271    log.debug("Ran command '%s'", cmd)
272    if out or err:
273        log.debug("out=%s\nerr=%s", out, err)
274
275    if proc.returncode != 0:
276        raise ValueError(err)
277
278
279def _fix_perms_chmod(dirname):
280    log.debug("Setting +x on %s", dirname)
281    mode = os.stat(dirname).st_mode
282    newmode = mode | stat.S_IXOTH
283    os.chmod(dirname, newmode)
284    if os.stat(dirname).st_mode != newmode:
285        # Trying to change perms on vfat at least doesn't work
286        # but also doesn't seem to error. Try and detect that
287        raise ValueError(  # pragma: no cover
288                _("Permissions on '%s' did not stick") % dirname)
289
290
291def set_dirs_searchable(dirlist, username):
292    useacl = True
293    errdict = {}
294    for dirname in dirlist:
295        if useacl:
296            try:
297                _fix_perms_acl(dirname, username)
298                continue
299            except Exception as e:
300                log.debug("setfacl failed: %s", e)
301                log.debug("trying chmod")
302                useacl = False
303
304        try:
305            # If we reach here, ACL setting failed, try chmod
306            _fix_perms_chmod(dirname)
307        except Exception as e:  # pragma: no cover
308            errdict[dirname] = str(e)
309
310    return errdict
311
312
313def _is_dir_searchable(dirname, uid, username):
314    """
315    Check if passed directory is searchable by uid
316    """
317    try:
318        statinfo = os.stat(dirname)
319    except OSError:  # pragma: no cover
320        return False
321
322    if uid == statinfo.st_uid:
323        flag = stat.S_IXUSR
324    elif uid == statinfo.st_gid:
325        flag = stat.S_IXGRP  # pragma: no cover
326    else:
327        flag = stat.S_IXOTH
328
329    if bool(statinfo.st_mode & flag):
330        return True
331
332    # Check POSIX ACL (since that is what we use to 'fix' access)
333    cmd = ["getfacl", dirname]
334    try:
335        proc = subprocess.Popen(cmd,
336                                stdout=subprocess.PIPE,
337                                stderr=subprocess.PIPE)
338        out, err = proc.communicate()
339    except OSError:  # pragma: no cover
340        log.debug("Didn't find the getfacl command.")
341        return False
342
343    if proc.returncode != 0:  # pragma: no cover
344        log.debug("Cmd '%s' failed: %s", cmd, err)
345        return False
346
347    pattern = "user:%s:..x" % username
348    return bool(re.search(pattern.encode("utf-8", "replace"), out))
349
350
351def is_path_searchable(path, uid, username):
352    """
353    Check each dir component of the passed path, see if they are
354    searchable by the uid/username, and return a list of paths
355    which aren't searchable
356    """
357    if os.path.isdir(path):
358        dirname = path
359        base = "-"
360    else:
361        dirname, base = os.path.split(path)
362
363    fixlist = []
364    while base:
365        if not _is_dir_searchable(dirname, uid, username):
366            fixlist.append(dirname)
367        dirname, base = os.path.split(dirname)
368
369    return fixlist
370
371
372##############################################
373# Classes for tracking storage media details #
374##############################################
375
376class _StorageBase(object):
377    """
378    Storage base class, defining the API used by DeviceDisk
379    """
380    def __init__(self, conn):
381        self._conn = conn
382        self._parent_pool_xml = None
383
384    def get_size(self):
385        raise NotImplementedError()
386    def get_dev_type(self):
387        raise NotImplementedError()
388    def get_driver_type(self):
389        raise NotImplementedError()
390    def get_vol_install(self):
391        raise NotImplementedError()
392    def get_vol_object(self):
393        raise NotImplementedError()
394    def get_parent_pool(self):
395        raise NotImplementedError()
396    def get_parent_pool_xml(self):
397        if not self._parent_pool_xml and self.get_parent_pool():
398            self._parent_pool_xml = StoragePool(self._conn,
399                parsexml=self.get_parent_pool().XMLDesc(0))
400        return self._parent_pool_xml
401    def validate(self):
402        raise NotImplementedError()
403    def get_path(self):
404        raise NotImplementedError()
405    def is_stub(self):
406        return False
407
408    # Storage creation routines
409    def is_size_conflict(self):
410        raise NotImplementedError()
411    def will_create_storage(self):
412        raise NotImplementedError()
413
414    def create(self, progresscb):
415        ignore = progresscb  # pragma: no cover
416        raise xmlutil.DevError(
417            "%s can't create storage" % self.__class__.__name__)
418
419
420class _StorageCreator(_StorageBase):
421    """
422    Base object for classes that will actually create storage on disk
423    """
424    def __init__(self, conn):
425        _StorageBase.__init__(self, conn)
426
427        self._pool = None
428        self._vol_install = None
429        self._path = None
430        self._size = None
431        self._dev_type = None
432
433
434    ##############
435    # Public API #
436    ##############
437
438    def create(self, progresscb):
439        raise NotImplementedError
440    def validate(self):
441        raise NotImplementedError
442    def get_size(self):
443        raise NotImplementedError
444
445    def get_path(self):
446        if self._vol_install and not self._path:
447            xmlobj = StoragePool(self._conn,
448                parsexml=self._vol_install.pool.XMLDesc(0))
449            if self.get_dev_type() == "network":
450                self._path = self._vol_install.name
451            else:
452                self._path = os.path.join(
453                        xmlobj.target_path, self._vol_install.name)
454        return self._path
455
456    def get_vol_install(self):
457        return self._vol_install
458    def get_vol_xml(self):
459        return self._vol_install
460
461    def get_dev_type(self):
462        if not self._dev_type:
463            self._dev_type = _get_dev_type(self._path, self._vol_install, None,
464                                           self.get_parent_pool_xml(),
465                                           self._conn.is_remote())
466        return self._dev_type
467
468    def get_driver_type(self):
469        if self._vol_install:
470            if self._vol_install.supports_format():
471                return self._vol_install.format
472        return "raw"
473
474    def will_create_storage(self):
475        return True
476    def get_vol_object(self):
477        return None
478    def get_parent_pool(self):
479        if self._vol_install:
480            return self._vol_install.pool
481        return None
482    def exists(self):
483        return False
484
485
486class ManagedStorageCreator(_StorageCreator):
487    """
488    Handles storage creation via libvirt APIs. All the actual creation
489    logic lives in StorageVolume, this is mostly about pulling out bits
490    from that class and mapping them to DeviceDisk elements
491    """
492    def __init__(self, conn, vol_install):
493        _StorageCreator.__init__(self, conn)
494
495        self._pool = vol_install.pool
496        self._vol_install = vol_install
497
498    def create(self, progresscb):
499        return self._vol_install.install(meter=progresscb)
500    def is_size_conflict(self):
501        return self._vol_install.is_size_conflict()
502    def validate(self):
503        return self._vol_install.validate()
504    def get_size(self):
505        return float(self._vol_install.capacity) / 1024.0 / 1024.0 / 1024.0
506
507
508class CloneStorageCreator(_StorageCreator):
509    """
510    Handles manually copying local files for Cloner
511
512    Many clone scenarios will use libvirt storage APIs, which will use
513    the ManagedStorageCreator
514    """
515    def __init__(self, conn, output_path, input_path, size, sparse):
516        _StorageCreator.__init__(self, conn)
517
518        self._path = output_path
519        self._output_path = output_path
520        self._input_path = input_path
521        self._size = size
522        self._sparse = sparse
523
524    def get_size(self):
525        return self._size
526
527    def is_size_conflict(self):
528        ret = False
529        msg = None
530        if self.get_dev_type() == "block":
531            avail = _get_size(self._path)  # pragma: no cover
532        else:
533            vfs = os.statvfs(os.path.dirname(os.path.abspath(self._path)))
534            avail = vfs.f_frsize * vfs.f_bavail
535        need = int(self._size) * 1024 * 1024 * 1024
536        if need > avail:  # pragma: no cover
537            if self._sparse:
538                msg = _("The filesystem will not have enough free space"
539                        " to fully allocate the sparse file when the guest"
540                        " is running.")
541            else:
542                ret = True
543                msg = _("There is not enough free space to create the disk.")
544
545
546            if msg:
547                msg += " "
548                msg += (_("%(mem1)s M requested > %(mem2)s M available") %
549                        {"mem1": (need // (1024 * 1024)),
550                         "mem2": (avail // (1024 * 1024))})
551        return (ret, msg)
552
553    def validate(self):
554        if self._size is None:  # pragma: no cover
555            raise ValueError(_("size is required for non-existent disk "
556                               "'%s'" % self.get_path()))
557
558        err, msg = self.is_size_conflict()
559        if err:
560            raise ValueError(msg)  # pragma: no cover
561        if msg:
562            log.warning(msg)  # pragma: no cover
563
564    def create(self, progresscb):
565        text = (_("Cloning %(srcfile)s") %
566                {'srcfile': os.path.basename(self._input_path)})
567
568        size_bytes = int(self.get_size() * 1024 * 1024 * 1024)
569        progresscb.start(filename=self._output_path, size=size_bytes,
570                         text=text)
571
572        # Plain file clone
573        self._clone_local(progresscb, size_bytes)
574
575    def _clone_local(self, meter, size_bytes):
576        if self._input_path == "/dev/null":  # pragma: no cover
577            # Not really sure why this check is here,
578            # but keeping for compat
579            log.debug("Source dev was /dev/null. Skipping")
580            return
581        if self._input_path == self._output_path:
582            log.debug("Source and destination are the same. Skipping.")
583            return
584
585        # If a destination file exists and sparse flag is True,
586        # this priority takes an existing file.
587
588        if (not os.path.exists(self._output_path) and self._sparse):
589            clone_block_size = 4096
590            sparse = True
591            fd = None
592            try:
593                fd = os.open(self._output_path, os.O_WRONLY | os.O_CREAT,
594                             0o640)
595                os.ftruncate(fd, size_bytes)
596            finally:
597                if fd:
598                    os.close(fd)
599        else:
600            clone_block_size = 1024 * 1024 * 10
601            sparse = False
602
603        log.debug("Local Cloning %s to %s, sparse=%s, block_size=%s",
604                      self._input_path, self._output_path,
605                      sparse, clone_block_size)
606
607        zeros = b'\0' * 4096
608
609        src_fd, dst_fd = None, None
610        try:
611            try:
612                src_fd = os.open(self._input_path, os.O_RDONLY)
613                dst_fd = os.open(self._output_path,
614                                 os.O_WRONLY | os.O_CREAT, 0o640)
615
616                i = 0
617                while 1:
618                    l = os.read(src_fd, clone_block_size)
619                    s = len(l)
620                    if s == 0:
621                        meter.end(size_bytes)
622                        break
623                    # check sequence of zeros
624                    if sparse and zeros == l:
625                        os.lseek(dst_fd, s, 1)
626                    else:
627                        b = os.write(dst_fd, l)
628                        if s != b:  # pragma: no cover
629                            meter.end(i)
630                            break
631                    i += s
632                    if i < size_bytes:
633                        meter.update(i)
634            except OSError as e:  # pragma: no cover
635                log.debug("Error while cloning", exc_info=True)
636                msg = (_("Error cloning diskimage "
637                         "%(inputpath)s to %(outputpath)s: %(error)s") %
638                         {"inputpath": self._input_path,
639                          "outputpath": self._output_path,
640                          "error": str(e)})
641                raise RuntimeError(msg) from None
642        finally:
643            if src_fd is not None:
644                os.close(src_fd)
645            if dst_fd is not None:
646                os.close(dst_fd)
647
648
649class StorageBackendStub(_StorageBase):
650    """
651    Class representing a storage path for a parsed XML disk, that we
652    don't want to do slow resolving of unless requested
653    """
654    def __init__(self, conn, path, dev_type, driver_type):
655        _StorageBase.__init__(self, conn)
656        self._path = path
657        self._dev_type = dev_type
658        self._driver_type = driver_type
659
660
661    def get_path(self):
662        return self._path
663    def get_vol_object(self):
664        return None
665    def get_vol_xml(self):
666        return None
667    def get_parent_pool(self):
668        return None
669    def get_size(self):
670        return 0
671    def exists(self):
672        return True
673    def get_dev_type(self):
674        return self._dev_type
675    def get_driver_type(self):
676        return self._driver_type
677
678    def validate(self):
679        return
680    def get_vol_install(self):
681        return None
682    def is_size_conflict(self):
683        return (False, None)
684    def is_stub(self):
685        return True
686    def will_create_storage(self):
687        return False
688
689
690class StorageBackend(_StorageBase):
691    """
692    Class that carries all the info about any existing storage that
693    the disk references
694    """
695    def __init__(self, conn, path, vol_object, parent_pool):
696        _StorageBase.__init__(self, conn)
697
698        self._vol_object = vol_object
699        self._parent_pool = parent_pool
700        self._path = path
701
702        if self._vol_object is not None:
703            self._path = None
704
705        if self._vol_object and not self._parent_pool:
706            raise xmlutil.DevError(
707                "parent_pool must be specified")
708
709        # Cached bits
710        self._vol_xml = None
711        self._parent_pool_xml = None
712        self._exists = None
713        self._size = None
714        self._dev_type = None
715
716
717    ##############
718    # Public API #
719    ##############
720
721    def get_path(self):
722        if self._vol_object:
723            return self.get_vol_xml().target_path
724        return self._path
725
726    def get_vol_object(self):
727        return self._vol_object
728    def get_vol_xml(self):
729        if self._vol_xml is None:
730            self._vol_xml = StorageVolume(self._conn,
731                parsexml=self._vol_object.XMLDesc(0))
732            self._vol_xml.pool = self._parent_pool
733        return self._vol_xml
734
735    def get_parent_pool(self):
736        return self._parent_pool
737
738    def get_size(self):
739        """
740        Return size of existing storage
741        """
742        if self._size is None:
743            ret = 0
744            if self._vol_object:
745                ret = self.get_vol_xml().capacity
746            elif self._path:
747                ret = _get_size(self._path)
748            self._size = (float(ret) / 1024.0 / 1024.0 / 1024.0)
749        return self._size
750
751    def exists(self):
752        if self._exists is None:
753            if self._vol_object:
754                self._exists = True
755            elif self._path is None:
756                self._exists = True
757            elif (not self.get_dev_type() == "network" and
758                  not self._conn.is_remote() and
759                  os.path.exists(self._path)):
760                self._exists = True
761            elif self._parent_pool:
762                self._exists = False
763            elif self.get_dev_type() == "network":
764                self._exists = True
765            elif (self._conn.is_remote() and
766                  not _can_auto_manage(self._path)):
767                # This allows users to pass /dev/sdX and we don't try to
768                # validate it exists on the remote connection, since
769                # autopooling /dev is perilous. Libvirt will error if
770                # the device doesn't exist.
771                self._exists = True
772            else:
773                self._exists = False
774        return self._exists
775
776    def get_dev_type(self):
777        """
778        Return disk 'type' value per storage settings
779        """
780        if self._dev_type is None:
781            vol_xml = None
782            if self._vol_object:
783                vol_xml = self.get_vol_xml()
784            self._dev_type = _get_dev_type(self._path, vol_xml, self._vol_object,
785                                           self.get_parent_pool_xml(),
786                                           self._conn.is_remote())
787        return self._dev_type
788
789    def get_driver_type(self):
790        if self._vol_object:
791            ret = self.get_vol_xml().format
792            if ret != "unknown":
793                return ret
794        return None
795
796    def validate(self):
797        return
798    def get_vol_install(self):
799        return None
800    def is_size_conflict(self):
801        return (False, None)
802    def will_create_storage(self):
803        return False
804