1# comps.py
2# Interface to libcomps.
3#
4# Copyright (C) 2013-2018 Red Hat, Inc.
5#
6# This copyrighted material is made available to anyone wishing to use,
7# modify, copy, or redistribute it subject to the terms and conditions of
8# the GNU General Public License v.2, or (at your option) any later version.
9# This program is distributed in the hope that it will be useful, but WITHOUT
10# ANY WARRANTY expressed or implied, including the implied warranties of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
12# Public License for more details.  You should have received a copy of the
13# GNU General Public License along with this program; if not, write to the
14# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
16# source code or documentation are not subject to the GNU General Public
17# License and may only be used or replicated with the express permission of
18# Red Hat, Inc.
19#
20
21from __future__ import absolute_import
22from __future__ import print_function
23from __future__ import unicode_literals
24
25import libdnf.transaction
26
27from dnf.exceptions import CompsError
28from dnf.i18n import _, ucd
29from functools import reduce
30
31import dnf.i18n
32import dnf.util
33import fnmatch
34import gettext
35import itertools
36import libcomps
37import locale
38import logging
39import operator
40import re
41import sys
42
43logger = logging.getLogger("dnf")
44
45# :api :binformat
46CONDITIONAL = libdnf.transaction.CompsPackageType_CONDITIONAL
47DEFAULT     = libdnf.transaction.CompsPackageType_DEFAULT
48MANDATORY   = libdnf.transaction.CompsPackageType_MANDATORY
49OPTIONAL    = libdnf.transaction.CompsPackageType_OPTIONAL
50
51ALL_TYPES = CONDITIONAL | DEFAULT | MANDATORY | OPTIONAL
52
53
54def _internal_comps_length(comps):
55    collections = (comps.categories, comps.groups, comps.environments)
56    return reduce(operator.__add__, map(len, collections))
57
58
59def _first_if_iterable(seq):
60    if seq is None:
61        return None
62    return dnf.util.first(seq)
63
64
65def _by_pattern(pattern, case_sensitive, sqn):
66    """Return items from sqn matching either exactly or glob-wise."""
67
68    pattern = dnf.i18n.ucd(pattern)
69    exact = {g for g in sqn if g.name == pattern or g.id == pattern}
70    if exact:
71        return exact
72
73    if case_sensitive:
74        match = re.compile(fnmatch.translate(pattern)).match
75    else:
76        match = re.compile(fnmatch.translate(pattern), flags=re.I).match
77
78    ret = set()
79    for g in sqn:
80        if match(g.id):
81            ret.add(g)
82        elif g.name is not None and match(g.name):
83            ret.add(g)
84        elif g.ui_name is not None and match(g.ui_name):
85            ret.add(g)
86
87    return ret
88
89
90def _fn_display_order(group):
91    return sys.maxsize if group.display_order is None else group.display_order
92
93
94def install_or_skip(install_fnc, grp_or_env_id, types, exclude=None,
95                    strict=True, exclude_groups=None):
96    """Either mark in persistor as installed given `grp_or_env` (group
97       or environment) or skip it (if it's already installed).
98       `install_fnc` has to be Solver._group_install
99       or Solver._environment_install.
100       """
101    try:
102        return install_fnc(grp_or_env_id, types, exclude, strict, exclude_groups)
103    except dnf.comps.CompsError as e:
104        logger.warning("%s, %s", ucd(e)[:-1], _("skipping."))
105
106
107class _Langs(object):
108
109    """Get all usable abbreviations for the current language."""
110
111    def __init__(self):
112        self.last_locale = None
113        self.cache = None
114
115    @staticmethod
116    def _dotted_locale_str():
117        lcl = locale.getlocale(locale.LC_MESSAGES)
118        if lcl == (None, None):
119            return 'C'
120        return '.'.join(lcl)
121
122    def get(self):
123        current_locale = self._dotted_locale_str()
124        if self.last_locale == current_locale:
125            return self.cache
126
127        self.cache = []
128        locales = [current_locale]
129        if current_locale != 'C':
130            locales.append('C')
131        for l in locales:
132            for nlang in gettext._expand_lang(l):
133                if nlang not in self.cache:
134                    self.cache.append(nlang)
135
136        self.last_locale = current_locale
137        return self.cache
138
139
140class CompsQuery(object):
141
142    AVAILABLE = 1
143    INSTALLED = 2
144
145    ENVIRONMENTS = 1
146    GROUPS = 2
147
148    def __init__(self, comps, history, kinds, status):
149        self.comps = comps
150        self.history = history
151        self.kinds = kinds
152        self.status = status
153
154    def _get_groups(self, available, installed):
155        result = set()
156        if self.status & self.AVAILABLE:
157            result.update({i.id for i in available})
158        if self.status & self.INSTALLED:
159            for i in installed:
160                group = i.getCompsGroupItem()
161                if not group:
162                    continue
163                result.add(group.getGroupId())
164        return result
165
166    def _get_envs(self, available, installed):
167        result = set()
168        if self.status & self.AVAILABLE:
169            result.update({i.id for i in available})
170        if self.status & self.INSTALLED:
171            for i in installed:
172                env = i.getCompsEnvironmentItem()
173                if not env:
174                    continue
175                result.add(env.getEnvironmentId())
176        return result
177
178    def get(self, *patterns):
179        res = dnf.util.Bunch()
180        res.environments = []
181        res.groups = []
182        for pat in patterns:
183            envs = grps = []
184            if self.kinds & self.ENVIRONMENTS:
185                available = self.comps.environments_by_pattern(pat)
186                installed = self.history.env.search_by_pattern(pat)
187                envs = self._get_envs(available, installed)
188                res.environments.extend(envs)
189            if self.kinds & self.GROUPS:
190                available = self.comps.groups_by_pattern(pat)
191                installed = self.history.group.search_by_pattern(pat)
192                grps = self._get_groups(available, installed)
193                res.groups.extend(grps)
194            if not envs and not grps:
195                if self.status == self.INSTALLED:
196                    msg = _("Module or Group '%s' is not installed.") % ucd(pat)
197                elif self.status == self.AVAILABLE:
198                    msg = _("Module or Group '%s' is not available.") % ucd(pat)
199                else:
200                    msg = _("Module or Group '%s' does not exist.") % ucd(pat)
201                raise CompsError(msg)
202        return res
203
204
205class Forwarder(object):
206    def __init__(self, iobj, langs):
207        self._i = iobj
208        self._langs = langs
209
210    def __getattr__(self, name):
211        return getattr(self._i, name)
212
213    def _ui_text(self, default, dct):
214        for l in self._langs.get():
215            t = dct.get(l)
216            if t is not None:
217                return t
218        return default
219
220    @property
221    def ui_description(self):
222        return self._ui_text(self.desc, self.desc_by_lang)
223
224    @property
225    def ui_name(self):
226        return self._ui_text(self.name, self.name_by_lang)
227
228class Category(Forwarder):
229    # :api
230    def __init__(self, iobj, langs, group_factory):
231        super(Category, self).__init__(iobj, langs)
232        self._group_factory = group_factory
233
234    def _build_group(self, grp_id):
235        grp = self._group_factory(grp_id.name)
236        if grp is None:
237            msg = "no group '%s' from category '%s'"
238            raise ValueError(msg % (grp_id.name, self.id))
239        return grp
240
241    def groups_iter(self):
242        for grp_id in self.group_ids:
243            yield self._build_group(grp_id)
244
245    @property
246    def groups(self):
247        return list(self.groups_iter())
248
249class Environment(Forwarder):
250    # :api
251
252    def __init__(self, iobj, langs, group_factory):
253        super(Environment, self).__init__(iobj, langs)
254        self._group_factory = group_factory
255
256    def _build_group(self, grp_id):
257        grp = self._group_factory(grp_id.name)
258        if grp is None:
259            msg = "no group '%s' from environment '%s'"
260            raise ValueError(msg % (grp_id.name, self.id))
261        return grp
262
263    def _build_groups(self, ids):
264        groups = []
265        for gi in ids:
266            try:
267                groups.append(self._build_group(gi))
268            except ValueError as e:
269                logger.error(e)
270
271        return groups
272
273    def groups_iter(self):
274        for grp_id in itertools.chain(self.group_ids, self.option_ids):
275            try:
276                yield self._build_group(grp_id)
277            except ValueError as e:
278                logger.error(e)
279
280    @property
281    def mandatory_groups(self):
282        return self._build_groups(self.group_ids)
283
284    @property
285    def optional_groups(self):
286        return self._build_groups(self.option_ids)
287
288class Group(Forwarder):
289    # :api
290    def __init__(self, iobj, langs, pkg_factory):
291        super(Group, self).__init__(iobj, langs)
292        self._pkg_factory = pkg_factory
293        self.selected = iobj.default
294
295    def _packages_of_type(self, type_):
296        return [pkg for pkg in self.packages if pkg.type == type_]
297
298    @property
299    def conditional_packages(self):
300        return self._packages_of_type(libcomps.PACKAGE_TYPE_CONDITIONAL)
301
302    @property
303    def default_packages(self):
304        return self._packages_of_type(libcomps.PACKAGE_TYPE_DEFAULT)
305
306    def packages_iter(self):
307        # :api
308        return map(self._pkg_factory, self.packages)
309
310    @property
311    def mandatory_packages(self):
312        return self._packages_of_type(libcomps.PACKAGE_TYPE_MANDATORY)
313
314    @property
315    def optional_packages(self):
316        return self._packages_of_type(libcomps.PACKAGE_TYPE_OPTIONAL)
317
318    @property
319    def visible(self):
320        return self._i.uservisible
321
322class Package(Forwarder):
323    """Represents comps package data. :api"""
324
325    _OPT_MAP = {
326        libcomps.PACKAGE_TYPE_CONDITIONAL : CONDITIONAL,
327        libcomps.PACKAGE_TYPE_DEFAULT     : DEFAULT,
328        libcomps.PACKAGE_TYPE_MANDATORY   : MANDATORY,
329        libcomps.PACKAGE_TYPE_OPTIONAL    : OPTIONAL,
330    }
331
332    def __init__(self, ipkg):
333        self._i = ipkg
334
335    @property
336    def name(self):
337        # :api
338        return self._i.name
339
340    @property
341    def option_type(self):
342        # :api
343        return self._OPT_MAP[self.type]
344
345class Comps(object):
346    # :api
347
348    def __init__(self):
349        self._i = libcomps.Comps()
350        self._langs = _Langs()
351
352    def __len__(self):
353        return _internal_comps_length(self._i)
354
355    def _build_category(self, icategory):
356        return Category(icategory, self._langs, self._group_by_id)
357
358    def _build_environment(self, ienvironment):
359        return Environment(ienvironment, self._langs, self._group_by_id)
360
361    def _build_group(self, igroup):
362        return Group(igroup, self._langs, self._build_package)
363
364    def _build_package(self, ipkg):
365        return Package(ipkg)
366
367    def _add_from_xml_filename(self, fn):
368        comps = libcomps.Comps()
369        try:
370            comps.fromxml_f(fn)
371        except libcomps.ParserError:
372            errors = comps.get_last_errors()
373            raise CompsError(' '.join(errors))
374        self._i += comps
375
376    @property
377    def categories(self):
378        # :api
379        return list(self.categories_iter())
380
381    def category_by_pattern(self, pattern, case_sensitive=False):
382        # :api
383        assert dnf.util.is_string_type(pattern)
384        cats = self.categories_by_pattern(pattern, case_sensitive)
385        return _first_if_iterable(cats)
386
387    def categories_by_pattern(self, pattern, case_sensitive=False):
388        # :api
389        assert dnf.util.is_string_type(pattern)
390        return _by_pattern(pattern, case_sensitive, self.categories)
391
392    def categories_iter(self):
393        # :api
394        return (self._build_category(c) for c in self._i.categories)
395
396    @property
397    def environments(self):
398        # :api
399        return sorted(self.environments_iter(), key=_fn_display_order)
400
401    def _environment_by_id(self, id):
402        assert dnf.util.is_string_type(id)
403        return dnf.util.first(g for g in self.environments_iter() if g.id == id)
404
405    def environment_by_pattern(self, pattern, case_sensitive=False):
406        # :api
407        assert dnf.util.is_string_type(pattern)
408        envs = self.environments_by_pattern(pattern, case_sensitive)
409        return _first_if_iterable(envs)
410
411    def environments_by_pattern(self, pattern, case_sensitive=False):
412        # :api
413        assert dnf.util.is_string_type(pattern)
414        envs = list(self.environments_iter())
415        found_envs = _by_pattern(pattern, case_sensitive, envs)
416        return sorted(found_envs, key=_fn_display_order)
417
418    def environments_iter(self):
419        # :api
420        return (self._build_environment(e) for e in self._i.environments)
421
422    @property
423    def groups(self):
424        # :api
425        return sorted(self.groups_iter(), key=_fn_display_order)
426
427    def _group_by_id(self, id_):
428        assert dnf.util.is_string_type(id_)
429        return dnf.util.first(g for g in self.groups_iter() if g.id == id_)
430
431    def group_by_pattern(self, pattern, case_sensitive=False):
432        # :api
433        assert dnf.util.is_string_type(pattern)
434        grps = self.groups_by_pattern(pattern, case_sensitive)
435        return _first_if_iterable(grps)
436
437    def groups_by_pattern(self, pattern, case_sensitive=False):
438        # :api
439        assert dnf.util.is_string_type(pattern)
440        grps = _by_pattern(pattern, case_sensitive, list(self.groups_iter()))
441        return sorted(grps, key=_fn_display_order)
442
443    def groups_iter(self):
444        # :api
445        return (self._build_group(g) for g in self._i.groups)
446
447class CompsTransPkg(object):
448    def __init__(self, pkg_or_name):
449        if dnf.util.is_string_type(pkg_or_name):
450            # from package name
451            self.basearchonly = False
452            self.name = pkg_or_name
453            self.optional = True
454            self.requires = None
455        elif isinstance(pkg_or_name, libdnf.transaction.CompsGroupPackage):
456            # from swdb package
457            # TODO:
458            self.basearchonly = False
459            # self.basearchonly = pkg_or_name.basearchonly
460            self.name = pkg_or_name.getName()
461            self.optional = pkg_or_name.getPackageType() & libcomps.PACKAGE_TYPE_OPTIONAL
462            # TODO:
463            self.requires = None
464            # self.requires = pkg_or_name.requires
465        else:
466            # from comps package
467            self.basearchonly = pkg_or_name.basearchonly
468            self.name = pkg_or_name.name
469            self.optional = pkg_or_name.type & libcomps.PACKAGE_TYPE_OPTIONAL
470            self.requires = pkg_or_name.requires
471
472    def __eq__(self, other):
473        return (self.name == other.name and
474                self.basearchonly == other.basearchonly and
475                self.optional == other.optional and
476                self.requires == other.requires)
477
478    def __str__(self):
479        return self.name
480
481    def __hash__(self):
482        return hash((self.name,
483                    self.basearchonly,
484                    self.optional,
485                    self.requires))
486
487class TransactionBunch(object):
488    def __init__(self):
489        self._install = set()
490        self._install_opt = set()
491        self._remove = set()
492        self._upgrade = set()
493
494    def __iadd__(self, other):
495        self._install.update(other._install)
496        self._install_opt.update(other._install_opt)
497        self._upgrade.update(other._upgrade)
498        self._remove = (self._remove | other._remove) - \
499            self._install - self._install_opt - self._upgrade
500        return self
501
502    def __len__(self):
503        return len(self.install) + len(self.install_opt) + len(self.upgrade) + len(self.remove)
504
505    @staticmethod
506    def _set_value(param, val):
507        for item in val:
508            if isinstance(item, CompsTransPkg):
509                param.add(item)
510            else:
511                param.add(CompsTransPkg(item))
512
513    @property
514    def install(self):
515        """
516        Packages to be installed with strict=True - transaction will
517        fail if they cannot be installed due to dependency errors etc.
518        """
519        return self._install
520
521    @install.setter
522    def install(self, value):
523        self._set_value(self._install, value)
524
525    @property
526    def install_opt(self):
527        """
528        Packages to be installed with strict=False - they will be
529        skipped if they cannot be installed
530        """
531        return self._install_opt
532
533    @install_opt.setter
534    def install_opt(self, value):
535        self._set_value(self._install_opt, value)
536
537    @property
538    def remove(self):
539        return self._remove
540
541    @remove.setter
542    def remove(self, value):
543        self._set_value(self._remove, value)
544
545    @property
546    def upgrade(self):
547        return self._upgrade
548
549    @upgrade.setter
550    def upgrade(self, value):
551        self._set_value(self._upgrade, value)
552
553
554class Solver(object):
555    def __init__(self, history, comps, reason_fn):
556        self.history = history
557        self.comps = comps
558        self._reason_fn = reason_fn
559
560    @staticmethod
561    def _mandatory_group_set(env):
562        return {grp.id for grp in env.mandatory_groups}
563
564    @staticmethod
565    def _full_package_set(grp):
566        return {pkg.getName() for pkg in grp.mandatory_packages +
567                grp.default_packages + grp.optional_packages +
568                grp.conditional_packages}
569
570    @staticmethod
571    def _pkgs_of_type(group, pkg_types, exclude=[]):
572        def filter(pkgs):
573            return [pkg for pkg in pkgs
574                    if pkg.name not in exclude]
575
576        pkgs = set()
577        if pkg_types & MANDATORY:
578            pkgs.update(filter(group.mandatory_packages))
579        if pkg_types & DEFAULT:
580            pkgs.update(filter(group.default_packages))
581        if pkg_types & OPTIONAL:
582            pkgs.update(filter(group.optional_packages))
583        if pkg_types & CONDITIONAL:
584            pkgs.update(filter(group.conditional_packages))
585        return pkgs
586
587    def _removable_pkg(self, pkg_name):
588        assert dnf.util.is_string_type(pkg_name)
589        return self.history.group.is_removable_pkg(pkg_name)
590
591    def _removable_grp(self, group_id):
592        assert dnf.util.is_string_type(group_id)
593        return self.history.env.is_removable_group(group_id)
594
595    def _environment_install(self, env_id, pkg_types, exclude, strict=True, exclude_groups=None):
596        assert dnf.util.is_string_type(env_id)
597        comps_env = self.comps._environment_by_id(env_id)
598        if not comps_env:
599            raise CompsError(_("Environment id '%s' does not exist.") % ucd(env_id))
600
601        swdb_env = self.history.env.new(env_id, comps_env.name, comps_env.ui_name, pkg_types)
602        self.history.env.install(swdb_env)
603
604        trans = TransactionBunch()
605        for comps_group in comps_env.mandatory_groups:
606            if exclude_groups and comps_group.id in exclude_groups:
607                continue
608            trans += self._group_install(comps_group.id, pkg_types, exclude, strict)
609            swdb_env.addGroup(comps_group.id, True, MANDATORY)
610
611        for comps_group in comps_env.optional_groups:
612            if exclude_groups and comps_group.id in exclude_groups:
613                continue
614            swdb_env.addGroup(comps_group.id, False, OPTIONAL)
615            # TODO: if a group is already installed, mark it as installed?
616        return trans
617
618    def _environment_remove(self, env_id):
619        assert dnf.util.is_string_type(env_id) is True
620        swdb_env = self.history.env.get(env_id)
621        if not swdb_env:
622            raise CompsError(_("Environment id '%s' is not installed.") % env_id)
623
624        self.history.env.remove(swdb_env)
625
626        trans = TransactionBunch()
627        group_ids = set([i.getGroupId() for i in swdb_env.getGroups()])
628        for group_id in group_ids:
629            if not self._removable_grp(group_id):
630                continue
631            trans += self._group_remove(group_id)
632        return trans
633
634    def _environment_upgrade(self, env_id):
635        assert dnf.util.is_string_type(env_id)
636        comps_env = self.comps._environment_by_id(env_id)
637        swdb_env = self.history.env.get(env_id)
638        if not swdb_env:
639            raise CompsError(_("Environment '%s' is not installed.") % env_id)
640        if not comps_env:
641            raise CompsError(_("Environment '%s' is not available.") % env_id)
642
643        old_set = set([i.getGroupId() for i in swdb_env.getGroups()])
644        pkg_types = swdb_env.getPackageTypes()
645
646        # create a new record for current transaction
647        swdb_env = self.history.env.new(comps_env.id, comps_env.name, comps_env.ui_name, pkg_types)
648
649        trans = TransactionBunch()
650        for comps_group in comps_env.mandatory_groups:
651            if comps_group.id in old_set:
652                if self.history.group.get(comps_group.id):
653                    # upgrade installed group
654                    trans += self._group_upgrade(comps_group.id)
655            else:
656                # install new group
657                trans += self._group_install(comps_group.id, pkg_types)
658            swdb_env.addGroup(comps_group.id, True, MANDATORY)
659
660        for comps_group in comps_env.optional_groups:
661            if comps_group.id in old_set and self.history.group.get(comps_group.id):
662                # upgrade installed group
663                trans += self._group_upgrade(comps_group.id)
664            swdb_env.addGroup(comps_group.id, False, OPTIONAL)
665            # TODO: if a group is already installed, mark it as installed?
666        self.history.env.upgrade(swdb_env)
667        return trans
668
669    def _group_install(self, group_id, pkg_types, exclude=None, strict=True, exclude_groups=None):
670        assert dnf.util.is_string_type(group_id)
671        comps_group = self.comps._group_by_id(group_id)
672        if not comps_group:
673            raise CompsError(_("Group id '%s' does not exist.") % ucd(group_id))
674
675        swdb_group = self.history.group.new(group_id, comps_group.name, comps_group.ui_name, pkg_types)
676        for i in comps_group.packages_iter():
677            swdb_group.addPackage(i.name, False, Package._OPT_MAP[i.type])
678        self.history.group.install(swdb_group)
679
680        trans = TransactionBunch()
681        # TODO: remove exclude
682        if strict:
683            trans.install.update(self._pkgs_of_type(comps_group, pkg_types, exclude=[]))
684        else:
685            trans.install_opt.update(self._pkgs_of_type(comps_group, pkg_types, exclude=[]))
686        return trans
687
688    def _group_remove(self, group_id):
689        assert dnf.util.is_string_type(group_id)
690        swdb_group = self.history.group.get(group_id)
691        if not swdb_group:
692            raise CompsError(_("Module or Group '%s' is not installed.") % group_id)
693        self.history.group.remove(swdb_group)
694        trans = TransactionBunch()
695        trans.remove = {pkg for pkg in swdb_group.getPackages() if self._removable_pkg(pkg.getName())}
696        return trans
697
698    def _group_upgrade(self, group_id):
699        assert dnf.util.is_string_type(group_id)
700        comps_group = self.comps._group_by_id(group_id)
701        swdb_group = self.history.group.get(group_id)
702        exclude = []
703
704        if not swdb_group:
705            argument = comps_group.ui_name if comps_group else group_id
706            raise CompsError(_("Module or Group '%s' is not installed.") % argument)
707        if not comps_group:
708            raise CompsError(_("Module or Group '%s' is not available.") % group_id)
709        pkg_types = swdb_group.getPackageTypes()
710        old_set = set([i.getName() for i in swdb_group.getPackages()])
711        new_set = self._pkgs_of_type(comps_group, pkg_types, exclude)
712
713        # create a new record for current transaction
714        swdb_group = self.history.group.new(group_id, comps_group.name, comps_group.ui_name, pkg_types)
715        for i in comps_group.packages_iter():
716            swdb_group.addPackage(i.name, False, Package._OPT_MAP[i.type])
717        self.history.group.upgrade(swdb_group)
718
719        trans = TransactionBunch()
720        trans.install = {pkg for pkg in new_set if pkg.name not in old_set}
721        trans.remove = {name for name in old_set
722                        if name not in [pkg.name for pkg in new_set]}
723        trans.upgrade = {pkg for pkg in new_set if pkg.name in old_set}
724        return trans
725
726    def _exclude_packages_from_installed_groups(self, base):
727        for group in self.persistor.groups:
728            p_grp = self.persistor.group(group)
729            if p_grp.installed:
730                installed_pkg_names = \
731                    set(p_grp.full_list) - set(p_grp.pkg_exclude)
732                installed_pkgs = base.sack.query().installed().filterm(name=installed_pkg_names)
733                for pkg in installed_pkgs:
734                    base._goal.install(pkg)
735