1# Copyright (c) 2009, Willow Garage, Inc.
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are met:
6#
7#     * Redistributions of source code must retain the above copyright
8#       notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above copyright
10#       notice, this list of conditions and the following disclaimer in the
11#       documentation and/or other materials provided with the distribution.
12#     * Neither the name of the Willow Garage, Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived from
14#       this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26# POSSIBILITY OF SUCH DAMAGE.
27
28# Author Tully Foote/tfoote@willowgarage.com, Ken Conley/kwc@willowgarage.com
29
30from __future__ import print_function
31
32import os
33import subprocess
34import traceback
35
36from rospkg.os_detect import OsDetect
37
38from .core import rd_debug, RosdepInternalError, InstallFailed, print_bold, InvalidData
39
40# kwc: InstallerContext is basically just a bunch of dictionaries with
41# defined lookup methods.  It really encompasses two facets of a
42# rosdep configuration: the pluggable nature of installers and
43# platforms, as well as the resolution of the operating system for a
44# specific machine.  It is possible to decouple those two notions,
45# though there are some touch points over how this interfaces with the
46# rospkg.os_detect library, i.e. how platforms can tweak these
47# detectors and how the higher-level APIs can override them.
48
49
50class InstallerContext(object):
51    """
52    :class:`InstallerContext` manages the context of execution for rosdep as it
53    relates to the installers, OS detectors, and other extensible
54    APIs.
55    """
56
57    def __init__(self, os_detect=None):
58        """
59        :param os_detect: (optional)
60        :class:`rospkg.os_detect.OsDetect` instance to use for
61          detecting platforms.  If `None`, default instance will be
62          used.
63        """
64        # platform configuration
65        self.installers = {}
66        self.os_installers = {}
67        self.default_os_installer = {}
68
69        # stores configuration of which value to use for the OS version key (version number or codename)
70        self.os_version_type = {}
71
72        # OS detection and override
73        if os_detect is None:
74            os_detect = OsDetect()
75        self.os_detect = os_detect
76        self.os_override = None
77
78        self.verbose = False
79
80    def set_verbose(self, verbose):
81        self.verbose = verbose
82
83    def set_os_override(self, os_name, os_version):
84        """
85        Override the OS detector with *os_name* and *os_version*.  See
86        :meth:`InstallerContext.detect_os`.
87
88        :param os_name: OS name value to use, ``str``
89        :param os_version: OS version value to use, ``str``
90        """
91        if self.verbose:
92            print('overriding OS to [%s:%s]' % (os_name, os_version))
93        self.os_override = os_name, os_version
94
95    def get_os_version_type(self, os_name):
96        return self.os_version_type.get(os_name, OsDetect.get_version)
97
98    def set_os_version_type(self, os_name, version_type):
99        if not hasattr(version_type, '__call__'):
100            raise ValueError('version type should be a method')
101        self.os_version_type[os_name] = version_type
102
103    def get_os_name_and_version(self):
104        """
105        Get the OS name and version key to use for resolution and
106        installation.  This will be the detected OS name/version
107        unless :meth:`InstallerContext.set_os_override()` has been
108        called.
109
110        :returns: (os_name, os_version), ``(str, str)``
111        """
112        if self.os_override:
113            return self.os_override
114        else:
115            os_name = self.os_detect.get_name()
116            os_key = self.get_os_version_type(os_name)
117            os_version = os_key(self.os_detect)
118            return os_name, os_version
119
120    def get_os_detect(self):
121        """
122        :returns os_detect: :class:`OsDetect` instance used for
123          detecting platforms.
124        """
125        return self.os_detect
126
127    def set_installer(self, installer_key, installer):
128        """
129        Set the installer to use for *installer_key*.  This will
130        replace any existing installer associated with the key.
131        *installer_key* should be the same key used for the
132        ``rosdep.yaml`` package manager key.  If *installer* is
133        ``None``, this will delete any existing associated installer
134        from this context.
135
136        :param installer_key: key/name to associate with installer, ``str``
137        :param installer: :class:`Installer` implementation, ``class``.
138        :raises: :exc:`TypeError` if *installer* is not a subclass of
139          :class:`Installer`
140        """
141        if installer is None:
142            del self.installers[installer_key]
143            return
144        if not isinstance(installer, Installer):
145            raise TypeError('installer must be a instance of Installer')
146        if self.verbose:
147            print('registering installer [%s]' % (installer_key))
148        self.installers[installer_key] = installer
149
150    def get_installer(self, installer_key):
151        """
152        :returns: :class:`Installer` class associated with *installer_key*.
153        :raises: :exc:`KeyError` If not associated installer
154        :raises: :exc:`InstallFailed` If installer cannot produce an install command (e.g. if installer is not installed)
155        """
156        return self.installers[installer_key]
157
158    def get_installer_keys(self):
159        """
160        :returns: list of registered installer keys
161        """
162        return self.installers.keys()
163
164    def get_os_keys(self):
165        """
166        :returns: list of OS keys that have registered with this context, ``[str]``
167        """
168        return self.os_installers.keys()
169
170    def add_os_installer_key(self, os_key, installer_key):
171        """
172        Register an installer for the specified OS.  This will fail
173        with a :exc:`KeyError` if no :class:`Installer` can be found
174        with the associated *installer_key*.
175
176        :param os_key: Key for OS
177        :param installer_key: Key for installer to add to OS
178        :raises: :exc:`KeyError`: if installer for *installer_key*
179          is not set.
180        """
181        # validate, will throw KeyError
182        self.get_installer(installer_key)
183        if self.verbose:
184            print('add installer [%s] to OS [%s]' % (installer_key, os_key))
185        if os_key in self.os_installers:
186            self.os_installers[os_key].append(installer_key)
187        else:
188            self.os_installers[os_key] = [installer_key]
189
190    def get_os_installer_keys(self, os_key):
191        """
192        Get list of installer keys registered for the specified OS.
193        These keys can be resolved by calling
194        :meth:`InstallerContext.get_installer`.
195
196        :param os_key: Key for OS
197        :raises: :exc:`KeyError`: if no information for OS *os_key* is registered.
198        """
199        if os_key in self.os_installers:
200            return self.os_installers[os_key][:]
201        else:
202            raise KeyError(os_key)
203
204    def set_default_os_installer_key(self, os_key, installer_key):
205        """
206        Set the default OS installer to use for OS.
207        :meth:`InstallerContext.add_os_installer` must have previously
208        been called with the same arguments.
209
210        :param os_key: Key for OS
211        :param installer_key: Key for installer to add to OS
212        :raises: :exc:`KeyError`: if installer for *installer_key*
213          is not set or if OS for *os_key* has no associated installers.
214        """
215        if os_key not in self.os_installers:
216            raise KeyError('unknown OS: %s' % (os_key))
217        if not hasattr(installer_key, '__call__'):
218            raise ValueError('version type should be a method')
219        if not installer_key(self.os_detect) in self.os_installers[os_key]:
220            raise KeyError('installer [%s] is not associated with OS [%s]. call add_os_installer_key() first' % (installer_key(self.os_detect), os_key))
221        if self.verbose:
222            print('set default installer [%s] for OS [%s]' % (installer_key(self.os_detect), os_key,))
223        self.default_os_installer[os_key] = installer_key
224
225    def get_default_os_installer_key(self, os_key):
226        """
227        Get the default OS installer key to use for OS, or ``None`` if
228        there is no default.
229
230        :param os_key: Key for OS
231        :returns: :class:`Installer`
232        :raises: :exc:`KeyError`: if no information for OS *os_key* is registered.
233        """
234        if os_key not in self.os_installers:
235            raise KeyError('unknown OS: %s' % (os_key))
236        try:
237            installer_key = self.default_os_installer[os_key](self.os_detect)
238            if installer_key not in self.os_installers[os_key]:
239                raise KeyError('installer [%s] is not associated with OS [%s]. call add_os_installer_key() first' % (installer_key, os_key))
240            # validate, will throw KeyError
241            self.get_installer(installer_key)
242            return installer_key
243        except KeyError:
244            return None
245
246
247class Installer(object):
248    """
249    The :class:`Installer` API is designed around opaque *resolved*
250    parameters. These parameters can be any type of sequence object,
251    but they must obey set arithmetic.  They should also implement
252    ``__str__()`` methods so they can be pretty printed.
253    """
254
255    def is_installed(self, resolved_item):
256        """
257        :param resolved: resolved installation item. NOTE: this is a single item,
258          not a list of items like the other APIs, ``opaque``.
259        :returns: ``True`` if all of the *resolved* items are installed on
260          the local system
261        """
262        raise NotImplementedError('is_installed', resolved_item)
263
264    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
265        """
266        :param resolved: list of resolved installation items, ``[opaque]``
267        :param interactive: If `False`, disable interactive prompts,
268          e.g. Pass through ``-y`` or equivalant to package manager.
269        :param reinstall: If `True`, install everything even if already installed
270        """
271        raise NotImplementedError('get_package_install_command', resolved, interactive, reinstall, quiet)
272
273    def get_depends(self, rosdep_args):
274        """
275        :returns: list of dependencies on other rosdep keys.  Only
276          necessary if the package manager doesn't handle
277          dependencies.
278        """
279        return []  # Default return empty list
280
281    def resolve(self, rosdep_args_dict):
282        """
283        :param rosdep_args_dict: argument dictionary to the rosdep rule for this package manager
284        :returns: [resolutions].  resolved objects should be printable to a user, but are otherwise opaque.
285        """
286        raise NotImplementedError('Base class resolve', rosdep_args_dict)
287
288    def unique(self, *resolved_rules):
289        """
290        Combine the resolved rules into a unique list.  This
291        is meant to combine the results of multiple calls to
292        :meth:`PackageManagerInstaller.resolve`.
293
294        Example::
295
296            resolved1 = installer.resolve(args1)
297            resolved2 = installer.resolve(args2)
298            resolved = installer.unique(resolved1, resolved2)
299
300        :param resolved_rules: resolved arguments.  Resolved
301          arguments must all be from this :class:`Installer` instance.
302        """
303        raise NotImplementedError('Base class unique', resolved_rules)
304
305
306class PackageManagerInstaller(Installer):
307    """
308    General form of a package manager :class:`Installer`
309    implementation that assumes:
310
311     - installer rosdep args spec is a list of package names stored with the key "packages"
312     - a detect function exists that can return a list of packages that are installed
313
314    Also, if *supports_depends* is set to ``True``:
315
316     - installer rosdep args spec can also include dependency specification with the key "depends"
317    """
318
319    def __init__(self, detect_fn, supports_depends=False):
320        """
321        :param supports_depends: package manager supports dependency key
322        :param detect_fn: function that for a given list of packages determines
323        the list of installed packages.
324        """
325        self.detect_fn = detect_fn
326        self.supports_depends = supports_depends
327        self.as_root = True
328        self.sudo_command = 'sudo -H' if os.geteuid() != 0 else ''
329
330    def elevate_priv(self, cmd):
331        """
332        Prepend *self.sudo_command* to the command if *self.as_root* is ``True``.
333
334        :param list cmd: list of strings comprising the command
335        :returns: a list of commands
336        """
337        return (self.sudo_command.split() if self.as_root else []) + cmd
338
339    def resolve(self, rosdep_args):
340        """
341        See :meth:`Installer.resolve()`
342        """
343        packages = None
344        if type(rosdep_args) == dict:
345            packages = rosdep_args.get('packages', [])
346            if isinstance(packages, str):
347                packages = packages.split()
348        elif isinstance(rosdep_args, str):
349            packages = rosdep_args.split(' ')
350        elif type(rosdep_args) == list:
351            packages = rosdep_args
352        else:
353            raise InvalidData('Invalid rosdep args: %s' % (rosdep_args))
354        return packages
355
356    def unique(self, *resolved_rules):
357        """
358        See :meth:`Installer.unique()`
359        """
360        s = set()
361        for resolved in resolved_rules:
362            s.update(resolved)
363        return sorted(list(s))
364
365    def get_packages_to_install(self, resolved, reinstall=False):
366        """
367        Return a list of packages (out of *resolved*) that still need to get
368        installed.
369        """
370        if reinstall:
371            return resolved
372        if not resolved:
373            return []
374        else:
375            detected = self.detect_fn(resolved)
376            return [x for x in resolved if x not in detected]
377
378    def is_installed(self, resolved_item):
379        """
380        Check if a given package was installed.
381        """
382        return not self.get_packages_to_install([resolved_item])
383
384    def get_version_strings(self):
385        """
386        Return a list of version information strings.
387
388        Where each string is of the form "<installer> <version string>".
389        For example, ["apt-get x.y.z"] or ["pip x.y.z", "setuptools x.y.z"].
390        """
391        raise NotImplementedError('subclasses must implement get_version_strings method')
392
393    def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False):
394        raise NotImplementedError('subclasses must implement', resolved, interactive, reinstall, quiet)
395
396    def get_depends(self, rosdep_args):
397        """
398        :returns: list of dependencies on other rosdep keys.  Only
399          necessary if the package manager doesn't handle
400          dependencies.
401        """
402        if self.supports_depends and type(rosdep_args) == dict:
403            return rosdep_args.get('depends', [])
404        return []  # Default return empty list
405
406
407def normalize_uninstalled_to_list(uninstalled):
408    uninstalled_dependencies = []
409    for pkg_or_list in [v for k, v in uninstalled]:
410        if isinstance(pkg_or_list, list):
411            for pkg in pkg_or_list:
412                uninstalled_dependencies.append(str(pkg))
413        else:
414            uninstalled_dependencies.append(str(pkg))
415    return uninstalled_dependencies
416
417
418class RosdepInstaller(object):
419
420    def __init__(self, installer_context, lookup):
421        self.installer_context = installer_context
422        self.lookup = lookup
423
424    def get_uninstalled(self, resources, implicit=False, verbose=False):
425        """
426        Get list of system dependencies that have not been installed
427        as well as a list of errors from performing the resolution.
428        This is a bulk API in order to provide performance
429        optimizations in checking install state.
430
431        :param resources: List of resource names (e.g. ROS package names), ``[str]]``
432        :param implicit: Install implicit (recursive) dependencies of
433            resources.  Default ``False``.
434
435        :returns: (uninstalled, errors), ``({str: [opaque]}, {str: ResolutionError})``.
436          Uninstalled is a dictionary with the installer_key as the key.
437        :raises: :exc:`RosdepInternalError`
438        """
439
440        installer_context = self.installer_context
441
442        # resolutions have been unique()d
443        if verbose:
444            print('resolving for resources [%s]' % (', '.join(resources)))
445        resolutions, errors = self.lookup.resolve_all(resources, installer_context, implicit=implicit)
446
447        # for each installer, figure out what is left to install
448        uninstalled = []
449        if resolutions == []:
450            return uninstalled, errors
451        for installer_key, resolved in resolutions:  # py3k
452            if verbose:
453                print('resolution: %s [%s]' % (installer_key, ', '.join([str(r) for r in resolved])))
454            try:
455                installer = installer_context.get_installer(installer_key)
456            except KeyError as e:  # lookup has to be buggy to cause this
457                raise RosdepInternalError(e)
458            try:
459                packages_to_install = installer.get_packages_to_install(resolved)
460            except Exception as e:
461                rd_debug(traceback.format_exc())
462                raise RosdepInternalError(e, message='Bad installer [%s]: %s' % (installer_key, e))
463
464            # only create key if there is something to do
465            if packages_to_install:
466                uninstalled.append((installer_key, packages_to_install))
467            if verbose:
468                print('uninstalled: [%s]' % (', '.join([str(p) for p in packages_to_install])))
469
470        return uninstalled, errors
471
472    def install(self, uninstalled, interactive=True, simulate=False,
473                continue_on_error=False, reinstall=False, verbose=False, quiet=False):
474        """
475        Install the uninstalled rosdeps.  This API is for the bulk
476        workflow of rosdep (see example below).  For a more targeted
477        install API, see :meth:`RosdepInstaller.install_resolved`.
478
479        :param uninstalled: uninstalled value from
480          :meth:`RosdepInstaller.get_uninstalled`.  Value is a
481          dictionary mapping installer key to a dictionary with resolution
482          data, ``{str: {str: vals}}``
483        :param interactive: If ``False``, suppress
484          interactive prompts (e.g. by passing '-y' to ``apt``).
485        :param simulate: If ``False`` simulate installation
486          without actually executing.
487        :param continue_on_error: If ``True``, continue installation
488          even if an install fails.  Otherwise, stop after first
489          installation failure.
490        :param reinstall: If ``True``, install dependencies if even
491          already installed (default ``False``).
492
493        :raises: :exc:`InstallFailed` if any rosdeps fail to install
494          and *continue_on_error* is ``False``.
495        :raises: :exc:`KeyError` If *uninstalled* value has invalid
496          installer keys
497
498        Example::
499
500            uninstalled, errors = installer.get_uninstalled(packages)
501            installer.install(uninstalled)
502        """
503        if verbose:
504            print(
505                'install options: reinstall[%s] simulate[%s] interactive[%s]' %
506                (reinstall, simulate, interactive)
507            )
508            uninstalled_list = normalize_uninstalled_to_list(uninstalled)
509            print('install: uninstalled keys are %s' % ', '.join(uninstalled_list))
510
511        # Squash uninstalled again, in case some dependencies were already installed
512        squashed_uninstalled = []
513        previous_installer_key = None
514        for installer_key, resolved in uninstalled:
515            if previous_installer_key != installer_key:
516                squashed_uninstalled.append((installer_key, []))
517                previous_installer_key = installer_key
518            squashed_uninstalled[-1][1].extend(resolved)
519
520        failures = []
521        for installer_key, resolved in squashed_uninstalled:
522            try:
523                self.install_resolved(installer_key, resolved, simulate=simulate,
524                                      interactive=interactive, reinstall=reinstall, continue_on_error=continue_on_error,
525                                      verbose=verbose, quiet=quiet)
526            except InstallFailed as e:
527                if not continue_on_error:
528                    raise
529                else:
530                    # accumulate errors
531                    failures.extend(e.failures)
532        if failures:
533            raise InstallFailed(failures=failures)
534
535    def install_resolved(self, installer_key, resolved, simulate=False, interactive=True,
536                         reinstall=False, continue_on_error=False, verbose=False, quiet=False):
537        """
538        Lower-level API for installing a rosdep dependency.  The
539        rosdep keys have already been resolved to *installer_key* and
540        *resolved* via :exc:`RosdepLookup` or other means.
541
542        :param installer_key: Key for installer to apply to *resolved*, ``str``
543        :param resolved: Opaque resolution list from :class:`RosdepLookup`.
544        :param interactive: If ``True``, allow interactive prompts (default ``True``)
545        :param simulate: If ``True``, don't execute installation commands, just print to screen.
546        :param reinstall: If ``True``, install dependencies if even
547          already installed (default ``False``).
548        :param verbose: If ``True``, print verbose output to screen (default ``False``)
549        :param quiet: If ``True``, supress output except for errors (default ``False``)
550
551        :raises: :exc:`InstallFailed` if any of *resolved* fail to install.
552        """
553        installer_context = self.installer_context
554        installer = installer_context.get_installer(installer_key)
555        command = installer.get_install_command(resolved, interactive=interactive, reinstall=reinstall, quiet=quiet)
556        if not command:
557            if verbose:
558                print('#No packages to install')
559            return
560
561        if simulate:
562            print('#[%s] Installation commands:' % (installer_key))
563            for sub_command in command:
564                if isinstance(sub_command[0], list):
565                    sub_cmd_len = len(sub_command)
566                    for i, cmd in enumerate(sub_command):
567                        print("  '%s' (alternative %d/%d)" % (' '.join(cmd), i + 1, sub_cmd_len))
568                else:
569                    print('  ' + ' '.join(sub_command))
570
571        # nothing left to do for simulation
572        if simulate:
573            return
574
575        def run_command(command, installer_key, failures, verbose):
576            # always echo commands to screen
577            print_bold('executing command [%s]' % ' '.join(command))
578            result = subprocess.call(command)
579            if verbose:
580                print('command return code [%s]: %s' % (' '.join(command), result))
581            if result != 0:
582                failures.append((installer_key, 'command [%s] failed' % (' '.join(command))))
583            return result
584
585        # run each install command set and collect errors
586        failures = []
587        for sub_command in command:
588            if isinstance(sub_command[0], list):  # list of alternatives
589                alt_failures = []
590                for alt_command in sub_command:
591                    result = run_command(alt_command, installer_key, alt_failures, verbose)
592                    if result == 0:  # one successsfull command is sufficient
593                        alt_failures = []  # clear failuers from other alternatives
594                        break
595                failures.extend(alt_failures)
596            else:
597                result = run_command(sub_command, installer_key, failures, verbose)
598            if result != 0:
599                if not continue_on_error:
600                    raise InstallFailed(failures=failures)
601
602        # test installation of each
603        for r in resolved:
604            if not installer.is_installed(r):
605                failures.append((installer_key, 'Failed to detect successful installation of [%s]' % (r)))
606        # finalize result
607        if failures:
608            raise InstallFailed(failures=failures)
609        elif verbose:
610            print('#successfully installed')
611