1# Software License Agreement (BSD License)
2#
3# Copyright (c) 2010, Willow Garage, Inc.
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions
8# are met:
9#
10#  * Redistributions of source code must retain the above copyright
11#    notice, this list of conditions and the following disclaimer.
12#  * Redistributions in binary form must reproduce the above
13#    copyright notice, this list of conditions and the following
14#    disclaimer in the documentation and/or other materials provided
15#    with the distribution.
16#  * Neither the name of Willow Garage, Inc. nor the names of its
17#    contributors may be used to endorse or promote products derived
18#    from this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31# POSSIBILITY OF SUCH DAMAGE.
32
33from __future__ import print_function
34import os
35import sys
36import textwrap
37import shutil
38import datetime
39import yaml
40from optparse import OptionParser, IndentedHelpFormatter
41
42from wstool.cli_common import get_info_list, get_info_table, \
43    get_info_table_raw_csv, ONLY_OPTION_VALID_ATTRS
44from wstool.common import samefile, select_element, select_elements, \
45    MultiProjectException, normalize_uri, string_diff
46from wstool.config_yaml import PathSpec, get_path_spec_from_yaml
47import wstool.multiproject_cmd as multiproject_cmd
48from wstool.ui import Ui
49
50# implementation of single CLI commands (extracted for use in several
51# overlapping scripts)
52
53# usage help
54__MULTIPRO_CMD_DICT__ = {
55    "help":     "provide help for commands",
56    "init":     "set up a directory as workspace",
57    "info":     "Overview of some entries",
58    "merge":    "merges your workspace with another config set",
59    "set":      "add or changes one entry from your workspace config",
60    "update":   "update or check out some of your config elements",
61    "remove":   "remove an entry from your workspace config, without deleting files",
62    "export":   "export a snapshot of the workspace",
63    "diff":     "print a diff over some SCM controlled entries",
64    "foreach":  "run shell command in given entries",
65    "status":   "print the change status of files in some SCM controlled entries",
66    "scrape":   "interactively add all found unmanaged VCS subfolders to workspace"
67}
68
69# usage help ordering and sections
70__MULTIPRO_CMD_HELP_LIST__ = ['help', 'init',
71                              None, 'set', 'merge', 'remove', 'scrape',
72                              None, 'update',
73                              None, 'info', 'export', 'status', 'diff', 'foreach']
74
75# command aliases
76__MULTIPRO_CMD_ALIASES__ = {'update': 'up',
77                            'remove': 'rm',
78                            'status': 'st',
79                            'diff': 'di'}
80
81
82def get_header(progname):
83    config_header = ("# THIS IS AN AUTOGENERATED FILE, LAST GENERATED USING %s ON %s\n"
84                     % (progname, datetime.date.today().isoformat()))
85    return config_header
86
87
88class IndentedHelpFormatterWithNL(IndentedHelpFormatter):
89    def format_description(self, description):
90        if not description:
91            return ""
92        desc_width = self.width - self.current_indent
93        indent = " " * self.current_indent
94        # the above is still the same
95        bits = description.split('\n')
96        formatted_bits = [
97            textwrap.fill(bit,
98                          desc_width,
99                          initial_indent=indent,
100                          subsequent_indent=indent)
101            for bit in bits]
102        result = "\n".join(formatted_bits) + "\n"
103        return result
104
105
106def _get_mode_from_options(parser, options):
107    mode = 'prompt'
108    if options.delete_changed:
109        mode = 'delete'
110    if options.abort_changed:
111        if mode == 'delete':
112            parser.error("delete-changed-uris is mutually exclusive with abort-changed-uris")
113        mode = 'abort'
114    if options.backup_changed != '':
115        if mode == 'delete':
116            parser.error("delete-changed-uris is mutually exclusive with backup-changed-uris")
117        if mode == 'abort':
118            parser.error("abort-changed-uris is mutually exclusive with backup-changed-uris")
119        mode = 'backup'
120    return mode
121
122
123def _get_element_diff(new_path_spec, config_old, extra_verbose=False):
124    """
125    :returns: a string telling what changed for element compared to old config
126    """
127    if new_path_spec is None or config_old is None:
128        return ''
129    output = [' %s' % new_path_spec.get_local_name()]
130    if extra_verbose:
131        old_element = None
132        if config_old is not None:
133            old_element = select_element(config_old.get_config_elements(),
134                                         new_path_spec.get_local_name())
135
136        if old_element is None:
137            if new_path_spec.get_scmtype() is not None:
138                output.append(
139                    "   \t%s  %s   %s" % (new_path_spec.get_scmtype(),
140                                          new_path_spec.get_uri(),
141                                          new_path_spec.get_version() or ''))
142        else:
143            old_path_spec = old_element.get_path_spec()
144            accessor_map = {PathSpec.get_scmtype: 'scmtype',
145                            PathSpec.get_version: 'version',
146                            PathSpec.get_revision: 'revision',
147                            PathSpec.get_current_revision: 'current revision',
148                            PathSpec.get_curr_uri: 'current_uri',
149                            PathSpec.get_uri: 'specified uri'}
150            for accessor, label in list(accessor_map.items()):
151                old_val = accessor(old_path_spec)
152                new_val = accessor(new_path_spec)
153                if old_val is not None and\
154                        old_val != new_val:
155                    diff = string_diff(old_val, new_val)
156                    output.append("  \t%s: %s -> %s;" % (label, old_val, diff))
157                elif old_val is None and\
158                        new_val is not None and\
159                        new_val != "" and\
160                        new_val != []:
161                    output.append("  %s = %s" % (label,
162                                                 new_val))
163    return ''.join(output)
164
165
166def prompt_merge(target_path,
167                 additional_uris,
168                 additional_specs,
169                 path_change_message=None,
170                 merge_strategy='KillAppend',
171                 confirmed=False,
172                 confirm=False,
173                 show_advanced=True,
174                 show_verbosity=True,
175                 config_filename=None,
176                 config=None,
177                 allow_other_element=True):
178    """
179    Prompts the user for the resolution of a merge. Without
180    further options, will prompt only if elements change. New
181    elements are just added without prompt.
182
183    :param target_path: Location of the config workspace
184    :param additional_uris: uris from which to load more elements
185    :param additional_specs: path specs for additional elements
186    :param path_change_message: Something to tell the user about elements order
187    :param merge_strategy: See Config.insert_element
188    :param confirmed: Never ask
189    :param confirm: Always ask, supercedes confirmed
190    :param config: None or a Config object for target path if available
191    :param show_advanced: if true allow to change merge strategy
192    :param show_verbosity: if true allows to change verbosity
193    :param allow_other_element: if False merge fails hwen it could cause other elements
194    :returns: tupel (Config or None if no change, bool path_changed)
195    """
196    if config is None:
197        config = multiproject_cmd.get_config(
198            target_path,
199            additional_uris=[],
200            config_filename=config_filename)
201    elif config.get_base_path() != target_path:
202        msg = "Config path does not match %s %s " % (config.get_base_path(),
203                                                     target_path)
204        raise MultiProjectException(msg)
205    local_names_old = [x.get_local_name() for x in config.get_config_elements()]
206
207    extra_verbose = confirmed or confirm
208    abort = False
209    last_merge_strategy = None
210    while not abort:
211
212        if (last_merge_strategy is None
213            or last_merge_strategy != merge_strategy):
214            if not config_filename:
215                # should never happen right now with rosinstall/rosws/wstool
216                # TODO Need a better way to work with clones of original config
217                raise ValueError('Cannot merge when no config filename is set')
218            newconfig = multiproject_cmd.get_config(
219                target_path,
220                additional_uris=[],
221                config_filename=config_filename)
222            config_actions = multiproject_cmd.add_uris(
223                config=newconfig,
224                additional_uris=additional_uris,
225                config_filename=None,
226                merge_strategy=merge_strategy,
227                allow_other_element=allow_other_element)
228            for path_spec in additional_specs:
229                action = newconfig.add_path_spec(path_spec, merge_strategy)
230                config_actions[path_spec.get_local_name()] = (action, path_spec)
231            last_merge_strategy = merge_strategy
232
233        local_names_new = [x.get_local_name() for x in newconfig.get_config_elements()]
234
235        path_changed = False
236        ask_user = False
237        output = ""
238        new_elements = []
239        changed_elements = []
240        discard_elements = []
241        for localname, (action, new_path_spec) in list(config_actions.items()):
242            index = -1
243            if localname in local_names_old:
244                index = local_names_old.index(localname)
245            if action == 'KillAppend':
246                ask_user = True
247                if (index > -1 and local_names_old[:index + 1] == local_names_new[:index + 1]):
248                    action = 'MergeReplace'
249                else:
250                    changed_elements.append(_get_element_diff(new_path_spec, config, extra_verbose))
251                    path_changed = True
252
253            if action == 'Append':
254                path_changed = True
255                new_elements.append(_get_element_diff(new_path_spec,
256                                                      config,
257                                                      extra_verbose))
258            elif action == 'MergeReplace':
259                changed_elements.append(_get_element_diff(new_path_spec,
260                                                          config,
261                                                          extra_verbose))
262                ask_user = True
263            elif action == 'MergeKeep':
264                discard_elements.append(_get_element_diff(new_path_spec,
265                                                          config,
266                                                          extra_verbose))
267                ask_user = True
268        if len(changed_elements) > 0:
269            output += "\n     Change details of element (Use --merge-keep or --merge-replace to change):\n"
270            if extra_verbose:
271                output += " %s\n" % ("\n".join(sorted(changed_elements)))
272            else:
273                output += " %s\n" % (", ".join(sorted(changed_elements)))
274        if len(new_elements) > 0:
275            output += "\n     Add new elements:\n"
276            if extra_verbose:
277                output += " %s\n" % ("\n".join(sorted(new_elements)))
278            else:
279                output += " %s\n" % (", ".join(sorted(new_elements)))
280
281        if local_names_old != local_names_new[:len(local_names_old)]:
282            old_order = ' '.join(reversed(local_names_old))
283            new_order = ' '.join(reversed(local_names_new))
284            output += "\n     %s " % path_change_message or "Element order change"
285            output += "(Use --merge-keep or --merge-replace to prevent) "
286            output += "from\n %s\n     to\n %s\n\n" % (old_order, new_order)
287            ask_user = True
288
289        if output == "":
290            return (None, False)
291        if not confirm and (confirmed or not ask_user):
292            print("     Performing actions: ")
293            print(output)
294            return (newconfig, path_changed)
295        else:
296            print(output)
297            showhelp = True
298            while(showhelp):
299                showhelp = False
300                prompt = "Continue: (y)es, (n)o"
301                if show_verbosity:
302                    prompt += ", (v)erbosity"
303                if show_advanced:
304                    prompt += ", (a)dvanced options"
305                prompt += ": "
306                mode_input = Ui.get_ui().get_input(prompt)
307                if mode_input == 'y':
308                    return (newconfig, path_changed)
309                elif mode_input == 'n':
310                    abort = True
311                elif show_advanced and mode_input == 'a':
312                    strategies = {'MergeKeep': "(k)eep",
313                                  'MergeReplace': "(s)witch in",
314                                  'KillAppend': "(a)ppending"}
315                    unselected = [v for k, v in
316                                  list(strategies.items())
317                                  if k != merge_strategy]
318                    print("""New entries will just be appended to the config and
319appear at the beginning of your ROS_PACKAGE_PATH. The merge strategy
320decides how to deal with entries having a duplicate localname or path.
321
322"(k)eep" means the existing entry will stay as it is, the new one will
323be discarded. Useful for getting additional elements from other
324workspaces without affecting your setup.
325
326"(s)witch in" means that the new entry will replace the old in the
327same position. Useful for upgrading/downgrading.
328
329"switch (a)ppend" means that the existing entry will be removed, and
330the new entry appended to the end of the list. This maintains order
331of elements in the order they were given.
332
333Switch append is the default.
334""")
335                    prompt = "Change Strategy %s: " % (", ".join(unselected))
336                    mode_input = Ui.get_ui().get_input(prompt)
337                    if mode_input == 's':
338                        merge_strategy = 'MergeReplace'
339                    elif mode_input == 'k':
340                        merge_strategy = 'MergeKeep'
341                    elif mode_input == 'a':
342                        merge_strategy = 'KillAppend'
343
344                elif show_verbosity and mode_input == 'v':
345                    extra_verbose = not extra_verbose
346        if abort:
347            print("No changes made.")
348        print('==========================================')
349    return (None, False)
350
351
352def list_usage(progname, description, command_keys, command_helps, command_aliases):
353    """
354    Constructs program usage for a list of commands with help and aliases.
355    Contructs in the order given in command keys. Newlines can be used for
356    command sections by adding None entries to command_keys list.
357    Only one alias allowed per command.
358
359    :param command_keys: list of keys or None to print help or empty lines
360    :param command_helps: dict{key: help}
361    :param command_aliases: dict{key: alias}
362    :returns: usage string (multiline)
363    """
364    dvars = {'prog': progname}
365    dvars.update(vars())
366    result = []
367    result.append(description % dvars)
368    for key in command_keys:
369        if key in command_aliases:
370            alias = ' (%s)' % command_aliases[key]
371        else:
372            alias = ''
373        if key is not None:
374            result.append(("%s%s" % (key, alias)).ljust(10) + '   \t' + command_helps[key])
375        else:
376            result.append('')
377    return '\n'.join(result)
378
379
380class MultiprojectCLI:
381
382    def __init__(self,
383                 progname,
384                 config_filename=None,
385                 allow_other_element=False,
386                 config_generator=None):
387        '''
388        creates the instance. Historically, rosinstall allowed "other"
389        elements that went into the ROS_PACKAGE_PATH, but were ignored
390        for vcs operations. A pure vcs tool has no use for such
391        elements.
392
393        :param progname: name to diplay in help
394        :param config_filename: filename of files maintaining workspaces (.rosinstall)
395        :param allow_other_element: bool, if True rosinstall semantics for "other" apply
396        :param config_generator: function that writes config file
397        '''
398        self.config_filename = config_filename
399        self.config_generator = config_generator or multiproject_cmd.cmd_persist_config
400        self.progname = progname
401        self.allow_other_element = allow_other_element
402
403    def cmd_init(self, argv):
404        if self.config_filename is None:
405            print('Error: Bug: config filename required for init')
406            return 1
407        parser = OptionParser(
408            usage="""usage: %s init [TARGET_PATH [SOURCE_PATH]]?""" % self.progname,
409            formatter=IndentedHelpFormatterWithNL(),
410            description=__MULTIPRO_CMD_DICT__["init"] + """
411
412%(prog)s init does the following:
413  1. Reads folder/file/web-uri SOURCE_PATH looking for a rosinstall yaml
414  2. Creates new %(cfg_file)s file at TARGET-PATH
415
416SOURCE_PATH can e.g. be a web uri or a rosinstall file with vcs entries only
417If PATH is not given, uses current dir.
418
419Examples:
420$ %(prog)s init ~/fuerte /opt/ros/fuerte
421""" % {'cfg_file': self.config_filename, 'prog': self.progname},
422                              epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
423        parser.add_option("--continue-on-error", dest="robust", default=False,
424                          help="Continue despite checkout errors",
425                          action="store_true")
426        parser.add_option("-j", "--parallel", dest="jobs", default=1,
427                          help="How many parallel threads to use for installing",
428                          action="store")
429        parser.add_option("--shallow", dest="shallow", default=False,
430                          help="Checkout only latest revision if possible",
431                          action="store_true")
432        (options, args) = parser.parse_args(argv)
433        if len(args) < 1:
434            target_path = '.'
435        else:
436            target_path = args[0]
437
438        if not os.path.isdir(target_path):
439            if not os.path.exists(target_path):
440                os.mkdir(target_path)
441            else:
442                print('Error: Cannot create in target path %s ' % target_path)
443
444        if os.path.exists(os.path.join(target_path, self.config_filename)):
445            print('Error: There already is a workspace config file %s at "%s". Use %s install/modify.' %
446                  (self.config_filename, target_path, self.progname))
447            return 1
448        if len(args) > 2:
449            parser.error('Too many arguments')
450
451        if len(args) == 2:
452            print('Using initial elements from: %s' % args[1])
453            config_uris = [args[1]]
454        else:
455            config_uris = []
456
457        config = multiproject_cmd.get_config(
458            basepath=target_path,
459            additional_uris=config_uris,
460            # catkin workspaces have no reasonable chaining semantics
461            # config_filename=self.config_filename
462            )
463        if config_uris and len(config.get_config_elements()) == 0:
464            sys.stderr.write('WARNING: Not using any element from %s\n' % config_uris[0])
465        for element in config.get_config_elements():
466            if not element.is_vcs_element():
467                raise MultiProjectException("wstool does not allow elements without vcs information. %s" % element)
468
469        # includes ROS specific files
470
471        if self.config_filename:
472            print("Writing %s" % os.path.join(config.get_base_path(), self.config_filename))
473        self.config_generator(config, self.config_filename, get_header(self.progname))
474
475        ## install or update each element
476        install_success = multiproject_cmd.cmd_install_or_update(
477            config,
478            robust=False,
479            shallow=options.shallow,
480            num_threads=int(options.jobs))
481
482        if not install_success:
483            print("Warning: installation encountered errors, but --continue-on-error was requested.  Look above for warnings.")
484        print("\nupdate complete.")
485        return 0
486
487    def cmd_merge(self, target_path, argv, config=None):
488        parser = OptionParser(
489            usage="usage: %s merge [URI] [OPTIONS]" % self.progname,
490            formatter=IndentedHelpFormatterWithNL(),
491            description=__MULTIPRO_CMD_DICT__["merge"] + """.
492
493The command merges config with given other rosinstall element sets, from files or web uris.
494
495The default workspace will be inferred from context, you can specify one using -t.
496
497By default, when an element in an additional URI has the same
498local-name as an existing element, the existing element will be
499replaced. In order to ensure the ordering of elements is as
500provided in the URI, use the option --merge-kill-append.
501
502Examples:
503$ %(prog)s merge someother.rosinstall
504
505You can use '-' to pipe in input, as an example:
506$ roslocate info robot_model | %(prog)s merge -
507""" % {'prog': self.progname},
508            epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
509        # same options as for multiproject
510        parser.add_option(
511            "-a", "--merge-kill-append", dest="merge_kill_append",
512            default=False,
513            help="merge by deleting given entry and appending new one",
514            action="store_true")
515        parser.add_option("-k", "--merge-keep", dest="merge_keep",
516                          default=False,
517                          help="merge by keeping existing entry and discarding new one",
518                          action="store_true")
519        parser.add_option("-r", "--merge-replace", dest="merge_replace",
520                          default=False,
521                          help="(default) merge by replacing given entry with new one maintaining ordering",
522                          action="store_true")
523        parser.add_option("-y", "--confirm-all", dest="confirm_all",
524                          default='',
525                          help="do not ask for confirmation unless strictly necessary",
526                          action="store_true")
527        # required here but used one layer above
528        parser.add_option(
529            "-t", "--target-workspace", dest="workspace", default=None,
530            help="which workspace to use",
531            action="store")
532        (options, args) = parser.parse_args(argv)
533
534        if len(args) > 1:
535            print("Error: Too many arguments.")
536            print(parser.usage)
537            return -1
538        if len(args) == 0:
539            print("Error: Too few arguments.")
540            print(parser.usage)
541            return -1
542
543        config_uris = args
544
545        specs = []
546        if config_uris[0] == '-':
547            pipedata = "".join(sys.stdin.readlines())
548            try:
549                yamldicts = yaml.safe_load(pipedata)
550            except yaml.YAMLError as e:
551                raise MultiProjectException(
552                    "Invalid yaml format: \n%s \n%s" % (pipedata, e))
553            if yamldicts is None:
554                parser.error("No Input read from stdin")
555            # cant have user interaction and piped input
556            options.confirm_all = True
557            specs.extend([get_path_spec_from_yaml(x) for x in yamldicts])
558            config_uris = []
559
560        merge_strategy = None
561        count_mergeoptions = 0
562        if options.merge_kill_append:
563            merge_strategy = 'KillAppend'
564            count_mergeoptions += 1
565        if options.merge_keep:
566            merge_strategy = 'MergeKeep'
567            count_mergeoptions += 1
568        if options.merge_replace:
569            merge_strategy = 'MergeReplace'
570            count_mergeoptions += 1
571        if count_mergeoptions > 1:
572            parser.error("You can only provide one merge-strategy")
573        # default option
574        if count_mergeoptions == 0:
575            merge_strategy = 'MergeReplace'
576        (newconfig, _) = prompt_merge(
577            target_path,
578            additional_uris=config_uris,
579            additional_specs=specs,
580            path_change_message="element order changed",
581            merge_strategy=merge_strategy,
582            confirmed=options.confirm_all,
583            config_filename=self.config_filename,
584            config=config,
585            allow_other_element=self.allow_other_element)
586        if newconfig is not None:
587            print("Config changed, maybe you need run %s update to update SCM entries." % self.progname)
588            print("Overwriting %s" % os.path.join(newconfig.get_base_path(), self.config_filename))
589            shutil.copy(os.path.join(newconfig.get_base_path(), self.config_filename), "%s.bak" % os.path.join(newconfig.get_base_path(), self.config_filename))
590            self.config_generator(newconfig, self.config_filename, get_header(self.progname))
591            print("\nupdate complete.")
592        else:
593            print("Merge caused no change, no new elements found")
594        return 0
595
596    def cmd_diff(self, target_path, argv, config=None):
597        parser = OptionParser(usage="usage: %s diff [localname]* " % self.progname,
598                              description=__MULTIPRO_CMD_DICT__["diff"],
599                              epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
600        # required here but used one layer above
601        parser.add_option("-t", "--target-workspace", dest="workspace",
602                          default=None,
603                          help="which workspace to use",
604                          action="store")
605        (_, args) = parser.parse_args(argv)
606
607        if config is None:
608            config = multiproject_cmd.get_config(
609                target_path,
610                additional_uris=[],
611                config_filename=self.config_filename)
612        elif config.get_base_path() != target_path:
613            raise MultiProjectException(
614                "Config path does not match %s %s " % (config.get_base_path(),
615                                                       target_path))
616
617        if len(args) > 0:
618            difflist = multiproject_cmd.cmd_diff(config, localnames=args)
619        else:
620            difflist = multiproject_cmd.cmd_diff(config)
621        alldiff = []
622        for entrydiff in difflist:
623            if entrydiff['diff'] is not None and entrydiff['diff'] != '':
624                alldiff.append(entrydiff['diff'])
625        result = '\n'.join(alldiff)
626        # result has no newline at end
627        if result:
628            print(result)
629
630        return False
631
632    def cmd_foreach(self, target_path, argv, config=None):
633        """Run shell commands in each repository."""
634        parser = OptionParser(
635            usage=('usage: %s foreach [[localname]* | [VCSFILTER]*]'
636                   ' [command] [OPTIONS]' % self.progname),
637            formatter=IndentedHelpFormatterWithNL(),
638            description=__MULTIPRO_CMD_DICT__['foreach'] + """.
639
640Example:
641$ %(progname)s foreach --git 'git status'
642""" % { 'progname': self.progname},
643            epilog='See: http://www.ros.org/wiki/rosinstall for details')
644        parser.add_option('--shell', default=False,
645                          help='use the shell as the program to execute',
646                          action='store_true')
647        parser.add_option('--no-stdout', dest='show_stdout',
648                          default=True,
649                          help='do not show stdout',
650                          action='store_false')
651        parser.add_option('--no-stderr', dest='show_stderr',
652                          default=True,
653                          help='do not show stderr',
654                          action='store_false')
655        parser.add_option("--git", dest="git", default=False,
656                          help="run command in git entries",
657                          action="store_true")
658        parser.add_option("--svn", dest="svn", default=False,
659                          help="run command in svn entries",
660                          action="store_true")
661        parser.add_option("--hg", dest="hg", default=False,
662                          help="run command in hg entries",
663                          action="store_true")
664        parser.add_option("--bzr", dest="bzr", default=False,
665                          help="run command in bzr entries",
666                          action="store_true")
667        parser.add_option("-m", "--timeout", dest="timeout",
668                          default=None,
669                          help="How long to wait for each repo before failing [seconds]",
670                          action="store", type=float)
671        parser.add_option("-j", "--parallel", dest="jobs",
672                          default=1,
673                          help="How many parallel threads to use for running the custom commands",
674                          action="store")
675        parser.add_option("-v", "--verbose", dest="verbose",
676                          default=False,
677                          help="Whether to print out more information",
678                          action="store_true")
679        # -t option required here for help but used one layer above
680        # see cli_common
681        parser.add_option("-t", "--target-workspace", dest="workspace",
682                          default=None,
683                          help="which workspace to use",
684                          action="store")
685        (options, args) = parser.parse_args(argv)
686
687        if args:
688            localnames, command = args[:-1], args[-1]
689            localnames = localnames if localnames else None
690        else:
691            print("Error: Too few arguments.")
692            print(parser.usage)
693            return -1
694
695        scm_types = []
696        if options.git:
697            scm_types.append('git')
698        if options.svn:
699            scm_types.append('svn')
700        if options.hg:
701            scm_types.append('hg')
702        if options.bzr:
703            scm_types.append('bzr')
704        if not scm_types:
705            scm_types = None
706
707        if localnames and scm_types:
708            sys.stderr.write("Error: Either localnames or scm-filters"
709                             " [--(git|svn|hg|bzr)] should be specified.\n")
710            return -1
711
712        if config is None:
713            config = multiproject_cmd.get_config(
714                target_path,
715                additional_uris=[],
716                config_filename=self.config_filename)
717        elif config.get_base_path() != target_path:
718            raise MultiProjectException('Config path does not match %s %s' %
719                                        (config.get_base_path(), target_path))
720
721        # run shell command
722        outputs = multiproject_cmd.cmd_foreach(config,
723                                               command=command,
724                                               localnames=localnames,
725                                               num_threads=int(options.jobs),
726                                               timeout=options.timeout,
727                                               scm_types=scm_types,
728                                               shell=options.shell,
729                                               verbose=options.verbose)
730
731        def add_localname_prefix(localname, lines):
732            return ['[%s] %s' % (localname, line) for line in lines]
733
734        for output in outputs:
735            localname = output['entry'].get_local_name()
736            rc = output['returncode']
737            if options.show_stdout:
738                if output['stdout'] is None:
739                    continue
740                lines = output['stdout'].strip().split('\n')
741                lines = add_localname_prefix(localname, lines)
742                sys.stdout.write('\n'.join(lines))
743                sys.stdout.write('\n')
744            if options.show_stderr:
745                lines = []
746                if output['stderr'] is not None:
747                    lines += output['stderr'].strip().split('\n')
748                if rc != 0:
749                    lines +=  ['Command failed with return code [%s]' % rc]
750                if not lines:
751                    continue
752                lines = add_localname_prefix(localname, lines)
753                sys.stderr.write('\n'.join(lines))
754                sys.stderr.write('\n')
755        return 0 if all([o['returncode'] == 0  for o in outputs]) else 1
756
757    def cmd_status(self, target_path, argv, config=None):
758        parser = OptionParser(usage="usage: %s status [localname]* " % self.progname,
759                              description=__MULTIPRO_CMD_DICT__["status"] +
760                              ". The status columns meanings are as the respective SCM defines them.",
761                              epilog="""See: http://www.ros.org/wiki/rosinstall for details""")
762        parser.add_option("-u", "--untracked", dest="untracked",
763                          default=False,
764                          help="Also shows untracked files",
765                          action="store_true")
766        # -t option required here for help but used one layer above, see cli_common
767        parser.add_option("-t", "--target-workspace", dest="workspace",
768                          default=None,
769                          help="which workspace to use",
770                          action="store")
771        (options, args) = parser.parse_args(argv)
772
773        if config is None:
774            config = multiproject_cmd.get_config(
775                target_path,
776                additional_uris=[],
777                config_filename=self.config_filename)
778        elif config.get_base_path() != target_path:
779            raise MultiProjectException(
780                "Config path does not match %s %s " % (config.get_base_path(),
781                                                       target_path))
782
783        if len(args) > 0:
784            statuslist = multiproject_cmd.cmd_status(config,
785                                                     localnames=args,
786                                                     untracked=options.untracked)
787        else:
788            statuslist = multiproject_cmd.cmd_status(config,
789                                                     untracked=options.untracked)
790        allstatus = []
791        for entrystatus in statuslist:
792            if entrystatus['status'] is not None:
793                allstatus.append(entrystatus['status'])
794        print(''.join(allstatus), end='')
795        return 0
796
797    def cmd_set(self, target_path, argv, config=None):
798        """
799        command for modifying/adding a single entry
800        :param target_path: where to look for config
801        :param config: config to use instead of parsing file anew
802        """
803        usage = ("usage: %s set [localname] [[SCM-URI] --(%ssvn|hg|git|bzr) [--version=VERSION]?]?" %
804                 (self.progname, 'detached|' if self.allow_other_element else ''))
805        parser = OptionParser(
806            usage=usage,
807            formatter=IndentedHelpFormatterWithNL(),
808            description=__MULTIPRO_CMD_DICT__["set"] + """
809The command will infer whether you want to add or modify an entry. If
810you modify, it will only change the details you provide, keeping
811those you did not provide. if you only provide a uri, will use the
812basename of it as localname unless such an element already exists.
813
814The command only changes the configuration, to checkout or update
815the element, run %(progname)s update afterwards.
816
817Examples:
818$ %(progname)s set robot_model --hg https://kforge.ros.org/robotmodel/robot_model
819$ %(progname)s set robot_model --version-new robot_model-1.7.1
820%(detached)s
821""" % { 'progname': self.progname,
822        'detached': '$ %s set robot_model --detached' % (self.progname
823                                                         if self.allow_other_element
824                                                         else '')},
825            epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
826        if self.allow_other_element:
827            parser.add_option("--detached", dest="detach", default=False,
828                              help="make an entry unmanaged (default for new element)",
829                              action="store_true")
830        parser.add_option("-v", "--version-new", dest="version", default=None,
831                          help="point SCM to this version",
832                          action="store")
833        parser.add_option("--git", dest="git", default=False,
834                          help="make an entry a git entry",
835                          action="store_true")
836        parser.add_option("--svn", dest="svn", default=False,
837                          help="make an entry a subversion entry",
838                          action="store_true")
839        parser.add_option("--hg", dest="hg", default=False,
840                          help="make an entry a mercurial entry",
841                          action="store_true")
842        parser.add_option("--bzr", dest="bzr", default=False,
843                          help="make an entry a bazaar entry",
844                          action="store_true")
845        parser.add_option("-y", "--confirm", dest="confirm", default='',
846                          help="Do not ask for confirmation",
847                          action="store_true")
848        parser.add_option("-u", "--update", dest="do_update", default=False,
849                          help="update repository after set",
850                          action="store_true")
851        # -t option required here for help but used one layer above, see cli_common
852        parser.add_option(
853            "-t", "--target-workspace", dest="workspace", default=None,
854            help="which workspace to use",
855            action="store")
856        (options, args) = parser.parse_args(argv)
857        if not self.allow_other_element:
858            options.detach = False
859
860        if len(args) > 2:
861            print("Error: Too many arguments.")
862            print(parser.usage)
863            return -1
864
865        if config is None:
866            config = multiproject_cmd.get_config(
867                target_path,
868                additional_uris=[],
869                config_filename=self.config_filename)
870        elif config.get_base_path() != target_path:
871            raise MultiProjectException(
872                "Config path does not match %s %s " % (config.get_base_path(),
873                                                       target_path))
874
875        scmtype = None
876        count_scms = 0
877        if options.git:
878            scmtype = 'git'
879            count_scms += 1
880        if options.svn:
881            scmtype = 'svn'
882            count_scms += 1
883        if options.hg:
884            scmtype = 'hg'
885            count_scms += 1
886        if options.bzr:
887            scmtype = 'bzr'
888            count_scms += 1
889        if options.detach:
890            count_scms += 1
891        if count_scms > 1:
892            parser.error(
893                "You cannot provide more than one scm provider option")
894
895        if len(args) == 0:
896            parser.error("Must provide a localname")
897
898        element = select_element(config.get_config_elements(), args[0])
899
900        uri = None
901        if len(args) == 2:
902            uri = args[1]
903        version = None
904        if options.version is not None:
905            version = options.version.strip("'\"")
906
907        # create spec object
908        if element is None:
909            if scmtype is None and not self.allow_other_element:
910                # for modification, not re-stating the scm type is
911                # okay, for new elements not
912                parser.error("You have to provide one scm provider option")
913            # asssume is insert, choose localname
914            localname = os.path.normpath(args[0])
915            rel_path = os.path.relpath(os.path.realpath(localname),
916                                       os.path.realpath(config.get_base_path()))
917            if os.path.isabs(localname):
918                # use shorter localname for folders inside workspace
919                if not rel_path.startswith('..'):
920                    localname = rel_path
921            else:
922                # got a relative path as localname, could point to a dir or be
923                # meant relative to workspace
924                if not samefile(os.getcwd(), config.get_base_path()):
925                    if os.path.isdir(localname):
926                        parser.error(
927                            "Cannot decide which one you want to add:\n%s\n%s" % (
928                                os.path.abspath(localname),
929                                os.path.join(config.get_base_path(), localname)))
930                    if not rel_path.startswith('..'):
931                        localname = rel_path
932
933            spec = PathSpec(local_name=localname,
934                            uri=normalize_uri(uri, config.get_base_path()),
935                            version=version,
936                            scmtype=scmtype)
937        else:
938            # modify
939            localname = element.get_local_name()
940            old_spec = element.get_path_spec()
941            if options.detach:
942                spec = PathSpec(local_name=localname)
943            else:
944                # '' evals to False, we do not want that
945                if version is None:
946                    version = old_spec.get_version()
947                spec = PathSpec(local_name=localname,
948                                uri=normalize_uri(uri or old_spec.get_uri(),
949                                                  config.get_base_path()),
950                                version=version,
951                                scmtype=scmtype or old_spec.get_scmtype(),
952                                path=old_spec.get_path())
953            if spec.get_legacy_yaml() == old_spec.get_legacy_yaml():
954                if not options.detach and spec.get_scmtype() is not None:
955                    parser.error(
956                        "Element %s already exists, did you mean --detached ?" % spec)
957                parser.error("Element %s already exists" % spec)
958
959        (newconfig, path_changed) = prompt_merge(
960            target_path,
961            additional_uris=[],
962            additional_specs=[spec],
963            merge_strategy='MergeReplace',
964            confirmed=options.confirm,
965            confirm=not options.confirm,
966            show_verbosity=False,
967            show_advanced=False,
968            config_filename=self.config_filename,
969            config=config,
970            allow_other_element=self.allow_other_element)
971
972        if newconfig is not None:
973            print("Overwriting %s" % os.path.join(
974                newconfig.get_base_path(), self.config_filename))
975            shutil.copy(
976                os.path.join(newconfig.get_base_path(), self.config_filename),
977                "%s.bak" % os.path.join(newconfig.get_base_path(), self.config_filename))
978            self.config_generator(newconfig, self.config_filename)
979            if options.do_update:
980                install_success = multiproject_cmd.cmd_install_or_update(
981                                        newconfig, localnames=[localname])
982                if not install_success:
983                    print("Warning: installation encountered errors.")
984                print("\nupdate complete.")
985            elif (spec.get_scmtype() is not None):
986                print("Config changed, remember to run '%s update %s' to update the folder from %s" %
987                      (self.progname, spec.get_local_name(), spec.get_scmtype()))
988        else:
989            print("New element %s could not be added, " % spec)
990            return 1
991        # auto-install not a good feature, maybe make an option
992        # for element in config.get_config_elements():
993        #   if element.get_local_name() == spec.get_local_name():
994        #     if element.is_vcs_element():
995        #       element.install(checkout=not os.path.exists(os.path.join(config.get_base_path(), spec.get_local_name())))
996        #       break
997        return 0
998
999    def cmd_update(self, target_path, argv, config=None):
1000        parser = OptionParser(usage="usage: %s update [localname]*" % self.progname,
1001                              formatter=IndentedHelpFormatterWithNL(),
1002                              description=__MULTIPRO_CMD_DICT__["update"] + """
1003
1004This command calls the SCM provider to pull changes from remote to
1005your local filesystem. In case the url has changed, the command will
1006ask whether to delete or backup the folder.
1007
1008Examples:
1009$ %(progname)s update -t ~/fuerte
1010$ %(progname)s update robot_model geometry
1011""" % {'progname': self.progname},
1012                              epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
1013        parser.add_option("--delete-changed-uris", dest="delete_changed",
1014                          default=False,
1015                          help="Delete the local copy of a directory before changing uri.",
1016                          action="store_true")
1017        parser.add_option("--abort-changed-uris", dest="abort_changed",
1018                          default=False,
1019                          help="Abort if changed uri detected",
1020                          action="store_true")
1021        parser.add_option("--continue-on-error", dest="robust",
1022                          default=False,
1023                          help="Continue despite checkout errors",
1024                          action="store_true")
1025        parser.add_option("--backup-changed-uris", dest="backup_changed",
1026                          default='',
1027                          help="backup the local copy of a directory before changing uri to this directory.",
1028                          action="store")
1029        parser.add_option("-m", "--timeout", dest="timeout",
1030                          default=None,
1031                          help="How long to wait for each repo before failing [seconds]",
1032                          action="store", type=float)
1033        parser.add_option("-j", "--parallel", dest="jobs",
1034                          default=1,
1035                          help="How many parallel threads to use for installing",
1036                          action="store")
1037        parser.add_option("-v", "--verbose", dest="verbose",
1038                          default=False,
1039                          help="Whether to print out more information",
1040                          action="store_true")
1041        # -t option required here for help but used one layer above, see cli_common
1042        parser.add_option("-t", "--target-workspace", dest="workspace",
1043                          default=None,
1044                          help="which workspace to use",
1045                          action="store")
1046        (options, args) = parser.parse_args(argv)
1047
1048        if config is None:
1049            config = multiproject_cmd.get_config(
1050                target_path,
1051                additional_uris=[],
1052                config_filename=self.config_filename)
1053        elif config.get_base_path() != target_path:
1054            raise MultiProjectException("Config path does not match %s %s " % (
1055                config.get_base_path(),
1056                target_path))
1057        success = True
1058        mode = _get_mode_from_options(parser, options)
1059        if args == []:
1060            # None means no filter, [] means filter all
1061            args = None
1062        if success:
1063            install_success = multiproject_cmd.cmd_install_or_update(
1064                config,
1065                localnames=args,
1066                backup_path=options.backup_changed,
1067                mode=mode,
1068                robust=options.robust,
1069                num_threads=int(options.jobs),
1070                timeout=options.timeout,
1071                verbose=options.verbose)
1072            if install_success or options.robust:
1073                return 0
1074        return 1
1075
1076    def cmd_remove(self, target_path, argv, config=None):
1077        parser = OptionParser(usage="usage: %s remove [localname]*" % self.progname,
1078                              formatter=IndentedHelpFormatterWithNL(),
1079                              description=__MULTIPRO_CMD_DICT__["remove"] + """
1080The command removes entries from your configuration file, it does not affect your filesystem.
1081""",
1082                              epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
1083        # -t option required here for help but used one layer above, see cli_common
1084        parser.add_option(
1085            "-t", "--target-workspace", dest="workspace", default=None,
1086            help="which workspace to use",
1087            action="store")
1088        (_, args) = parser.parse_args(argv)
1089        if len(args) < 1:
1090            print("Error: Too few arguments.")
1091            print(parser.usage)
1092            return -1
1093
1094        if config is None:
1095            config = multiproject_cmd.get_config(
1096                target_path,
1097                additional_uris=[],
1098                config_filename=self.config_filename)
1099        elif config.get_base_path() != target_path:
1100            raise MultiProjectException(
1101                "Config path does not match %s %s " % (config.get_base_path(),
1102                                                       target_path))
1103        success = True
1104        elements = select_elements(config, args)
1105        for element in elements:
1106            if not config.remove_element(element.get_local_name()):
1107                success = False
1108                print("Bug: No such element %s in config, aborting without changes" %
1109                      (element.get_local_name()))
1110                break
1111        if success:
1112            print("Overwriting %s" % os.path.join(config.get_base_path(),
1113                                                  self.config_filename))
1114            shutil.copy(os.path.join(config.get_base_path(),
1115                                     self.config_filename),
1116                        "%s.bak" % os.path.join(config.get_base_path(),
1117                                                self.config_filename))
1118            self.config_generator(config, self.config_filename)
1119            print("Removed entries %s" % args)
1120
1121        return 0
1122
1123    def cmd_snapshot(self, target_path, argv, config=None):
1124        parser = OptionParser(
1125            usage="usage: %s info [localname]* [OPTIONS]" % self.progname,
1126            formatter=IndentedHelpFormatterWithNL(),
1127            description=__MULTIPRO_CMD_DICT__["export"] + """
1128Exports the current workspace.
1129
1130The --exact option will cause the output to contain the exact commit uuid for
1131each version-controlled entry. The --spec option tells wstool to look at the
1132workspace .rosinstall for versioning info instead of the workspace.
1133
1134Examples:
1135$ %(prog)s export
1136$ %(prog)s export -t ~/ros/fuerte
1137$ %(prog)s export --exact
1138""" % {'prog': self.progname, 'opts': ONLY_OPTION_VALID_ATTRS},
1139            epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
1140        parser.add_option(
1141            "-o", "--output", dest="output_filename", default=None,
1142            help="Write the .rosinstall export to the specified file",
1143            action="store")
1144        parser.add_option(
1145            "-t", "--target-workspace", dest="workspace", default=None,
1146            help="which workspace to use",
1147            action="store")
1148        parser.add_option(
1149            "--exact", dest="exact", default=False, action="store_true",
1150            help="export exact commit hashes instead of branch names")
1151        parser.add_option(
1152            "--spec", dest="spec", default=False, action="store_true",
1153            help="export version from workspace spec instead of current")
1154
1155        (options, _) = parser.parse_args(argv)
1156
1157        if config is None:
1158            config = multiproject_cmd.get_config(
1159                target_path,
1160                additional_uris=[],
1161                config_filename=self.config_filename)
1162        elif config.get_base_path() != target_path:
1163            raise MultiProjectException("Config path does not match %s %s " %
1164                                        (config.get_base_path(), target_path))
1165
1166        # TODO: Check for workspace differences and issue warnings?
1167
1168        fname = options.output_filename
1169        if fname:
1170            fname = os.path.abspath(fname)
1171            print("Writing %s" % fname)
1172        self.config_generator(config, fname, get_header(self.progname),
1173                              spec=options.spec, exact=options.exact,
1174                              vcs_only=True)
1175
1176        return 0
1177
1178    def cmd_info(self, target_path, argv, reverse=True, config=None):
1179        parser = OptionParser(
1180            usage="usage: %s info [localname]* [OPTIONS]" % self.progname,
1181            formatter=IndentedHelpFormatterWithNL(),
1182            description=__MULTIPRO_CMD_DICT__["info"] + """
1183
1184The Status (S) column shows
1185 x  for missing
1186 L  for uncommited (local) changes
1187 V  for difference in version and/or remote URI
1188 C  for difference in local and remote versions
1189
1190The 'Version-Spec' column shows what tag, branch or revision was given
1191in the .rosinstall file. The 'UID' column shows the unique ID of the
1192current (and specified) version. The 'URI' column shows the configured
1193URL of the repo.
1194
1195If status is V, the difference between what was specified and what is
1196real is shown in the respective column. For SVN entries, the url is
1197split up according to standard layout (trunk/tags/branches).
1198
1199When given one localname, just show the data of one element in list form.
1200This also has the generic properties element which is usually empty.
1201
1202The --only option accepts keywords: %(opts)s
1203
1204Examples:
1205$ %(prog)s info -t ~/ros/fuerte
1206$ %(prog)s info robot_model
1207$ %(prog)s info --yaml
1208$ %(prog)s info --only=path,cur_uri,cur_revision robot_model geometry
1209""" % {'prog': self.progname, 'opts': ONLY_OPTION_VALID_ATTRS},
1210            epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
1211        parser.add_option(
1212            "--root", dest="show_ws_root", default=False,
1213            help="Show workspace root path",
1214            action="store_true")
1215        parser.add_option(
1216            "--data-only", dest="data_only", default=False,
1217            help="Does not provide explanations",
1218            action="store_true")
1219        parser.add_option(
1220            "-s", "--short", dest="short", default=False,
1221            help="Shows simplified version info table.",
1222            action="store_true")
1223        parser.add_option(
1224            "--only", dest="only", default=False,
1225            help="Shows comma-separated lists of only given comma-separated attribute(s).",
1226            action="store")
1227        parser.add_option(
1228            "--yaml", dest="yaml", default=False,
1229            help="Shows only version of single entry. Intended for scripting.",
1230            action="store_true")
1231        parser.add_option(
1232            "--fetch", dest="fetch", default=False,
1233            help="When used, retrieves version information from remote (takes longer).",
1234            action="store_true")
1235        parser.add_option(
1236            "-u", "--untracked", dest="untracked",
1237            default=False,
1238            help="Also show untracked files as modifications",
1239            action="store_true")
1240        # -t option required here for help but used one layer above, see cli_common
1241        parser.add_option(
1242            "-t", "--target-workspace", dest="workspace", default=None,
1243            help="which workspace to use",
1244            action="store")
1245        parser.add_option(
1246            "-m", "--managed-only", dest="unmanaged", default=True,
1247            help="only show managed elements",
1248            action="store_false")
1249        (options, args) = parser.parse_args(argv)
1250
1251        if config is None:
1252            config = multiproject_cmd.get_config(
1253                target_path,
1254                additional_uris=[],
1255                config_filename=self.config_filename)
1256        elif config.get_base_path() != target_path:
1257            raise MultiProjectException("Config path does not match %s %s " %
1258                                        (config.get_base_path(), target_path))
1259
1260        if options.show_ws_root:
1261            print(config.get_base_path())
1262            return 0
1263
1264        if args == []:
1265            args = None
1266
1267        if options.only:
1268            only_options = options.only.split(",")
1269            if only_options == '':
1270                parser.error('No valid options given')
1271            lines = get_info_table_raw_csv(config,
1272                                           parser,
1273                                           properties=only_options,
1274                                           localnames=args)
1275            print('\n'.join(lines))
1276            return 0
1277        elif options.yaml:
1278            # TODO: Not sure what this does, used to be cmd_snapshot,
1279            # but that command was not implemented.
1280            source_aggregate = multiproject_cmd.cmd_snapshot(config,
1281                                                             localnames=args)
1282            print(yaml.safe_dump(source_aggregate, default_flow_style=None), end='')
1283            return 0
1284
1285        # this call takes long, as it invokes scms.
1286        outputs = multiproject_cmd.cmd_info(config,
1287                                            localnames=args,
1288                                            untracked=options.untracked,
1289                                            fetch=options.fetch)
1290        if args and len(args) == 1:
1291            # if only one element selected, print just one line
1292            print(get_info_list(config.get_base_path(),
1293                                outputs[0],
1294                                options.data_only))
1295            return 0
1296
1297        columns = None
1298        if options.short:
1299            columns = ['localname', 'status', 'version']
1300
1301        header = 'workspace: %s' % (target_path)
1302        print(header)
1303        table = get_info_table(config.get_base_path(),
1304                               outputs,
1305                               options.data_only,
1306                               reverse=reverse,
1307                               selected_headers=columns)
1308        if table is not None and table != '':
1309           print("\n%s" % table)
1310
1311        if options.unmanaged:
1312            outputs2 = multiproject_cmd.cmd_find_unmanaged_repos(config)
1313            table2 = get_info_table(config.get_base_path(),
1314                                   outputs2,
1315                                   options.data_only,
1316                                   reverse=reverse,
1317                                   unmanaged=True,
1318                                   selected_headers=columns)
1319            if table2 is not None and table2 != '':
1320                print("\nAlso detected these repositories in the workspace, add using '%s scrape' or '%s set':\n\n%s" % (self.progname, self.progname, table2))
1321
1322        return 0
1323
1324    def cmd_scrape(self, target_path, argv, config=None):
1325        """
1326        command for adding yet unamanaged repos under workspace root to managed repos.
1327        :param target_path: where to look for config
1328        :param config: config to use instead of parsing file anew
1329        """
1330        usage = ("usage: %s scrape [OPTIONS]" % self.progname)
1331        parser = OptionParser(
1332            usage=usage,
1333            description=__MULTIPRO_CMD_DICT__["scrape"],
1334            epilog="See: http://www.ros.org/wiki/rosinstall for details\n")
1335        parser.add_option("-y", "--confirm", dest="confirm", default='',
1336                          help="Do not ask for confirmation",
1337                          action="store_true")
1338        # -t option required here for help but used one layer above, see cli_common
1339        parser.add_option(
1340            "-t", "--target-workspace", dest="workspace", default=None,
1341            help="which workspace to use",
1342            action="store")
1343        (options, args) = parser.parse_args(argv)
1344
1345        if config is None:
1346            config = multiproject_cmd.get_config(
1347                target_path,
1348                additional_uris=[],
1349                config_filename=self.config_filename)
1350        elif config.get_base_path() != target_path:
1351            raise MultiProjectException(
1352                "Config path does not match %s %s " % (config.get_base_path(),
1353                                                       target_path))
1354
1355        elems = multiproject_cmd.cmd_find_unmanaged_repos(config)
1356        if not elems:
1357            raise MultiProjectException(
1358                "No unmanaged repos found below '%s'" % (config.get_base_path()))
1359        for elem in elems:
1360            elem_abs_path = os.path.join(config.get_base_path(), elem['localname'])
1361            if os.path.isdir(elem_abs_path):
1362                args = [elem_abs_path, elem['scm'], elem['uri']]
1363                if (options.confirm):
1364                    args.append('-y')
1365                self.cmd_set(target_path, args)
1366        return 0
1367