1# Software License Agreement (BSD License) 2# 3# Copyright (c) 2010, Willow Garage, Inc. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 10# * Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# * Redistributions in binary form must reproduce the above 13# copyright notice, this list of conditions and the following 14# disclaimer in the documentation and/or other materials provided 15# with the distribution. 16# * Neither the name of Willow Garage, Inc. nor the names of its 17# contributors may be used to endorse or promote products derived 18# from this software without specific prior written permission. 19# 20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 30# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31# POSSIBILITY OF SUCH DAMAGE. 32 33""" 34Representation/model of rosdistro format. 35""" 36 37import os 38import re 39import string 40try: 41 from urllib.request import urlopen 42except ImportError: 43 from urllib2 import urlopen 44import yaml 45 46from .common import ResourceNotFound 47from .environment import get_etc_ros_dir 48 49TARBALL_URI_EVAL = 'http://svn.code.sf.net/p/ros-dry-releases/code/download/stacks/$STACK_NAME/$STACK_NAME-$STACK_VERSION/$STACK_NAME-$STACK_VERSION.tar.bz2' 50TARBALL_VERSION_EVAL = '$STACK_NAME-$STACK_VERSION' 51 52 53class InvalidDistro(Exception): 54 """ 55 Distro file data does not match specification. 56 """ 57 pass 58 59 60def distro_uri(distro_name): 61 """ 62 Get distro URI of main ROS distribution files. 63 64 :param distro_name: name of distro, e.g. 'diamondback' 65 :returns: the SVN/HTTP URL of the specified distro. This function should only be used 66 with the main distros. 67 """ 68 return "http://svn.code.sf.net/p/ros-dry-releases/code/trunk/distros/%s.rosdistro" % (distro_name) 69 70def expand_rule(rule, stack_name, stack_ver, release_name): 71 s = rule.replace('$STACK_NAME', stack_name) 72 if stack_ver: 73 s = s.replace('$STACK_VERSION', stack_ver) 74 s = s.replace('$RELEASE_NAME', release_name) 75 return s 76 77 78class DistroStack(object): 79 """Stores information about a stack release""" 80 81 def __init__(self, stack_name, stack_version, release_name, rules): 82 """ 83 :param stack_name: Name of stack 84 :param stack_version: Version number of stack. 85 :param release_name: name of distribution release. Necessary for rule expansion. 86 :param rules: raw '_rules' data. Will be converted into appropriate vcs config instance. 87 """ 88 self.name = stack_name 89 self.version = stack_version 90 self.release_name = release_name 91 self._rules = rules 92 self.repo = rules.get('repo', None) 93 self.vcs_config = load_vcs_config(self._rules, self._expand_rule) 94 95 def _expand_rule(self, rule): 96 """ 97 Perform variable substitution on stack rule. 98 """ 99 return expand_rule(rule, self.name, self.version, self.release_name) 100 101 def __eq__(self, other): 102 try: 103 return self.name == other.name and \ 104 self.version == other.version and \ 105 self.vcs_config == other.vcs_config 106 except AttributeError: 107 return False 108 109 110class Variant(object): 111 """ 112 A variant defines a specific set of stacks ("metapackage", in Debian 113 parlance). For example, "base", "pr2". These variants can extend 114 another variant. 115 """ 116 117 def __init__(self, variant_name, extends, stack_names, stack_names_implicit): 118 """ 119 :param variant_name: name of variant to load from distro file, ``str`` 120 :param stack_names_implicit: full list of stacks implicitly included in this variant, ``[str]`` 121 :param raw_data: raw rosdistro data for this variant 122 """ 123 self.name = variant_name 124 self.extends = extends 125 self._stack_names = stack_names 126 self._stack_names_implicit = stack_names_implicit 127 128 def get_stack_names(self, implicit=True): 129 if implicit: 130 return self._stack_names_implicit 131 else: 132 return self._stack_names 133 134 # stack_names includes implicit stack names. Use get_stack_names() 135 # to get explicit only 136 stack_names = property(get_stack_names) 137 138 139class Distro(object): 140 """ 141 Store information in a rosdistro file. 142 """ 143 144 def __init__(self, stacks, variants, release_name, version, raw_data): 145 """ 146 :param stacks: dictionary mapping stack names to :class:`DistroStack` instances 147 :param variants: dictionary mapping variant names to :class:`Variant` instances 148 :param release_name: name of release, e.g. 'diamondback' 149 :param version: version number of release 150 :param raw_data: raw dictionary representation of a distro 151 """ 152 self._stacks = stacks 153 self.variants = variants 154 self.release_name = release_name 155 self.version = version 156 self.raw_data = raw_data 157 158 def get_stacks(self, released=False): 159 """ 160 :param released: only included released stacks 161 :returns: dictionary of stack names to :class:`DistroStack` instances in 162 this distro. 163 """ 164 if released: 165 return self._get_released_stacks() 166 else: 167 return self._stacks.copy() 168 169 def _get_released_stacks(self): 170 retval = {} 171 for s, obj in self._stacks.items(): 172 if obj.version: 173 retval[s] = obj 174 return retval 175 176 # gets map of all stacks 177 stacks = property(get_stacks) 178 # gets maps of released stacks 179 released_stacks = property(_get_released_stacks) 180 181 182def load_distro(source_uri): 183 """ 184 :param source_uri: source URI of distro file, or path to distro 185 file. Filename has precedence in resolution. 186 187 :raises: :exc:`InvalidDistro` If distro file is invalid 188 :raises: :exc:`ResourceNotFound` If file at *source_uri* is not found 189 """ 190 try: 191 # parse rosdistro yaml 192 if os.path.isfile(source_uri): 193 # load rosdistro file 194 with open(source_uri) as f: 195 raw_data = yaml.load(f.read()) 196 else: 197 try: 198 request = urlopen(source_uri) 199 except Exception as e: 200 raise ResourceNotFound('%s (%s)' % (str(e), source_uri)) 201 try: 202 raw_data = yaml.load(request) 203 except ValueError: 204 raise ResourceNotFound(source_uri) 205 if not type(raw_data) == dict: 206 raise InvalidDistro("Distro must be a dictionary: %s" % (source_uri)) 207 except yaml.YAMLError as e: 208 raise InvalidDistro(str(e)) 209 210 try: 211 version = _distro_version(raw_data.get('version', '0')) 212 release_name = raw_data['release'] 213 stacks = _load_distro_stacks(raw_data, release_name) 214 variants = _load_variants(raw_data.get('variants', {}), stacks) 215 return Distro(stacks, variants, release_name, version, raw_data) 216 except KeyError as e: 217 raise InvalidDistro("distro is missing required '%s' key" % (str(e))) 218 219 220def _load_variants(raw_data, stacks): 221 if not raw_data: 222 return {} 223 all_variants_raw_data = {} 224 for v in raw_data: 225 if type(v) != dict or len(v.keys()) != 1: 226 raise InvalidDistro("invalid variant spec: %s" % v) 227 variant_name = list(v.keys())[0] 228 all_variants_raw_data[variant_name] = v[variant_name] 229 variants = {} 230 for variant_name in all_variants_raw_data.keys(): 231 variants[variant_name] = _load_variant(variant_name, all_variants_raw_data) 232 233 # Disabling validation to support variants which include wet packages. 234 # validate 235 # for stack_name in variants[variant_name].get_stack_names(implicit=False): 236 # if stack_name not in stacks: 237 # raise InvalidDistro("variant [%s] refers to non-existent stack [%s]"%(variant_name, stack_name)) 238 return variants 239 240 241def _load_variant(variant_name, all_variants_raw_data): 242 variant_raw_data = all_variants_raw_data[variant_name] 243 stack_names_implicit = list(variant_raw_data.get('stacks', [])) 244 extends = variant_raw_data.get('extends', []) 245 if isinstance(extends, str): 246 extends = [extends] 247 for e in extends: 248 parent_variant = _load_variant(e, all_variants_raw_data) 249 stack_names_implicit = parent_variant.get_stack_names(implicit=True) + stack_names_implicit 250 return Variant(variant_name, extends, variant_raw_data.get('stacks', []), stack_names_implicit) 251 252 253def _load_distro_stacks(distro_doc, release_name): 254 """ 255 :param distro_doc: dictionary form of rosdistro file, `dict` 256 :returns: dictionary of stack names to :class:`DistroStack` instances, `{str : DistroStack}` 257 :raises: :exc:`InvalidDistro` if distro_doc format is invalid 258 """ 259 260 # load stacks and expand out uri rules 261 stacks = {} 262 try: 263 stack_props = distro_doc['stacks'] 264 stack_props = stack_props or {} 265 stack_names = [x for x in stack_props.keys() if not x[0] == '_'] 266 except KeyError: 267 raise InvalidDistro("distro is missing required 'stacks' key") 268 for stack_name in stack_names: 269 stack_version = stack_props[stack_name].get('version', None) 270 rules = _get_rules(distro_doc, stack_name) 271 if not rules: 272 raise InvalidDistro("no VCS rules for stack [%s]" % (stack_name)) 273 stacks[stack_name] = DistroStack(stack_name, stack_version, release_name, rules) 274 return stacks 275 276 277def _distro_version(version_val): 278 """ 279 Parse distro version value, converting SVN revision to version value if necessary 280 """ 281 version_val = str(version_val) 282 # check for no keyword sub 283 if version_val == '$Revision$': 284 return 0 285 m = re.search('\$Revision:\s*([0-9]*)\s*\$', version_val) 286 if m is not None: 287 version_val = 'r' + m.group(1) 288 289 # Check that is a valid version string 290 valid = string.ascii_letters + string.digits + '.+~' 291 if False in (c in valid for c in version_val): 292 raise InvalidDistro("Version string %s not valid" % version_val) 293 return version_val 294 295 296def distro_to_rosinstall(distro, branch, variant_name=None, implicit=True, released_only=True, anonymous=True): 297 """ 298 :param branch: branch to convert for 299 :param variant_name: if not None, only include stacks in the specified variant. 300 :param implicit: if variant_name is provided, include full (recursive) dependencies of variant, default True 301 :param released_only: only included released stacks, default True. 302 :param anonymous: create for anonymous access rules 303 :returns: rosinstall data in Python list format, ``[dict]`` 304 305 :raises: :exc:`KeyError` If branch is invalid or if distro is mis-configured 306 """ 307 variant = distro.variants.get(variant_name, None) 308 if variant_name: 309 stack_names = set(variant.get_stack_names(implicit=implicit)) 310 else: 311 stack_names = distro.released_stacks.keys() 312 rosinstall_data = [] 313 for s in stack_names: 314 if released_only and s not in distro.released_stacks: 315 continue 316 rosinstall_data.extend(distro.stacks[s].vcs_config.to_rosinstall(s, branch, anonymous)) 317 return rosinstall_data 318 319################################################################################ 320 321 322def _get_rules(distro_doc, stack_name): 323 """ 324 Retrieve rules from distro_doc for specified stack. This operates on 325 the raw distro dictionary document. 326 327 :param distro_doc: rosdistro document, ``dict`` 328 :param stack_name: name of stack to get rules for, ``str`` 329 """ 330 # top-level named section 331 named_rules_d = distro_doc.get('_rules', {}) 332 333 # other rules to search 334 rules_d = [distro_doc.get('stacks', {}), 335 distro_doc.get('stacks', {}).get(stack_name, {})] 336 rules_d = [d for d in rules_d if '_rules' in d] 337 338 # last rules wins 339 if not rules_d: 340 return None 341 rules_d = rules_d[-1] 342 343 update_r = rules_d.get('_rules', {}) 344 if type(update_r) == str: 345 try: 346 update_r = named_rules_d[update_r] 347 except KeyError: 348 raise InvalidDistro("no _rules named [%s]" % (update_r)) 349 if not type(update_r) == dict: 350 raise InvalidDistro("invalid rules: %s %s" % (update_r, type(update_r))) 351 return update_r 352 353################################################################################ 354 355 356class VcsConfig(object): 357 """ 358 Base representation of a rosdistro VCS rules configuration. 359 """ 360 361 def __init__(self, type_): 362 self.type = type_ 363 self.tarball_url = self.tarball_version = None 364 365 def to_rosinstall(self, local_name, branch, anonymous): 366 uri, version_tag = self.get_branch(branch, anonymous) 367 if branch == 'release-tar': 368 type_ = 'tar' 369 else: 370 type_ = self.type 371 if version_tag: 372 return [{type_: {"uri": uri, 'local-name': local_name, 'version': version_tag}}] 373 else: 374 return [({type_: {"uri": uri, 'local-name': local_name}})] 375 376 def load(self, rules, rule_eval): 377 """ 378 Initialize fields of this class based on the raw rosdistro 379 *rules* data after applying *rule_eval* function (e.g. to 380 replace variables in rules). 381 382 :param rules: raw rosdistro rules entry, ``dict`` 383 :param rule_eval: function to evaluate rule values, ``fn(str) -> str`` 384 """ 385 self.tarball_url = rule_eval(TARBALL_URI_EVAL) 386 self.tarball_version = rule_eval(TARBALL_VERSION_EVAL) 387 388 def get_branch(self, branch, anonymous): 389 """ 390 :raises: :exc:`ValueError` If branch is invalid 391 """ 392 if branch == 'release-tar': 393 return self.tarball_url, self.tarball_version 394 else: 395 raise ValueError(branch) 396 397 def __eq__(self, other): 398 return self.type == other.type and \ 399 self.tarball_url == other.tarball_url 400 401 402class DvcsConfig(VcsConfig): 403 """ 404 Configuration information for a distributed VCS-style repository. 405 406 Configuration fields: 407 408 * ``repo_uri``: base URI of repo 409 * ``dev_branch``: git branch the code is developed 410 * ``distro_tag``: a tag of the latest released code for a specific ROS distribution 411 * ``release_tag``: a tag of the code for a specific release 412 """ 413 414 def __init__(self, type_): 415 super(DvcsConfig, self).__init__(type_) 416 self.repo_uri = self.anon_repo_uri = None 417 self.dev_branch = self.distro_tag = self.release_tag = None 418 419 def load(self, rules, rule_eval): 420 super(DvcsConfig, self).load(rules, rule_eval) 421 422 self.repo_uri = rule_eval(rules['uri']) 423 if 'anon-uri' in rules: 424 self.anon_repo_uri = rule_eval(rules['anon-uri']) 425 else: 426 self.anon_repo_uri = self.repo_uri 427 self.dev_branch = rule_eval(rules['dev-branch']) 428 self.distro_tag = rule_eval(rules['distro-tag']) 429 self.release_tag = rule_eval(rules['release-tag']) 430 431 def get_branch(self, branch, anonymous): 432 """ 433 :raises: :exc:`KeyError` Invalid branch parameter 434 """ 435 if branch == 'release-tar': 436 return super(DvcsConfig, self).get_branch(branch, anonymous) 437 elif branch == 'devel': 438 version_tag = self.dev_branch 439 elif branch == 'distro': 440 version_tag = self.distro_tag 441 elif branch == 'release': 442 version_tag = self.release_tag 443 else: 444 raise ValueError("invalid branch spec [%s]" % (branch)) 445 # occurs, for example, with unreleased stacks. Only devel is valid 446 if version_tag is None: 447 raise ValueError("branch [%s] is not available for this config" % (branch)) 448 if anonymous: 449 return self.anon_repo_uri, version_tag 450 else: 451 return self.repo_uri, version_tag 452 453 def __eq__(self, other): 454 return super(DvcsConfig, self).__eq__(other) and \ 455 self.repo_uri == other.repo_uri and \ 456 self.anon_repo_uri == other.anon_repo_uri and \ 457 self.dev_branch == other.dev_branch and \ 458 self.release_tag == other.release_tag and \ 459 self.distro_tag == other.distro_tag 460 461 462class GitConfig(DvcsConfig): 463 """ 464 Configuration information about an GIT repository. See parent class :class:`DvcsConfig` for more API information. 465 """ 466 467 def __init__(self): 468 super(GitConfig, self).__init__('git') 469 470 471class HgConfig(DvcsConfig): 472 """ 473 Configuration information about a Mercurial repository. See parent class :class:`DvcsConfig` for more API information. 474 """ 475 476 def __init__(self): 477 super(HgConfig, self).__init__('hg') 478 479 480class BzrConfig(DvcsConfig): 481 """ 482 Configuration information about an BZR repository. See parent class :class:`DvcsConfig` for more API information. 483 """ 484 485 def __init__(self): 486 super(BzrConfig, self).__init__('bzr') 487 488 489class SvnConfig(VcsConfig): 490 """ 491 Configuration information about an SVN repository. 492 493 Configuration fields: 494 495 * ``dev``: where the code is developed 496 * ``distro_tag``: a tag of the code for a specific ROS distribution 497 * ``release_tag``: a tag of the code for a specific release 498 """ 499 500 def __init__(self): 501 super(SvnConfig, self).__init__('svn') 502 self.dev = None 503 self.distro_tag = None 504 self.release_tag = None 505 506 # anonymously readable version of URLs above. Some repos have 507 # separate URLs for read-only vs. writable versions of repo 508 # and many tools need to be able to read repos without 509 # providing credentials. 510 self.anon_dev = None 511 self.anon_distro_tag = None 512 self.anon_release_tag = None 513 514 def load(self, rules, rule_eval): 515 super(SvnConfig, self).load(rules, rule_eval) 516 for k in ['dev', 'distro-tag', 'release-tag']: 517 if k not in rules: 518 raise KeyError("svn rules missing required %s key: %s" % (k, rules)) 519 self.dev = rule_eval(rules['dev']) 520 self.distro_tag = rule_eval(rules['distro-tag']) 521 self.release_tag = rule_eval(rules['release-tag']) 522 523 # specify urls that are safe to anonymously read 524 # from. Users must supply a complete set. 525 if 'anon-dev' in rules: 526 self.anon_dev = rule_eval(rules['anon-dev']) 527 self.anon_distro_tag = rule_eval(rules['anon-distro-tag']) 528 self.anon_release_tag = rule_eval(rules['anon-release-tag']) 529 else: 530 # if no login credentials, assume that anonymous is 531 # same as normal keys. 532 self.anon_dev = self.dev 533 self.anon_distro_tag = self.distro_tag 534 self.anon_release_tag = self.release_tag 535 536 def get_branch(self, branch, anonymous): 537 """ 538 :raises: :exc:`ValueError` If branch is invalid 539 """ 540 if branch == 'release-tar': 541 return super(SvnConfig, self).get_branch(branch, anonymous) 542 else: 543 key_map = dict(devel='dev', distro='distro_tag', release='release_tag') 544 if branch not in key_map: 545 raise KeyError("invalid branch spec [%s]" % (branch)) 546 attr_name = key_map[branch] 547 if anonymous: 548 attr_name = 'anon_' + attr_name 549 uri = getattr(self, attr_name) 550 # occurs, for example, with unreleased stacks. Only devel is valid 551 if uri is None: 552 raise ValueError("branch [%s] is not available for this config" % (branch)) 553 return uri, None 554 555 def __eq__(self, other): 556 return super(SvnConfig, self).__eq__(other) and \ 557 self.dev == other.dev and \ 558 self.distro_tag == other.distro_tag and \ 559 self.release_tag == other.release_tag and \ 560 self.anon_dev == other.anon_dev and \ 561 self.anon_distro_tag == other.anon_distro_tag and \ 562 self.anon_release_tag == other.anon_release_tag 563 564 565_vcs_configs = { 566 'svn': SvnConfig, 567 'git': GitConfig, 568 'hg': HgConfig, 569 'bzr': BzrConfig, 570} 571 572 573def get_vcs_configs(): 574 """ 575 :returns: Dictionary of supported :class:`VcsConfig` instances. 576 Key is the VCS type name, e.g. 'svn'. ``{str: VcsConfig}`` 577 """ 578 return _vcs_configs.copy() 579 580 581def load_vcs_config(rules, rule_eval): 582 """ 583 Factory for creating :class:`VcsConfig` subclass based on 584 rosdistro _rules data. 585 586 :param rules: rosdistro rules data 587 :param rules_eval: Function to apply to rule values, e.g. to 588 convert variables. ``fn(str)->str`` 589 :returns: :class:`VcsConfig` subclass instance with interpreted rules data. 590 """ 591 vcs_config = None 592 for k, clazz in _vcs_configs.items(): 593 if k in rules: 594 vcs_config = clazz() 595 vcs_config.load(rules[k], rule_eval) 596 break 597 return vcs_config 598 599 600def _current_distro_electric_parse_roscore(roscore_file): 601 if not os.path.exists(roscore_file): 602 return None 603 import xml.dom.minidom 604 try: 605 dom = xml.dom.minidom.parse(roscore_file) 606 tags = dom.getElementsByTagName("param") 607 for t in tags: 608 if t.hasAttribute('name') and t.getAttribute('name') == 'rosdistro': 609 return t.getAttribute('value') 610 except: 611 return None 612 613 614# for < fuerte, retrieve from roscore file 615def _current_distro_electric(env=None): 616 if env is None: 617 env = os.environ 618 from . import RosPack, get_ros_paths 619 rospack = RosPack(get_ros_paths(env)) 620 # there's some chance that the location of this file changes in the future 621 try: 622 roscore_file = os.path.join(rospack.get_path('roslaunch'), 'roscore.xml') 623 return _current_distro_electric_parse_roscore(roscore_file) 624 except: 625 return None 626 627 628def current_distro_codename(env=None): 629 """ 630 Get the currently active ROS distribution codename, e.g. 'fuerte' 631 632 :param env: override os.environ, ``dict`` 633 """ 634 if env is None: 635 env = os.environ 636 637 # ROS_DISTRO is only used in ros catkin buildspace. It is not 638 # meant to be well publicized and thus is not declared in 639 # rospkg.environment. 640 if 'ROS_DISTRO' in env: 641 return env['ROS_DISTRO'] 642 643 # check for /etc/ros/distro file 644 distro_name = None 645 etc_ros = get_etc_ros_dir(env=env) 646 distro_file = os.path.join(etc_ros, 'distro') 647 if os.path.isfile(distro_file): 648 with open(distro_file, 'r') as f: 649 distro_name = f.read().strip() 650 651 # fallback logic for pre-Fuerte 652 if distro_name is None: 653 distro_name = _current_distro_electric(env=env) 654 655 return distro_name 656