1# Copyright (c) 2012, Willow Garage, Inc. 2# All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution. 12# * Neither the name of the Willow Garage, Inc. nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# Author Ken Conley/kwc@willowgarage.com 29 30from __future__ import print_function 31 32import os 33import sys 34import yaml 35try: 36 from urllib.request import urlopen 37 from urllib.error import URLError 38except ImportError: 39 from urllib2 import urlopen 40 from urllib2 import URLError 41try: 42 import cPickle as pickle 43except ImportError: 44 import pickle 45 46from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file 47from .core import InvalidData, DownloadFailure, CachePermissionError 48from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data 49from .meta import MetaDatabase 50 51try: 52 import urlparse 53except ImportError: 54 import urllib.parse as urlparse # py3k 55 56try: 57 import httplib 58except ImportError: 59 import http.client as httplib # py3k 60 61import rospkg 62import rospkg.distro 63 64from .loader import RosdepLoader 65from .rosdistrohelper import get_index, get_index_url 66 67# default file to download with 'init' command in order to bootstrap 68# rosdep 69DEFAULT_SOURCES_LIST_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/sources.list.d/20-default.list' 70 71# seconds to wait before aborting download of rosdep data 72DOWNLOAD_TIMEOUT = 15.0 73 74SOURCES_LIST_DIR = 'sources.list.d' 75SOURCES_CACHE_DIR = 'sources.cache' 76 77# name of index file for sources cache 78CACHE_INDEX = 'index' 79 80# extension for binary cache 81SOURCE_PATH_ENV = 'ROSDEP_SOURCE_PATH' 82 83 84def get_sources_list_dirs(source_list_dir): 85 if SOURCE_PATH_ENV in os.environ: 86 sdirs = os.environ[SOURCE_PATH_ENV].split(os.pathsep) 87 else: 88 sdirs = [source_list_dir] 89 for p in list(sdirs): 90 if not os.path.exists(p): 91 sdirs.remove(p) 92 return sdirs 93 94 95def get_sources_list_dir(): 96 # base of where we read config files from 97 # TODO: windows 98 if 0: 99 # we can't use etc/ros because environment config does not carry over under sudo 100 etc_ros = rospkg.get_etc_ros_dir() 101 else: 102 etc_ros = '/usr/local/etc/ros' 103 # compute default system wide sources directory 104 sys_sources_list_dir = os.path.join(etc_ros, 'rosdep', SOURCES_LIST_DIR) 105 sources_list_dirs = get_sources_list_dirs(sys_sources_list_dir) 106 if sources_list_dirs: 107 return sources_list_dirs[0] 108 else: 109 return sys_sources_list_dir 110 111 112def get_default_sources_list_file(): 113 return os.path.join(get_sources_list_dir(), '20-default.list') 114 115 116def get_sources_cache_dir(): 117 ros_home = rospkg.get_ros_home() 118 return os.path.join(ros_home, 'rosdep', SOURCES_CACHE_DIR) 119 120 121# Default rosdep.yaml format. For now this is the only valid type and 122# is specified for future compatibility. 123TYPE_YAML = 'yaml' 124# git-buildpackage repo list 125TYPE_GBPDISTRO = 'gbpdistro' 126VALID_TYPES = [TYPE_YAML, TYPE_GBPDISTRO] 127 128 129class DataSource(object): 130 131 def __init__(self, type_, url, tags, origin=None): 132 """ 133 :param type_: data source type, e.g. TYPE_YAML, TYPE_GBPDISTRO 134 135 :param url: URL of data location. For file resources, must 136 start with the file:// scheme. For remote resources, URL 137 must include a path. 138 139 :param tags: tags for matching data source to configurations 140 :param origin: filename or other indicator of where data came from for debugging. 141 142 :raises: :exc:`ValueError` if parameters do not validate 143 """ 144 # validate inputs 145 if type_ not in VALID_TYPES: 146 raise ValueError('type must be one of [%s]' % (','.join(VALID_TYPES))) 147 parsed = urlparse.urlparse(url) 148 if not parsed.scheme or (parsed.scheme != 'file' and not parsed.netloc) or parsed.path in ('', '/'): 149 raise ValueError('url must be a fully-specified URL with scheme, hostname, and path: %s' % (str(url))) 150 if not type(tags) == list: 151 raise ValueError('tags must be a list: %s' % (str(tags))) 152 153 self.type = type_ 154 self.tags = tags 155 156 self.url = url 157 self.origin = origin 158 159 def __eq__(self, other): 160 return isinstance(other, DataSource) and \ 161 self.type == other.type and \ 162 self.tags == other.tags and \ 163 self.url == other.url and \ 164 self.origin == other.origin 165 166 def __str__(self): 167 if self.origin: 168 return '[%s]:\n%s %s %s' % (self.origin, self.type, self.url, ' '.join(self.tags)) 169 else: 170 return '%s %s %s' % (self.type, self.url, ' '.join(self.tags)) 171 172 def __repr__(self): 173 return repr((self.type, self.url, self.tags, self.origin)) 174 175 176class RosDistroSource(DataSource): 177 178 def __init__(self, distro): 179 self.type = TYPE_GBPDISTRO 180 self.tags = [distro] 181 # In this case self.url is a list if REP-143 is being used 182 self.url = get_index().distributions[distro]['distribution'] 183 self.origin = None 184 185# create function we can pass in as model to parse_source_data. The 186# function emulates the CachedDataSource constructor but does the 187# necessary full filepath calculation and loading of data. 188 189 190def cache_data_source_loader(sources_cache_dir, verbose=False): 191 def create_model(type_, uri, tags, origin=None): 192 # compute the filename has from the URL 193 filename = compute_filename_hash(uri) 194 filepath = os.path.join(sources_cache_dir, filename) 195 pickle_filepath = filepath + PICKLE_CACHE_EXT 196 if os.path.exists(pickle_filepath): 197 if verbose: 198 print('loading cached data source:\n\t%s\n\t%s' % (uri, pickle_filepath), file=sys.stderr) 199 with open(pickle_filepath, 'rb') as f: 200 rosdep_data = pickle.loads(f.read()) 201 elif os.path.exists(filepath): 202 if verbose: 203 print('loading cached data source:\n\t%s\n\t%s' % (uri, filepath), file=sys.stderr) 204 with open(filepath) as f: 205 rosdep_data = yaml.safe_load(f.read()) 206 else: 207 rosdep_data = {} 208 return CachedDataSource(type_, uri, tags, rosdep_data, origin=filepath) 209 return create_model 210 211 212class CachedDataSource(object): 213 214 def __init__(self, type_, url, tags, rosdep_data, origin=None): 215 """ 216 Stores data source and loaded rosdep data for that source. 217 218 NOTE: this is not a subclass of DataSource, though it's API is 219 duck-type compatible with the DataSource API. 220 """ 221 self.source = DataSource(type_, url, tags, origin=origin) 222 self.rosdep_data = rosdep_data 223 224 def __eq__(self, other): 225 try: 226 return self.source == other.source and \ 227 self.rosdep_data == other.rosdep_data 228 except AttributeError: 229 return False 230 231 def __str__(self): 232 return '%s\n%s' % (self.source, self.rosdep_data) 233 234 def __repr__(self): 235 return repr((self.type, self.url, self.tags, self.rosdep_data, self.origin)) 236 237 @property 238 def type(self): 239 """ 240 :returns: data source type 241 """ 242 return self.source.type 243 244 @property 245 def url(self): 246 """ 247 :returns: data source URL 248 """ 249 return self.source.url 250 251 @property 252 def tags(self): 253 """ 254 :returns: data source tags 255 """ 256 return self.source.tags 257 258 @property 259 def origin(self): 260 """ 261 :returns: data source origin, if set, or ``None`` 262 """ 263 return self.source.origin 264 265 266class DataSourceMatcher(object): 267 268 def __init__(self, tags): 269 self.tags = tags 270 271 def matches(self, rosdep_data_source): 272 """ 273 Check if the datasource matches this configuration. 274 275 :param rosdep_data_source: :class:`DataSource` 276 """ 277 # all of the rosdep_data_source tags must be in our matcher tags 278 return not any(set(rosdep_data_source.tags) - set(self.tags)) 279 280 @staticmethod 281 def create_default(os_override=None): 282 """ 283 Create a :class:`DataSourceMatcher` to match the current 284 configuration. 285 286 :param os_override: (os_name, os_codename) tuple to override 287 OS detection 288 :returns: :class:`DataSourceMatcher` 289 """ 290 distro_name = rospkg.distro.current_distro_codename() 291 if os_override is None: 292 os_detect = rospkg.os_detect.OsDetect() 293 os_name, os_version, os_codename = os_detect.detect_os() 294 else: 295 os_name, os_codename = os_override 296 tags = [t for t in (distro_name, os_name, os_codename) if t] 297 return DataSourceMatcher(tags) 298 299 300def download_rosdep_data(url): 301 """ 302 :raises: :exc:`DownloadFailure` If data cannot be 303 retrieved (e.g. 404, bad YAML format, server down). 304 """ 305 try: 306 f = urlopen(url, timeout=DOWNLOAD_TIMEOUT) 307 text = f.read() 308 f.close() 309 data = yaml.safe_load(text) 310 if type(data) != dict: 311 raise DownloadFailure('rosdep data from [%s] is not a YAML dictionary' % (url)) 312 return data 313 except (URLError, httplib.HTTPException) as e: 314 raise DownloadFailure(str(e) + ' (%s)' % url) 315 except yaml.YAMLError as e: 316 raise DownloadFailure(str(e)) 317 318 319def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): 320 """ 321 Download (and validate) contents of default sources list. 322 323 :param url: override URL of default sources list file 324 :return: raw sources list data, ``str`` 325 :raises: :exc:`DownloadFailure` If data cannot be 326 retrieved (e.g. 404, bad YAML format, server down). 327 :raises: :exc:`urllib2.URLError` If data cannot be 328 retrieved (e.g. 404, server down). 329 """ 330 try: 331 f = urlopen(url, timeout=DOWNLOAD_TIMEOUT) 332 except (URLError, httplib.HTTPException) as e: 333 raise URLError(str(e) + ' (%s)' % url) 334 data = f.read().decode() 335 f.close() 336 if not data: 337 raise DownloadFailure('cannot download defaults file from %s : empty contents' % url) 338 # parse just for validation 339 try: 340 parse_sources_data(data) 341 except InvalidData as e: 342 raise DownloadFailure( 343 'The content downloaded from %s failed to pass validation.' 344 ' It is likely that the source is invalid unless the data was corrupted during the download.' 345 ' The contents were:{{{%s}}} The error raised was: %s' % (url, data, e)) 346 return data 347 348 349def parse_sources_data(data, origin='<string>', model=None): 350 """ 351 Parse sources file format (tags optional):: 352 353 # comments and empty lines allowed 354 <type> <uri> [tags] 355 356 e.g.:: 357 358 yaml http://foo/rosdep.yaml fuerte lucid ubuntu 359 360 If tags are specified, *all* tags must match the current 361 configuration for the sources data to be used. 362 363 :param data: data in sources file format 364 :param model: model to load data into. Defaults to :class:`DataSource` 365 366 :returns: List of data sources, [:class:`DataSource`] 367 :raises: :exc:`InvalidData` 368 """ 369 if model is None: 370 model = DataSource 371 372 sources = [] 373 for line in data.split('\n'): 374 line = line.strip() 375 # ignore empty lines or comments 376 if not line or line.startswith('#'): 377 continue 378 splits = line.split(' ') 379 if len(splits) < 2: 380 raise InvalidData('invalid line:\n%s' % (line), origin=origin) 381 type_ = splits[0] 382 url = splits[1] 383 tags = splits[2:] 384 try: 385 sources.append(model(type_, url, tags, origin=origin)) 386 except ValueError as e: 387 raise InvalidData('line:\n\t%s\n%s' % (line, e), origin=origin) 388 return sources 389 390 391def parse_sources_file(filepath): 392 """ 393 Parse file on disk 394 395 :returns: List of data sources, [:class:`DataSource`] 396 :raises: :exc:`InvalidData` If any error occurs reading 397 file, so an I/O error, non-existent file, or invalid format. 398 """ 399 try: 400 with open(filepath, 'r') as f: 401 return parse_sources_data(f.read(), origin=filepath) 402 except IOError as e: 403 raise InvalidData('I/O error reading sources file: %s' % (str(e)), origin=filepath) 404 405 406def parse_sources_list(sources_list_dir=None): 407 """ 408 Parse data stored in on-disk sources list directory into a list of 409 :class:`DataSource` for processing. 410 411 :returns: List of data sources, [:class:`DataSource`]. If there is 412 no sources list dir, this returns an empty list. 413 :raises: :exc:`InvalidData` 414 :raises: :exc:`OSError` if *sources_list_dir* cannot be read. 415 :raises: :exc:`IOError` if *sources_list_dir* cannot be read. 416 """ 417 if sources_list_dir is None: 418 sources_list_dir = get_sources_list_dir() 419 sources_list_dirs = get_sources_list_dirs(sources_list_dir) 420 421 filelist = [] 422 for sdir in sources_list_dirs: 423 filelist += sorted([os.path.join(sdir, f) for f in os.listdir(sdir) if f.endswith('.list')]) 424 sources_list = [] 425 for f in filelist: 426 sources_list.extend(parse_sources_file(f)) 427 return sources_list 428 429 430def _generate_key_from_urls(urls): 431 # urls may be a list of urls or a single string 432 try: 433 assert isinstance(urls, (list, basestring)) 434 except NameError: 435 assert isinstance(urls, (list, str)) 436 # We join the urls by the '^' character because it is not allowed in urls 437 return '^'.join(urls if isinstance(urls, list) else [urls]) 438 439 440def update_sources_list(sources_list_dir=None, sources_cache_dir=None, 441 success_handler=None, error_handler=None, 442 skip_eol_distros=False): 443 """ 444 Re-downloaded data from remote sources and store in cache. Also 445 update the cache index based on current sources. 446 447 :param sources_list_dir: override source list directory 448 :param sources_cache_dir: override sources cache directory 449 :param success_handler: fn(DataSource) to call if a particular 450 source loads successfully. This hook is mainly for printing 451 errors to console. 452 :param error_handler: fn(DataSource, DownloadFailure) to call 453 if a particular source fails. This hook is mainly for 454 printing errors to console. 455 :param skip_eol_distros: skip downloading sources for EOL distros 456 457 :returns: list of (`DataSource`, cache_file_path) pairs for cache 458 files that were updated, ``[str]`` 459 :raises: :exc:`InvalidData` If any of the sources list files is invalid 460 :raises: :exc:`OSError` if *sources_list_dir* cannot be read. 461 :raises: :exc:`IOError` If *sources_list_dir* cannot be read or cache data cannot be written 462 """ 463 if sources_cache_dir is None: 464 sources_cache_dir = get_sources_cache_dir() 465 466 sources = parse_sources_list(sources_list_dir=sources_list_dir) 467 retval = [] 468 for source in list(sources): 469 try: 470 if source.type == TYPE_YAML: 471 rosdep_data = download_rosdep_data(source.url) 472 elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 473 if not source.tags[0] in ['electric', 'fuerte']: 474 print('Ignore legacy gbpdistro "%s"' % source.tags[0]) 475 sources.remove(source) 476 continue # do not store this entry in the cache 477 rosdep_data = download_gbpdistro_as_rosdep_data(source.url) 478 retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) 479 if success_handler is not None: 480 success_handler(source) 481 except DownloadFailure as e: 482 if error_handler is not None: 483 error_handler(source, e) 484 485 # Additional sources for ros distros 486 # In compliance with REP137 and REP143 487 python_versions = {} 488 489 print('Query rosdistro index %s' % get_index_url()) 490 for dist_name in sorted(get_index().distributions.keys()): 491 distribution = get_index().distributions[dist_name] 492 if skip_eol_distros: 493 if distribution.get('distribution_status') == 'end-of-life': 494 print('Skip end-of-life distro "%s"' % dist_name) 495 continue 496 print('Add distro "%s"' % dist_name) 497 rds = RosDistroSource(dist_name) 498 rosdep_data = get_gbprepo_as_rosdep_data(dist_name) 499 # Store Python version from REP153 500 if distribution.get('python_version'): 501 python_versions[dist_name] = distribution.get('python_version') 502 # dist_files can either be a string (single filename) or a list (list of filenames) 503 dist_files = distribution['distribution'] 504 key = _generate_key_from_urls(dist_files) 505 retval.append((rds, write_cache_file(sources_cache_dir, key, rosdep_data))) 506 sources.append(rds) 507 508 # cache metadata that isn't a source list 509 MetaDatabase().set('ROS_PYTHON_VERSION', python_versions) 510 511 # Create a combined index of *all* the sources. We do all the 512 # sources regardless of failures because a cache from a previous 513 # attempt may still exist. We have to do this cache index so that 514 # loads() see consistent data. 515 if not os.path.exists(sources_cache_dir): 516 os.makedirs(sources_cache_dir) 517 cache_index = os.path.join(sources_cache_dir, CACHE_INDEX) 518 data = "#autogenerated by rosdep, do not edit. use 'rosdep update' instead\n" 519 for source in sources: 520 url = _generate_key_from_urls(source.url) 521 data += 'yaml %s %s\n' % (url, ' '.join(source.tags)) 522 write_atomic(cache_index, data) 523 # mainly for debugging and testing 524 return retval 525 526 527def load_cached_sources_list(sources_cache_dir=None, verbose=False): 528 """ 529 Load cached data based on the sources list. 530 531 :returns: list of :class:`CachedDataSource` instance with raw 532 rosdep data loaded. 533 :raises: :exc:`OSError` if cache cannot be read 534 :raises: :exc:`IOError` if cache cannot be read 535 """ 536 if sources_cache_dir is None: 537 sources_cache_dir = get_sources_cache_dir() 538 cache_index = os.path.join(sources_cache_dir, 'index') 539 if not os.path.exists(cache_index): 540 if verbose: 541 print('no cache index present, not loading cached sources', file=sys.stderr) 542 return [] 543 with open(cache_index, 'r') as f: 544 cache_data = f.read() 545 # the loader does all the work 546 model = cache_data_source_loader(sources_cache_dir, verbose=verbose) 547 return parse_sources_data(cache_data, origin=cache_index, model=model) 548 549 550class SourcesListLoader(RosdepLoader): 551 """ 552 SourcesList loader implements the general RosdepLoader API. This 553 implementation is fairly simple as there is only one view the 554 source list loader can create. It is also a bit degenerate as it 555 is not capable of mapping resource names to views, thus any 556 resource-name-based API fails or returns nothing interesting. 557 558 This loader should not be used directly; instead, it is more 559 useful composed with other higher-level implementations, like the 560 :class:`rosdep2.rospkg_loader.RospkgLoader`. The general intent 561 is to compose it with another loader by making all of the other 562 loader's views depends on all the views in this loader. 563 """ 564 565 ALL_VIEW_KEY = 'sources.list' 566 567 def __init__(self, sources): 568 """ 569 :param sources: cached sources list entries, [:class:`CachedDataSource`] 570 """ 571 self.sources = sources 572 573 @staticmethod 574 def create_default(matcher=None, sources_cache_dir=None, os_override=None, verbose=False): 575 """ 576 :param matcher: override DataSourceMatcher. Defaults to 577 DataSourceMatcher.create_default(). 578 :param sources_cache_dir: override location of sources cache 579 """ 580 if matcher is None: 581 matcher = DataSourceMatcher.create_default(os_override=os_override) 582 if verbose: 583 print('using matcher with tags [%s]' % (', '.join(matcher.tags)), file=sys.stderr) 584 585 sources = load_cached_sources_list(sources_cache_dir=sources_cache_dir, verbose=verbose) 586 if verbose: 587 print('loaded %s sources' % (len(sources)), file=sys.stderr) 588 sources = [x for x in sources if matcher.matches(x)] 589 if verbose: 590 print('%s sources match current tags' % (len(sources)), file=sys.stderr) 591 return SourcesListLoader(sources) 592 593 def load_view(self, view_name, rosdep_db, verbose=False): 594 """ 595 Load view data into rosdep_db. If the view has already been 596 loaded into rosdep_db, this method does nothing. 597 598 :param view_name: name of ROS stack to load, ``str`` 599 :param rosdep_db: database to load stack data into, :class:`RosdepDatabase` 600 601 :raises: :exc:`InvalidData` 602 """ 603 if rosdep_db.is_loaded(view_name): 604 return 605 source = self.get_source(view_name) 606 if verbose: 607 print('loading view [%s] with sources.list loader' % (view_name), file=sys.stderr) 608 view_dependencies = self.get_view_dependencies(view_name) 609 rosdep_db.set_view_data(view_name, source.rosdep_data, view_dependencies, view_name) 610 611 def get_loadable_resources(self): 612 return [] 613 614 def get_loadable_views(self): 615 return [x.url for x in self.sources] 616 617 def get_view_dependencies(self, view_name): 618 # use dependencies to implement precedence 619 if view_name != SourcesListLoader.ALL_VIEW_KEY: 620 # if the view_name matches one of our sources, return 621 # empty list as none of our sources has deps. 622 if any([x for x in self.sources if view_name == x.url]): 623 return [] 624 625 # not one of our views, so it depends on everything we provide 626 return [x.url for x in self.sources] 627 628 def get_source(self, view_name): 629 matches = [x for x in self.sources if x.url == view_name] 630 if matches: 631 return matches[0] 632 else: 633 raise rospkg.ResourceNotFound(view_name) 634 635 def get_rosdeps(self, resource_name, implicit=True): 636 """ 637 Always raises as SourceListLoader defines no concrete resources with rosdeps. 638 639 :raises: :exc:`rospkg.ResourceNotFound` 640 """ 641 raise rospkg.ResourceNotFound(resource_name) 642 643 def get_view_key(self, resource_name): 644 """ 645 Always raises as SourceListLoader defines no concrete resources with rosdeps. 646 647 :returns: Name of view that *resource_name* is in, ``None`` if no associated view. 648 :raises: :exc:`rospkg.ResourceNotFound` if *resource_name* cannot be found. 649 """ 650 raise rospkg.ResourceNotFound(resource_name) 651