1# Software License Agreement (BSD License) 2# 3# Copyright (c) 2009, Willow Garage, Inc. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 10# * Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# * Redistributions in binary form must reproduce the above 13# copyright notice, this list of conditions and the following 14# disclaimer in the documentation and/or other materials provided 15# with the distribution. 16# * Neither the name of Willow Garage, Inc. nor the names of its 17# contributors may be used to endorse or promote products derived 18# from this software without specific prior written permission. 19# 20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 23# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 24# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 30# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31# POSSIBILITY OF SUCH DAMAGE. 32 33import os 34import yaml 35from vcstools.common import urlopen_netrc 36from wstool.common import MultiProjectException 37 38__REPOTYPES__ = ['svn', 'bzr', 'hg', 'git', 'tar'] 39__ALLTYPES__ = __REPOTYPES__ + ['other', 'setup-file'] 40 41## The Path spec is a lightweight object to transport the 42## specification of a config element between functions, 43## independently of yaml structure. 44## Specifications are persisted in yaml, this file deals 45## with manipulations of any such structures representing configs as 46## yaml. 47## get_path_spec_from_yaml turns yaml into path_spec, and pathspec 48## get_legacy_yaml returns yaml. 49 50 51def get_yaml_from_uri(uri): 52 """reads and parses yaml from a local file or remote uri""" 53 stream = None 54 try: 55 try: 56 if os.path.isfile(uri): 57 try: 58 stream = open(uri, 'r') 59 except IOError as ioe: 60 raise MultiProjectException( 61 "Unable open file [%s]: %s" % (uri, ioe)) 62 else: 63 try: 64 stream = urlopen_netrc(uri) 65 except IOError as ioe2: 66 raise MultiProjectException( 67 "Unable to download URL [%s]: %s" % (uri, ioe2)) 68 except ValueError as vae: 69 raise MultiProjectException( 70 "Is not a local file, nor a valid URL [%s] : %s" % (uri, vae)) 71 if not stream: 72 raise MultiProjectException("couldn't load config uri %s" % uri) 73 try: 74 yamldata = yaml.safe_load(stream) 75 except yaml.YAMLError as yame: 76 raise MultiProjectException( 77 "Invalid multiproject yaml format in [%s]: %s" % (uri, yame)) 78 79 # we want a list or a dict, but pyyaml parses xml as string 80 if type(yamldata) == 'str': 81 raise MultiProjectException( 82 "Invalid multiproject yaml format in [%s]: %s" % (uri, yamldata)) 83 finally: 84 if stream is not None: 85 stream.close() 86 return yamldata 87 88 89def get_path_specs_from_uri(uri, config_filename=None, as_is=False): 90 """ 91 Builds a list of PathSpec elements from several types of input 92 locations, "uris". 93 The function treats other workspace folders/files as special uris 94 to prevent mutual conflicts. 95 96 :param uri: a folder, a file, or a web url 97 :param config_filename: name for files to be treated special 98 as other workspaces 99 :param as_is: do not rewrite, used for loading the current 100 workspace config without rewriting 101 """ 102 if os.path.isdir(uri): 103 if (config_filename is not None and 104 os.path.isfile(os.path.join(uri, config_filename))): 105 106 uri = os.path.join(uri, config_filename) 107 else: 108 # plain folders returned as themselves 109 return [PathSpec(local_name=uri)] 110 yaml_spec = get_yaml_from_uri(uri) 111 if yaml_spec is None: 112 return [] 113 specs = [get_path_spec_from_yaml(x) for x in yaml_spec] 114 115 if (config_filename is not None and 116 not as_is and 117 os.path.isfile(uri) and 118 os.path.basename(uri) == config_filename): 119 120 # treat config files and folders with such files special 121 # to prevent 2 workspaces from interacting 122 specs = rewrite_included_source(specs, os.path.dirname(uri)) 123 return specs 124 125 126def rewrite_included_source(source_path_specs, source_dir): 127 """ 128 assumes source_path_specs is the contents of a config file in 129 another directory source dir. It rewrites all elements, by changing 130 any relative path relative to source dir and changing vcs 131 types to non-vcs types types, to prevent two environments from 132 conflicting 133 """ 134 for index, pathspec in enumerate(source_path_specs): 135 local_name = os.path.normpath(os.path.join(source_dir, 136 pathspec.get_local_name())) 137 pathspec.set_local_name(local_name) 138 if pathspec.get_path() is not None: 139 path = os.path.normpath( 140 os.path.join(source_dir, pathspec.get_path())) 141 pathspec.set_path(path) 142 pathspec.detach_vcs_info() 143 source_path_specs[index] = pathspec 144 return source_path_specs 145 146 147def aggregate_from_uris(config_uris, config_filename=None, allow_other_element=True): 148 """ 149 Builds a List of PathSpec from a list of location strings (uri, 150 paths). If locations is a folder, attempts to find config_filename 151 in it, and use "folder/config_filename" instead(rewriting element 152 path and stripping scm nature), else add folder as PathSpec. 153 Anything else, parse yaml at location, and add a PathSpec for each 154 element. 155 156 :param config_uris: source of yaml 157 :param config_filename: file to use when given a folder 158 :param allow_other_element: if False, discards elements 159 to be added without SCM information 160 """ 161 aggregate_source_yaml = [] 162 # build up a merged list of config elements from all given config_uris 163 if config_uris is None: 164 return [] 165 for loop_uri in config_uris: 166 source_path_specs = get_path_specs_from_uri( 167 loop_uri, config_filename) 168 # allow duplicates, dealt with in Config class 169 if not allow_other_element: 170 for spec in source_path_specs: 171 if not spec.get_scmtype(): 172 raise MultiProjectException( 173 "Forbidden non-SCM element: %s (%s)" % 174 (spec.get_local_name(), spec.get_legacy_type())) 175 aggregate_source_yaml.extend(source_path_specs) 176 return aggregate_source_yaml 177 178 179class PathSpec: 180 def __init__(self, 181 # localname is used as ID, currently also is used as path 182 local_name, 183 scmtype=None, 184 uri=None, 185 version=None, 186 curr_version=None, 187 tags=None, 188 revision=None, 189 currevision=None, 190 remote_revision=None, 191 path=None, 192 curr_uri=None): 193 """ 194 Fills in local properties based on dict, unifies different syntaxes 195 :param local-name: to be unique within config, filesystem path to folder 196 :param scmtype: one of __ALLTYPES__ 197 :param uri: uri from config file 198 :param version: version label from config file (branchname, tagname, sha-id) 199 :param cur_version: version information label(s) from VCS (branchname, remote, tracking branch) 200 :param tags: arbirtrary meta-information (used for ROS package indexing) 201 :param revision: unique id of label stored in version 202 :param currrevision: unique id of actual version in file system 203 :param path: path to folder (currently equivalent to local_name) 204 :param curr_uri: actual remote uri used in local checkout 205 """ 206 self._local_name = local_name 207 self._path = path 208 self._uri = uri 209 self._curr_uri = curr_uri 210 self._version = version 211 self._curr_version = curr_version 212 self._scmtype = scmtype 213 self._tags = tags or [] 214 self._revision = revision 215 self._currevision = currevision 216 self._remote_revision = remote_revision 217 218 def __str__(self): 219 return str(self.get_legacy_yaml()) 220 221 def __repr__(self): 222 return "PathSpec(%s)" % self.__str__() 223 224 def __eq__(self, other): 225 if isinstance(other, self.__class__): 226 return self.__dict__ == other.__dict__ 227 else: 228 return False 229 230 def __ne__(self, other): 231 return not self.__eq__(other) 232 233 def detach_vcs_info(self): 234 """if wrapper has VCS information, remove it to make it a plain folder""" 235 if self._scmtype is not None: 236 self._scmtype = None 237 self._uri = None 238 self._version = None 239 self._curr_version = None 240 self._revision = None 241 self._currevision = None 242 self._remote_revision = None 243 244 def get_legacy_type(self): 245 """return one of __ALLTYPES__""" 246 if self._scmtype is not None: 247 return self._scmtype 248 elif self._tags is not None and 'setup-file' in self._tags: 249 return 'setup-file' 250 return 'other' 251 252 def get_legacy_yaml(self, spec=True, exact=False): 253 """ 254 :param spec: If True, the version information will come from the 255 workspace .rosinstall. If False, the version information will come 256 from the current work trees. 257 :param exact: If True, the versions will be set to the exact commit 258 UUIDs. If False, the version name will be used, which might be a 259 branch name aut cetera. 260 261 return something like 262 {hg: {local-name: common, 263 version: common-1.0.2, 264 uri: https://kforge.org/common/}} 265 """ 266 # TODO switch to new syntax 267 properties = {'local-name': self._local_name} 268 if spec: 269 if self._uri is not None: 270 properties['uri'] = self._uri 271 if exact: 272 if self._revision is not None: 273 properties['version'] = self._revision 274 else: 275 if self._version is not None: 276 properties['version'] = self._version 277 else: 278 if self._curr_uri is not None: 279 properties['uri'] = self._curr_uri 280 if exact: 281 if self._currevision is not None: 282 properties['version'] = self._currevision 283 284 else: 285 if self._curr_version is not None: 286 properties['version'] = self._curr_version 287 288 if self._tags is not None: 289 for tag in self._tags: 290 if tag != 'setup-file' and tag != []: 291 if type(tag) == dict: 292 properties.update(tag) 293 else: 294 properties[tag] = None 295 yaml_dict = {self.get_legacy_type(): properties} 296 return yaml_dict 297 298 def get_local_name(self): 299 return self._local_name 300 301 def set_local_name(self, local_name): 302 self._local_name = local_name 303 304 def get_path(self): 305 return self._path 306 307 def set_path(self, path): 308 self._path = path 309 310 def get_tags(self): 311 return self._tags 312 313 def get_scmtype(self): 314 return self._scmtype 315 316 def get_version(self): 317 return self._version 318 319 def get_curr_version(self): 320 return self._curr_version 321 322 def get_revision(self): 323 return self._revision 324 325 def get_current_revision(self): 326 return self._currevision 327 328 def get_remote_revision(self): 329 return self._remote_revision 330 331 def get_uri(self): 332 return self._uri 333 334 def get_curr_uri(self): 335 return self._curr_uri 336 337 338def get_path_spec_from_yaml(yaml_dict): 339 """ 340 Fills in local properties based on dict, unifies different syntaxes 341 """ 342 local_name = None 343 uri = None 344 version = None 345 scmtype = None 346 tags = [] 347 if type(yaml_dict) != dict: 348 raise MultiProjectException( 349 "Yaml for each element must be in YAML dict form: %s " % yaml_dict) 350 # old syntax: 351# - hg: {local-name: common_rosdeps, 352# version: common_rosdeps-1.0.2, 353# uri: https://kforge.ros.org/common/rosdepcore} 354# - setup-file: {local-name: /opt/ros/fuerte/setup.sh} 355# - other: {local-name: /opt/ros/fuerte/share/ros} 356# - other: {local-name: /opt/ros/fuerte/share} 357# - other: {local-name: /opt/ros/fuerte/stacks} 358 if yaml_dict is None or len(yaml_dict) == 0: 359 raise MultiProjectException("no element in yaml dict.") 360 if len(yaml_dict) > 1: 361 raise MultiProjectException( 362 "too many keys in element dict %s" % (list(yaml_dict.keys()))) 363 if not list(yaml_dict.keys())[0] in __ALLTYPES__: 364 raise MultiProjectException( 365 "Unknown element type '%s'" % (list(yaml_dict.keys())[0])) 366 firstkey = list(yaml_dict.keys())[0] 367 if firstkey in __REPOTYPES__: 368 scmtype = list(yaml_dict.keys())[0] 369 if firstkey == 'setup-file': 370 tags.append('setup-file') 371 values = yaml_dict[firstkey] 372 if values is not None: 373 for key, value in list(values.items()): 374 if key == "local-name": 375 local_name = value 376 elif key == "meta": 377 tags.append({key: value}) 378 elif key == "uri": 379 uri = value 380 elif key == "version": 381 version = value 382 else: 383 raise MultiProjectException( 384 "Unknown key %s in %s" % (key, yaml_dict)) 385 # global validation 386 if local_name is None: 387 raise MultiProjectException( 388 "Config element without a local-name: %s" % (yaml_dict)) 389 if scmtype != None: 390 if uri is None: 391 raise MultiProjectException( 392 "scm type without declared uri in %s" % (values)) 393 # local_name is fixed, path may be normalized, made absolute, etc. 394 path = local_name 395 return PathSpec(local_name=local_name, 396 path=path, 397 scmtype=scmtype, 398 uri=uri, 399 version=version, 400 tags=tags) 401 402 403def generate_config_yaml(config, filename, header, pretty=False, 404 sort_with_localname=False, spec=True, 405 exact=False, vcs_only=False): 406 """ 407 Writes file filename with header first and then the config as YAML. 408 409 :param config: The configuration containing all the entries to be included 410 in the generated YAML. 411 :param filename: If filename is not an absolute path, it will be assumed to 412 be relative to config.get_base_path(). If filename is None, the output will 413 be sent to stdout instead of a file. 414 :param header: A header to be included with the generated config YAML. 415 :param pretty: If True, the generated config YAML will be printed in 416 long-form YAML. If false, the default flow style will be used instead. 417 :param sort_with_localname: If true, config entries will be sorted by their 418 localname fields. If false, the order will be as passed in through config. 419 :param spec: If True, the version information will come from the workspace 420 .rosinstall. If False, the version information will come from the current 421 work trees. 422 :param exact: If True, the versions will be set to the exact commit UUIDs. 423 If False, the version name will be used, which might be a branch name 424 aut cetera. 425 :param vcs_only: If True, the generated config YAML will include only 426 version-controlled entries. If False, all entries in current workspace will 427 be included. 428 """ 429 if not os.path.exists(config.get_base_path()): 430 os.makedirs(config.get_base_path()) 431 432 content = "" 433 if header: 434 content += header 435 436 # Do a pass-through if just pulling versioning information straight from 437 # the .rosinstall 438 passthrough = spec and not exact 439 items = config.get_source(not passthrough, vcs_only) 440 if sort_with_localname: 441 items = sorted(items, key=lambda x: x.get_local_name()) 442 items = [x.get_legacy_yaml(spec, exact) for x in items] 443 444 if items: 445 if pretty: 446 content += yaml.safe_dump(items, allow_unicode=True, 447 default_flow_style=False) 448 else: 449 content += yaml.safe_dump(items, default_flow_style=None) 450 451 if filename: 452 config_filepath = filename if os.path.isabs(filename) else \ 453 os.path.realpath(os.path.join(config.get_base_path(), filename)) 454 455 with open(config_filepath, 'w+b') as f: 456 f.write(content.encode('UTF-8')) 457 else: 458 print(content) 459