1# Copyright 2014-2017 Insight Software Consortium. 2# Copyright 2004-2009 Roman Yakovenko. 3# Distributed under the Boost Software License, Version 1.0. 4# See http://www.boost.org/LICENSE_1_0.txt 5 6import os 7import timeit 8 9import pygccxml.declarations 10 11from . import source_reader 12from . import declarations_cache 13from . import declarations_joiner 14from .. import utils 15 16 17class COMPILATION_MODE(object): 18 ALL_AT_ONCE = 'all at once' 19 FILE_BY_FILE = 'file by file' 20 21 22class file_configuration_t(object): 23 24 """ 25 source code location configuration. 26 27 The class instance uses "variant" interface to represent the following 28 data: 29 30 1) path to a C++ source file 31 32 2) path to GCC-XML generated XML file 33 34 3) path to a C++ source file and path to GCC-XML generated file 35 36 In this case, if XML file does not exists, it will be created. Next 37 time you will ask to parse the source file, the XML file will be used 38 instead. 39 40 Small tip: you can setup your makefile to delete XML files every time, 41 the relevant source file was changed. 42 43 4) Python string, that contains valid C++ code 44 45 46 There are few functions, that will help you to construct 47 :class:`file_configuration_t` object: 48 49 * :func:`create_source_fc` 50 51 * :func:`create_gccxml_fc` 52 53 * :func:`create_cached_source_fc` 54 55 * :func:`create_text_fc` 56 57 """ 58 59 class CONTENT_TYPE(object): 60 STANDARD_SOURCE_FILE = 'standard source file' 61 CACHED_SOURCE_FILE = 'cached source file' 62 GCCXML_GENERATED_FILE = 'gccxml generated file' 63 TEXT = 'text' 64 65 def __init__( 66 self, 67 data, 68 start_with_declarations=None, 69 content_type=CONTENT_TYPE.STANDARD_SOURCE_FILE, 70 cached_source_file=None): 71 object.__init__(self) 72 self.__data = data 73 if not start_with_declarations: 74 start_with_declarations = [] 75 self.__start_with_declarations = start_with_declarations 76 self.__content_type = content_type 77 self.__cached_source_file = cached_source_file 78 if not self.__cached_source_file \ 79 and self.__content_type == self.CONTENT_TYPE.CACHED_SOURCE_FILE: 80 self.__cached_source_file = self.__data + '.xml' 81 82 @property 83 def data(self): 84 return self.__data 85 86 @property 87 def start_with_declarations(self): 88 return self.__start_with_declarations 89 90 @property 91 def content_type(self): 92 return self.__content_type 93 94 @property 95 def cached_source_file(self): 96 return self.__cached_source_file 97 98 99def create_text_fc(text): 100 """ 101 Creates :class:`parser.file_configuration_t` instance, configured to 102 contain Python string, that contains valid C++ code 103 104 :param text: C++ code 105 :type text: str 106 107 :rtype: :class:`parser.file_configuration_t` 108 """ 109 110 return file_configuration_t( 111 data=text, 112 content_type=file_configuration_t.CONTENT_TYPE.TEXT) 113 114 115def create_source_fc(header): 116 """ 117 Creates :class:`parser.file_configuration_t` instance, configured to 118 contain path to C++ source file 119 120 :param header: path to C++ source file 121 :type header: str 122 123 :rtype: :class:`parser.file_configuration_t` 124 """ 125 126 return file_configuration_t( 127 data=header, 128 content_type=file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE) 129 130 131def create_gccxml_fc(xml_file): 132 """ 133 Creates :class:`parser.file_configuration_t` instance, configured to 134 contain path to GCC-XML generated XML file. 135 136 :param xml_file: path to GCC-XML generated XML file 137 :type xml_file: str 138 139 :rtype: :class:`parser.file_configuration_t` 140 """ 141 142 return file_configuration_t( 143 data=xml_file, 144 content_type=file_configuration_t.CONTENT_TYPE.GCCXML_GENERATED_FILE) 145 146 147def create_cached_source_fc(header, cached_source_file): 148 """ 149 Creates :class:`parser.file_configuration_t` instance, configured to 150 contain path to GCC-XML generated XML file and C++ source file. If XML file 151 does not exists, it will be created and used for parsing. If XML file 152 exists, it will be used for parsing. 153 154 :param header: path to C++ source file 155 :type header: str 156 157 :param cached_source_file: path to GCC-XML generated XML file 158 :type cached_source_file: str 159 160 :rtype: :class:`parser.file_configuration_t` 161 """ 162 163 return file_configuration_t( 164 data=header, 165 cached_source_file=cached_source_file, 166 content_type=file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE) 167 168 169class project_reader_t(object): 170 171 """parses header files and returns the contained declarations""" 172 173 def __init__(self, config, cache=None, decl_factory=None): 174 """ 175 :param config: GCCXML configuration 176 :type config: :class:xml_generator_configuration_t 177 178 :param cache: declaration cache, by default a cache functionality will 179 not be used 180 :type cache: :class:`cache_base_t` instance or `str` 181 182 :param decl_factory: declaration factory 183 :type decl_factory: :class:`decl_factory_t` 184 """ 185 186 self.__config = config 187 self.__dcache = None 188 if isinstance(cache, declarations_cache.cache_base_t): 189 self.__dcache = cache 190 elif utils.is_str(cache): 191 self.__dcache = declarations_cache.file_cache_t(cache) 192 else: 193 self.__dcache = declarations_cache.dummy_cache_t() 194 self.__decl_factory = decl_factory 195 if not decl_factory: 196 self.__decl_factory = pygccxml.declarations.decl_factory_t() 197 198 self.logger = utils.loggers.cxx_parser 199 self.__xml_generator_from_xml_file = None 200 201 @property 202 def xml_generator_from_xml_file(self): 203 """ 204 Configuration object containing information about the xml generator 205 read from the xml file. 206 207 Returns: 208 utils.xml_generators: configuration object 209 """ 210 return self.__xml_generator_from_xml_file 211 212 @staticmethod 213 def get_os_file_names(files): 214 """ 215 returns file names 216 217 :param files: list of strings and\\or :class:`file_configuration_t` 218 instances. 219 :type files: list 220 """ 221 222 fnames = [] 223 for f in files: 224 if utils.is_str(f): 225 fnames.append(f) 226 elif isinstance(f, file_configuration_t): 227 if f.content_type in ( 228 file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE, 229 file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE): 230 231 fnames.append(f.data) 232 else: 233 pass 234 return fnames 235 236 def read_files( 237 self, 238 files, 239 compilation_mode=COMPILATION_MODE.FILE_BY_FILE): 240 """ 241 parses a set of files 242 243 :param files: list of strings and\\or :class:`file_configuration_t` 244 instances. 245 :type files: list 246 247 :param compilation_mode: determines whether the files are parsed 248 individually or as one single chunk 249 :type compilation_mode: :class:`COMPILATION_MODE` 250 :rtype: [:class:`declaration_t`] 251 """ 252 253 if compilation_mode == COMPILATION_MODE.ALL_AT_ONCE \ 254 and len(files) == len(self.get_os_file_names(files)): 255 return self.__parse_all_at_once(files) 256 else: 257 if compilation_mode == COMPILATION_MODE.ALL_AT_ONCE: 258 msg = ''.join([ 259 "Unable to parse files using ALL_AT_ONCE mode. ", 260 "There is some file configuration that is not file. ", 261 "pygccxml.parser.project_reader_t switches to ", 262 "FILE_BY_FILE mode."]) 263 self.logger.warning(msg) 264 return self.__parse_file_by_file(files) 265 266 def __parse_file_by_file(self, files): 267 namespaces = [] 268 config = self.__config.clone() 269 self.logger.debug("Reading project files: file by file") 270 for prj_file in files: 271 272 if isinstance(prj_file, file_configuration_t): 273 del config.start_with_declarations[:] 274 config.start_with_declarations.extend( 275 prj_file.start_with_declarations) 276 header = prj_file.data 277 content_type = prj_file.content_type 278 else: 279 config = self.__config 280 header = prj_file 281 content_type = \ 282 file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE 283 284 reader = source_reader.source_reader_t( 285 config, 286 self.__dcache, 287 self.__decl_factory) 288 289 if content_type == \ 290 file_configuration_t.CONTENT_TYPE.STANDARD_SOURCE_FILE: 291 self.logger.info('Parsing source file "%s" ... ', header) 292 decls = reader.read_file(header) 293 elif content_type == \ 294 file_configuration_t.CONTENT_TYPE.GCCXML_GENERATED_FILE: 295 self.logger.info('Parsing xml file "%s" ... ', header) 296 decls = reader.read_xml_file(header) 297 elif content_type == \ 298 file_configuration_t.CONTENT_TYPE.CACHED_SOURCE_FILE: 299 # TODO: raise error when header file does not exist 300 if not os.path.exists(prj_file.cached_source_file): 301 dir_ = os.path.split(prj_file.cached_source_file)[0] 302 if dir_ and not os.path.exists(dir_): 303 os.makedirs(dir_) 304 self.logger.info( 305 'Creating xml file "%s" from source file "%s" ... ', 306 prj_file.cached_source_file, header) 307 reader.create_xml_file(header, prj_file.cached_source_file) 308 self.logger.info( 309 'Parsing xml file "%s" ... ', 310 prj_file.cached_source_file) 311 decls = reader.read_xml_file(prj_file.cached_source_file) 312 else: 313 decls = reader.read_string(header) 314 self.__xml_generator_from_xml_file = \ 315 reader.xml_generator_from_xml_file 316 namespaces.append(decls) 317 318 self.logger.debug("Flushing cache... ") 319 start_time = timeit.default_timer() 320 self.__dcache.flush() 321 self.logger.debug( 322 "Cache has been flushed in %.1f secs", 323 (timeit.default_timer() - start_time)) 324 answer = [] 325 self.logger.debug("Joining namespaces ...") 326 for file_nss in namespaces: 327 answer = self._join_top_namespaces(answer, file_nss) 328 self.logger.debug("Joining declarations ...") 329 for ns in answer: 330 if isinstance(ns, pygccxml.declarations.namespace_t): 331 declarations_joiner.join_declarations(ns) 332 leaved_classes = self._join_class_hierarchy(answer) 333 types = self.__declarated_types(answer) 334 self.logger.debug("Relinking declared types ...") 335 self._relink_declarated_types(leaved_classes, types) 336 declarations_joiner.bind_aliases( 337 pygccxml.declarations.make_flatten(answer)) 338 return answer 339 340 def __parse_all_at_once(self, files): 341 config = self.__config.clone() 342 self.logger.debug("Reading project files: all at once") 343 header_content = [] 344 for header in files: 345 if isinstance(header, file_configuration_t): 346 del config.start_with_declarations[:] 347 config.start_with_declarations.extend( 348 header.start_with_declarations) 349 header_content.append( 350 '#include "%s" %s' % 351 (header.data, os.linesep)) 352 else: 353 header_content.append( 354 '#include "%s" %s' % 355 (header, os.linesep)) 356 return self.read_string(''.join(header_content)) 357 358 def read_string(self, content): 359 """Parse a string containing C/C++ source code. 360 361 :param content: C/C++ source code. 362 :type content: str 363 :rtype: Declarations 364 """ 365 reader = source_reader.source_reader_t( 366 self.__config, 367 None, 368 self.__decl_factory) 369 decls = reader.read_string(content) 370 self.__xml_generator_from_xml_file = reader.xml_generator_from_xml_file 371 return decls 372 373 def read_xml(self, file_configuration): 374 """parses C++ code, defined on the file_configurations and returns 375 GCCXML generated file content""" 376 377 xml_file_path = None 378 delete_xml_file = True 379 fc = file_configuration 380 reader = source_reader.source_reader_t( 381 self.__config, 382 None, 383 self.__decl_factory) 384 try: 385 if fc.content_type == fc.CONTENT_TYPE.STANDARD_SOURCE_FILE: 386 self.logger.info('Parsing source file "%s" ... ', fc.data) 387 xml_file_path = reader.create_xml_file(fc.data) 388 elif fc.content_type == \ 389 file_configuration_t.CONTENT_TYPE.GCCXML_GENERATED_FILE: 390 self.logger.info('Parsing xml file "%s" ... ', fc.data) 391 xml_file_path = fc.data 392 delete_xml_file = False 393 elif fc.content_type == fc.CONTENT_TYPE.CACHED_SOURCE_FILE: 394 # TODO: raise error when header file does not exist 395 if not os.path.exists(fc.cached_source_file): 396 dir_ = os.path.split(fc.cached_source_file)[0] 397 if dir_ and not os.path.exists(dir_): 398 os.makedirs(dir_) 399 self.logger.info( 400 'Creating xml file "%s" from source file "%s" ... ', 401 fc.cached_source_file, fc.data) 402 xml_file_path = reader.create_xml_file( 403 fc.data, 404 fc.cached_source_file) 405 else: 406 xml_file_path = fc.cached_source_file 407 else: 408 xml_file_path = reader.create_xml_file_from_string(fc.data) 409 with open(xml_file_path, "r") as xml_file: 410 xml = xml_file.read() 411 utils.remove_file_no_raise(xml_file_path, self.__config) 412 self.__xml_generator_from_xml_file = \ 413 reader.xml_generator_from_xml_file 414 return xml 415 finally: 416 if xml_file_path and delete_xml_file: 417 utils.remove_file_no_raise(xml_file_path, self.__config) 418 419 @staticmethod 420 def _join_top_namespaces(main_ns_list, other_ns_list): 421 answer = main_ns_list[:] 422 for other_ns in other_ns_list: 423 main_ns = pygccxml.declarations.find_declaration( 424 answer, 425 decl_type=pygccxml.declarations.namespace_t, 426 name=other_ns._name, 427 recursive=False) 428 if main_ns: 429 main_ns.take_parenting(other_ns) 430 else: 431 answer.append(other_ns) 432 return answer 433 434 @staticmethod 435 def _create_key(decl): 436 return ( 437 decl.location.as_tuple(), 438 tuple(pygccxml.declarations.declaration_path(decl))) 439 440 def _join_class_hierarchy(self, namespaces): 441 classes = [ 442 decl for decl in pygccxml.declarations.make_flatten(namespaces) 443 if isinstance(decl, pygccxml.declarations.class_t)] 444 leaved_classes = {} 445 # selecting classes to leave 446 for class_ in classes: 447 key = self._create_key(class_) 448 if key not in leaved_classes: 449 leaved_classes[key] = class_ 450 # replacing base and derived classes with those that should be leave 451 # also this loop will add missing derived classes to the base 452 for class_ in classes: 453 leaved_class = leaved_classes[self._create_key(class_)] 454 for base_info in class_.bases: 455 leaved_base = leaved_classes[ 456 self._create_key(base_info.related_class)] 457 # treating base class hierarchy of leaved_class 458 leaved_base_info = pygccxml.declarations.hierarchy_info_t( 459 related_class=leaved_base, access=base_info.access) 460 if leaved_base_info not in leaved_class.bases: 461 leaved_class.bases.append(leaved_base_info) 462 else: 463 index = leaved_class.bases.index(leaved_base_info) 464 leaved_class.bases[ 465 index].related_class = leaved_base_info.related_class 466 # treating derived class hierarchy of leaved_base 467 leaved_derived_for_base_info = \ 468 pygccxml.declarations.hierarchy_info_t( 469 related_class=leaved_class, 470 access=base_info.access) 471 if leaved_derived_for_base_info not in leaved_base.derived: 472 leaved_base.derived.append(leaved_derived_for_base_info) 473 else: 474 index = leaved_base.derived.index( 475 leaved_derived_for_base_info) 476 leaved_base.derived[index].related_class = \ 477 leaved_derived_for_base_info.related_class 478 for derived_info in class_.derived: 479 leaved_derived = leaved_classes[ 480 self._create_key( 481 derived_info.related_class)] 482 # treating derived class hierarchy of leaved_class 483 leaved_derived_info = pygccxml.declarations.hierarchy_info_t( 484 related_class=leaved_derived, access=derived_info.access) 485 if leaved_derived_info not in leaved_class.derived: 486 leaved_class.derived.append(leaved_derived_info) 487 # treating base class hierarchy of leaved_derived 488 leaved_base_for_derived_info = \ 489 pygccxml.declarations.hierarchy_info_t( 490 related_class=leaved_class, 491 access=derived_info.access) 492 if leaved_base_for_derived_info not in leaved_derived.bases: 493 leaved_derived.bases.append(leaved_base_for_derived_info) 494 # this loops remove instance we from parent.declarations 495 for class_ in classes: 496 key = self._create_key(class_) 497 if id(leaved_classes[key]) == id(class_): 498 continue 499 else: 500 if class_.parent: 501 declarations = class_.parent.declarations 502 else: 503 # yes, we are talking about global class that doesn't 504 # belong to any namespace. Usually is compiler generated 505 # top level classes 506 declarations = namespaces 507 declarations_ids = [id(decl) for decl in declarations] 508 del declarations[declarations_ids.index(id(class_))] 509 return leaved_classes 510 511 @staticmethod 512 def _create_name_key(decl): 513 # Not all declarations have a mangled name with castxml 514 # we can only rely on the name 515 if decl.mangled is not None: 516 # gccxml 517 return decl.location.as_tuple(), decl.mangled 518 519 # castxml 520 return decl.location.as_tuple(), decl.name 521 522 def _relink_declarated_types(self, leaved_classes, declarated_types): 523 524 mangled_leaved_classes = {} 525 for leaved_class in leaved_classes.values(): 526 mangled_leaved_classes[ 527 self._create_name_key(leaved_class)] = leaved_class 528 529 for decl_wrapper_type in declarated_types: 530 # it is possible, that cache contains reference to dropped class 531 # We need to clear it 532 decl_wrapper_type.cache.reset() 533 if isinstance( 534 decl_wrapper_type.declaration, 535 pygccxml.declarations.class_t): 536 key = self._create_key(decl_wrapper_type.declaration) 537 if key in leaved_classes: 538 decl_wrapper_type.declaration = leaved_classes[key] 539 else: 540 541 name = decl_wrapper_type.declaration._name 542 if name == "": 543 # Happens with gcc5, castxml + std=c++11 544 # See issue #45 545 continue 546 if name.startswith("__vmi_class_type_info_pseudo"): 547 continue 548 if name == "rebind<std::__tree_node" + \ 549 "<std::basic_string<char>, void *> >": 550 continue 551 552 msg = [] 553 msg.append( 554 "Unable to find out actual class definition: '%s'." % 555 decl_wrapper_type.declaration._name) 556 msg.append(( 557 "Class definition has been changed from one " + 558 "compilation to an other.")) 559 msg.append(( 560 "Why did it happen to me? Here is a short list " + 561 "of reasons: ")) 562 msg.append(( 563 " 1. There are different preprocessor " + 564 "definitions applied on same file during compilation")) 565 msg.append(" 2. Bug in pygccxml.") 566 raise Exception(os.linesep.join(msg)) 567 elif isinstance( 568 decl_wrapper_type.declaration, 569 pygccxml.declarations.class_declaration_t): 570 key = self._create_name_key(decl_wrapper_type.declaration) 571 if key in mangled_leaved_classes: 572 decl_wrapper_type.declaration = mangled_leaved_classes[key] 573 574 @staticmethod 575 def __declarated_types(namespaces): 576 def get_from_type(cpptype): 577 if not cpptype: 578 return [] 579 elif isinstance(cpptype, pygccxml.declarations.fundamental_t): 580 return [] 581 elif isinstance(cpptype, pygccxml.declarations.declarated_t): 582 return [cpptype] 583 elif isinstance(cpptype, pygccxml.declarations.compound_t): 584 return get_from_type(cpptype.base) 585 elif isinstance(cpptype, pygccxml.declarations.calldef_type_t): 586 types = get_from_type(cpptype.return_type) 587 for arg in cpptype.arguments_types: 588 types.extend(get_from_type(arg)) 589 return types 590 assert isinstance( 591 cpptype, 592 (pygccxml.declarations.unknown_t, 593 pygccxml.declarations.ellipsis_t)) 594 return [] 595 types = [] 596 for decl in pygccxml.declarations.make_flatten(namespaces): 597 if isinstance(decl, pygccxml.declarations.calldef_t): 598 types.extend(get_from_type(decl.function_type())) 599 elif isinstance( 600 decl, (pygccxml.declarations.typedef_t, 601 pygccxml.declarations.variable_t)): 602 types.extend(get_from_type(decl.decl_type)) 603 return types 604