1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this file, 3# You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from StringIO import StringIO 6import json 7import fnmatch 8import os 9import shutil 10import sys 11import types 12 13from .ini import read_ini 14from .filters import ( 15 DEFAULT_FILTERS, 16 enabled, 17 exists as _exists, 18 filterlist, 19) 20 21__all__ = ['ManifestParser', 'TestManifest', 'convert'] 22 23relpath = os.path.relpath 24string = (basestring,) 25 26 27# path normalization 28 29def normalize_path(path): 30 """normalize a relative path""" 31 if sys.platform.startswith('win'): 32 return path.replace('/', os.path.sep) 33 return path 34 35 36def denormalize_path(path): 37 """denormalize a relative path""" 38 if sys.platform.startswith('win'): 39 return path.replace(os.path.sep, '/') 40 return path 41 42 43# objects for parsing manifests 44 45class ManifestParser(object): 46 """read .ini manifests""" 47 48 def __init__(self, manifests=(), defaults=None, strict=True, rootdir=None, 49 finder=None, handle_defaults=True): 50 """Creates a ManifestParser from the given manifest files. 51 52 :param manifests: An iterable of file paths or file objects corresponding 53 to manifests. If a file path refers to a manifest file that 54 does not exist, an IOError is raised. 55 :param defaults: Variables to pre-define in the environment for evaluating 56 expressions in manifests. 57 :param strict: If False, the provided manifests may contain references to 58 listed (test) files that do not exist without raising an 59 IOError during reading, and certain errors in manifests 60 are not considered fatal. Those errors include duplicate 61 section names, redefining variables, and defining empty 62 variables. 63 :param rootdir: The directory used as the basis for conversion to and from 64 relative paths during manifest reading. 65 :param finder: If provided, this finder object will be used for filesystem 66 interactions. Finder objects are part of the mozpack package, 67 documented at 68 http://gecko.readthedocs.org/en/latest/python/mozpack.html#module-mozpack.files 69 :param handle_defaults: If not set, do not propagate manifest defaults to individual 70 test objects. Callers are expected to manage per-manifest 71 defaults themselves via the manifest_defaults member 72 variable in this case. 73 """ 74 self._defaults = defaults or {} 75 self._ancestor_defaults = {} 76 self.tests = [] 77 self.manifest_defaults = {} 78 self.strict = strict 79 self.rootdir = rootdir 80 self.relativeRoot = None 81 self.finder = finder 82 self._handle_defaults = handle_defaults 83 if manifests: 84 self.read(*manifests) 85 86 def path_exists(self, path): 87 if self.finder: 88 return self.finder.get(path) is not None 89 return os.path.exists(path) 90 91 # methods for reading manifests 92 93 def _read(self, root, filename, defaults, defaults_only=False, parentmanifest=None): 94 """ 95 Internal recursive method for reading and parsing manifests. 96 Stores all found tests in self.tests 97 :param root: The base path 98 :param filename: File object or string path for the base manifest file 99 :param defaults: Options that apply to all items 100 :param defaults_only: If True will only gather options, not include 101 tests. Used for upstream parent includes 102 (default False) 103 :param parentmanifest: Filename of the parent manifest (default None) 104 """ 105 def read_file(type): 106 include_file = section.split(type, 1)[-1] 107 include_file = normalize_path(include_file) 108 if not os.path.isabs(include_file): 109 include_file = os.path.join(here, include_file) 110 if not self.path_exists(include_file): 111 message = "Included file '%s' does not exist" % include_file 112 if self.strict: 113 raise IOError(message) 114 else: 115 sys.stderr.write("%s\n" % message) 116 return 117 return include_file 118 119 # get directory of this file if not file-like object 120 if isinstance(filename, string): 121 # If we're using mercurial as our filesystem via a finder 122 # during manifest reading, the getcwd() calls that happen 123 # with abspath calls will not be meaningful, so absolute 124 # paths are required. 125 if self.finder: 126 assert os.path.isabs(filename) 127 filename = os.path.abspath(filename) 128 if self.finder: 129 fp = self.finder.get(filename) 130 else: 131 fp = open(filename) 132 here = os.path.dirname(filename) 133 else: 134 fp = filename 135 filename = here = None 136 defaults['here'] = here 137 138 # Rootdir is needed for relative path calculation. Precompute it for 139 # the microoptimization used below. 140 if self.rootdir is None: 141 rootdir = "" 142 else: 143 assert os.path.isabs(self.rootdir) 144 rootdir = self.rootdir + os.path.sep 145 146 # read the configuration 147 sections = read_ini(fp=fp, variables=defaults, strict=self.strict, 148 handle_defaults=self._handle_defaults) 149 self.manifest_defaults[filename] = defaults 150 151 parent_section_found = False 152 153 # get the tests 154 for section, data in sections: 155 # In case of defaults only, no other section than parent: has to 156 # be processed. 157 if defaults_only and not section.startswith('parent:'): 158 continue 159 160 # read the parent manifest if specified 161 if section.startswith('parent:'): 162 parent_section_found = True 163 164 include_file = read_file('parent:') 165 if include_file: 166 self._read(root, include_file, {}, True) 167 continue 168 169 # a file to include 170 # TODO: keep track of included file structure: 171 # self.manifests = {'manifest.ini': 'relative/path.ini'} 172 if section.startswith('include:'): 173 include_file = read_file('include:') 174 if include_file: 175 include_defaults = data.copy() 176 self._read(root, include_file, include_defaults, parentmanifest=filename) 177 continue 178 179 # otherwise an item 180 # apply ancestor defaults, while maintaining current file priority 181 data = dict(self._ancestor_defaults.items() + data.items()) 182 183 test = data 184 test['name'] = section 185 186 # Will be None if the manifest being read is a file-like object. 187 test['manifest'] = filename 188 189 # determine the path 190 path = test.get('path', section) 191 _relpath = path 192 if '://' not in path: # don't futz with URLs 193 path = normalize_path(path) 194 if here and not os.path.isabs(path): 195 # Profiling indicates 25% of manifest parsing is spent 196 # in this call to normpath, but almost all calls return 197 # their argument unmodified, so we avoid the call if 198 # '..' if not present in the path. 199 path = os.path.join(here, path) 200 if '..' in path: 201 path = os.path.normpath(path) 202 203 # Microoptimization, because relpath is quite expensive. 204 # We know that rootdir is an absolute path or empty. If path 205 # starts with rootdir, then path is also absolute and the tail 206 # of the path is the relative path (possibly non-normalized, 207 # when here is unknown). 208 # For this to work rootdir needs to be terminated with a path 209 # separator, so that references to sibling directories with 210 # a common prefix don't get misscomputed (e.g. /root and 211 # /rootbeer/file). 212 # When the rootdir is unknown, the relpath needs to be left 213 # unchanged. We use an empty string as rootdir in that case, 214 # which leaves relpath unchanged after slicing. 215 if path.startswith(rootdir): 216 _relpath = path[len(rootdir):] 217 else: 218 _relpath = relpath(path, rootdir) 219 220 test['path'] = path 221 test['relpath'] = _relpath 222 223 if parentmanifest is not None: 224 # If a test was included by a parent manifest we may need to 225 # indicate that in the test object for the sake of identifying 226 # a test, particularly in the case a test file is included by 227 # multiple manifests. 228 test['ancestor-manifest'] = parentmanifest 229 230 # append the item 231 self.tests.append(test) 232 233 # if no parent: section was found for defaults-only, only read the 234 # defaults section of the manifest without interpreting variables 235 if defaults_only and not parent_section_found: 236 sections = read_ini(fp=fp, variables=defaults, defaults_only=True, 237 strict=self.strict) 238 (section, self._ancestor_defaults) = sections[0] 239 240 def read(self, *filenames, **defaults): 241 """ 242 read and add manifests from file paths or file-like objects 243 244 filenames -- file paths or file-like objects to read as manifests 245 defaults -- default variables 246 """ 247 248 # ensure all files exist 249 missing = [filename for filename in filenames 250 if isinstance(filename, string) and not self.path_exists(filename)] 251 if missing: 252 raise IOError('Missing files: %s' % ', '.join(missing)) 253 254 # default variables 255 _defaults = defaults.copy() or self._defaults.copy() 256 _defaults.setdefault('here', None) 257 258 # process each file 259 for filename in filenames: 260 # set the per file defaults 261 defaults = _defaults.copy() 262 here = None 263 if isinstance(filename, string): 264 here = os.path.dirname(os.path.abspath(filename)) 265 defaults['here'] = here # directory of master .ini file 266 267 if self.rootdir is None: 268 # set the root directory 269 # == the directory of the first manifest given 270 self.rootdir = here 271 272 self._read(here, filename, defaults) 273 274 # methods for querying manifests 275 276 def query(self, *checks, **kw): 277 """ 278 general query function for tests 279 - checks : callable conditions to test if the test fulfills the query 280 """ 281 tests = kw.get('tests', None) 282 if tests is None: 283 tests = self.tests 284 retval = [] 285 for test in tests: 286 for check in checks: 287 if not check(test): 288 break 289 else: 290 retval.append(test) 291 return retval 292 293 def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs): 294 # TODO: pass a dict instead of kwargs since you might hav 295 # e.g. 'inverse' as a key in the dict 296 297 # TODO: tags should just be part of kwargs with None values 298 # (None == any is kinda weird, but probably still better) 299 300 # fix up tags 301 if tags: 302 tags = set(tags) 303 else: 304 tags = set() 305 306 # make some check functions 307 if inverse: 308 def has_tags(test): 309 return not tags.intersection(test.keys()) 310 311 def dict_query(test): 312 for key, value in kwargs.items(): 313 if test.get(key) == value: 314 return False 315 return True 316 else: 317 def has_tags(test): 318 return tags.issubset(test.keys()) 319 320 def dict_query(test): 321 for key, value in kwargs.items(): 322 if test.get(key) != value: 323 return False 324 return True 325 326 # query the tests 327 tests = self.query(has_tags, dict_query, tests=tests) 328 329 # if a key is given, return only a list of that key 330 # useful for keys like 'name' or 'path' 331 if _key: 332 return [test[_key] for test in tests] 333 334 # return the tests 335 return tests 336 337 def manifests(self, tests=None): 338 """ 339 return manifests in order in which they appear in the tests 340 """ 341 if tests is None: 342 # Make sure to return all the manifests, even ones without tests. 343 return self.manifest_defaults.keys() 344 345 manifests = [] 346 for test in tests: 347 manifest = test.get('manifest') 348 if not manifest: 349 continue 350 if manifest not in manifests: 351 manifests.append(manifest) 352 return manifests 353 354 def paths(self): 355 return [i['path'] for i in self.tests] 356 357 # methods for auditing 358 359 def missing(self, tests=None): 360 """ 361 return list of tests that do not exist on the filesystem 362 """ 363 if tests is None: 364 tests = self.tests 365 existing = list(_exists(tests, {})) 366 return [t for t in tests if t not in existing] 367 368 def check_missing(self, tests=None): 369 missing = self.missing(tests=tests) 370 if missing: 371 missing_paths = [test['path'] for test in missing] 372 if self.strict: 373 raise IOError("Strict mode enabled, test paths must exist. " 374 "The following test(s) are missing: %s" % 375 json.dumps(missing_paths, indent=2)) 376 print >> sys.stderr, "Warning: The following test(s) are missing: %s" % \ 377 json.dumps(missing_paths, indent=2) 378 return missing 379 380 def verifyDirectory(self, directories, pattern=None, extensions=None): 381 """ 382 checks what is on the filesystem vs what is in a manifest 383 returns a 2-tuple of sets: 384 (missing_from_filesystem, missing_from_manifest) 385 """ 386 387 files = set([]) 388 if isinstance(directories, basestring): 389 directories = [directories] 390 391 # get files in directories 392 for directory in directories: 393 for dirpath, dirnames, filenames in os.walk(directory, topdown=True): 394 395 # only add files that match a pattern 396 if pattern: 397 filenames = fnmatch.filter(filenames, pattern) 398 399 # only add files that have one of the extensions 400 if extensions: 401 filenames = [filename for filename in filenames 402 if os.path.splitext(filename)[-1] in extensions] 403 404 files.update([os.path.join(dirpath, filename) for filename in filenames]) 405 406 paths = set(self.paths()) 407 missing_from_filesystem = paths.difference(files) 408 missing_from_manifest = files.difference(paths) 409 return (missing_from_filesystem, missing_from_manifest) 410 411 # methods for output 412 413 def write(self, fp=sys.stdout, rootdir=None, 414 global_tags=None, global_kwargs=None, 415 local_tags=None, local_kwargs=None): 416 """ 417 write a manifest given a query 418 global and local options will be munged to do the query 419 globals will be written to the top of the file 420 locals (if given) will be written per test 421 """ 422 423 # open file if `fp` given as string 424 close = False 425 if isinstance(fp, string): 426 fp = file(fp, 'w') 427 close = True 428 429 # root directory 430 if rootdir is None: 431 rootdir = self.rootdir 432 433 # sanitize input 434 global_tags = global_tags or set() 435 local_tags = local_tags or set() 436 global_kwargs = global_kwargs or {} 437 local_kwargs = local_kwargs or {} 438 439 # create the query 440 tags = set([]) 441 tags.update(global_tags) 442 tags.update(local_tags) 443 kwargs = {} 444 kwargs.update(global_kwargs) 445 kwargs.update(local_kwargs) 446 447 # get matching tests 448 tests = self.get(tags=tags, **kwargs) 449 450 # print the .ini manifest 451 if global_tags or global_kwargs: 452 print >> fp, '[DEFAULT]' 453 for tag in global_tags: 454 print >> fp, '%s =' % tag 455 for key, value in global_kwargs.items(): 456 print >> fp, '%s = %s' % (key, value) 457 print >> fp 458 459 for test in tests: 460 test = test.copy() # don't overwrite 461 462 path = test['name'] 463 if not os.path.isabs(path): 464 path = test['path'] 465 if self.rootdir: 466 path = relpath(test['path'], self.rootdir) 467 path = denormalize_path(path) 468 print >> fp, '[%s]' % path 469 470 # reserved keywords: 471 reserved = ['path', 'name', 'here', 'manifest', 'relpath', 'ancestor-manifest'] 472 for key in sorted(test.keys()): 473 if key in reserved: 474 continue 475 if key in global_kwargs: 476 continue 477 if key in global_tags and not test[key]: 478 continue 479 print >> fp, '%s = %s' % (key, test[key]) 480 print >> fp 481 482 if close: 483 # close the created file 484 fp.close() 485 486 def __str__(self): 487 fp = StringIO() 488 self.write(fp=fp) 489 value = fp.getvalue() 490 return value 491 492 def copy(self, directory, rootdir=None, *tags, **kwargs): 493 """ 494 copy the manifests and associated tests 495 - directory : directory to copy to 496 - rootdir : root directory to copy to (if not given from manifests) 497 - tags : keywords the tests must have 498 - kwargs : key, values the tests must match 499 """ 500 # XXX note that copy does *not* filter the tests out of the 501 # resulting manifest; it just stupidly copies them over. 502 # ideally, it would reread the manifests and filter out the 503 # tests that don't match *tags and **kwargs 504 505 # destination 506 if not os.path.exists(directory): 507 os.path.makedirs(directory) 508 else: 509 # sanity check 510 assert os.path.isdir(directory) 511 512 # tests to copy 513 tests = self.get(tags=tags, **kwargs) 514 if not tests: 515 return # nothing to do! 516 517 # root directory 518 if rootdir is None: 519 rootdir = self.rootdir 520 521 # copy the manifests + tests 522 manifests = [relpath(manifest, rootdir) for manifest in self.manifests()] 523 for manifest in manifests: 524 destination = os.path.join(directory, manifest) 525 dirname = os.path.dirname(destination) 526 if not os.path.exists(dirname): 527 os.makedirs(dirname) 528 else: 529 # sanity check 530 assert os.path.isdir(dirname) 531 shutil.copy(os.path.join(rootdir, manifest), destination) 532 533 missing = self.check_missing(tests) 534 tests = [test for test in tests if test not in missing] 535 for test in tests: 536 if os.path.isabs(test['name']): 537 continue 538 source = test['path'] 539 destination = os.path.join(directory, relpath(test['path'], rootdir)) 540 shutil.copy(source, destination) 541 # TODO: ensure that all of the tests are below the from_dir 542 543 def update(self, from_dir, rootdir=None, *tags, **kwargs): 544 """ 545 update the tests as listed in a manifest from a directory 546 - from_dir : directory where the tests live 547 - rootdir : root directory to copy to (if not given from manifests) 548 - tags : keys the tests must have 549 - kwargs : key, values the tests must match 550 """ 551 552 # get the tests 553 tests = self.get(tags=tags, **kwargs) 554 555 # get the root directory 556 if not rootdir: 557 rootdir = self.rootdir 558 559 # copy them! 560 for test in tests: 561 if not os.path.isabs(test['name']): 562 _relpath = relpath(test['path'], rootdir) 563 source = os.path.join(from_dir, _relpath) 564 if not os.path.exists(source): 565 message = "Missing test: '%s' does not exist!" 566 if self.strict: 567 raise IOError(message) 568 print >> sys.stderr, message + " Skipping." 569 continue 570 destination = os.path.join(rootdir, _relpath) 571 shutil.copy(source, destination) 572 573 # directory importers 574 575 @classmethod 576 def _walk_directories(cls, directories, callback, pattern=None, ignore=()): 577 """ 578 internal function to import directories 579 """ 580 581 if isinstance(pattern, basestring): 582 patterns = [pattern] 583 else: 584 patterns = pattern 585 ignore = set(ignore) 586 587 if not patterns: 588 def accept_filename(filename): 589 return True 590 else: 591 def accept_filename(filename): 592 for pattern in patterns: 593 if fnmatch.fnmatch(filename, pattern): 594 return True 595 596 if not ignore: 597 def accept_dirname(dirname): 598 return True 599 else: 600 def accept_dirname(dirname): 601 return dirname not in ignore 602 603 rootdirectories = directories[:] 604 seen_directories = set() 605 for rootdirectory in rootdirectories: 606 # let's recurse directories using list 607 directories = [os.path.realpath(rootdirectory)] 608 while directories: 609 directory = directories.pop(0) 610 if directory in seen_directories: 611 # eliminate possible infinite recursion due to 612 # symbolic links 613 continue 614 seen_directories.add(directory) 615 616 files = [] 617 subdirs = [] 618 for name in sorted(os.listdir(directory)): 619 path = os.path.join(directory, name) 620 if os.path.isfile(path): 621 # os.path.isfile follow symbolic links, we don't 622 # need to handle them here. 623 if accept_filename(name): 624 files.append(name) 625 continue 626 elif os.path.islink(path): 627 # eliminate symbolic links 628 path = os.path.realpath(path) 629 630 # we must have a directory here 631 if accept_dirname(name): 632 subdirs.append(name) 633 # this subdir is added for recursion 634 directories.insert(0, path) 635 636 # here we got all subdirs and files filtered, we can 637 # call the callback function if directory is not empty 638 if subdirs or files: 639 callback(rootdirectory, directory, subdirs, files) 640 641 @classmethod 642 def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), 643 overwrite=False): 644 """ 645 walks directories and writes manifests of name `filename` in-place; 646 returns `cls` instance populated with the given manifests 647 648 filename -- filename of manifests to write 649 pattern -- shell pattern (glob) or patterns of filenames to match 650 ignore -- directory names to ignore 651 overwrite -- whether to overwrite existing files of given name 652 """ 653 654 manifest_dict = {} 655 656 if os.path.basename(filename) != filename: 657 raise IOError("filename should not include directory name") 658 659 # no need to hit directories more than once 660 _directories = directories 661 directories = [] 662 for directory in _directories: 663 if directory not in directories: 664 directories.append(directory) 665 666 def callback(directory, dirpath, dirnames, filenames): 667 """write a manifest for each directory""" 668 669 manifest_path = os.path.join(dirpath, filename) 670 if (dirnames or filenames) and not (os.path.exists(manifest_path) and overwrite): 671 with file(manifest_path, 'w') as manifest: 672 for dirname in dirnames: 673 print >> manifest, '[include:%s]' % os.path.join(dirname, filename) 674 for _filename in filenames: 675 print >> manifest, '[%s]' % _filename 676 677 # add to list of manifests 678 manifest_dict.setdefault(directory, manifest_path) 679 680 # walk the directories to gather files 681 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) 682 # get manifests 683 manifests = [manifest_dict[directory] for directory in _directories] 684 685 # create a `cls` instance with the manifests 686 return cls(manifests=manifests) 687 688 @classmethod 689 def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None): 690 """ 691 convert directories to a simple manifest; returns ManifestParser instance 692 693 pattern -- shell pattern (glob) or patterns of filenames to match 694 ignore -- directory names to ignore 695 write -- filename or file-like object of manifests to write; 696 if `None` then a StringIO instance will be created 697 relative_to -- write paths relative to this path; 698 if false then the paths are absolute 699 """ 700 701 # determine output 702 opened_manifest_file = None # name of opened manifest file 703 absolute = not relative_to # whether to output absolute path names as names 704 if isinstance(write, string): 705 opened_manifest_file = write 706 write = file(write, 'w') 707 if write is None: 708 write = StringIO() 709 710 # walk the directories, generating manifests 711 def callback(directory, dirpath, dirnames, filenames): 712 713 # absolute paths 714 filenames = [os.path.join(dirpath, filename) 715 for filename in filenames] 716 # ensure new manifest isn't added 717 filenames = [filename for filename in filenames 718 if filename != opened_manifest_file] 719 # normalize paths 720 if not absolute and relative_to: 721 filenames = [relpath(filename, relative_to) 722 for filename in filenames] 723 724 # write to manifest 725 print >> write, '\n'.join(['[%s]' % denormalize_path(filename) 726 for filename in filenames]) 727 728 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) 729 730 if opened_manifest_file: 731 # close file 732 write.close() 733 manifests = [opened_manifest_file] 734 else: 735 # manifests/write is a file-like object; 736 # rewind buffer 737 write.flush() 738 write.seek(0) 739 manifests = [write] 740 741 # make a ManifestParser instance 742 return cls(manifests=manifests) 743 744convert = ManifestParser.from_directories 745 746 747class TestManifest(ManifestParser): 748 """ 749 apply logic to manifests; this is your integration layer :) 750 specific harnesses may subclass from this if they need more logic 751 """ 752 753 def __init__(self, *args, **kwargs): 754 ManifestParser.__init__(self, *args, **kwargs) 755 self.filters = filterlist(DEFAULT_FILTERS) 756 self.last_used_filters = [] 757 758 def active_tests(self, exists=True, disabled=True, filters=None, **values): 759 """ 760 Run all applied filters on the set of tests. 761 762 :param exists: filter out non-existing tests (default True) 763 :param disabled: whether to return disabled tests (default True) 764 :param values: keys and values to filter on (e.g. `os = linux mac`) 765 :param filters: list of filters to apply to the tests 766 :returns: list of test objects that were not filtered out 767 """ 768 tests = [i.copy() for i in self.tests] # shallow copy 769 770 # mark all tests as passing 771 for test in tests: 772 test['expected'] = test.get('expected', 'pass') 773 774 # make a copy so original doesn't get modified 775 fltrs = self.filters[:] 776 if exists: 777 if self.strict: 778 self.check_missing(tests) 779 else: 780 fltrs.append(_exists) 781 782 if not disabled: 783 fltrs.append(enabled) 784 785 if filters: 786 fltrs += filters 787 788 self.last_used_filters = fltrs[:] 789 for fn in fltrs: 790 tests = fn(tests, values) 791 return list(tests) 792 793 def test_paths(self): 794 return [test['path'] for test in self.active_tests()] 795 796 def fmt_filters(self, filters=None): 797 filters = filters or self.last_used_filters 798 names = [] 799 for f in filters: 800 if isinstance(f, types.FunctionType): 801 names.append(f.__name__) 802 else: 803 names.append(str(f)) 804 return ', '.join(names) 805