1# Copyright 2015-2019, Damian Johnson and The Tor Project 2# See LICENSE for licensing information 3 4""" 5Helper functions for testing. 6 7Our **stylistic_issues**, **pyflakes_issues**, and **type_check_issues** 8respect a 'exclude_paths' in our test config, excluding any absolute paths 9matching those regexes. Issue strings can start or end with an asterisk 10to match just against the prefix or suffix. For instance... 11 12:: 13 14 exclude_paths .*/stem/test/data/.* 15 16.. versionadded:: 1.2.0 17 18:: 19 20 TimedTestRunner - test runner that tracks test runtimes 21 test_runtimes - provides runtime of tests excuted through TimedTestRunners 22 clean_orphaned_pyc - delete *.pyc files without corresponding *.py 23 24 is_pyflakes_available - checks if pyflakes is available 25 is_pycodestyle_available - checks if pycodestyle is available 26 27 pyflakes_issues - static checks for problems via pyflakes 28 stylistic_issues - checks for PEP8 and other stylistic issues 29""" 30 31import collections 32import linecache 33import multiprocessing 34import os 35import re 36import threading 37import time 38import traceback 39import unittest 40 41import stem.prereq 42import stem.util.conf 43import stem.util.enum 44import stem.util.system 45 46CONFIG = stem.util.conf.config_dict('test', { 47 'pep8.ignore': [], # TODO: drop with stem 2.x, legacy alias for pycodestyle.ignore 48 'pycodestyle.ignore': [], 49 'pyflakes.ignore': [], 50 'exclude_paths': [], 51}) 52 53TEST_RUNTIMES = {} 54ASYNC_TESTS = {} 55 56AsyncStatus = stem.util.enum.UppercaseEnum('PENDING', 'RUNNING', 'FINISHED') 57AsyncResult = collections.namedtuple('AsyncResult', 'type msg') 58 59# TODO: Providing a copy of SkipTest that works with python 2.6. This will be 60# dropped when we remove python 2.6 support. 61 62if stem.prereq._is_python_26(): 63 class SkipTest(Exception): 64 'Notes that the test was skipped.' 65else: 66 SkipTest = unittest.case.SkipTest 67 68 69def assert_equal(expected, actual, msg = None): 70 """ 71 Function form of a TestCase's assertEqual. 72 73 .. versionadded:: 1.6.0 74 75 :param object expected: expected value 76 :param object actual: actual value 77 :param str msg: message if assertion fails 78 79 :raises: **AssertionError** if values aren't equal 80 """ 81 82 if expected != actual: 83 raise AssertionError("Expected '%s' but was '%s'" % (expected, actual) if msg is None else msg) 84 85 86def assert_in(expected, actual, msg = None): 87 """ 88 Asserts that a given value is within this content. 89 90 .. versionadded:: 1.6.0 91 92 :param object expected: expected value 93 :param object actual: actual value 94 :param str msg: message if assertion fails 95 96 :raises: **AssertionError** if the expected value isn't in the actual 97 """ 98 99 if expected not in actual: 100 raise AssertionError("Expected '%s' to be within '%s'" % (expected, actual) if msg is None else msg) 101 102 103def skip(msg): 104 """ 105 Function form of a TestCase's skipTest. 106 107 .. versionadded:: 1.6.0 108 109 :param str msg: reason test is being skipped 110 111 :raises: **unittest.case.SkipTest** for this reason 112 """ 113 114 raise SkipTest(msg) 115 116 117def asynchronous(func): 118 test = stem.util.test_tools.AsyncTest(func) 119 ASYNC_TESTS[test.name] = test 120 return test.method 121 122 123class AsyncTest(object): 124 """ 125 Test that's run asychronously. These are functions (no self reference) 126 performed like the following... 127 128 :: 129 130 class MyTest(unittest.TestCase): 131 @staticmethod 132 def run_tests(): 133 MyTest.test_addition = stem.util.test_tools.AsyncTest(MyTest.test_addition).method 134 135 @staticmethod 136 def test_addition(): 137 if 1 + 1 != 2: 138 raise AssertionError('tisk, tisk') 139 140 MyTest.run() 141 142 .. versionadded:: 1.6.0 143 """ 144 145 def __init__(self, runner, args = None, threaded = False): 146 self.name = '%s.%s' % (runner.__module__, runner.__name__) 147 148 self._runner = runner 149 self._runner_args = args 150 self._threaded = threaded 151 152 self.method = lambda test: self.result(test) # method that can be mixed into TestCases 153 154 self._process = None 155 self._process_pipe = None 156 self._process_lock = threading.RLock() 157 158 self._result = None 159 self._status = AsyncStatus.PENDING 160 161 def run(self, *runner_args, **kwargs): 162 if stem.prereq._is_python_26(): 163 return # not supported under python 2.6 164 165 def _wrapper(conn, runner, args): 166 os.nice(12) 167 168 try: 169 runner(*args) if args else runner() 170 conn.send(AsyncResult('success', None)) 171 except AssertionError as exc: 172 conn.send(AsyncResult('failure', str(exc))) 173 except SkipTest as exc: 174 conn.send(AsyncResult('skipped', str(exc))) 175 except: 176 conn.send(AsyncResult('error', traceback.format_exc())) 177 finally: 178 conn.close() 179 180 with self._process_lock: 181 if self._status == AsyncStatus.PENDING: 182 if runner_args: 183 self._runner_args = runner_args 184 185 if 'threaded' in kwargs: 186 self._threaded = kwargs['threaded'] 187 188 self._process_pipe, child_pipe = multiprocessing.Pipe() 189 190 if self._threaded: 191 self._process = threading.Thread( 192 target = _wrapper, 193 args = (child_pipe, self._runner, self._runner_args), 194 name = 'Background test of %s' % self.name, 195 ) 196 197 self._process.setDaemon(True) 198 else: 199 self._process = multiprocessing.Process(target = _wrapper, args = (child_pipe, self._runner, self._runner_args)) 200 201 self._process.start() 202 self._status = AsyncStatus.RUNNING 203 204 def pid(self): 205 with self._process_lock: 206 return self._process.pid if (self._process and not self._threaded) else None 207 208 def join(self): 209 self.result(None) 210 211 def result(self, test): 212 if stem.prereq._is_python_26(): 213 return # not supported under python 2.6 214 215 with self._process_lock: 216 if self._status == AsyncStatus.PENDING: 217 self.run() 218 219 if self._status == AsyncStatus.RUNNING: 220 self._result = self._process_pipe.recv() 221 self._process.join() 222 self._status = AsyncStatus.FINISHED 223 224 if test and self._result.type == 'failure': 225 test.fail(self._result.msg) 226 elif test and self._result.type == 'error': 227 test.fail(self._result.msg) 228 elif test and self._result.type == 'skipped': 229 test.skipTest(self._result.msg) 230 231 232class Issue(collections.namedtuple('Issue', ['line_number', 'message', 'line'])): 233 """ 234 Issue encountered by pyflakes or pycodestyle. 235 236 :var int line_number: line number the issue occured on 237 :var str message: description of the issue 238 :var str line: content of the line the issue is about 239 """ 240 241 242class TimedTestRunner(unittest.TextTestRunner): 243 """ 244 Test runner that tracks the runtime of individual tests. When tests are run 245 with this their runtimes are made available through 246 :func:`stem.util.test_tools.test_runtimes`. 247 248 .. versionadded:: 1.6.0 249 """ 250 251 def run(self, test): 252 for t in test._tests: 253 original_type = type(t) 254 255 class _TestWrapper(original_type): 256 def run(self, result = None): 257 start_time = time.time() 258 result = super(type(self), self).run(result) 259 TEST_RUNTIMES[self.id()] = time.time() - start_time 260 return result 261 262 # TODO: remove and drop unnecessary 'returns' when dropping python 2.6 263 # support 264 265 def skipTest(self, message): 266 if not stem.prereq._is_python_26(): 267 return super(original_type, self).skipTest(message) 268 269 # TODO: remove when dropping python 2.6 support 270 271 def assertItemsEqual(self, expected, actual): 272 if stem.prereq._is_python_26(): 273 self.assertEqual(set(expected), set(actual)) 274 else: 275 return super(original_type, self).assertItemsEqual(expected, actual) 276 277 def assertRaisesWith(self, exc_type, exc_msg, func, *args, **kwargs): 278 """ 279 Asserts the given invokation raises the expected excepiton. This is 280 similar to unittest's assertRaises and assertRaisesRegexp, but checks 281 for an exact match. 282 283 This method is **not** being vended to external users and may be 284 changed without notice. If you want this method to be part of our 285 vended API then please let us know. 286 """ 287 288 return self.assertRaisesRegexp(exc_type, '^%s$' % re.escape(exc_msg), func, *args, **kwargs) 289 290 def assertRaisesRegexp(self, exc_type, exc_msg, func, *args, **kwargs): 291 if stem.prereq._is_python_26(): 292 try: 293 func(*args, **kwargs) 294 self.fail('Expected a %s to be raised but nothing was' % exc_type) 295 except exc_type as exc: 296 self.assertTrue(re.search(exc_msg, str(exc), re.MULTILINE)) 297 else: 298 return super(original_type, self).assertRaisesRegexp(exc_type, exc_msg, func, *args, **kwargs) 299 300 def id(self): 301 return '%s.%s.%s' % (original_type.__module__, original_type.__name__, self._testMethodName) 302 303 def __str__(self): 304 return '%s (%s.%s)' % (self._testMethodName, original_type.__module__, original_type.__name__) 305 306 t.__class__ = _TestWrapper 307 308 return super(TimedTestRunner, self).run(test) 309 310 311def test_runtimes(): 312 """ 313 Provides the runtimes of tests executed through TimedTestRunners. 314 315 :returns: **dict** of fully qualified test names to floats for the runtime in 316 seconds 317 318 .. versionadded:: 1.6.0 319 """ 320 321 return dict(TEST_RUNTIMES) 322 323 324def clean_orphaned_pyc(paths): 325 """ 326 Deletes any file with a \\*.pyc extention without a corresponding \\*.py. This 327 helps to address a common gotcha when deleting python files... 328 329 * You delete module 'foo.py' and run the tests to ensure that you haven't 330 broken anything. They pass, however there *are* still some 'import foo' 331 statements that still work because the bytecode (foo.pyc) is still around. 332 333 * You push your change. 334 335 * Another developer clones our repository and is confused because we have a 336 bunch of ImportErrors. 337 338 :param list paths: paths to search for orphaned pyc files 339 340 :returns: list of absolute paths that were deleted 341 """ 342 343 orphaned_pyc = [] 344 345 for path in paths: 346 for pyc_path in stem.util.system.files_with_suffix(path, '.pyc'): 347 py_path = pyc_path[:-1] 348 349 # If we're running python 3 then the *.pyc files are no longer bundled 350 # with the *.py. Rather, they're in a __pycache__ directory. 351 352 pycache = '%s__pycache__%s' % (os.path.sep, os.path.sep) 353 354 if pycache in pyc_path: 355 directory, pycache_filename = pyc_path.split(pycache, 1) 356 357 if not pycache_filename.endswith('.pyc'): 358 continue # should look like 'test_tools.cpython-32.pyc' 359 360 py_path = os.path.join(directory, pycache_filename.split('.')[0] + '.py') 361 362 if not os.path.exists(py_path): 363 orphaned_pyc.append(pyc_path) 364 os.remove(pyc_path) 365 366 return orphaned_pyc 367 368 369def is_pyflakes_available(): 370 """ 371 Checks if pyflakes is availalbe. 372 373 :returns: **True** if we can use pyflakes and **False** otherwise 374 """ 375 376 return _module_exists('pyflakes.api') and _module_exists('pyflakes.reporter') 377 378 379def is_pycodestyle_available(): 380 """ 381 Checks if pycodestyle is availalbe. 382 383 :returns: **True** if we can use pycodestyle and **False** otherwise 384 """ 385 386 if _module_exists('pycodestyle'): 387 import pycodestyle 388 elif _module_exists('pep8'): 389 import pep8 as pycodestyle 390 else: 391 return False 392 393 return hasattr(pycodestyle, 'BaseReport') 394 395 396def stylistic_issues(paths, check_newlines = False, check_exception_keyword = False, prefer_single_quotes = False): 397 """ 398 Checks for stylistic issues that are an issue according to the parts of PEP8 399 we conform to. You can suppress pycodestyle issues by making a 'test' 400 configuration that sets 'pycodestyle.ignore'. 401 402 For example, with a 'test/settings.cfg' of... 403 404 :: 405 406 # pycodestyle compliance issues that we're ignoreing... 407 # 408 # * E111 and E121 four space indentations 409 # * E501 line is over 79 characters 410 411 pycodestyle.ignore E111 412 pycodestyle.ignore E121 413 pycodestyle.ignore E501 414 415 pycodestyle.ignore run_tests.py => E402: import stem.util.enum 416 417 ... you can then run tests with... 418 419 :: 420 421 import stem.util.conf 422 423 test_config = stem.util.conf.get_config('test') 424 test_config.load('test/settings.cfg') 425 426 issues = stylistic_issues('my_project') 427 428 .. versionchanged:: 1.3.0 429 Renamed from get_stylistic_issues() to stylistic_issues(). The old name 430 still works as an alias, but will be dropped in Stem version 2.0.0. 431 432 .. versionchanged:: 1.4.0 433 Changing tuples in return value to be namedtuple instances, and adding the 434 line that had the issue. 435 436 .. versionchanged:: 1.4.0 437 Added the prefer_single_quotes option. 438 439 .. versionchanged:: 1.6.0 440 Changed 'pycodestyle.ignore' code snippets to only need to match against 441 the prefix. 442 443 :param list paths: paths to search for stylistic issues 444 :param bool check_newlines: check that we have standard newlines (\\n), not 445 windows (\\r\\n) nor classic mac (\\r) 446 :param bool check_exception_keyword: checks that we're using 'as' for 447 exceptions rather than a comma 448 :param bool prefer_single_quotes: standardize on using single rather than 449 double quotes for strings, when reasonable 450 451 :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances 452 """ 453 454 issues = {} 455 456 ignore_rules = [] 457 ignore_for_file = [] 458 ignore_all_for_files = [] 459 460 for rule in CONFIG['pycodestyle.ignore'] + CONFIG['pep8.ignore']: 461 if '=>' in rule: 462 path, rule_entry = rule.split('=>', 1) 463 464 if ':' in rule_entry: 465 rule, code = rule_entry.split(':', 1) 466 ignore_for_file.append((path.strip(), rule.strip(), code.strip())) 467 elif rule_entry.strip() == '*': 468 ignore_all_for_files.append(path.strip()) 469 else: 470 ignore_rules.append(rule) 471 472 def is_ignored(path, rule, code): 473 for ignored_path, ignored_rule, ignored_code in ignore_for_file: 474 if path.endswith(ignored_path) and ignored_rule == rule and code.strip().startswith(ignored_code): 475 return True 476 477 for ignored_path in ignore_all_for_files: 478 if path.endswith(ignored_path): 479 return True 480 481 return False 482 483 if is_pycodestyle_available(): 484 if _module_exists('pep8'): 485 import pep8 as pycodestyle 486 else: 487 import pycodestyle 488 489 class StyleReport(pycodestyle.BaseReport): 490 def init_file(self, filename, lines, expected, line_offset): 491 super(StyleReport, self).init_file(filename, lines, expected, line_offset) 492 493 if not check_newlines and not check_exception_keyword and not prefer_single_quotes: 494 return 495 496 is_block_comment = False 497 498 for ignored_path in ignore_all_for_files: 499 if filename.endswith(ignored_path): 500 return 501 502 for index, line in enumerate(lines): 503 content = line.split('#', 1)[0].strip() 504 505 if check_newlines and '\r' in line: 506 issues.setdefault(filename, []).append(Issue(index + 1, 'contains a windows newline', line)) 507 508 if not content: 509 continue # blank line 510 511 if '"""' in content: 512 is_block_comment = not is_block_comment 513 514 if check_exception_keyword and content.startswith('except') and content.endswith(', exc:'): 515 # Python 2.6 - 2.7 supports two forms for exceptions... 516 # 517 # except ValueError, exc: 518 # except ValueError as exc: 519 # 520 # The former is the old method and no longer supported in python 3 521 # going forward. 522 523 # TODO: This check only works if the exception variable is called 524 # 'exc'. We should generalize this via a regex so other names work 525 # too. 526 527 issues.setdefault(filename, []).append(Issue(index + 1, "except clause should use 'as', not comma", line)) 528 529 if prefer_single_quotes and not is_block_comment: 530 if '"' in content and "'" not in content and '"""' not in content and not content.endswith('\\'): 531 # Checking if the line already has any single quotes since that 532 # usually means double quotes are preferable for the content (for 533 # instance "I'm hungry"). Also checking for '\' at the end since 534 # that can indicate a multi-line string. 535 536 issues.setdefault(filename, []).append(Issue(index + 1, 'use single rather than double quotes', line)) 537 538 def error(self, line_number, offset, text, check): 539 code = super(StyleReport, self).error(line_number, offset, text, check) 540 541 if code: 542 line = linecache.getline(self.filename, line_number) 543 544 if not is_ignored(self.filename, code, line): 545 issues.setdefault(self.filename, []).append(Issue(line_number, text, line)) 546 547 style_checker = pycodestyle.StyleGuide(ignore = ignore_rules, reporter = StyleReport) 548 style_checker.check_files(list(_python_files(paths))) 549 550 return issues 551 552 553def pyflakes_issues(paths): 554 """ 555 Performs static checks via pyflakes. False positives can be ignored via 556 'pyflakes.ignore' entries in our 'test' config. For instance... 557 558 :: 559 560 pyflakes.ignore stem/util/test_tools.py => 'pyflakes' imported but unused 561 pyflakes.ignore stem/util/test_tools.py => 'pycodestyle' imported but unused 562 563 .. versionchanged:: 1.3.0 564 Renamed from get_pyflakes_issues() to pyflakes_issues(). The old name 565 still works as an alias, but will be dropped in Stem version 2.0.0. 566 567 .. versionchanged:: 1.4.0 568 Changing tuples in return value to be namedtuple instances, and adding the 569 line that had the issue. 570 571 .. versionchanged:: 1.5.0 572 Support matching against prefix or suffix issue strings. 573 574 :param list paths: paths to search for problems 575 576 :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances 577 """ 578 579 issues = {} 580 581 if is_pyflakes_available(): 582 import pyflakes.api 583 import pyflakes.reporter 584 585 class Reporter(pyflakes.reporter.Reporter): 586 def __init__(self): 587 self._ignored_issues = {} 588 589 for line in CONFIG['pyflakes.ignore']: 590 path, issue = line.split('=>') 591 self._ignored_issues.setdefault(path.strip(), []).append(issue.strip()) 592 593 def unexpectedError(self, filename, msg): 594 self._register_issue(filename, None, msg, None) 595 596 def syntaxError(self, filename, msg, lineno, offset, text): 597 self._register_issue(filename, lineno, msg, text) 598 599 def flake(self, msg): 600 self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args, None) 601 602 def _is_ignored(self, path, issue): 603 # Paths in pyflakes_ignore are relative, so we need to check to see if our 604 # path ends with any of them. 605 606 for ignored_path, ignored_issues in self._ignored_issues.items(): 607 if path.endswith(ignored_path): 608 if issue in ignored_issues: 609 return True 610 611 for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]: 612 if issue.startswith(prefix): 613 return True 614 615 for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]: 616 if issue.endswith(suffix): 617 return True 618 619 return False 620 621 def _register_issue(self, path, line_number, issue, line): 622 if not self._is_ignored(path, issue): 623 if path and line_number and not line: 624 line = linecache.getline(path, line_number).strip() 625 626 issues.setdefault(path, []).append(Issue(line_number, issue, line)) 627 628 reporter = Reporter() 629 630 for path in _python_files(paths): 631 pyflakes.api.checkPath(path, reporter) 632 633 return issues 634 635 636def _module_exists(module_name): 637 """ 638 Checks if a module exists. 639 640 :param str module_name: module to check existance of 641 642 :returns: **True** if module exists and **False** otherwise 643 """ 644 645 try: 646 __import__(module_name) 647 return True 648 except ImportError: 649 return False 650 651 652def _python_files(paths): 653 for path in paths: 654 for file_path in stem.util.system.files_with_suffix(path, '.py'): 655 skip = False 656 657 for exclude_path in CONFIG['exclude_paths']: 658 if re.match(exclude_path, file_path): 659 skip = True 660 break 661 662 if not skip: 663 yield file_path 664 665 666# TODO: drop with stem 2.x 667# We renamed our methods to drop a redundant 'get_*' prefix, so alias the old 668# names for backward compatability, and account for pep8 being renamed to 669# pycodestyle. 670 671get_stylistic_issues = stylistic_issues 672get_pyflakes_issues = pyflakes_issues 673is_pep8_available = is_pycodestyle_available 674