1""" 2TestCommon.py: a testing framework for commands and scripts 3 with commonly useful error handling 4 5The TestCommon module provides a simple, high-level interface for writing 6tests of executable commands and scripts, especially commands and scripts 7that interact with the file system. All methods throw exceptions and 8exit on failure, with useful error messages. This makes a number of 9explicit checks unnecessary, making the test scripts themselves simpler 10to write and easier to read. 11 12The TestCommon class is a subclass of the TestCmd class. In essence, 13TestCommon is a wrapper that handles common TestCmd error conditions in 14useful ways. You can use TestCommon directly, or subclass it for your 15program and add additional (or override) methods to tailor it to your 16program's specific needs. Alternatively, the TestCommon class serves 17as a useful example of how to define your own TestCmd subclass. 18 19As a subclass of TestCmd, TestCommon provides access to all of the 20variables and methods from the TestCmd module. Consequently, you can 21use any variable or method documented in the TestCmd module without 22having to explicitly import TestCmd. 23 24A TestCommon environment object is created via the usual invocation: 25 26 import TestCommon 27 test = TestCommon.TestCommon() 28 29You can use all of the TestCmd keyword arguments when instantiating a 30TestCommon object; see the TestCmd documentation for details. 31 32Here is an overview of the methods and keyword arguments that are 33provided by the TestCommon class: 34 35 test.must_be_writable('file1', ['file2', ...]) 36 37 test.must_contain('file', 'required text\n') 38 39 test.must_contain_all_lines(output, lines, ['title', find]) 40 41 test.must_contain_any_line(output, lines, ['title', find]) 42 43 test.must_exist('file1', ['file2', ...]) 44 45 test.must_match('file', "expected contents\n") 46 47 test.must_not_be_writable('file1', ['file2', ...]) 48 49 test.must_not_contain('file', 'banned text\n') 50 51 test.must_not_contain_any_line(output, lines, ['title', find]) 52 53 test.must_not_exist('file1', ['file2', ...]) 54 55 test.run(options = "options to be prepended to arguments", 56 stdout = "expected standard output from the program", 57 stderr = "expected error output from the program", 58 status = expected_status, 59 match = match_function) 60 61The TestCommon module also provides the following variables 62 63 TestCommon.python_executable 64 TestCommon.exe_suffix 65 TestCommon.obj_suffix 66 TestCommon.shobj_prefix 67 TestCommon.shobj_suffix 68 TestCommon.lib_prefix 69 TestCommon.lib_suffix 70 TestCommon.dll_prefix 71 TestCommon.dll_suffix 72 73""" 74 75# Copyright 2000-2010 Steven Knight 76# This module is free software, and you may redistribute it and/or modify 77# it under the same terms as Python itself, so long as this copyright message 78# and disclaimer are retained in their original form. 79# 80# IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, 81# SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF 82# THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 83# DAMAGE. 84# 85# THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT 86# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 87# PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, 88# AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, 89# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 90 91__author__ = "Steven Knight <knight at baldmt dot com>" 92__revision__ = "TestCommon.py 0.37.D001 2010/01/11 16:55:50 knight" 93__version__ = "0.37" 94 95import copy 96import os 97import os.path 98import stat 99import string 100import sys 101import types 102import UserList 103 104from TestCmd import * 105from TestCmd import __all__ 106 107__all__.extend([ 'TestCommon', 108 'exe_suffix', 109 'obj_suffix', 110 'shobj_prefix', 111 'shobj_suffix', 112 'lib_prefix', 113 'lib_suffix', 114 'dll_prefix', 115 'dll_suffix', 116 ]) 117 118# Variables that describe the prefixes and suffixes on this system. 119if sys.platform == 'win32': 120 exe_suffix = '.exe' 121 obj_suffix = '.obj' 122 shobj_suffix = '.obj' 123 shobj_prefix = '' 124 lib_prefix = '' 125 lib_suffix = '.lib' 126 dll_prefix = '' 127 dll_suffix = '.dll' 128elif sys.platform == 'cygwin': 129 exe_suffix = '.exe' 130 obj_suffix = '.o' 131 shobj_suffix = '.os' 132 shobj_prefix = '' 133 lib_prefix = 'lib' 134 lib_suffix = '.a' 135 dll_prefix = '' 136 dll_suffix = '.dll' 137elif string.find(sys.platform, 'irix') != -1: 138 exe_suffix = '' 139 obj_suffix = '.o' 140 shobj_suffix = '.o' 141 shobj_prefix = '' 142 lib_prefix = 'lib' 143 lib_suffix = '.a' 144 dll_prefix = 'lib' 145 dll_suffix = '.so' 146elif string.find(sys.platform, 'darwin') != -1: 147 exe_suffix = '' 148 obj_suffix = '.o' 149 shobj_suffix = '.os' 150 shobj_prefix = '' 151 lib_prefix = 'lib' 152 lib_suffix = '.a' 153 dll_prefix = 'lib' 154 dll_suffix = '.dylib' 155elif string.find(sys.platform, 'sunos') != -1: 156 exe_suffix = '' 157 obj_suffix = '.o' 158 shobj_suffix = '.os' 159 shobj_prefix = 'so_' 160 lib_prefix = 'lib' 161 lib_suffix = '.a' 162 dll_prefix = 'lib' 163 dll_suffix = '.dylib' 164else: 165 exe_suffix = '' 166 obj_suffix = '.o' 167 shobj_suffix = '.os' 168 shobj_prefix = '' 169 lib_prefix = 'lib' 170 lib_suffix = '.a' 171 dll_prefix = 'lib' 172 dll_suffix = '.so' 173 174def is_List(e): 175 return type(e) is types.ListType \ 176 or isinstance(e, UserList.UserList) 177 178def is_writable(f): 179 mode = os.stat(f)[stat.ST_MODE] 180 return mode & stat.S_IWUSR 181 182def separate_files(flist): 183 existing = [] 184 missing = [] 185 for f in flist: 186 if os.path.exists(f): 187 existing.append(f) 188 else: 189 missing.append(f) 190 return existing, missing 191 192if os.name == 'posix': 193 def _failed(self, status = 0): 194 if self.status is None or status is None: 195 return None 196 return _status(self) != status 197 def _status(self): 198 return self.status 199elif os.name == 'nt': 200 def _failed(self, status = 0): 201 return not (self.status is None or status is None) and \ 202 self.status != status 203 def _status(self): 204 return self.status 205 206class TestCommon(TestCmd): 207 208 # Additional methods from the Perl Test::Cmd::Common module 209 # that we may wish to add in the future: 210 # 211 # $test->subdir('subdir', ...); 212 # 213 # $test->copy('src_file', 'dst_file'); 214 215 def __init__(self, **kw): 216 """Initialize a new TestCommon instance. This involves just 217 calling the base class initialization, and then changing directory 218 to the workdir. 219 """ 220 apply(TestCmd.__init__, [self], kw) 221 os.chdir(self.workdir) 222 223 def must_be_writable(self, *files): 224 """Ensures that the specified file(s) exist and are writable. 225 An individual file can be specified as a list of directory names, 226 in which case the pathname will be constructed by concatenating 227 them. Exits FAILED if any of the files does not exist or is 228 not writable. 229 """ 230 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 231 existing, missing = separate_files(files) 232 unwritable = filter(lambda x, iw=is_writable: not iw(x), existing) 233 if missing: 234 print "Missing files: `%s'" % string.join(missing, "', `") 235 if unwritable: 236 print "Unwritable files: `%s'" % string.join(unwritable, "', `") 237 self.fail_test(missing + unwritable) 238 239 def must_contain(self, file, required, mode = 'rb'): 240 """Ensures that the specified file contains the required text. 241 """ 242 file_contents = self.read(file, mode) 243 contains = (string.find(file_contents, required) != -1) 244 if not contains: 245 print "File `%s' does not contain required string." % file 246 print self.banner('Required string ') 247 print required 248 print self.banner('%s contents ' % file) 249 print file_contents 250 self.fail_test(not contains) 251 252 def must_contain_all_lines(self, output, lines, title=None, find=None): 253 """Ensures that the specified output string (first argument) 254 contains all of the specified lines (second argument). 255 256 An optional third argument can be used to describe the type 257 of output being searched, and only shows up in failure output. 258 259 An optional fourth argument can be used to supply a different 260 function, of the form "find(line, output), to use when searching 261 for lines in the output. 262 """ 263 if find is None: 264 find = lambda o, l: string.find(o, l) != -1 265 missing = [] 266 for line in lines: 267 if not find(output, line): 268 missing.append(line) 269 270 if missing: 271 if title is None: 272 title = 'output' 273 sys.stdout.write("Missing expected lines from %s:\n" % title) 274 for line in missing: 275 sys.stdout.write(' ' + repr(line) + '\n') 276 sys.stdout.write(self.banner(title + ' ')) 277 sys.stdout.write(output) 278 self.fail_test() 279 280 def must_contain_any_line(self, output, lines, title=None, find=None): 281 """Ensures that the specified output string (first argument) 282 contains at least one of the specified lines (second argument). 283 284 An optional third argument can be used to describe the type 285 of output being searched, and only shows up in failure output. 286 287 An optional fourth argument can be used to supply a different 288 function, of the form "find(line, output), to use when searching 289 for lines in the output. 290 """ 291 if find is None: 292 find = lambda o, l: string.find(o, l) != -1 293 for line in lines: 294 if find(output, line): 295 return 296 297 if title is None: 298 title = 'output' 299 sys.stdout.write("Missing any expected line from %s:\n" % title) 300 for line in lines: 301 sys.stdout.write(' ' + repr(line) + '\n') 302 sys.stdout.write(self.banner(title + ' ')) 303 sys.stdout.write(output) 304 self.fail_test() 305 306 def must_contain_lines(self, lines, output, title=None): 307 # Deprecated; retain for backwards compatibility. 308 return self.must_contain_all_lines(output, lines, title) 309 310 def must_exist(self, *files): 311 """Ensures that the specified file(s) must exist. An individual 312 file be specified as a list of directory names, in which case the 313 pathname will be constructed by concatenating them. Exits FAILED 314 if any of the files does not exist. 315 """ 316 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 317 missing = filter(lambda x: not os.path.exists(x), files) 318 if missing: 319 print "Missing files: `%s'" % string.join(missing, "', `") 320 self.fail_test(missing) 321 322 def must_match(self, file, expect, mode = 'rb'): 323 """Matches the contents of the specified file (first argument) 324 against the expected contents (second argument). The expected 325 contents are a list of lines or a string which will be split 326 on newlines. 327 """ 328 file_contents = self.read(file, mode) 329 try: 330 self.fail_test(not self.match(file_contents, expect)) 331 except KeyboardInterrupt: 332 raise 333 except: 334 print "Unexpected contents of `%s'" % file 335 self.diff(expect, file_contents, 'contents ') 336 raise 337 338 def must_not_contain(self, file, banned, mode = 'rb'): 339 """Ensures that the specified file doesn't contain the banned text. 340 """ 341 file_contents = self.read(file, mode) 342 contains = (string.find(file_contents, banned) != -1) 343 if contains: 344 print "File `%s' contains banned string." % file 345 print self.banner('Banned string ') 346 print banned 347 print self.banner('%s contents ' % file) 348 print file_contents 349 self.fail_test(contains) 350 351 def must_not_contain_any_line(self, output, lines, title=None, find=None): 352 """Ensures that the specified output string (first argument) 353 does not contain any of the specified lines (second argument). 354 355 An optional third argument can be used to describe the type 356 of output being searched, and only shows up in failure output. 357 358 An optional fourth argument can be used to supply a different 359 function, of the form "find(line, output), to use when searching 360 for lines in the output. 361 """ 362 if find is None: 363 find = lambda o, l: string.find(o, l) != -1 364 unexpected = [] 365 for line in lines: 366 if find(output, line): 367 unexpected.append(line) 368 369 if unexpected: 370 if title is None: 371 title = 'output' 372 sys.stdout.write("Unexpected lines in %s:\n" % title) 373 for line in unexpected: 374 sys.stdout.write(' ' + repr(line) + '\n') 375 sys.stdout.write(self.banner(title + ' ')) 376 sys.stdout.write(output) 377 self.fail_test() 378 379 def must_not_contain_lines(self, lines, output, title=None): 380 return self.must_not_contain_any_line(output, lines, title) 381 382 def must_not_exist(self, *files): 383 """Ensures that the specified file(s) must not exist. 384 An individual file be specified as a list of directory names, in 385 which case the pathname will be constructed by concatenating them. 386 Exits FAILED if any of the files exists. 387 """ 388 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 389 existing = filter(os.path.exists, files) 390 if existing: 391 print "Unexpected files exist: `%s'" % string.join(existing, "', `") 392 self.fail_test(existing) 393 394 395 def must_not_be_writable(self, *files): 396 """Ensures that the specified file(s) exist and are not writable. 397 An individual file can be specified as a list of directory names, 398 in which case the pathname will be constructed by concatenating 399 them. Exits FAILED if any of the files does not exist or is 400 writable. 401 """ 402 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 403 existing, missing = separate_files(files) 404 writable = filter(is_writable, existing) 405 if missing: 406 print "Missing files: `%s'" % string.join(missing, "', `") 407 if writable: 408 print "Writable files: `%s'" % string.join(writable, "', `") 409 self.fail_test(missing + writable) 410 411 def _complete(self, actual_stdout, expected_stdout, 412 actual_stderr, expected_stderr, status, match): 413 """ 414 Post-processes running a subcommand, checking for failure 415 status and displaying output appropriately. 416 """ 417 if _failed(self, status): 418 expect = '' 419 if status != 0: 420 expect = " (expected %s)" % str(status) 421 print "%s returned %s%s" % (self.program, str(_status(self)), expect) 422 print self.banner('STDOUT ') 423 print actual_stdout 424 print self.banner('STDERR ') 425 print actual_stderr 426 self.fail_test() 427 if not expected_stdout is None and not match(actual_stdout, expected_stdout): 428 self.diff(expected_stdout, actual_stdout, 'STDOUT ') 429 if actual_stderr: 430 print self.banner('STDERR ') 431 print actual_stderr 432 self.fail_test() 433 if not expected_stderr is None and not match(actual_stderr, expected_stderr): 434 print self.banner('STDOUT ') 435 print actual_stdout 436 self.diff(expected_stderr, actual_stderr, 'STDERR ') 437 self.fail_test() 438 439 def start(self, program = None, 440 interpreter = None, 441 arguments = None, 442 universal_newlines = None, 443 **kw): 444 """ 445 Starts a program or script for the test environment. 446 447 This handles the "options" keyword argument and exceptions. 448 """ 449 try: 450 options = kw['options'] 451 del kw['options'] 452 except KeyError: 453 pass 454 else: 455 if options: 456 if arguments is None: 457 arguments = options 458 else: 459 arguments = options + " " + arguments 460 try: 461 return apply(TestCmd.start, 462 (self, program, interpreter, arguments, universal_newlines), 463 kw) 464 except KeyboardInterrupt: 465 raise 466 except Exception, e: 467 print self.banner('STDOUT ') 468 try: 469 print self.stdout() 470 except IndexError: 471 pass 472 print self.banner('STDERR ') 473 try: 474 print self.stderr() 475 except IndexError: 476 pass 477 cmd_args = self.command_args(program, interpreter, arguments) 478 sys.stderr.write('Exception trying to execute: %s\n' % cmd_args) 479 raise e 480 481 def finish(self, popen, stdout = None, stderr = '', status = 0, **kw): 482 """ 483 Finishes and waits for the process being run under control of 484 the specified popen argument. Additional arguments are similar 485 to those of the run() method: 486 487 stdout The expected standard output from 488 the command. A value of None means 489 don't test standard output. 490 491 stderr The expected error output from 492 the command. A value of None means 493 don't test error output. 494 495 status The expected exit status from the 496 command. A value of None means don't 497 test exit status. 498 """ 499 apply(TestCmd.finish, (self, popen,), kw) 500 match = kw.get('match', self.match) 501 self._complete(self.stdout(), stdout, 502 self.stderr(), stderr, status, match) 503 504 def run(self, options = None, arguments = None, 505 stdout = None, stderr = '', status = 0, **kw): 506 """Runs the program under test, checking that the test succeeded. 507 508 The arguments are the same as the base TestCmd.run() method, 509 with the addition of: 510 511 options Extra options that get appended to the beginning 512 of the arguments. 513 514 stdout The expected standard output from 515 the command. A value of None means 516 don't test standard output. 517 518 stderr The expected error output from 519 the command. A value of None means 520 don't test error output. 521 522 status The expected exit status from the 523 command. A value of None means don't 524 test exit status. 525 526 By default, this expects a successful exit (status = 0), does 527 not test standard output (stdout = None), and expects that error 528 output is empty (stderr = ""). 529 """ 530 if options: 531 if arguments is None: 532 arguments = options 533 else: 534 arguments = options + " " + arguments 535 kw['arguments'] = arguments 536 try: 537 match = kw['match'] 538 del kw['match'] 539 except KeyError: 540 match = self.match 541 apply(TestCmd.run, [self], kw) 542 self._complete(self.stdout(), stdout, 543 self.stderr(), stderr, status, match) 544 545 def skip_test(self, message="Skipping test.\n"): 546 """Skips a test. 547 548 Proper test-skipping behavior is dependent on the external 549 TESTCOMMON_PASS_SKIPS environment variable. If set, we treat 550 the skip as a PASS (exit 0), and otherwise treat it as NO RESULT. 551 In either case, we print the specified message as an indication 552 that the substance of the test was skipped. 553 554 (This was originally added to support development under Aegis. 555 Technically, skipping a test is a NO RESULT, but Aegis would 556 treat that as a test failure and prevent the change from going to 557 the next step. Since we ddn't want to force anyone using Aegis 558 to have to install absolutely every tool used by the tests, we 559 would actually report to Aegis that a skipped test has PASSED 560 so that the workflow isn't held up.) 561 """ 562 if message: 563 sys.stdout.write(message) 564 sys.stdout.flush() 565 pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS') 566 if pass_skips in [None, 0, '0']: 567 # skip=1 means skip this function when showing where this 568 # result came from. They only care about the line where the 569 # script called test.skip_test(), not the line number where 570 # we call test.no_result(). 571 self.no_result(skip=1) 572 else: 573 # We're under the development directory for this change, 574 # so this is an Aegis invocation; pass the test (exit 0). 575 self.pass_test() 576 577# Local Variables: 578# tab-width:4 579# indent-tabs-mode:nil 580# End: 581# vim: set expandtab tabstop=4 shiftwidth=4: 582