1"""Notebook Javascript Test Controller 2 3This module runs one or more subprocesses which will actually run the Javascript 4test suite. 5""" 6 7# Copyright (c) Jupyter Development Team. 8# Distributed under the terms of the Modified BSD License. 9 10import argparse 11import json 12import multiprocessing.pool 13import os 14import re 15import requests 16import signal 17import sys 18import subprocess 19import time 20from io import BytesIO 21from threading import Thread, Lock, Event 22 23from unittest.mock import patch 24 25from jupyter_core.paths import jupyter_runtime_dir 26from ipython_genutils.py3compat import bytes_to_str, which 27from notebook._sysinfo import get_sys_info 28from ipython_genutils.tempdir import TemporaryDirectory 29 30from subprocess import TimeoutExpired 31def popen_wait(p, timeout): 32 return p.wait(timeout) 33 34NOTEBOOK_SHUTDOWN_TIMEOUT = 10 35 36have = {} 37have['casperjs'] = bool(which('casperjs')) 38have['phantomjs'] = bool(which('phantomjs')) 39have['slimerjs'] = bool(which('slimerjs')) 40 41class StreamCapturer(Thread): 42 daemon = True # Don't hang if main thread crashes 43 started = False 44 def __init__(self, echo=False): 45 super().__init__() 46 self.echo = echo 47 self.streams = [] 48 self.buffer = BytesIO() 49 self.readfd, self.writefd = os.pipe() 50 self.buffer_lock = Lock() 51 self.stop = Event() 52 53 def run(self): 54 self.started = True 55 56 while not self.stop.is_set(): 57 chunk = os.read(self.readfd, 1024) 58 59 with self.buffer_lock: 60 self.buffer.write(chunk) 61 if self.echo: 62 sys.stdout.write(bytes_to_str(chunk)) 63 64 os.close(self.readfd) 65 os.close(self.writefd) 66 67 def reset_buffer(self): 68 with self.buffer_lock: 69 self.buffer.truncate(0) 70 self.buffer.seek(0) 71 72 def get_buffer(self): 73 with self.buffer_lock: 74 return self.buffer.getvalue() 75 76 def ensure_started(self): 77 if not self.started: 78 self.start() 79 80 def halt(self): 81 """Safely stop the thread.""" 82 if not self.started: 83 return 84 85 self.stop.set() 86 os.write(self.writefd, b'\0') # Ensure we're not locked in a read() 87 self.join() 88 89 90class TestController(object): 91 """Run tests in a subprocess 92 """ 93 #: str, test group to be executed. 94 section = None 95 #: list, command line arguments to be executed 96 cmd = None 97 #: dict, extra environment variables to set for the subprocess 98 env = None 99 #: list, TemporaryDirectory instances to clear up when the process finishes 100 dirs = None 101 #: subprocess.Popen instance 102 process = None 103 #: str, process stdout+stderr 104 stdout = None 105 106 def __init__(self): 107 self.cmd = [] 108 self.env = {} 109 self.dirs = [] 110 111 def setup(self): 112 """Create temporary directories etc. 113 114 This is only called when we know the test group will be run. Things 115 created here may be cleaned up by self.cleanup(). 116 """ 117 pass 118 119 def launch(self, buffer_output=False, capture_output=False): 120 # print('*** ENV:', self.env) # dbg 121 # print('*** CMD:', self.cmd) # dbg 122 env = os.environ.copy() 123 env.update(self.env) 124 if buffer_output: 125 capture_output = True 126 self.stdout_capturer = c = StreamCapturer(echo=not buffer_output) 127 c.start() 128 stdout = c.writefd if capture_output else None 129 stderr = subprocess.STDOUT if capture_output else None 130 self.process = subprocess.Popen(self.cmd, stdout=stdout, 131 stderr=stderr, env=env) 132 133 def wait(self): 134 self.process.wait() 135 self.stdout_capturer.halt() 136 self.stdout = self.stdout_capturer.get_buffer() 137 return self.process.returncode 138 139 def print_extra_info(self): 140 """Print extra information about this test run. 141 142 If we're running in parallel and showing the concise view, this is only 143 called if the test group fails. Otherwise, it's called before the test 144 group is started. 145 146 The base implementation does nothing, but it can be overridden by 147 subclasses. 148 """ 149 return 150 151 def cleanup_process(self): 152 """Cleanup on exit by killing any leftover processes.""" 153 subp = self.process 154 if subp is None or (subp.poll() is not None): 155 return # Process doesn't exist, or is already dead. 156 157 try: 158 print('Cleaning up stale PID: %d' % subp.pid) 159 subp.kill() 160 except: # (OSError, WindowsError) ? 161 # This is just a best effort, if we fail or the process was 162 # really gone, ignore it. 163 pass 164 else: 165 for i in range(10): 166 if subp.poll() is None: 167 time.sleep(0.1) 168 else: 169 break 170 171 if subp.poll() is None: 172 # The process did not die... 173 print('... failed. Manual cleanup may be required.') 174 175 def cleanup(self): 176 "Kill process if it's still alive, and clean up temporary directories" 177 self.cleanup_process() 178 for td in self.dirs: 179 td.cleanup() 180 181 __del__ = cleanup 182 183 184def get_js_test_dir(): 185 import notebook.tests as t 186 return os.path.join(os.path.dirname(t.__file__), '') 187 188def all_js_groups(): 189 import glob 190 test_dir = get_js_test_dir() 191 all_subdirs = glob.glob(test_dir + '[!_]*/') 192 return [os.path.relpath(x, test_dir) for x in all_subdirs] 193 194class JSController(TestController): 195 """Run CasperJS tests """ 196 197 requirements = ['casperjs'] 198 199 def __init__(self, section, xunit=True, engine='phantomjs', url=None): 200 """Create new test runner.""" 201 TestController.__init__(self) 202 self.engine = engine 203 self.section = section 204 self.xunit = xunit 205 self.url = url 206 # run with a base URL that would be escaped, 207 # to test that we don't double-escape URLs 208 self.base_url = '/a@b/' 209 self.slimer_failure = re.compile('^FAIL.*', flags=re.MULTILINE) 210 js_test_dir = get_js_test_dir() 211 includes = '--includes=' + os.path.join(js_test_dir,'util.js') 212 test_cases = os.path.join(js_test_dir, self.section) 213 self.cmd = ['casperjs', 'test', includes, test_cases, '--engine=%s' % self.engine] 214 215 def setup(self): 216 self.ipydir = TemporaryDirectory() 217 self.config_dir = TemporaryDirectory() 218 self.nbdir = TemporaryDirectory() 219 self.home = TemporaryDirectory() 220 self.env = { 221 'HOME': self.home.name, 222 'JUPYTER_CONFIG_DIR': self.config_dir.name, 223 'IPYTHONDIR': self.ipydir.name, 224 } 225 self.dirs.append(self.ipydir) 226 self.dirs.append(self.home) 227 self.dirs.append(self.config_dir) 228 self.dirs.append(self.nbdir) 229 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir1', u'sub ∂ir 1a'))) 230 os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir2', u'sub ∂ir 1b'))) 231 232 if self.xunit: 233 self.add_xunit() 234 235 # If a url was specified, use that for the testing. 236 if self.url: 237 try: 238 alive = requests.get(self.url).status_code == 200 239 except: 240 alive = False 241 242 if alive: 243 self.cmd.append("--url=%s" % self.url) 244 else: 245 raise Exception('Could not reach "%s".' % self.url) 246 else: 247 # start the ipython notebook, so we get the port number 248 self.server_port = 0 249 self._init_server() 250 if self.server_port: 251 self.cmd.append('--url=http://localhost:%i%s' % (self.server_port, self.base_url)) 252 else: 253 # don't launch tests if the server didn't start 254 self.cmd = [sys.executable, '-c', 'raise SystemExit(1)'] 255 256 def add_xunit(self): 257 xunit_file = os.path.abspath(self.section.replace('/','.') + '.xunit.xml') 258 self.cmd.append('--xunit=%s' % xunit_file) 259 260 def launch(self, buffer_output): 261 # If the engine is SlimerJS, we need to buffer the output because 262 # SlimerJS does not support exit codes, so CasperJS always returns 0. 263 if self.engine == 'slimerjs' and not buffer_output: 264 return super().launch(capture_output=True) 265 266 else: 267 return super().launch(buffer_output=buffer_output) 268 269 def wait(self, *pargs, **kwargs): 270 """Wait for the JSController to finish""" 271 ret = super().wait(*pargs, **kwargs) 272 # If this is a SlimerJS controller, check the captured stdout for 273 # errors. Otherwise, just return the return code. 274 if self.engine == 'slimerjs': 275 stdout = bytes_to_str(self.stdout) 276 if ret != 0: 277 # This could still happen e.g. if it's stopped by SIGINT 278 return ret 279 return bool(self.slimer_failure.search(stdout)) 280 else: 281 return ret 282 283 def print_extra_info(self): 284 print("Running tests with notebook directory %r" % self.nbdir.name) 285 286 @property 287 def will_run(self): 288 should_run = all(have[a] for a in self.requirements + [self.engine]) 289 return should_run 290 291 def _init_server(self): 292 "Start the notebook server in a separate process" 293 self.server_command = command = [sys.executable, 294 '-m', 'notebook', 295 '--no-browser', 296 '--notebook-dir', self.nbdir.name, 297 '--NotebookApp.token=', 298 '--NotebookApp.base_url=%s' % self.base_url, 299 ] 300 # ipc doesn't work on Windows, and darwin has crazy-long temp paths, 301 # which run afoul of ipc's maximum path length. 302 if sys.platform.startswith('linux'): 303 command.append('--KernelManager.transport=ipc') 304 self.stream_capturer = c = StreamCapturer() 305 c.start() 306 env = os.environ.copy() 307 env.update(self.env) 308 self.server = subprocess.Popen(command, 309 stdout = c.writefd, 310 stderr = subprocess.STDOUT, 311 cwd=self.nbdir.name, 312 env=env, 313 ) 314 with patch.dict('os.environ', {'HOME': self.home.name}): 315 runtime_dir = jupyter_runtime_dir() 316 self.server_info_file = os.path.join(runtime_dir, 317 'nbserver-%i.json' % self.server.pid 318 ) 319 self._wait_for_server() 320 321 def _wait_for_server(self): 322 """Wait 30 seconds for the notebook server to start""" 323 for i in range(300): 324 if self.server.poll() is not None: 325 return self._failed_to_start() 326 if os.path.exists(self.server_info_file): 327 try: 328 self._load_server_info() 329 except ValueError: 330 # If the server is halfway through writing the file, we may 331 # get invalid JSON; it should be ready next iteration. 332 pass 333 else: 334 return 335 time.sleep(0.1) 336 print("Notebook server-info file never arrived: %s" % self.server_info_file, 337 file=sys.stderr 338 ) 339 340 def _failed_to_start(self): 341 """Notebook server exited prematurely""" 342 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace') 343 print("Notebook failed to start: ", file=sys.stderr) 344 print(self.server_command) 345 print(captured, file=sys.stderr) 346 347 def _load_server_info(self): 348 """Notebook server started, load connection info from JSON""" 349 with open(self.server_info_file) as f: 350 info = json.load(f) 351 self.server_port = info['port'] 352 353 def cleanup(self): 354 if hasattr(self, 'server'): 355 try: 356 self.server.terminate() 357 except OSError: 358 # already dead 359 pass 360 # wait 10s for the server to shutdown 361 try: 362 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT) 363 except TimeoutExpired: 364 # server didn't terminate, kill it 365 try: 366 print("Failed to terminate notebook server, killing it.", 367 file=sys.stderr 368 ) 369 self.server.kill() 370 except OSError: 371 # already dead 372 pass 373 # wait another 10s 374 try: 375 popen_wait(self.server, NOTEBOOK_SHUTDOWN_TIMEOUT) 376 except TimeoutExpired: 377 print("Notebook server still running (%s)" % self.server_info_file, 378 file=sys.stderr 379 ) 380 381 self.stream_capturer.halt() 382 TestController.cleanup(self) 383 384 385def prepare_controllers(options): 386 """Returns two lists of TestController instances, those to run, and those 387 not to run.""" 388 testgroups = options.testgroups 389 if not testgroups: 390 testgroups = all_js_groups() 391 392 engine = 'slimerjs' if options.slimerjs else 'phantomjs' 393 c_js = [JSController(name, xunit=options.xunit, engine=engine, url=options.url) for name in testgroups] 394 395 controllers = c_js 396 to_run = [c for c in controllers if c.will_run] 397 not_run = [c for c in controllers if not c.will_run] 398 return to_run, not_run 399 400def do_run(controller, buffer_output=True): 401 """Setup and run a test controller. 402 403 If buffer_output is True, no output is displayed, to avoid it appearing 404 interleaved. In this case, the caller is responsible for displaying test 405 output on failure. 406 407 Returns 408 ------- 409 controller : TestController 410 The same controller as passed in, as a convenience for using map() type 411 APIs. 412 exitcode : int 413 The exit code of the test subprocess. Non-zero indicates failure. 414 """ 415 try: 416 try: 417 controller.setup() 418 if not buffer_output: 419 controller.print_extra_info() 420 controller.launch(buffer_output=buffer_output) 421 except Exception: 422 import traceback 423 traceback.print_exc() 424 return controller, 1 # signal failure 425 426 exitcode = controller.wait() 427 return controller, exitcode 428 429 except KeyboardInterrupt: 430 return controller, -signal.SIGINT 431 finally: 432 controller.cleanup() 433 434def report(): 435 """Return a string with a summary report of test-related variables.""" 436 inf = get_sys_info() 437 out = [] 438 def _add(name, value): 439 out.append((name, value)) 440 441 _add('Python version', inf['sys_version'].replace('\n','')) 442 _add('sys.executable', inf['sys_executable']) 443 _add('Platform', inf['platform']) 444 445 width = max(len(n) for (n,v) in out) 446 out = ["{:<{width}}: {}\n".format(n, v, width=width) for (n,v) in out] 447 448 avail = [] 449 not_avail = [] 450 451 for k, is_avail in have.items(): 452 if is_avail: 453 avail.append(k) 454 else: 455 not_avail.append(k) 456 457 if avail: 458 out.append('\nTools and libraries available at test time:\n') 459 avail.sort() 460 out.append(' ' + ' '.join(avail)+'\n') 461 462 if not_avail: 463 out.append('\nTools and libraries NOT available at test time:\n') 464 not_avail.sort() 465 out.append(' ' + ' '.join(not_avail)+'\n') 466 467 return ''.join(out) 468 469def run_jstestall(options): 470 """Run the entire Javascript test suite. 471 472 This function constructs TestControllers and runs them in subprocesses. 473 474 Parameters 475 ---------- 476 477 All parameters are passed as attributes of the options object. 478 479 testgroups : list of str 480 Run only these sections of the test suite. If empty, run all the available 481 sections. 482 483 fast : int or None 484 Run the test suite in parallel, using n simultaneous processes. If None 485 is passed, one process is used per CPU core. Default 1 (i.e. sequential) 486 487 inc_slow : bool 488 Include slow tests. By default, these tests aren't run. 489 490 slimerjs : bool 491 Use slimerjs if it's installed instead of phantomjs for casperjs tests. 492 493 url : unicode 494 Address:port to use when running the JS tests. 495 496 xunit : bool 497 Produce Xunit XML output. This is written to multiple foo.xunit.xml files. 498 499 extra_args : list 500 Extra arguments to pass to the test subprocesses, e.g. '-v' 501 """ 502 to_run, not_run = prepare_controllers(options) 503 504 def justify(ltext, rtext, width=70, fill='-'): 505 ltext += ' ' 506 rtext = (' ' + rtext).rjust(width - len(ltext), fill) 507 return ltext + rtext 508 509 # Run all test runners, tracking execution time 510 failed = [] 511 t_start = time.time() 512 513 print() 514 if options.fast == 1: 515 # This actually means sequential, i.e. with 1 job 516 for controller in to_run: 517 print('Test group:', controller.section) 518 sys.stdout.flush() # Show in correct order when output is piped 519 controller, res = do_run(controller, buffer_output=False) 520 if res: 521 failed.append(controller) 522 if res == -signal.SIGINT: 523 print("Interrupted") 524 break 525 print() 526 527 else: 528 # Run tests concurrently 529 try: 530 pool = multiprocessing.pool.ThreadPool(options.fast) 531 for (controller, res) in pool.imap_unordered(do_run, to_run): 532 res_string = 'OK' if res == 0 else 'FAILED' 533 print(justify('Test group: ' + controller.section, res_string)) 534 if res: 535 controller.print_extra_info() 536 print(bytes_to_str(controller.stdout)) 537 failed.append(controller) 538 if res == -signal.SIGINT: 539 print("Interrupted") 540 break 541 except KeyboardInterrupt: 542 return 543 544 for controller in not_run: 545 print(justify('Test group: ' + controller.section, 'NOT RUN')) 546 547 t_end = time.time() 548 t_tests = t_end - t_start 549 nrunners = len(to_run) 550 nfail = len(failed) 551 # summarize results 552 print('_'*70) 553 print('Test suite completed for system with the following information:') 554 print(report()) 555 took = "Took %.3fs." % t_tests 556 print('Status: ', end='') 557 if not failed: 558 print('OK (%d test groups).' % nrunners, took) 559 else: 560 # If anything went wrong, point out what command to rerun manually to 561 # see the actual errors and individual summary 562 failed_sections = [c.section for c in failed] 563 print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, 564 nrunners, ', '.join(failed_sections)), took) 565 print() 566 print('You may wish to rerun these, with:') 567 print(' python -m notebook.jstest', *failed_sections) 568 print() 569 570 if failed: 571 # Ensure that our exit code indicates failure 572 sys.exit(1) 573 574argparser = argparse.ArgumentParser(description='Run Jupyter Notebook Javascript tests') 575argparser.add_argument('testgroups', nargs='*', 576 help='Run specified groups of tests. If omitted, run ' 577 'all tests.') 578argparser.add_argument('--slimerjs', action='store_true', 579 help="Use slimerjs if it's installed instead of phantomjs for casperjs tests.") 580argparser.add_argument('--url', help="URL to use for the JS tests.") 581argparser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, 582 help='Run test sections in parallel. This starts as many ' 583 'processes as you have cores, or you can specify a number.') 584argparser.add_argument('--xunit', action='store_true', 585 help='Produce Xunit XML results') 586argparser.add_argument('--subproc-streams', default='capture', 587 help="What to do with stdout/stderr from subprocesses. " 588 "'capture' (default), 'show' and 'discard' are the options.") 589 590def default_options(): 591 """Get an argparse Namespace object with the default arguments, to pass to 592 :func:`run_iptestall`. 593 """ 594 options = argparser.parse_args([]) 595 options.extra_args = [] 596 return options 597 598def main(): 599 try: 600 ix = sys.argv.index('--') 601 except ValueError: 602 to_parse = sys.argv[1:] 603 extra_args = [] 604 else: 605 to_parse = sys.argv[1:ix] 606 extra_args = sys.argv[ix+1:] 607 608 options = argparser.parse_args(to_parse) 609 options.extra_args = extra_args 610 611 run_jstestall(options) 612 613 614if __name__ == '__main__': 615 main() 616