1"""An Application for launching a kernel""" 2 3# Copyright (c) IPython Development Team. 4# Distributed under the terms of the Modified BSD License. 5 6import atexit 7import os 8import sys 9import errno 10import signal 11import traceback 12import logging 13 14import tornado 15from tornado import ioloop 16 17import zmq 18from zmq.eventloop import ioloop as zmq_ioloop 19from zmq.eventloop.zmqstream import ZMQStream 20 21from IPython.core.application import ( 22 BaseIPythonApplication, base_flags, base_aliases, catch_config_error 23) 24from IPython.core.profiledir import ProfileDir 25from IPython.core.shellapp import ( 26 InteractiveShellApp, shell_flags, shell_aliases 27) 28from ipython_genutils.path import filefind, ensure_dir_exists 29from traitlets import ( 30 Any, Instance, Dict, Unicode, Integer, Bool, DottedObjectName, Type, default 31) 32from ipython_genutils.importstring import import_item 33from jupyter_core.paths import jupyter_runtime_dir 34from jupyter_client import write_connection_file 35from jupyter_client.connect import ConnectionFileMixin 36 37# local imports 38from .iostream import IOPubThread 39from .heartbeat import Heartbeat 40from .ipkernel import IPythonKernel 41from .parentpoller import ParentPollerUnix, ParentPollerWindows 42from jupyter_client.session import ( 43 Session, session_flags, session_aliases, 44) 45from .zmqshell import ZMQInteractiveShell 46 47#----------------------------------------------------------------------------- 48# Flags and Aliases 49#----------------------------------------------------------------------------- 50 51kernel_aliases = dict(base_aliases) 52kernel_aliases.update({ 53 'ip' : 'IPKernelApp.ip', 54 'hb' : 'IPKernelApp.hb_port', 55 'shell' : 'IPKernelApp.shell_port', 56 'iopub' : 'IPKernelApp.iopub_port', 57 'stdin' : 'IPKernelApp.stdin_port', 58 'control' : 'IPKernelApp.control_port', 59 'f' : 'IPKernelApp.connection_file', 60 'transport': 'IPKernelApp.transport', 61}) 62 63kernel_flags = dict(base_flags) 64kernel_flags.update({ 65 'no-stdout' : ( 66 {'IPKernelApp' : {'no_stdout' : True}}, 67 "redirect stdout to the null device"), 68 'no-stderr' : ( 69 {'IPKernelApp' : {'no_stderr' : True}}, 70 "redirect stderr to the null device"), 71 'pylab' : ( 72 {'IPKernelApp' : {'pylab' : 'auto'}}, 73 """Pre-load matplotlib and numpy for interactive use with 74 the default matplotlib backend."""), 75 'trio-loop' : ( 76 {'InteractiveShell' : {'trio_loop' : False}}, 77 'Enable Trio as main event loop.' 78 ), 79}) 80 81# inherit flags&aliases for any IPython shell apps 82kernel_aliases.update(shell_aliases) 83kernel_flags.update(shell_flags) 84 85# inherit flags&aliases for Sessions 86kernel_aliases.update(session_aliases) 87kernel_flags.update(session_flags) 88 89_ctrl_c_message = """\ 90NOTE: When using the `ipython kernel` entry point, Ctrl-C will not work. 91 92To exit, you will have to explicitly quit this process, by either sending 93"quit" from a client, or using Ctrl-\\ in UNIX-like environments. 94 95To read more about this, see https://github.com/ipython/ipython/issues/2049 96 97""" 98 99#----------------------------------------------------------------------------- 100# Application class for starting an IPython Kernel 101#----------------------------------------------------------------------------- 102 103class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, 104 ConnectionFileMixin): 105 name='ipython-kernel' 106 aliases = Dict(kernel_aliases) 107 flags = Dict(kernel_flags) 108 classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] 109 # the kernel class, as an importstring 110 kernel_class = Type('ipykernel.ipkernel.IPythonKernel', 111 klass='ipykernel.kernelbase.Kernel', 112 help="""The Kernel subclass to be used. 113 114 This should allow easy re-use of the IPKernelApp entry point 115 to configure and launch kernels other than IPython's own. 116 """).tag(config=True) 117 kernel = Any() 118 poller = Any() # don't restrict this even though current pollers are all Threads 119 heartbeat = Instance(Heartbeat, allow_none=True) 120 121 context = Any() 122 shell_socket = Any() 123 control_socket = Any() 124 stdin_socket = Any() 125 iopub_socket = Any() 126 iopub_thread = Any() 127 128 ports = Dict() 129 130 subcommands = { 131 'install': ( 132 'ipykernel.kernelspec.InstallIPythonKernelSpecApp', 133 'Install the IPython kernel' 134 ), 135 } 136 137 # connection info: 138 connection_dir = Unicode() 139 140 @default('connection_dir') 141 def _default_connection_dir(self): 142 return jupyter_runtime_dir() 143 144 @property 145 def abs_connection_file(self): 146 if os.path.basename(self.connection_file) == self.connection_file: 147 return os.path.join(self.connection_dir, self.connection_file) 148 else: 149 return self.connection_file 150 151 # streams, etc. 152 no_stdout = Bool(False, help="redirect stdout to the null device").tag(config=True) 153 no_stderr = Bool(False, help="redirect stderr to the null device").tag(config=True) 154 trio_loop = Bool(False, help="Set main event loop.").tag(config=True) 155 quiet = Bool(True, help="Only send stdout/stderr to output stream").tag(config=True) 156 outstream_class = DottedObjectName('ipykernel.iostream.OutStream', 157 help="The importstring for the OutStream factory").tag(config=True) 158 displayhook_class = DottedObjectName('ipykernel.displayhook.ZMQDisplayHook', 159 help="The importstring for the DisplayHook factory").tag(config=True) 160 161 # polling 162 parent_handle = Integer(int(os.environ.get('JPY_PARENT_PID') or 0), 163 help="""kill this process if its parent dies. On Windows, the argument 164 specifies the HANDLE of the parent process, otherwise it is simply boolean. 165 """).tag(config=True) 166 interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), 167 help="""ONLY USED ON WINDOWS 168 Interrupt this process when the parent is signaled. 169 """).tag(config=True) 170 171 def init_crash_handler(self): 172 sys.excepthook = self.excepthook 173 174 def excepthook(self, etype, evalue, tb): 175 # write uncaught traceback to 'real' stderr, not zmq-forwarder 176 traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) 177 178 def init_poller(self): 179 if sys.platform == 'win32': 180 if self.interrupt or self.parent_handle: 181 self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) 182 elif self.parent_handle and self.parent_handle != 1: 183 # PID 1 (init) is special and will never go away, 184 # only be reassigned. 185 # Parent polling doesn't work if ppid == 1 to start with. 186 self.poller = ParentPollerUnix() 187 188 def _try_bind_socket(self, s, port): 189 iface = '%s://%s' % (self.transport, self.ip) 190 if self.transport == 'tcp': 191 if port <= 0: 192 port = s.bind_to_random_port(iface) 193 else: 194 s.bind("tcp://%s:%i" % (self.ip, port)) 195 elif self.transport == 'ipc': 196 if port <= 0: 197 port = 1 198 path = "%s-%i" % (self.ip, port) 199 while os.path.exists(path): 200 port = port + 1 201 path = "%s-%i" % (self.ip, port) 202 else: 203 path = "%s-%i" % (self.ip, port) 204 s.bind("ipc://%s" % path) 205 return port 206 207 def _bind_socket(self, s, port): 208 try: 209 win_in_use = errno.WSAEADDRINUSE 210 except AttributeError: 211 win_in_use = None 212 213 # Try up to 100 times to bind a port when in conflict to avoid 214 # infinite attempts in bad setups 215 max_attempts = 1 if port else 100 216 for attempt in range(max_attempts): 217 try: 218 return self._try_bind_socket(s, port) 219 except zmq.ZMQError as ze: 220 # Raise if we have any error not related to socket binding 221 if ze.errno != errno.EADDRINUSE and ze.errno != win_in_use: 222 raise 223 if attempt == max_attempts - 1: 224 raise 225 226 def write_connection_file(self): 227 """write connection info to JSON file""" 228 cf = self.abs_connection_file 229 self.log.debug("Writing connection file: %s", cf) 230 write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, 231 shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, 232 iopub_port=self.iopub_port, control_port=self.control_port) 233 234 def cleanup_connection_file(self): 235 cf = self.abs_connection_file 236 self.log.debug("Cleaning up connection file: %s", cf) 237 try: 238 os.remove(cf) 239 except (IOError, OSError): 240 pass 241 242 self.cleanup_ipc_files() 243 244 def init_connection_file(self): 245 if not self.connection_file: 246 self.connection_file = "kernel-%s.json"%os.getpid() 247 try: 248 self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) 249 except IOError: 250 self.log.debug("Connection file not found: %s", self.connection_file) 251 # This means I own it, and I'll create it in this directory: 252 ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700) 253 # Also, I will clean it up: 254 atexit.register(self.cleanup_connection_file) 255 return 256 try: 257 self.load_connection_file() 258 except Exception: 259 self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) 260 self.exit(1) 261 262 def init_sockets(self): 263 # Create a context, a session, and the kernel sockets. 264 self.log.info("Starting the kernel at pid: %i", os.getpid()) 265 assert self.context is None, "init_sockets cannot be called twice!" 266 self.context = context = zmq.Context() 267 atexit.register(self.close) 268 269 self.shell_socket = context.socket(zmq.ROUTER) 270 self.shell_socket.linger = 1000 271 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) 272 self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) 273 274 self.stdin_socket = context.socket(zmq.ROUTER) 275 self.stdin_socket.linger = 1000 276 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) 277 self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) 278 279 self.control_socket = context.socket(zmq.ROUTER) 280 self.control_socket.linger = 1000 281 self.control_port = self._bind_socket(self.control_socket, self.control_port) 282 self.log.debug("control ROUTER Channel on port: %i" % self.control_port) 283 284 if hasattr(zmq, 'ROUTER_HANDOVER'): 285 # set router-handover to workaround zeromq reconnect problems 286 # in certain rare circumstances 287 # see ipython/ipykernel#270 and zeromq/libzmq#2892 288 self.shell_socket.router_handover = \ 289 self.control_socket.router_handover = \ 290 self.stdin_socket.router_handover = 1 291 292 self.init_iopub(context) 293 294 def init_iopub(self, context): 295 self.iopub_socket = context.socket(zmq.PUB) 296 self.iopub_socket.linger = 1000 297 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) 298 self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) 299 self.configure_tornado_logger() 300 self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) 301 self.iopub_thread.start() 302 # backward-compat: wrap iopub socket API in background thread 303 self.iopub_socket = self.iopub_thread.background_socket 304 305 def init_heartbeat(self): 306 """start the heart beating""" 307 # heartbeat doesn't share context, because it mustn't be blocked 308 # by the GIL, which is accessed by libzmq when freeing zero-copy messages 309 hb_ctx = zmq.Context() 310 self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) 311 self.hb_port = self.heartbeat.port 312 self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) 313 self.heartbeat.start() 314 315 def close(self): 316 """Close zmq sockets in an orderly fashion""" 317 # un-capture IO before we start closing channels 318 self.reset_io() 319 self.log.info("Cleaning up sockets") 320 if self.heartbeat: 321 self.log.debug("Closing heartbeat channel") 322 self.heartbeat.context.term() 323 if self.iopub_thread: 324 self.log.debug("Closing iopub channel") 325 self.iopub_thread.stop() 326 self.iopub_thread.close() 327 for channel in ('shell', 'control', 'stdin'): 328 self.log.debug("Closing %s channel", channel) 329 socket = getattr(self, channel + "_socket", None) 330 if socket and not socket.closed: 331 socket.close() 332 self.log.debug("Terminating zmq context") 333 self.context.term() 334 self.log.debug("Terminated zmq context") 335 336 def log_connection_info(self): 337 """display connection info, and store ports""" 338 basename = os.path.basename(self.connection_file) 339 if basename == self.connection_file or \ 340 os.path.dirname(self.connection_file) == self.connection_dir: 341 # use shortname 342 tail = basename 343 else: 344 tail = self.connection_file 345 lines = [ 346 "To connect another client to this kernel, use:", 347 " --existing %s" % tail, 348 ] 349 # log connection info 350 # info-level, so often not shown. 351 # frontends should use the %connect_info magic 352 # to see the connection info 353 for line in lines: 354 self.log.info(line) 355 # also raw print to the terminal if no parent_handle (`ipython kernel`) 356 # unless log-level is CRITICAL (--quiet) 357 if not self.parent_handle and self.log_level < logging.CRITICAL: 358 print(_ctrl_c_message, file=sys.__stdout__) 359 for line in lines: 360 print(line, file=sys.__stdout__) 361 362 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, 363 stdin=self.stdin_port, hb=self.hb_port, 364 control=self.control_port) 365 366 def init_blackhole(self): 367 """redirects stdout/stderr to devnull if necessary""" 368 if self.no_stdout or self.no_stderr: 369 blackhole = open(os.devnull, 'w') 370 if self.no_stdout: 371 sys.stdout = sys.__stdout__ = blackhole 372 if self.no_stderr: 373 sys.stderr = sys.__stderr__ = blackhole 374 375 def init_io(self): 376 """Redirect input streams and set a display hook.""" 377 if self.outstream_class: 378 outstream_factory = import_item(str(self.outstream_class)) 379 if sys.stdout is not None: 380 sys.stdout.flush() 381 382 e_stdout = None if self.quiet else sys.__stdout__ 383 e_stderr = None if self.quiet else sys.__stderr__ 384 385 sys.stdout = outstream_factory(self.session, self.iopub_thread, 386 'stdout', 387 echo=e_stdout) 388 if sys.stderr is not None: 389 sys.stderr.flush() 390 sys.stderr = outstream_factory(self.session, self.iopub_thread, 391 'stderr', 392 echo=e_stderr) 393 if self.displayhook_class: 394 displayhook_factory = import_item(str(self.displayhook_class)) 395 self.displayhook = displayhook_factory(self.session, self.iopub_socket) 396 sys.displayhook = self.displayhook 397 398 self.patch_io() 399 400 def reset_io(self): 401 """restore original io 402 403 restores state after init_io 404 """ 405 sys.stdout = sys.__stdout__ 406 sys.stderr = sys.__stderr__ 407 sys.displayhook = sys.__displayhook__ 408 409 def patch_io(self): 410 """Patch important libraries that can't handle sys.stdout forwarding""" 411 try: 412 import faulthandler 413 except ImportError: 414 pass 415 else: 416 # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible 417 # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): 418 # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable 419 420 # change default file to __stderr__ from forwarded stderr 421 faulthandler_enable = faulthandler.enable 422 def enable(file=sys.__stderr__, all_threads=True, **kwargs): 423 return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) 424 425 faulthandler.enable = enable 426 427 if hasattr(faulthandler, 'register'): 428 faulthandler_register = faulthandler.register 429 def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): 430 return faulthandler_register(signum, file=file, all_threads=all_threads, 431 chain=chain, **kwargs) 432 faulthandler.register = register 433 434 def init_signal(self): 435 signal.signal(signal.SIGINT, signal.SIG_IGN) 436 437 def init_kernel(self): 438 """Create the Kernel object itself""" 439 shell_stream = ZMQStream(self.shell_socket) 440 control_stream = ZMQStream(self.control_socket) 441 442 kernel_factory = self.kernel_class.instance 443 444 kernel = kernel_factory(parent=self, session=self.session, 445 control_stream=control_stream, 446 shell_streams=[shell_stream, control_stream], 447 iopub_thread=self.iopub_thread, 448 iopub_socket=self.iopub_socket, 449 stdin_socket=self.stdin_socket, 450 log=self.log, 451 profile_dir=self.profile_dir, 452 user_ns=self.user_ns, 453 ) 454 kernel.record_ports({ 455 name + '_port': port for name, port in self.ports.items() 456 }) 457 self.kernel = kernel 458 459 # Allow the displayhook to get the execution count 460 self.displayhook.get_execution_count = lambda: kernel.execution_count 461 462 def init_gui_pylab(self): 463 """Enable GUI event loop integration, taking pylab into account.""" 464 465 # Register inline backend as default 466 # this is higher priority than matplotlibrc, 467 # but lower priority than anything else (mpl.use() for instance). 468 # This only affects matplotlib >= 1.5 469 if not os.environ.get('MPLBACKEND'): 470 os.environ['MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline' 471 472 # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` 473 # to ensure that any exception is printed straight to stderr. 474 # Normally _showtraceback associates the reply with an execution, 475 # which means frontends will never draw it, as this exception 476 # is not associated with any execute request. 477 478 shell = self.shell 479 _showtraceback = shell._showtraceback 480 try: 481 # replace error-sending traceback with stderr 482 def print_tb(etype, evalue, stb): 483 print ("GUI event loop or pylab initialization failed", 484 file=sys.stderr) 485 print (shell.InteractiveTB.stb2text(stb), file=sys.stderr) 486 shell._showtraceback = print_tb 487 InteractiveShellApp.init_gui_pylab(self) 488 finally: 489 shell._showtraceback = _showtraceback 490 491 def init_shell(self): 492 self.shell = getattr(self.kernel, 'shell', None) 493 if self.shell: 494 self.shell.configurables.append(self) 495 496 def configure_tornado_logger(self): 497 """ Configure the tornado logging.Logger. 498 499 Must set up the tornado logger or else tornado will call 500 basicConfig for the root logger which makes the root logger 501 go to the real sys.stderr instead of the capture streams. 502 This function mimics the setup of logging.basicConfig. 503 """ 504 logger = logging.getLogger('tornado') 505 handler = logging.StreamHandler() 506 formatter = logging.Formatter(logging.BASIC_FORMAT) 507 handler.setFormatter(formatter) 508 logger.addHandler(handler) 509 510 def _init_asyncio_patch(self): 511 """set default asyncio policy to be compatible with tornado 512 513 Tornado 6 (at least) is not compatible with the default 514 asyncio implementation on Windows 515 516 Pick the older SelectorEventLoopPolicy on Windows 517 if the known-incompatible default policy is in use. 518 519 do this as early as possible to make it a low priority and overrideable 520 521 ref: https://github.com/tornadoweb/tornado/issues/2608 522 523 FIXME: if/when tornado supports the defaults in asyncio, 524 remove and bump tornado requirement for py38 525 """ 526 if sys.platform.startswith("win") and sys.version_info >= (3, 8) and tornado.version_info < (6, 1): 527 import asyncio 528 try: 529 from asyncio import ( 530 WindowsProactorEventLoopPolicy, 531 WindowsSelectorEventLoopPolicy, 532 ) 533 except ImportError: 534 pass 535 # not affected 536 else: 537 if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: 538 # WindowsProactorEventLoopPolicy is not compatible with tornado 6 539 # fallback to the pre-3.8 default of Selector 540 asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) 541 542 def init_pdb(self): 543 """Replace pdb with IPython's version that is interruptible. 544 545 With the non-interruptible version, stopping pdb() locks up the kernel in a 546 non-recoverable state. 547 """ 548 import pdb 549 from IPython.core import debugger 550 if hasattr(debugger, "InterruptiblePdb"): 551 # Only available in newer IPython releases: 552 debugger.Pdb = debugger.InterruptiblePdb 553 pdb.Pdb = debugger.Pdb 554 pdb.set_trace = debugger.set_trace 555 556 @catch_config_error 557 def initialize(self, argv=None): 558 self._init_asyncio_patch() 559 super(IPKernelApp, self).initialize(argv) 560 if self.subapp is not None: 561 return 562 563 self.init_pdb() 564 self.init_blackhole() 565 self.init_connection_file() 566 self.init_poller() 567 self.init_sockets() 568 self.init_heartbeat() 569 # writing/displaying connection info must be *after* init_sockets/heartbeat 570 self.write_connection_file() 571 # Log connection info after writing connection file, so that the connection 572 # file is definitely available at the time someone reads the log. 573 self.log_connection_info() 574 self.init_io() 575 try: 576 self.init_signal() 577 except: 578 # Catch exception when initializing signal fails, eg when running the 579 # kernel on a separate thread 580 if self.log_level < logging.CRITICAL: 581 self.log.error("Unable to initialize signal:", exc_info=True) 582 self.init_kernel() 583 # shell init steps 584 self.init_path() 585 self.init_shell() 586 if self.shell: 587 self.init_gui_pylab() 588 self.init_extensions() 589 self.init_code() 590 # flush stdout/stderr, so that anything written to these streams during 591 # initialization do not get associated with the first execution request 592 sys.stdout.flush() 593 sys.stderr.flush() 594 595 def start(self): 596 if self.subapp is not None: 597 return self.subapp.start() 598 if self.poller is not None: 599 self.poller.start() 600 self.kernel.start() 601 self.io_loop = ioloop.IOLoop.current() 602 if self.trio_loop: 603 from ipykernel.trio_runner import TrioRunner 604 tr = TrioRunner() 605 tr.initialize(self.kernel, self.io_loop) 606 try: 607 tr.run() 608 except KeyboardInterrupt: 609 pass 610 else: 611 try: 612 self.io_loop.start() 613 except KeyboardInterrupt: 614 pass 615 616 617launch_new_instance = IPKernelApp.launch_instance 618 619 620def main(): 621 """Run an IPKernel as an application""" 622 app = IPKernelApp.instance() 623 app.initialize() 624 app.start() 625 626 627if __name__ == '__main__': 628 main() 629