1"""A ZMQ-based subclass of InteractiveShell. 2 3This code is meant to ease the refactoring of the base InteractiveShell into 4something with a cleaner architecture for 2-process use, without actually 5breaking InteractiveShell itself. So we're doing something a bit ugly, where 6we subclass and override what we want to fix. Once this is working well, we 7can go back to the base class and refactor the code for a cleaner inheritance 8implementation that doesn't rely on so much monkeypatching. 9 10But this lets us maintain a fully working IPython as we develop the new 11machinery. This should thus be thought of as scaffolding. 12""" 13 14# Copyright (c) IPython Development Team. 15# Distributed under the terms of the Modified BSD License. 16 17import os 18import sys 19import time 20import warnings 21from threading import local 22 23from tornado import ioloop 24 25from IPython.core.interactiveshell import ( 26 InteractiveShell, InteractiveShellABC 27) 28from IPython.core import page 29from IPython.core.autocall import ZMQExitAutocall 30from IPython.core.displaypub import DisplayPublisher 31from IPython.core.error import UsageError 32from IPython.core.magics import MacroToEdit, CodeMagics 33from IPython.core.magic import magics_class, line_magic, Magics 34from IPython.core import payloadpage 35from IPython.core.usage import default_banner 36from IPython.display import display, Javascript 37from ipykernel import ( 38 get_connection_file, get_connection_info, connect_qtconsole 39) 40from IPython.utils import openpy 41from ipykernel.jsonutil import json_clean, encode_images 42from IPython.utils.process import arg_split, system 43from ipython_genutils import py3compat 44from traitlets import ( 45 Instance, Type, Dict, CBool, CBytes, Any, default, observe 46) 47from ipykernel.displayhook import ZMQShellDisplayHook 48 49from jupyter_core.paths import jupyter_runtime_dir 50from jupyter_client.session import extract_header, Session 51 52try: 53 # available since ipyparallel 5.0.0 54 from ipyparallel.engine.datapub import ZMQDataPublisher 55except ImportError: 56 # Deprecated since ipykernel 4.3.0 57 from ipykernel.datapub import ZMQDataPublisher 58 59#----------------------------------------------------------------------------- 60# Functions and classes 61#----------------------------------------------------------------------------- 62 63class ZMQDisplayPublisher(DisplayPublisher): 64 """A display publisher that publishes data using a ZeroMQ PUB socket.""" 65 66 session = Instance(Session, allow_none=True) 67 pub_socket = Any(allow_none=True) 68 parent_header = Dict({}) 69 topic = CBytes(b'display_data') 70 71 # thread_local: 72 # An attribute used to ensure the correct output message 73 # is processed. See ipykernel Issue 113 for a discussion. 74 _thread_local = Any() 75 76 def set_parent(self, parent): 77 """Set the parent for outbound messages.""" 78 self.parent_header = extract_header(parent) 79 80 def _flush_streams(self): 81 """flush IO Streams prior to display""" 82 sys.stdout.flush() 83 sys.stderr.flush() 84 85 @default('_thread_local') 86 def _default_thread_local(self): 87 """Initialize our thread local storage""" 88 return local() 89 90 @property 91 def _hooks(self): 92 if not hasattr(self._thread_local, 'hooks'): 93 # create new list for a new thread 94 self._thread_local.hooks = [] 95 return self._thread_local.hooks 96 97 def publish(self, data, metadata=None, source=None, transient=None, 98 update=False, 99 ): 100 """Publish a display-data message 101 102 Parameters 103 ---------- 104 data: dict 105 A mime-bundle dict, keyed by mime-type. 106 metadata: dict, optional 107 Metadata associated with the data. 108 transient: dict, optional, keyword-only 109 Transient data that may only be relevant during a live display, 110 such as display_id. 111 Transient data should not be persisted to documents. 112 update: bool, optional, keyword-only 113 If True, send an update_display_data message instead of display_data. 114 """ 115 self._flush_streams() 116 if metadata is None: 117 metadata = {} 118 if transient is None: 119 transient = {} 120 self._validate_data(data, metadata) 121 content = {} 122 content['data'] = encode_images(data) 123 content['metadata'] = metadata 124 content['transient'] = transient 125 126 msg_type = 'update_display_data' if update else 'display_data' 127 128 # Use 2-stage process to send a message, 129 # in order to put it through the transform 130 # hooks before potentially sending. 131 msg = self.session.msg( 132 msg_type, json_clean(content), 133 parent=self.parent_header 134 ) 135 136 # Each transform either returns a new 137 # message or None. If None is returned, 138 # the message has been 'used' and we return. 139 for hook in self._hooks: 140 msg = hook(msg) 141 if msg is None: 142 return 143 144 self.session.send( 145 self.pub_socket, msg, ident=self.topic, 146 ) 147 148 def clear_output(self, wait=False): 149 """Clear output associated with the current execution (cell). 150 151 Parameters 152 ---------- 153 wait: bool (default: False) 154 If True, the output will not be cleared immediately, 155 instead waiting for the next display before clearing. 156 This reduces bounce during repeated clear & display loops. 157 158 """ 159 content = dict(wait=wait) 160 self._flush_streams() 161 self.session.send( 162 self.pub_socket, 'clear_output', content, 163 parent=self.parent_header, ident=self.topic, 164 ) 165 166 def register_hook(self, hook): 167 """ 168 Registers a hook with the thread-local storage. 169 170 Parameters 171 ---------- 172 hook : Any callable object 173 174 Returns 175 ------- 176 Either a publishable message, or `None`. 177 178 The DisplayHook objects must return a message from 179 the __call__ method if they still require the 180 `session.send` method to be called after transformation. 181 Returning `None` will halt that execution path, and 182 session.send will not be called. 183 """ 184 self._hooks.append(hook) 185 186 def unregister_hook(self, hook): 187 """ 188 Un-registers a hook with the thread-local storage. 189 190 Parameters 191 ---------- 192 hook: Any callable object which has previously been 193 registered as a hook. 194 195 Returns 196 ------- 197 bool - `True` if the hook was removed, `False` if it wasn't 198 found. 199 """ 200 try: 201 self._hooks.remove(hook) 202 return True 203 except ValueError: 204 return False 205 206 207@magics_class 208class KernelMagics(Magics): 209 #------------------------------------------------------------------------ 210 # Magic overrides 211 #------------------------------------------------------------------------ 212 # Once the base class stops inheriting from magic, this code needs to be 213 # moved into a separate machinery as well. For now, at least isolate here 214 # the magics which this class needs to implement differently from the base 215 # class, or that are unique to it. 216 217 _find_edit_target = CodeMagics._find_edit_target 218 219 @line_magic 220 def edit(self, parameter_s='', last_call=['','']): 221 """Bring up an editor and execute the resulting code. 222 223 Usage: 224 %edit [options] [args] 225 226 %edit runs an external text editor. You will need to set the command for 227 this editor via the ``TerminalInteractiveShell.editor`` option in your 228 configuration file before it will work. 229 230 This command allows you to conveniently edit multi-line code right in 231 your IPython session. 232 233 If called without arguments, %edit opens up an empty editor with a 234 temporary file and will execute the contents of this file when you 235 close it (don't forget to save it!). 236 237 Options: 238 239 -n <number> 240 Open the editor at a specified line number. By default, the IPython 241 editor hook uses the unix syntax 'editor +N filename', but you can 242 configure this by providing your own modified hook if your favorite 243 editor supports line-number specifications with a different syntax. 244 245 -p 246 Call the editor with the same data as the previous time it was used, 247 regardless of how long ago (in your current session) it was. 248 249 -r 250 Use 'raw' input. This option only applies to input taken from the 251 user's history. By default, the 'processed' history is used, so that 252 magics are loaded in their transformed version to valid Python. If 253 this option is given, the raw input as typed as the command line is 254 used instead. When you exit the editor, it will be executed by 255 IPython's own processor. 256 257 Arguments: 258 259 If arguments are given, the following possibilities exist: 260 261 - The arguments are numbers or pairs of colon-separated numbers (like 262 1 4:8 9). These are interpreted as lines of previous input to be 263 loaded into the editor. The syntax is the same of the %macro command. 264 265 - If the argument doesn't start with a number, it is evaluated as a 266 variable and its contents loaded into the editor. You can thus edit 267 any string which contains python code (including the result of 268 previous edits). 269 270 - If the argument is the name of an object (other than a string), 271 IPython will try to locate the file where it was defined and open the 272 editor at the point where it is defined. You can use ``%edit function`` 273 to load an editor exactly at the point where 'function' is defined, 274 edit it and have the file be executed automatically. 275 276 If the object is a macro (see %macro for details), this opens up your 277 specified editor with a temporary file containing the macro's data. 278 Upon exit, the macro is reloaded with the contents of the file. 279 280 Note: opening at an exact line is only supported under Unix, and some 281 editors (like kedit and gedit up to Gnome 2.8) do not understand the 282 '+NUMBER' parameter necessary for this feature. Good editors like 283 (X)Emacs, vi, jed, pico and joe all do. 284 285 - If the argument is not found as a variable, IPython will look for a 286 file with that name (adding .py if necessary) and load it into the 287 editor. It will execute its contents with execfile() when you exit, 288 loading any code in the file into your interactive namespace. 289 290 Unlike in the terminal, this is designed to use a GUI editor, and we do 291 not know when it has closed. So the file you edit will not be 292 automatically executed or printed. 293 294 Note that %edit is also available through the alias %ed. 295 """ 296 297 opts,args = self.parse_options(parameter_s, 'prn:') 298 299 try: 300 filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call) 301 except MacroToEdit: 302 # TODO: Implement macro editing over 2 processes. 303 print("Macro editing not yet implemented in 2-process model.") 304 return 305 306 # Make sure we send to the client an absolute path, in case the working 307 # directory of client and kernel don't match 308 filename = os.path.abspath(filename) 309 310 payload = { 311 'source' : 'edit_magic', 312 'filename' : filename, 313 'line_number' : lineno 314 } 315 self.shell.payload_manager.write_payload(payload) 316 317 # A few magics that are adapted to the specifics of using pexpect and a 318 # remote terminal 319 320 @line_magic 321 def clear(self, arg_s): 322 """Clear the terminal.""" 323 if os.name == 'posix': 324 self.shell.system("clear") 325 else: 326 self.shell.system("cls") 327 328 if os.name == 'nt': 329 # This is the usual name in windows 330 cls = line_magic('cls')(clear) 331 332 # Terminal pagers won't work over pexpect, but we do have our own pager 333 334 @line_magic 335 def less(self, arg_s): 336 """Show a file through the pager. 337 338 Files ending in .py are syntax-highlighted.""" 339 if not arg_s: 340 raise UsageError('Missing filename.') 341 342 if arg_s.endswith('.py'): 343 cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False)) 344 else: 345 cont = open(arg_s).read() 346 page.page(cont) 347 348 more = line_magic('more')(less) 349 350 # Man calls a pager, so we also need to redefine it 351 if os.name == 'posix': 352 @line_magic 353 def man(self, arg_s): 354 """Find the man page for the given command and display in pager.""" 355 page.page(self.shell.getoutput('man %s | col -b' % arg_s, 356 split=False)) 357 358 @line_magic 359 def connect_info(self, arg_s): 360 """Print information for connecting other clients to this kernel 361 362 It will print the contents of this session's connection file, as well as 363 shortcuts for local clients. 364 365 In the simplest case, when called from the most recently launched kernel, 366 secondary clients can be connected, simply with: 367 368 $> jupyter <app> --existing 369 370 """ 371 372 try: 373 connection_file = get_connection_file() 374 info = get_connection_info(unpack=False) 375 except Exception as e: 376 warnings.warn("Could not get connection info: %r" % e) 377 return 378 379 # if it's in the default dir, truncate to basename 380 if jupyter_runtime_dir() == os.path.dirname(connection_file): 381 connection_file = os.path.basename(connection_file) 382 383 384 print (info + '\n') 385 print ("Paste the above JSON into a file, and connect with:\n" 386 " $> jupyter <app> --existing <file>\n" 387 "or, if you are local, you can connect with just:\n" 388 " $> jupyter <app> --existing {0}\n" 389 "or even just:\n" 390 " $> jupyter <app> --existing\n" 391 "if this is the most recent Jupyter kernel you have started.".format( 392 connection_file 393 ) 394 ) 395 396 @line_magic 397 def qtconsole(self, arg_s): 398 """Open a qtconsole connected to this kernel. 399 400 Useful for connecting a qtconsole to running notebooks, for better 401 debugging. 402 """ 403 404 # %qtconsole should imply bind_kernel for engines: 405 # FIXME: move to ipyparallel Kernel subclass 406 if 'ipyparallel' in sys.modules: 407 from ipyparallel import bind_kernel 408 bind_kernel() 409 410 try: 411 connect_qtconsole(argv=arg_split(arg_s, os.name=='posix')) 412 except Exception as e: 413 warnings.warn("Could not start qtconsole: %r" % e) 414 return 415 416 @line_magic 417 def autosave(self, arg_s): 418 """Set the autosave interval in the notebook (in seconds). 419 420 The default value is 120, or two minutes. 421 ``%autosave 0`` will disable autosave. 422 423 This magic only has an effect when called from the notebook interface. 424 It has no effect when called in a startup file. 425 """ 426 427 try: 428 interval = int(arg_s) 429 except ValueError as e: 430 raise UsageError("%%autosave requires an integer, got %r" % arg_s) from e 431 432 # javascript wants milliseconds 433 milliseconds = 1000 * interval 434 display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds), 435 include=['application/javascript'] 436 ) 437 if interval: 438 print("Autosaving every %i seconds" % interval) 439 else: 440 print("Autosave disabled") 441 442 443class ZMQInteractiveShell(InteractiveShell): 444 """A subclass of InteractiveShell for ZMQ.""" 445 446 displayhook_class = Type(ZMQShellDisplayHook) 447 display_pub_class = Type(ZMQDisplayPublisher) 448 data_pub_class = Type(ZMQDataPublisher) 449 kernel = Any() 450 parent_header = Any() 451 452 @default('banner1') 453 def _default_banner1(self): 454 return default_banner 455 456 # Override the traitlet in the parent class, because there's no point using 457 # readline for the kernel. Can be removed when the readline code is moved 458 # to the terminal frontend. 459 readline_use = CBool(False) 460 # autoindent has no meaning in a zmqshell, and attempting to enable it 461 # will print a warning in the absence of readline. 462 autoindent = CBool(False) 463 464 exiter = Instance(ZMQExitAutocall) 465 466 @default('exiter') 467 def _default_exiter(self): 468 return ZMQExitAutocall(self) 469 470 @observe('exit_now') 471 def _update_exit_now(self, change): 472 """stop eventloop when exit_now fires""" 473 if change['new']: 474 loop = self.kernel.io_loop 475 loop.call_later(0.1, loop.stop) 476 if self.kernel.eventloop: 477 exit_hook = getattr(self.kernel.eventloop, 'exit_hook', None) 478 if exit_hook: 479 exit_hook(self.kernel) 480 481 keepkernel_on_exit = None 482 483 # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no 484 # interactive input being read; we provide event loop support in ipkernel 485 def enable_gui(self, gui): 486 from .eventloops import enable_gui as real_enable_gui 487 try: 488 real_enable_gui(gui) 489 self.active_eventloop = gui 490 except ValueError as e: 491 raise UsageError("%s" % e) from e 492 493 def init_environment(self): 494 """Configure the user's environment.""" 495 env = os.environ 496 # These two ensure 'ls' produces nice coloring on BSD-derived systems 497 env['TERM'] = 'xterm-color' 498 env['CLICOLOR'] = '1' 499 # Since normal pagers don't work at all (over pexpect we don't have 500 # single-key control of the subprocess), try to disable paging in 501 # subprocesses as much as possible. 502 env['PAGER'] = 'cat' 503 env['GIT_PAGER'] = 'cat' 504 505 def init_hooks(self): 506 super(ZMQInteractiveShell, self).init_hooks() 507 self.set_hook('show_in_pager', page.as_hook(payloadpage.page), 99) 508 509 def init_data_pub(self): 510 """Delay datapub init until request, for deprecation warnings""" 511 pass 512 513 @property 514 def data_pub(self): 515 if not hasattr(self, '_data_pub'): 516 warnings.warn("InteractiveShell.data_pub is deprecated outside IPython parallel.", 517 DeprecationWarning, stacklevel=2) 518 519 self._data_pub = self.data_pub_class(parent=self) 520 self._data_pub.session = self.display_pub.session 521 self._data_pub.pub_socket = self.display_pub.pub_socket 522 return self._data_pub 523 524 @data_pub.setter 525 def data_pub(self, pub): 526 self._data_pub = pub 527 528 def ask_exit(self): 529 """Engage the exit actions.""" 530 self.exit_now = (not self.keepkernel_on_exit) 531 payload = dict( 532 source='ask_exit', 533 keepkernel=self.keepkernel_on_exit, 534 ) 535 self.payload_manager.write_payload(payload) 536 537 def run_cell(self, *args, **kwargs): 538 self._last_traceback = None 539 return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) 540 541 def _showtraceback(self, etype, evalue, stb): 542 # try to preserve ordering of tracebacks and print statements 543 sys.stdout.flush() 544 sys.stderr.flush() 545 546 exc_content = { 547 'traceback' : stb, 548 'ename' : str(etype.__name__), 549 'evalue' : py3compat.safe_unicode(evalue), 550 } 551 552 dh = self.displayhook 553 # Send exception info over pub socket for other clients than the caller 554 # to pick up 555 topic = None 556 if dh.topic: 557 topic = dh.topic.replace(b'execute_result', b'error') 558 559 exc_msg = dh.session.send(dh.pub_socket, 'error', json_clean(exc_content), 560 dh.parent_header, ident=topic) 561 562 # FIXME - Once we rely on Python 3, the traceback is stored on the 563 # exception object, so we shouldn't need to store it here. 564 self._last_traceback = stb 565 566 def set_next_input(self, text, replace=False): 567 """Send the specified text to the frontend to be presented at the next 568 input cell.""" 569 payload = dict( 570 source='set_next_input', 571 text=text, 572 replace=replace, 573 ) 574 self.payload_manager.write_payload(payload) 575 576 def set_parent(self, parent): 577 """Set the parent header for associating output with its triggering input""" 578 self.parent_header = parent 579 self.displayhook.set_parent(parent) 580 self.display_pub.set_parent(parent) 581 if hasattr(self, '_data_pub'): 582 self.data_pub.set_parent(parent) 583 try: 584 sys.stdout.set_parent(parent) 585 except AttributeError: 586 pass 587 try: 588 sys.stderr.set_parent(parent) 589 except AttributeError: 590 pass 591 592 def get_parent(self): 593 return self.parent_header 594 595 def init_magics(self): 596 super(ZMQInteractiveShell, self).init_magics() 597 self.register_magics(KernelMagics) 598 self.magics_manager.register_alias('ed', 'edit') 599 600 def enable_matplotlib(self, gui=None): 601 gui, backend = super(ZMQInteractiveShell, self).enable_matplotlib(gui) 602 603 from ipykernel.pylab.backend_inline import configure_inline_support 604 605 configure_inline_support(self, backend) 606 607 return gui, backend 608 609 def init_virtualenv(self): 610 # Overridden not to do virtualenv detection, because it's probably 611 # not appropriate in a kernel. To use a kernel in a virtualenv, install 612 # it inside the virtualenv. 613 # https://ipython.readthedocs.io/en/latest/install/kernel_install.html 614 pass 615 616 def system_piped(self, cmd): 617 """Call the given cmd in a subprocess, piping stdout/err 618 619 Parameters 620 ---------- 621 cmd : str 622 Command to execute (can not end in '&', as background processes are 623 not supported. Should not be a command that expects input 624 other than simple text. 625 """ 626 if cmd.rstrip().endswith('&'): 627 # this is *far* from a rigorous test 628 # We do not support backgrounding processes because we either use 629 # pexpect or pipes to read from. Users can always just call 630 # os.system() or use ip.system=ip.system_raw 631 # if they really want a background process. 632 raise OSError("Background processes not supported.") 633 634 # we explicitly do NOT return the subprocess status code, because 635 # a non-None value would trigger :func:`sys.displayhook` calls. 636 # Instead, we store the exit_code in user_ns. 637 # Also, protect system call from UNC paths on Windows here too 638 # as is done in InteractiveShell.system_raw 639 if sys.platform == 'win32': 640 cmd = self.var_expand(cmd, depth=1) 641 from IPython.utils._process_win32 import AvoidUNCPath 642 with AvoidUNCPath() as path: 643 if path is not None: 644 cmd = 'pushd %s &&%s' % (path, cmd) 645 self.user_ns['_exit_code'] = system(cmd) 646 else: 647 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1)) 648 649 # Ensure new system_piped implementation is used 650 system = system_piped 651 652 653InteractiveShellABC.register(ZMQInteractiveShell) 654