1"""A ZMQ-based subclass of InteractiveShell. 2 3This code is meant to ease the refactoring of the base InteractiveShell into 4something with a cleaner architecture for 2-process use, without actually 5breaking InteractiveShell itself. So we're doing something a bit ugly, where 6we subclass and override what we want to fix. Once this is working well, we 7can go back to the base class and refactor the code for a cleaner inheritance 8implementation that doesn't rely on so much monkeypatching. 9 10But this lets us maintain a fully working IPython as we develop the new 11machinery. This should thus be thought of as scaffolding. 12""" 13 14# Copyright (c) IPython Development Team. 15# Distributed under the terms of the Modified BSD License. 16 17import os 18import sys 19import warnings 20from threading import local 21 22from IPython.core.interactiveshell import ( 23 InteractiveShell, InteractiveShellABC 24) 25from IPython.core import page 26from IPython.core.autocall import ZMQExitAutocall 27from IPython.core.displaypub import DisplayPublisher 28from IPython.core.error import UsageError 29from IPython.core.magics import MacroToEdit, CodeMagics 30from IPython.core.magic import magics_class, line_magic, Magics 31from IPython.core import payloadpage 32from IPython.core.usage import default_banner 33from IPython.display import display, Javascript 34from ipykernel import ( 35 get_connection_file, get_connection_info, connect_qtconsole 36) 37from IPython.utils import openpy 38from ipykernel.jsonutil import json_clean, encode_images 39from IPython.utils.process import arg_split, system 40from traitlets import ( 41 Instance, Type, Dict, CBool, CBytes, Any, default, observe 42) 43from ipykernel.displayhook import ZMQShellDisplayHook 44 45from jupyter_core.paths import jupyter_runtime_dir 46from jupyter_client.session import extract_header, Session 47 48#----------------------------------------------------------------------------- 49# Functions and classes 50#----------------------------------------------------------------------------- 51 52 53class ZMQDisplayPublisher(DisplayPublisher): 54 """A display publisher that publishes data using a ZeroMQ PUB socket.""" 55 56 session = Instance(Session, allow_none=True) 57 pub_socket = Any(allow_none=True) 58 parent_header = Dict({}) 59 topic = CBytes(b'display_data') 60 61 # thread_local: 62 # An attribute used to ensure the correct output message 63 # is processed. See ipykernel Issue 113 for a discussion. 64 _thread_local = Any() 65 66 def set_parent(self, parent): 67 """Set the parent for outbound messages.""" 68 self.parent_header = extract_header(parent) 69 70 def _flush_streams(self): 71 """flush IO Streams prior to display""" 72 sys.stdout.flush() 73 sys.stderr.flush() 74 75 @default('_thread_local') 76 def _default_thread_local(self): 77 """Initialize our thread local storage""" 78 return local() 79 80 @property 81 def _hooks(self): 82 if not hasattr(self._thread_local, 'hooks'): 83 # create new list for a new thread 84 self._thread_local.hooks = [] 85 return self._thread_local.hooks 86 87 def publish( 88 self, 89 data, 90 metadata=None, 91 transient=None, 92 update=False, 93 ): 94 """Publish a display-data message 95 96 Parameters 97 ---------- 98 data : dict 99 A mime-bundle dict, keyed by mime-type. 100 metadata : dict, optional 101 Metadata associated with the data. 102 transient : dict, optional, keyword-only 103 Transient data that may only be relevant during a live display, 104 such as display_id. 105 Transient data should not be persisted to documents. 106 update : bool, optional, keyword-only 107 If True, send an update_display_data message instead of display_data. 108 """ 109 self._flush_streams() 110 if metadata is None: 111 metadata = {} 112 if transient is None: 113 transient = {} 114 self._validate_data(data, metadata) 115 content = {} 116 content['data'] = encode_images(data) 117 content['metadata'] = metadata 118 content['transient'] = transient 119 120 msg_type = 'update_display_data' if update else 'display_data' 121 122 # Use 2-stage process to send a message, 123 # in order to put it through the transform 124 # hooks before potentially sending. 125 msg = self.session.msg( 126 msg_type, json_clean(content), 127 parent=self.parent_header 128 ) 129 130 # Each transform either returns a new 131 # message or None. If None is returned, 132 # the message has been 'used' and we return. 133 for hook in self._hooks: 134 msg = hook(msg) 135 if msg is None: 136 return 137 138 self.session.send( 139 self.pub_socket, msg, ident=self.topic, 140 ) 141 142 def clear_output(self, wait=False): 143 """Clear output associated with the current execution (cell). 144 145 Parameters 146 ---------- 147 wait : bool (default: False) 148 If True, the output will not be cleared immediately, 149 instead waiting for the next display before clearing. 150 This reduces bounce during repeated clear & display loops. 151 152 """ 153 content = dict(wait=wait) 154 self._flush_streams() 155 self.session.send( 156 self.pub_socket, 'clear_output', content, 157 parent=self.parent_header, ident=self.topic, 158 ) 159 160 def register_hook(self, hook): 161 """ 162 Registers a hook with the thread-local storage. 163 164 Parameters 165 ---------- 166 hook : Any callable object 167 168 Returns 169 ------- 170 Either a publishable message, or `None`. 171 The DisplayHook objects must return a message from 172 the __call__ method if they still require the 173 `session.send` method to be called after transformation. 174 Returning `None` will halt that execution path, and 175 session.send will not be called. 176 """ 177 self._hooks.append(hook) 178 179 def unregister_hook(self, hook): 180 """ 181 Un-registers a hook with the thread-local storage. 182 183 Parameters 184 ---------- 185 hook : Any callable object which has previously been 186 registered as a hook. 187 188 Returns 189 ------- 190 bool - `True` if the hook was removed, `False` if it wasn't 191 found. 192 """ 193 try: 194 self._hooks.remove(hook) 195 return True 196 except ValueError: 197 return False 198 199 200@magics_class 201class KernelMagics(Magics): 202 #------------------------------------------------------------------------ 203 # Magic overrides 204 #------------------------------------------------------------------------ 205 # Once the base class stops inheriting from magic, this code needs to be 206 # moved into a separate machinery as well. For now, at least isolate here 207 # the magics which this class needs to implement differently from the base 208 # class, or that are unique to it. 209 210 _find_edit_target = CodeMagics._find_edit_target 211 212 @line_magic 213 def edit(self, parameter_s='', last_call=['','']): 214 """Bring up an editor and execute the resulting code. 215 216 Usage: 217 %edit [options] [args] 218 219 %edit runs an external text editor. You will need to set the command for 220 this editor via the ``TerminalInteractiveShell.editor`` option in your 221 configuration file before it will work. 222 223 This command allows you to conveniently edit multi-line code right in 224 your IPython session. 225 226 If called without arguments, %edit opens up an empty editor with a 227 temporary file and will execute the contents of this file when you 228 close it (don't forget to save it!). 229 230 Options: 231 232 -n <number> 233 Open the editor at a specified line number. By default, the IPython 234 editor hook uses the unix syntax 'editor +N filename', but you can 235 configure this by providing your own modified hook if your favorite 236 editor supports line-number specifications with a different syntax. 237 238 -p 239 Call the editor with the same data as the previous time it was used, 240 regardless of how long ago (in your current session) it was. 241 242 -r 243 Use 'raw' input. This option only applies to input taken from the 244 user's history. By default, the 'processed' history is used, so that 245 magics are loaded in their transformed version to valid Python. If 246 this option is given, the raw input as typed as the command line is 247 used instead. When you exit the editor, it will be executed by 248 IPython's own processor. 249 250 Arguments: 251 252 If arguments are given, the following possibilities exist: 253 254 - The arguments are numbers or pairs of colon-separated numbers (like 255 1 4:8 9). These are interpreted as lines of previous input to be 256 loaded into the editor. The syntax is the same of the %macro command. 257 258 - If the argument doesn't start with a number, it is evaluated as a 259 variable and its contents loaded into the editor. You can thus edit 260 any string which contains python code (including the result of 261 previous edits). 262 263 - If the argument is the name of an object (other than a string), 264 IPython will try to locate the file where it was defined and open the 265 editor at the point where it is defined. You can use ``%edit function`` 266 to load an editor exactly at the point where 'function' is defined, 267 edit it and have the file be executed automatically. 268 269 If the object is a macro (see %macro for details), this opens up your 270 specified editor with a temporary file containing the macro's data. 271 Upon exit, the macro is reloaded with the contents of the file. 272 273 Note: opening at an exact line is only supported under Unix, and some 274 editors (like kedit and gedit up to Gnome 2.8) do not understand the 275 '+NUMBER' parameter necessary for this feature. Good editors like 276 (X)Emacs, vi, jed, pico and joe all do. 277 278 - If the argument is not found as a variable, IPython will look for a 279 file with that name (adding .py if necessary) and load it into the 280 editor. It will execute its contents with execfile() when you exit, 281 loading any code in the file into your interactive namespace. 282 283 Unlike in the terminal, this is designed to use a GUI editor, and we do 284 not know when it has closed. So the file you edit will not be 285 automatically executed or printed. 286 287 Note that %edit is also available through the alias %ed. 288 """ 289 290 opts,args = self.parse_options(parameter_s, 'prn:') 291 292 try: 293 filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call) 294 except MacroToEdit: 295 # TODO: Implement macro editing over 2 processes. 296 print("Macro editing not yet implemented in 2-process model.") 297 return 298 299 # Make sure we send to the client an absolute path, in case the working 300 # directory of client and kernel don't match 301 filename = os.path.abspath(filename) 302 303 payload = { 304 'source' : 'edit_magic', 305 'filename' : filename, 306 'line_number' : lineno 307 } 308 self.shell.payload_manager.write_payload(payload) 309 310 # A few magics that are adapted to the specifics of using pexpect and a 311 # remote terminal 312 313 @line_magic 314 def clear(self, arg_s): 315 """Clear the terminal.""" 316 if os.name == 'posix': 317 self.shell.system("clear") 318 else: 319 self.shell.system("cls") 320 321 if os.name == 'nt': 322 # This is the usual name in windows 323 cls = line_magic('cls')(clear) 324 325 # Terminal pagers won't work over pexpect, but we do have our own pager 326 327 @line_magic 328 def less(self, arg_s): 329 """Show a file through the pager. 330 331 Files ending in .py are syntax-highlighted.""" 332 if not arg_s: 333 raise UsageError('Missing filename.') 334 335 if arg_s.endswith('.py'): 336 cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False)) 337 else: 338 cont = open(arg_s).read() 339 page.page(cont) 340 341 more = line_magic('more')(less) 342 343 # Man calls a pager, so we also need to redefine it 344 if os.name == 'posix': 345 @line_magic 346 def man(self, arg_s): 347 """Find the man page for the given command and display in pager.""" 348 page.page(self.shell.getoutput('man %s | col -b' % arg_s, 349 split=False)) 350 351 @line_magic 352 def connect_info(self, arg_s): 353 """Print information for connecting other clients to this kernel 354 355 It will print the contents of this session's connection file, as well as 356 shortcuts for local clients. 357 358 In the simplest case, when called from the most recently launched kernel, 359 secondary clients can be connected, simply with: 360 361 $> jupyter <app> --existing 362 363 """ 364 365 try: 366 connection_file = get_connection_file() 367 info = get_connection_info(unpack=False) 368 except Exception as e: 369 warnings.warn("Could not get connection info: %r" % e) 370 return 371 372 # if it's in the default dir, truncate to basename 373 if jupyter_runtime_dir() == os.path.dirname(connection_file): 374 connection_file = os.path.basename(connection_file) 375 376 377 print (info + '\n') 378 print ("Paste the above JSON into a file, and connect with:\n" 379 " $> jupyter <app> --existing <file>\n" 380 "or, if you are local, you can connect with just:\n" 381 " $> jupyter <app> --existing {0}\n" 382 "or even just:\n" 383 " $> jupyter <app> --existing\n" 384 "if this is the most recent Jupyter kernel you have started.".format( 385 connection_file 386 ) 387 ) 388 389 @line_magic 390 def qtconsole(self, arg_s): 391 """Open a qtconsole connected to this kernel. 392 393 Useful for connecting a qtconsole to running notebooks, for better 394 debugging. 395 """ 396 397 # %qtconsole should imply bind_kernel for engines: 398 # FIXME: move to ipyparallel Kernel subclass 399 if 'ipyparallel' in sys.modules: 400 from ipyparallel import bind_kernel 401 bind_kernel() 402 403 try: 404 connect_qtconsole(argv=arg_split(arg_s, os.name=='posix')) 405 except Exception as e: 406 warnings.warn("Could not start qtconsole: %r" % e) 407 return 408 409 @line_magic 410 def autosave(self, arg_s): 411 """Set the autosave interval in the notebook (in seconds). 412 413 The default value is 120, or two minutes. 414 ``%autosave 0`` will disable autosave. 415 416 This magic only has an effect when called from the notebook interface. 417 It has no effect when called in a startup file. 418 """ 419 420 try: 421 interval = int(arg_s) 422 except ValueError as e: 423 raise UsageError("%%autosave requires an integer, got %r" % arg_s) from e 424 425 # javascript wants milliseconds 426 milliseconds = 1000 * interval 427 display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds), 428 include=['application/javascript'] 429 ) 430 if interval: 431 print("Autosaving every %i seconds" % interval) 432 else: 433 print("Autosave disabled") 434 435 436class ZMQInteractiveShell(InteractiveShell): 437 """A subclass of InteractiveShell for ZMQ.""" 438 439 displayhook_class = Type(ZMQShellDisplayHook) 440 display_pub_class = Type(ZMQDisplayPublisher) 441 data_pub_class = Any() 442 kernel = Any() 443 parent_header = Any() 444 445 @default('banner1') 446 def _default_banner1(self): 447 return default_banner 448 449 # Override the traitlet in the parent class, because there's no point using 450 # readline for the kernel. Can be removed when the readline code is moved 451 # to the terminal frontend. 452 readline_use = CBool(False) 453 # autoindent has no meaning in a zmqshell, and attempting to enable it 454 # will print a warning in the absence of readline. 455 autoindent = CBool(False) 456 457 exiter = Instance(ZMQExitAutocall) 458 459 @default('exiter') 460 def _default_exiter(self): 461 return ZMQExitAutocall(self) 462 463 @observe('exit_now') 464 def _update_exit_now(self, change): 465 """stop eventloop when exit_now fires""" 466 if change['new']: 467 if hasattr(self.kernel, 'io_loop'): 468 loop = self.kernel.io_loop 469 loop.call_later(0.1, loop.stop) 470 if self.kernel.eventloop: 471 exit_hook = getattr(self.kernel.eventloop, 'exit_hook', None) 472 if exit_hook: 473 exit_hook(self.kernel) 474 475 keepkernel_on_exit = None 476 477 # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no 478 # interactive input being read; we provide event loop support in ipkernel 479 def enable_gui(self, gui): 480 from .eventloops import enable_gui as real_enable_gui 481 try: 482 real_enable_gui(gui) 483 self.active_eventloop = gui 484 except ValueError as e: 485 raise UsageError("%s" % e) from e 486 487 def init_environment(self): 488 """Configure the user's environment.""" 489 env = os.environ 490 # These two ensure 'ls' produces nice coloring on BSD-derived systems 491 env['TERM'] = 'xterm-color' 492 env['CLICOLOR'] = '1' 493 # Since normal pagers don't work at all (over pexpect we don't have 494 # single-key control of the subprocess), try to disable paging in 495 # subprocesses as much as possible. 496 env['PAGER'] = 'cat' 497 env['GIT_PAGER'] = 'cat' 498 499 def init_hooks(self): 500 super(ZMQInteractiveShell, self).init_hooks() 501 self.set_hook('show_in_pager', page.as_hook(payloadpage.page), 99) 502 503 def init_data_pub(self): 504 """Delay datapub init until request, for deprecation warnings""" 505 pass 506 507 @property 508 def data_pub(self): 509 if not hasattr(self, '_data_pub'): 510 warnings.warn("InteractiveShell.data_pub is deprecated outside IPython parallel.", 511 DeprecationWarning, stacklevel=2) 512 513 self._data_pub = self.data_pub_class(parent=self) 514 self._data_pub.session = self.display_pub.session 515 self._data_pub.pub_socket = self.display_pub.pub_socket 516 return self._data_pub 517 518 @data_pub.setter 519 def data_pub(self, pub): 520 self._data_pub = pub 521 522 def ask_exit(self): 523 """Engage the exit actions.""" 524 self.exit_now = (not self.keepkernel_on_exit) 525 payload = dict( 526 source='ask_exit', 527 keepkernel=self.keepkernel_on_exit, 528 ) 529 self.payload_manager.write_payload(payload) 530 531 def run_cell(self, *args, **kwargs): 532 self._last_traceback = None 533 return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) 534 535 def _showtraceback(self, etype, evalue, stb): 536 # try to preserve ordering of tracebacks and print statements 537 sys.stdout.flush() 538 sys.stderr.flush() 539 540 exc_content = { 541 'traceback' : stb, 542 'ename' : str(etype.__name__), 543 'evalue' : str(evalue), 544 } 545 546 dh = self.displayhook 547 # Send exception info over pub socket for other clients than the caller 548 # to pick up 549 topic = None 550 if dh.topic: 551 topic = dh.topic.replace(b'execute_result', b'error') 552 553 dh.session.send( 554 dh.pub_socket, 555 "error", 556 json_clean(exc_content), 557 dh.parent_header, 558 ident=topic, 559 ) 560 561 # FIXME - Once we rely on Python 3, the traceback is stored on the 562 # exception object, so we shouldn't need to store it here. 563 self._last_traceback = stb 564 565 def set_next_input(self, text, replace=False): 566 """Send the specified text to the frontend to be presented at the next 567 input cell.""" 568 payload = dict( 569 source='set_next_input', 570 text=text, 571 replace=replace, 572 ) 573 self.payload_manager.write_payload(payload) 574 575 def set_parent(self, parent): 576 """Set the parent header for associating output with its triggering input""" 577 self.parent_header = parent 578 self.displayhook.set_parent(parent) 579 self.display_pub.set_parent(parent) 580 if hasattr(self, '_data_pub'): 581 self.data_pub.set_parent(parent) 582 try: 583 sys.stdout.set_parent(parent) 584 except AttributeError: 585 pass 586 try: 587 sys.stderr.set_parent(parent) 588 except AttributeError: 589 pass 590 591 def get_parent(self): 592 return self.parent_header 593 594 def init_magics(self): 595 super(ZMQInteractiveShell, self).init_magics() 596 self.register_magics(KernelMagics) 597 self.magics_manager.register_alias('ed', 'edit') 598 599 def init_virtualenv(self): 600 # Overridden not to do virtualenv detection, because it's probably 601 # not appropriate in a kernel. To use a kernel in a virtualenv, install 602 # it inside the virtualenv. 603 # https://ipython.readthedocs.io/en/latest/install/kernel_install.html 604 pass 605 606 def system_piped(self, cmd): 607 """Call the given cmd in a subprocess, piping stdout/err 608 609 Parameters 610 ---------- 611 cmd : str 612 Command to execute (can not end in '&', as background processes are 613 not supported. Should not be a command that expects input 614 other than simple text. 615 """ 616 if cmd.rstrip().endswith('&'): 617 # this is *far* from a rigorous test 618 # We do not support backgrounding processes because we either use 619 # pexpect or pipes to read from. Users can always just call 620 # os.system() or use ip.system=ip.system_raw 621 # if they really want a background process. 622 raise OSError("Background processes not supported.") 623 624 # we explicitly do NOT return the subprocess status code, because 625 # a non-None value would trigger :func:`sys.displayhook` calls. 626 # Instead, we store the exit_code in user_ns. 627 # Also, protect system call from UNC paths on Windows here too 628 # as is done in InteractiveShell.system_raw 629 if sys.platform == 'win32': 630 cmd = self.var_expand(cmd, depth=1) 631 from IPython.utils._process_win32 import AvoidUNCPath 632 with AvoidUNCPath() as path: 633 if path is not None: 634 cmd = 'pushd %s &&%s' % (path, cmd) 635 self.user_ns['_exit_code'] = system(cmd) 636 else: 637 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1)) 638 639 # Ensure new system_piped implementation is used 640 system = system_piped 641 642 643InteractiveShellABC.register(ZMQInteractiveShell) 644