1# -*- coding: utf-8 -*- 2 3"""Code for maintaining the background process and for running 4user programs 5 6Commands get executed via shell, this way the command line in the 7shell becomes kind of title for the execution. 8 9""" 10 11 12import collections 13import logging 14import os.path 15import re 16import shlex 17import signal 18import subprocess 19import traceback 20 21import sys 22import time 23import tkinter as tk 24import warnings 25from logging import debug 26from threading import Thread 27from time import sleep 28from tkinter import messagebox, ttk 29from typing import Any, List, Optional, Set, Union, Callable # @UnusedImport; @UnusedImport 30 31import thonny 32from thonny import THONNY_USER_DIR, common, get_runner, get_shell, get_workbench 33from thonny.common import ( 34 BackendEvent, 35 CommandToBackend, 36 DebuggerCommand, 37 DebuggerResponse, 38 EOFCommand, 39 InlineCommand, 40 InputSubmission, 41 ToplevelCommand, 42 ToplevelResponse, 43 UserError, 44 is_same_path, 45 normpath_with_actual_case, 46 parse_message, 47 path_startswith, 48 serialize_message, 49 update_system_path, 50 MessageFromBackend, 51 universal_relpath, 52) 53from thonny.editors import ( 54 get_current_breakpoints, 55 get_saved_current_script_filename, 56 is_remote_path, 57 is_local_path, 58 get_target_dirname_from_editor_filename, 59 extract_target_path, 60) 61from thonny.languages import tr 62from thonny.misc_utils import ( 63 construct_cmd_line, 64 inside_flatpak, 65 running_on_mac_os, 66 running_on_windows, 67 show_command_not_available_in_flatpak_message, 68) 69from thonny.ui_utils import CommonDialogEx, select_sequence, show_dialog 70from thonny.workdlg import WorkDialog 71 72logger = logging.getLogger(__name__) 73 74WINDOWS_EXE = "python.exe" 75OUTPUT_MERGE_THRESHOLD = 1000 76 77RUN_COMMAND_LABEL = "" # init later when gettext is ready 78RUN_COMMAND_CAPTION = "" 79EDITOR_CONTENT_TOKEN = "$EDITOR_CONTENT" 80 81EXPECTED_TERMINATION_CODE = 123 82 83INTERRUPT_SEQUENCE = "<Control-c>" 84 85ANSI_CODE_TERMINATOR = re.compile("[@-~]") 86 87# other components may turn it on in order to avoid grouping output lines into one event 88io_animation_required = False 89 90_console_allocated = False 91 92 93class Runner: 94 def __init__(self) -> None: 95 get_workbench().set_default("run.auto_cd", True) 96 97 self._init_commands() 98 self._state = "starting" 99 self._proxy = None # type: BackendProxy 100 self._publishing_events = False 101 self._polling_after_id = None 102 self._postponed_commands = [] # type: List[CommandToBackend] 103 104 def _remove_obsolete_jedi_copies(self) -> None: 105 # Thonny 2.1 used to copy jedi in order to make it available 106 # for the backend. Get rid of it now 107 for item in os.listdir(THONNY_USER_DIR): 108 if item.startswith("jedi_0."): 109 import shutil 110 111 shutil.rmtree(os.path.join(THONNY_USER_DIR, item), True) 112 113 def start(self) -> None: 114 global _console_allocated 115 try: 116 self._check_alloc_console() 117 _console_allocated = True 118 except Exception: 119 logger.exception("Problem allocating console") 120 _console_allocated = False 121 122 self.restart_backend(False, True) 123 # temporary 124 self._remove_obsolete_jedi_copies() 125 126 def _init_commands(self) -> None: 127 global RUN_COMMAND_CAPTION, RUN_COMMAND_LABEL 128 129 RUN_COMMAND_LABEL = tr("Run current script") 130 RUN_COMMAND_CAPTION = tr("Run") 131 132 get_workbench().set_default("run.run_in_terminal_python_repl", False) 133 get_workbench().set_default("run.run_in_terminal_keep_open", True) 134 135 try: 136 import thonny.plugins.debugger # @UnusedImport 137 138 debugger_available = True 139 except ImportError: 140 debugger_available = False 141 142 get_workbench().add_command( 143 "run_current_script", 144 "run", 145 RUN_COMMAND_LABEL, 146 caption=RUN_COMMAND_CAPTION, 147 handler=self.cmd_run_current_script, 148 default_sequence="<F5>", 149 extra_sequences=[select_sequence("<Control-r>", "<Command-r>")], 150 tester=self.cmd_run_current_script_enabled, 151 group=10, 152 image="run-current-script", 153 include_in_toolbar=not (get_workbench().in_simple_mode() and debugger_available), 154 show_extra_sequences=True, 155 ) 156 157 get_workbench().add_command( 158 "run_current_script_in_terminal", 159 "run", 160 tr("Run current script in terminal"), 161 caption="RunT", 162 handler=self._cmd_run_current_script_in_terminal, 163 default_sequence="<Control-t>", 164 extra_sequences=["<<CtrlTInText>>"], 165 tester=self._cmd_run_current_script_in_terminal_enabled, 166 group=35, 167 image="terminal", 168 ) 169 170 get_workbench().add_command( 171 "restart", 172 "run", 173 tr("Stop/Restart backend"), 174 caption=tr("Stop"), 175 handler=self.cmd_stop_restart, 176 default_sequence="<Control-F2>", 177 group=100, 178 image="stop", 179 include_in_toolbar=True, 180 ) 181 182 get_workbench().add_command( 183 "interrupt", 184 "run", 185 tr("Interrupt execution"), 186 handler=self._cmd_interrupt, 187 tester=self._cmd_interrupt_enabled, 188 default_sequence=INTERRUPT_SEQUENCE, 189 skip_sequence_binding=True, # Sequence will be bound differently 190 group=100, 191 bell_when_denied=False, 192 ) 193 get_workbench().bind(INTERRUPT_SEQUENCE, self._cmd_interrupt_with_shortcut, True) 194 195 get_workbench().add_command( 196 "ctrld", 197 "run", 198 tr("Send EOF / Soft reboot"), 199 self.ctrld, 200 self.ctrld_enabled, 201 group=100, 202 default_sequence="<Control-d>", 203 extra_sequences=["<<CtrlDInText>>"], 204 ) 205 206 get_workbench().add_command( 207 "disconnect", 208 "run", 209 tr("Disconnect"), 210 self.disconnect, 211 self.disconnect_enabled, 212 group=100, 213 ) 214 215 def get_state(self) -> str: 216 """State is one of "running", "waiting_debugger_command", "waiting_toplevel_command" """ 217 return self._state 218 219 def _set_state(self, state: str) -> None: 220 if self._state != state: 221 logging.debug("Runner state changed: %s ==> %s" % (self._state, state)) 222 self._state = state 223 224 def is_running(self): 225 return self._state == "running" 226 227 def is_waiting(self): 228 return self._state.startswith("waiting") 229 230 def is_waiting_toplevel_command(self): 231 return self._state == "waiting_toplevel_command" 232 233 def is_waiting_debugger_command(self): 234 return self._state == "waiting_debugger_command" 235 236 def get_sys_path(self) -> List[str]: 237 return self._proxy.get_sys_path() 238 239 def send_command(self, cmd: CommandToBackend) -> None: 240 if self._proxy is None: 241 return 242 243 if self._publishing_events: 244 # allow all event handlers to complete before sending the commands 245 # issued by first event handlers 246 self._postpone_command(cmd) 247 return 248 249 # First sanity check 250 if ( 251 isinstance(cmd, ToplevelCommand) 252 and not self.is_waiting_toplevel_command() 253 and cmd.name not in ["Reset", "Run", "Debug"] 254 or isinstance(cmd, DebuggerCommand) 255 and not self.is_waiting_debugger_command() 256 ): 257 get_workbench().bell() 258 logging.warning( 259 "RUNNER: Command %s was attempted at state %s" % (cmd, self.get_state()) 260 ) 261 return 262 263 # Attach extra info 264 if "debug" in cmd.name.lower(): 265 cmd["breakpoints"] = get_current_breakpoints() 266 267 if "id" not in cmd: 268 cmd["id"] = generate_command_id() 269 270 cmd["local_cwd"] = get_workbench().get_local_cwd() 271 272 # Offer the command 273 logging.debug("RUNNER Sending: %s, %s", cmd.name, cmd) 274 response = self._proxy.send_command(cmd) 275 276 if response == "discard": 277 return None 278 elif response == "postpone": 279 self._postpone_command(cmd) 280 return 281 else: 282 assert response is None 283 get_workbench().event_generate("CommandAccepted", command=cmd) 284 285 if isinstance(cmd, (ToplevelCommand, DebuggerCommand)): 286 self._set_state("running") 287 288 if cmd.name[0].isupper(): 289 # This may be only logical restart, which does not look like restart to the runner 290 get_workbench().event_generate("BackendRestart", full=False) 291 292 def send_command_and_wait(self, cmd: CommandToBackend, dialog_title: str) -> MessageFromBackend: 293 dlg = InlineCommandDialog(get_workbench(), cmd, title=dialog_title + " ...") 294 show_dialog(dlg) 295 return dlg.response 296 297 def _postpone_command(self, cmd: CommandToBackend) -> None: 298 # in case of InlineCommands, discard older same type command 299 if isinstance(cmd, InlineCommand): 300 for older_cmd in self._postponed_commands: 301 if older_cmd.name == cmd.name: 302 self._postponed_commands.remove(older_cmd) 303 304 if len(self._postponed_commands) > 10: 305 logging.warning("Can't pile up too many commands. This command will be just ignored") 306 else: 307 self._postponed_commands.append(cmd) 308 309 def _send_postponed_commands(self) -> None: 310 todo = self._postponed_commands 311 self._postponed_commands = [] 312 313 for cmd in todo: 314 logging.debug("Sending postponed command: %s", cmd) 315 self.send_command(cmd) 316 317 def send_program_input(self, data: str) -> None: 318 assert self.is_running() 319 self._proxy.send_program_input(data) 320 321 def execute_script( 322 self, 323 script_path: str, 324 args: List[str], 325 working_directory: Optional[str] = None, 326 command_name: str = "Run", 327 ) -> None: 328 329 if self._proxy.get_cwd() != working_directory: 330 # create compound command 331 # start with %cd 332 cd_cmd_line = construct_cd_command(working_directory) + "\n" 333 else: 334 # create simple command 335 cd_cmd_line = "" 336 337 rel_filename = universal_relpath(script_path, working_directory) 338 cmd_parts = ["%" + command_name, rel_filename] + args 339 exe_cmd_line = construct_cmd_line(cmd_parts, [EDITOR_CONTENT_TOKEN]) + "\n" 340 341 # submit to shell (shell will execute it) 342 get_shell().submit_magic_command(cd_cmd_line + exe_cmd_line) 343 344 def execute_editor_content(self, command_name, args): 345 get_shell().submit_magic_command( 346 construct_cmd_line( 347 ["%" + command_name, "-c", EDITOR_CONTENT_TOKEN] + args, [EDITOR_CONTENT_TOKEN] 348 ) 349 ) 350 351 def execute_current(self, command_name: str) -> None: 352 """ 353 This method's job is to create a command for running/debugging 354 current file/script and submit it to shell 355 """ 356 357 if not self.is_waiting_toplevel_command(): 358 self.restart_backend(True, False, 2) 359 360 filename = get_saved_current_script_filename() 361 362 if not filename: 363 # user has cancelled file saving 364 return 365 366 if ( 367 is_remote_path(filename) 368 and not self._proxy.can_run_remote_files() 369 or is_local_path(filename) 370 and not self._proxy.can_run_local_files() 371 ): 372 self.execute_editor_content(command_name, self._get_active_arguments()) 373 else: 374 if get_workbench().get_option("run.auto_cd") and command_name[0].isupper(): 375 working_directory = get_target_dirname_from_editor_filename(filename) 376 else: 377 working_directory = self._proxy.get_cwd() 378 379 if is_local_path(filename): 380 target_path = filename 381 else: 382 target_path = extract_target_path(filename) 383 self.execute_script( 384 target_path, self._get_active_arguments(), working_directory, command_name 385 ) 386 387 def _get_active_arguments(self): 388 if get_workbench().get_option("view.show_program_arguments"): 389 args_str = get_workbench().get_option("run.program_arguments") 390 get_workbench().log_program_arguments_string(args_str) 391 return shlex.split(args_str) 392 else: 393 return [] 394 395 def cmd_run_current_script_enabled(self) -> bool: 396 return ( 397 get_workbench().get_editor_notebook().get_current_editor() is not None 398 and "run" in get_runner().get_supported_features() 399 ) 400 401 def _cmd_run_current_script_in_terminal_enabled(self) -> bool: 402 return ( 403 self._proxy 404 and "run_in_terminal" in self._proxy.get_supported_features() 405 and self.cmd_run_current_script_enabled() 406 ) 407 408 def cmd_run_current_script(self) -> None: 409 if get_workbench().in_simple_mode(): 410 get_workbench().hide_view("VariablesView") 411 self.execute_current("Run") 412 413 def _cmd_run_current_script_in_terminal(self) -> None: 414 if inside_flatpak(): 415 show_command_not_available_in_flatpak_message() 416 return 417 418 filename = get_saved_current_script_filename() 419 if not filename: 420 return 421 422 self._proxy.run_script_in_terminal( 423 filename, 424 self._get_active_arguments(), 425 get_workbench().get_option("run.run_in_terminal_python_repl"), 426 get_workbench().get_option("run.run_in_terminal_keep_open"), 427 ) 428 429 def _cmd_interrupt(self) -> None: 430 if self._proxy is not None: 431 if _console_allocated: 432 self._proxy.interrupt() 433 else: 434 messagebox.showerror( 435 "No console", 436 "Can't interrupt as console was not allocated.\n\nUse Stop/Restart instead.", 437 master=self, 438 ) 439 else: 440 logging.warning("User tried interrupting without proxy") 441 442 def _cmd_interrupt_with_shortcut(self, event=None): 443 if not self._cmd_interrupt_enabled(): 444 return None 445 446 if not running_on_mac_os(): # on Mac Ctrl+C is not used for Copy. 447 # Disable Ctrl+C interrupt in editor and shell, when some text is selected 448 # (assuming user intended to copy instead of interrupting) 449 widget = get_workbench().focus_get() 450 if isinstance(widget, tk.Text): 451 if len(widget.tag_ranges("sel")) > 0: 452 # this test is reliable, unlike selection_get below 453 return None 454 elif isinstance(widget, (tk.Listbox, ttk.Entry, tk.Entry, tk.Spinbox)): 455 try: 456 selection = widget.selection_get() 457 if isinstance(selection, str) and len(selection) > 0: 458 # Assuming user meant to copy, not interrupt 459 # (IDLE seems to follow same logic) 460 461 # NB! This is not perfect, as in Linux the selection can be in another app 462 # ie. there may be no selection in Thonny actually. 463 # In other words, Ctrl+C interrupt may be dropped without reason 464 # when given inside the widgets listed above. 465 return None 466 except Exception: 467 # widget either doesn't have selection_get or it 468 # gave error (can happen without selection on Ubuntu) 469 pass 470 471 self._cmd_interrupt() 472 return "break" 473 474 def _cmd_interrupt_enabled(self) -> bool: 475 return self._proxy and self._proxy.is_connected() 476 477 def cmd_stop_restart(self) -> None: 478 if get_workbench().in_simple_mode(): 479 get_workbench().hide_view("VariablesView") 480 481 self.restart_backend(True) 482 483 def disconnect(self): 484 proxy = self.get_backend_proxy() 485 assert hasattr(proxy, "disconnect") 486 proxy.disconnect() 487 488 def disconnect_enabled(self): 489 return hasattr(self.get_backend_proxy(), "disconnect") 490 491 def ctrld(self): 492 proxy = self.get_backend_proxy() 493 if not proxy: 494 return 495 496 if get_shell().has_pending_input(): 497 messagebox.showerror( 498 "Can't perform this action", 499 "Ctrl+D only has effect on an empty line / prompt.\n" 500 + "Submit current input (press ENTER) and try again", 501 master=self, 502 ) 503 return 504 505 proxy.send_command(EOFCommand()) 506 self._set_state("running") 507 508 def ctrld_enabled(self): 509 proxy = self.get_backend_proxy() 510 return proxy and proxy.is_connected() 511 512 def _poll_backend_messages(self) -> None: 513 """I chose polling instead of event_generate in listener thread, 514 because event_generate across threads is not reliable 515 http://www.thecodingforums.com/threads/more-on-tk-event_generate-and-threads.359615/ 516 """ 517 self._polling_after_id = None 518 if self._pull_backend_messages() is False: 519 return 520 521 self._polling_after_id = get_workbench().after(20, self._poll_backend_messages) 522 523 def _pull_backend_messages(self): 524 # Don't process too many messages in single batch, allow screen updates 525 # and user actions between batches. 526 # Mostly relevant when backend prints a lot quickly. 527 msg_count = 0 528 max_msg_count = 10 529 while self._proxy is not None and msg_count < max_msg_count: 530 try: 531 msg = self._proxy.fetch_next_message() 532 if not msg: 533 break 534 logging.debug( 535 "RUNNER GOT: %s, %s in state: %s", msg.event_type, msg, self.get_state() 536 ) 537 538 msg_count += 1 539 except BackendTerminatedError as exc: 540 self._report_backend_crash(exc) 541 self.destroy_backend() 542 return False 543 544 if msg.get("SystemExit", False): 545 self.restart_backend(True) 546 return False 547 548 # change state 549 if isinstance(msg, ToplevelResponse): 550 self._set_state("waiting_toplevel_command") 551 elif isinstance(msg, DebuggerResponse): 552 self._set_state("waiting_debugger_command") 553 else: 554 "other messages don't affect the state" 555 556 # Publish the event 557 # NB! This may cause another command to be sent before we get to postponed commands. 558 try: 559 self._publishing_events = True 560 class_event_type = type(msg).__name__ 561 get_workbench().event_generate(class_event_type, event=msg) # more general event 562 if msg.event_type != class_event_type: 563 # more specific event 564 get_workbench().event_generate(msg.event_type, event=msg) 565 finally: 566 self._publishing_events = False 567 568 # TODO: is it necessary??? 569 # https://stackoverflow.com/a/13520271/261181 570 # get_workbench().update() 571 572 self._send_postponed_commands() 573 574 def _report_backend_crash(self, exc: Exception) -> None: 575 returncode = getattr(exc, "returncode", "?") 576 err = "Backend terminated or disconnected." 577 578 try: 579 faults_file = os.path.join(THONNY_USER_DIR, "backend_faults.log") 580 if os.path.exists(faults_file): 581 with open(faults_file, encoding="ASCII") as fp: 582 err += fp.read() 583 except Exception: 584 logging.exception("Failed retrieving backend faults") 585 586 err = err.strip() + " Use 'Stop/Restart' to restart.\n" 587 588 if returncode != EXPECTED_TERMINATION_CODE: 589 get_workbench().event_generate("ProgramOutput", stream_name="stderr", data="\n" + err) 590 591 get_workbench().become_active_window(False) 592 593 def restart_backend(self, clean: bool, first: bool = False, wait: float = 0) -> None: 594 """Recreate (or replace) backend proxy / backend process.""" 595 596 if not first: 597 get_shell().restart() 598 get_shell().update_idletasks() 599 600 self.destroy_backend() 601 backend_name = get_workbench().get_option("run.backend_name") 602 if backend_name not in get_workbench().get_backends(): 603 raise UserError( 604 "Can't find backend '{}'. Please select another backend from options".format( 605 backend_name 606 ) 607 ) 608 609 backend_class = get_workbench().get_backends()[backend_name].proxy_class 610 self._set_state("running") 611 self._proxy = None 612 self._proxy = backend_class(clean) 613 614 self._poll_backend_messages() 615 616 if wait: 617 start_time = time.time() 618 while not self.is_waiting_toplevel_command() and time.time() - start_time <= wait: 619 # self._pull_backend_messages() 620 # TODO: update in a loop can be slow on Mac https://core.tcl-lang.org/tk/tktview/f642d7c0f4 621 get_workbench().update() 622 sleep(0.01) 623 624 get_workbench().event_generate("BackendRestart", full=True) 625 626 def destroy_backend(self) -> None: 627 if self._polling_after_id is not None: 628 get_workbench().after_cancel(self._polling_after_id) 629 self._polling_after_id = None 630 631 self._postponed_commands = [] 632 if self._proxy: 633 self._proxy.destroy() 634 self._proxy = None 635 636 get_workbench().event_generate("BackendTerminated") 637 638 def get_local_executable(self) -> Optional[str]: 639 if self._proxy is None: 640 return None 641 else: 642 return self._proxy.get_local_executable() 643 644 def get_backend_proxy(self) -> "BackendProxy": 645 return self._proxy 646 647 def _check_alloc_console(self) -> None: 648 if sys.executable.endswith("pythonw.exe"): 649 # These don't have console allocated. 650 # Console is required for sending interrupts. 651 652 # AllocConsole would be easier but flashes console window 653 654 import ctypes 655 656 kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) 657 658 exe = sys.executable.replace("pythonw.exe", "python.exe") 659 660 cmd = [exe, "-c", "print('Hi!'); input()"] 661 child = subprocess.Popen( 662 cmd, 663 stdin=subprocess.PIPE, 664 stdout=subprocess.PIPE, 665 stderr=subprocess.PIPE, 666 shell=True, 667 ) 668 child.stdout.readline() 669 result = kernel32.AttachConsole(child.pid) 670 if not result: 671 err = ctypes.get_last_error() 672 logging.info("Could not allocate console. Error code: " + str(err)) 673 child.stdin.write(b"\n") 674 try: 675 child.stdin.flush() 676 except Exception: 677 # May happen eg. when installation path has "&" in it 678 # See https://bitbucket.org/plas/thonny/issues/508/cant-allocate-windows-console-when 679 # Without flush the console window becomes visible, but Thonny can be still used 680 logger.exception("Problem with finalizing console allocation") 681 682 def ready_for_remote_file_operations(self, show_message=False): 683 if not self._proxy or not self.supports_remote_files(): 684 return False 685 686 ready = self._proxy.ready_for_remote_file_operations() 687 688 if not ready and show_message: 689 if not self._proxy.is_connected(): 690 msg = "Device is not connected" 691 else: 692 msg = ( 693 "Device is busy -- can't perform this action now." 694 + "\nPlease wait or cancel current work and try again!" 695 ) 696 messagebox.showerror("Can't complete", msg, master=get_workbench()) 697 698 return ready 699 700 def get_supported_features(self) -> Set[str]: 701 if self._proxy is None: 702 return set() 703 else: 704 return self._proxy.get_supported_features() 705 706 def supports_remote_files(self): 707 if self._proxy is None: 708 return False 709 else: 710 return self._proxy.supports_remote_files() 711 712 def supports_remote_directories(self): 713 if self._proxy is None: 714 return False 715 else: 716 return self._proxy.supports_remote_directories() 717 718 def get_node_label(self): 719 if self._proxy is None: 720 return "Back-end" 721 else: 722 return self._proxy.get_node_label() 723 724 def using_venv(self) -> bool: 725 from thonny.plugins.cpython import CPythonProxy 726 727 return isinstance(self._proxy, CPythonProxy) and self._proxy._in_venv 728 729 730class BackendProxy: 731 """Communicates with backend process. 732 733 All communication methods must be non-blocking, 734 ie. suitable for calling from GUI thread.""" 735 736 # backend_name will be overwritten on Workbench.add_backend 737 # Subclasses don't need to worry about it. 738 backend_name = None 739 backend_description = None 740 741 def __init__(self, clean: bool) -> None: 742 """Initializes (or starts the initialization of) the backend process. 743 744 Backend is considered ready when the runner gets a ToplevelResponse 745 with attribute "welcome_text" from fetch_next_message. 746 """ 747 748 def send_command(self, cmd: CommandToBackend) -> Optional[str]: 749 """Send the command to backend. Return None, 'discard' or 'postpone'""" 750 raise NotImplementedError() 751 752 def send_program_input(self, data: str) -> None: 753 """Send input data to backend""" 754 raise NotImplementedError() 755 756 def fetch_next_message(self): 757 """Read next message from the queue or None if queue is empty""" 758 raise NotImplementedError() 759 760 def run_script_in_terminal(self, script_path, args, interactive, keep_open): 761 raise NotImplementedError() 762 763 def get_sys_path(self): 764 "backend's sys.path" 765 return [] 766 767 def get_backend_name(self): 768 return type(self).backend_name 769 770 def get_pip_gui_class(self): 771 return None 772 773 def interrupt(self): 774 """Tries to interrupt current command without resetting the backend""" 775 pass 776 777 def destroy(self): 778 """Called when Thonny no longer needs this instance 779 (Thonny gets closed or new backend gets selected) 780 """ 781 pass 782 783 def is_connected(self): 784 return True 785 786 def get_local_executable(self): 787 """Return system command for invoking current interpreter""" 788 return None 789 790 def get_supported_features(self): 791 return {"run"} 792 793 def get_node_label(self): 794 """Used as files caption if back-end has separate files""" 795 return "Back-end" 796 797 def get_full_label(self): 798 """Used in pip GUI title""" 799 return self.get_node_label() 800 801 def supports_remote_files(self): 802 """Whether remote file browser should be presented with this back-end""" 803 return False 804 805 def uses_local_filesystem(self): 806 """Whether it runs code from local files""" 807 return True 808 809 def supports_remote_directories(self): 810 return False 811 812 def supports_trash(self): 813 return True 814 815 def can_run_remote_files(self): 816 raise NotImplementedError() 817 818 def can_run_local_files(self): 819 raise NotImplementedError() 820 821 def ready_for_remote_file_operations(self): 822 return False 823 824 def get_cwd(self): 825 return None 826 827 def get_clean_description(self): 828 return self.backend_description 829 830 @classmethod 831 def get_current_switcher_configuration(cls): 832 """returns the dict of configuration entries that distinguish current backend conf from other 833 items in the backend switcher""" 834 return {"run.backend_name": cls.backend_name} 835 836 @classmethod 837 def get_switcher_entries(cls): 838 """ 839 Each returned entry creates one item in the backend switcher menu. 840 """ 841 return [(cls.get_current_switcher_configuration(), cls.backend_description)] 842 843 def has_custom_system_shell(self): 844 return False 845 846 def open_custom_system_shell(self): 847 raise NotImplementedError() 848 849 850class SubprocessProxy(BackendProxy): 851 def __init__(self, clean: bool, executable: Optional[str] = None) -> None: 852 super().__init__(clean) 853 854 if executable: 855 self._executable = executable 856 else: 857 self._executable = get_interpreter_for_subprocess() 858 859 if ".." in self._executable: 860 self._executable = os.path.normpath(self._executable) 861 862 if not os.path.isfile(self._executable): 863 raise UserError( 864 "Interpreter '%s' does not exist. Please check the configuration!" 865 % self._executable 866 ) 867 self._welcome_text = "" 868 869 self._proc = None 870 self._terminated_readers = 0 871 self._response_queue = None 872 self._sys_path = [] 873 self._usersitepackages = None 874 self._gui_update_loop_id = None 875 self._in_venv = None 876 self._cwd = self._get_initial_cwd() # pylint: disable=assignment-from-none 877 self._start_background_process(clean=clean) 878 879 def _get_initial_cwd(self): 880 return None 881 882 def _get_environment(self): 883 env = get_environment_for_python_subprocess(self._executable) 884 # variables controlling communication with the back-end process 885 env["PYTHONIOENCODING"] = "utf-8" 886 887 # because cmd line option -u won't reach child processes 888 # see https://github.com/thonny/thonny/issues/808 889 env["PYTHONUNBUFFERED"] = "1" 890 891 # Let back-end know about plug-ins 892 env["THONNY_USER_DIR"] = THONNY_USER_DIR 893 env["THONNY_FRONTEND_SYS_PATH"] = repr(sys.path) 894 895 env["THONNY_LANGUAGE"] = get_workbench().get_option("general.language") 896 897 if thonny.in_debug_mode(): 898 env["THONNY_DEBUG"] = "1" 899 elif "THONNY_DEBUG" in env: 900 del env["THONNY_DEBUG"] 901 return env 902 903 def _start_background_process(self, clean=None, extra_args=[]): 904 # deque, because in one occasion I need to put messages back 905 self._response_queue = collections.deque() 906 907 if not os.path.exists(self._executable): 908 raise UserError( 909 "Interpreter (%s) not found. Please recheck corresponding option!" 910 % self._executable 911 ) 912 913 cmd_line = ( 914 [ 915 self._executable, 916 "-u", # unbuffered IO 917 "-B", # don't write pyo/pyc files 918 # (to avoid problems when using different Python versions without write permissions) 919 ] 920 + self._get_launcher_with_args() 921 + extra_args 922 ) 923 924 creationflags = 0 925 if running_on_windows(): 926 creationflags = subprocess.CREATE_NEW_PROCESS_GROUP 927 928 debug("Starting the backend: %s %s", cmd_line, get_workbench().get_local_cwd()) 929 930 extra_params = {} 931 if sys.version_info >= (3, 6): 932 extra_params["encoding"] = "utf-8" 933 934 self._proc = subprocess.Popen( 935 cmd_line, 936 bufsize=0, 937 stdin=subprocess.PIPE, 938 stdout=subprocess.PIPE, 939 stderr=subprocess.PIPE, 940 cwd=self._get_launch_cwd(), 941 env=self._get_environment(), 942 universal_newlines=True, 943 creationflags=creationflags, 944 **extra_params 945 ) 946 947 # setup asynchronous output listeners 948 self._terminated_readers = 0 949 Thread(target=self._listen_stdout, args=(self._proc.stdout,), daemon=True).start() 950 Thread(target=self._listen_stderr, args=(self._proc.stderr,), daemon=True).start() 951 952 def _get_launch_cwd(self): 953 return self.get_cwd() if self.uses_local_filesystem() else None 954 955 def _get_launcher_with_args(self): 956 raise NotImplementedError() 957 958 def send_command(self, cmd: CommandToBackend) -> Optional[str]: 959 """Send the command to backend. Return None, 'discard' or 'postpone'""" 960 if isinstance(cmd, ToplevelCommand) and cmd.name[0].isupper(): 961 self._clear_environment() 962 963 if isinstance(cmd, ToplevelCommand): 964 # required by SshCPythonBackend for creating fresh target process 965 cmd["expected_cwd"] = self._cwd 966 967 method_name = "_cmd_" + cmd.name 968 969 if hasattr(self, method_name): 970 getattr(self, method_name)(cmd) 971 else: 972 self._send_msg(cmd) 973 974 def _send_msg(self, msg): 975 self._proc.stdin.write(serialize_message(msg) + "\n") 976 self._proc.stdin.flush() 977 978 def _clear_environment(self): 979 pass 980 981 def send_program_input(self, data): 982 self._send_msg(InputSubmission(data)) 983 984 def process_is_alive(self): 985 return self._proc is not None and self._proc.poll() is None 986 987 def is_terminated(self): 988 return not self.process_is_alive() 989 990 def is_connected(self): 991 return self.process_is_alive() 992 993 def get_sys_path(self): 994 return self._sys_path 995 996 def destroy(self): 997 self._close_backend() 998 999 def _close_backend(self): 1000 if self._proc is not None and self._proc.poll() is None: 1001 self._proc.kill() 1002 1003 self._proc = None 1004 self._response_queue = None 1005 1006 def _listen_stdout(self, stdout): 1007 # debug("... started listening to stdout") 1008 # will be called from separate thread 1009 1010 # allow self._response_queue to be replaced while processing 1011 message_queue = self._response_queue 1012 1013 def publish_as_msg(data): 1014 msg = parse_message(data) 1015 if "cwd" in msg: 1016 self.cwd = msg["cwd"] 1017 message_queue.append(msg) 1018 1019 if len(message_queue) > 10: 1020 # Probably backend runs an infinite/long print loop. 1021 # Throttle message throughput in order to keep GUI thread responsive. 1022 while len(message_queue) > 0: 1023 sleep(0.005) 1024 1025 while True: 1026 try: 1027 data = stdout.readline() 1028 except IOError: 1029 sleep(0.1) 1030 continue 1031 1032 # debug("... read some stdout data", repr(data)) 1033 if data == "": 1034 break 1035 else: 1036 try: 1037 publish_as_msg(data) 1038 except Exception: 1039 # Can mean the line was from subprocess, 1040 # which can't be captured by stream faking. 1041 # NB! If subprocess printed it without linebreak, 1042 # then the suffix can be thonny message 1043 1044 parts = data.rsplit(common.MESSAGE_MARKER, maxsplit=1) 1045 1046 # print first part as it is 1047 message_queue.append( 1048 BackendEvent("ProgramOutput", data=parts[0], stream_name="stdout") 1049 ) 1050 1051 if len(parts) == 2: 1052 second_part = common.MESSAGE_MARKER + parts[1] 1053 try: 1054 publish_as_msg(second_part) 1055 except Exception: 1056 # just print ... 1057 message_queue.append( 1058 BackendEvent( 1059 "ProgramOutput", data=second_part, stream_name="stdout" 1060 ) 1061 ) 1062 1063 self._terminated_readers += 1 1064 1065 def _listen_stderr(self, stderr): 1066 # stderr is used only for debugger debugging 1067 while True: 1068 data = stderr.readline() 1069 if data == "": 1070 break 1071 else: 1072 self._response_queue.append( 1073 BackendEvent("ProgramOutput", stream_name="stderr", data=data) 1074 ) 1075 1076 self._terminated_readers += 1 1077 1078 def _store_state_info(self, msg): 1079 if "cwd" in msg: 1080 self._cwd = msg["cwd"] 1081 self._publish_cwd(msg["cwd"]) 1082 1083 if msg.get("welcome_text"): 1084 self._welcome_text = msg["welcome_text"] 1085 1086 if "in_venv" in msg: 1087 self._in_venv = msg["in_venv"] 1088 1089 if "sys_path" in msg: 1090 self._sys_path = msg["sys_path"] 1091 1092 if "usersitepackages" in msg: 1093 self._usersitepackages = msg["usersitepackages"] 1094 1095 if "prefix" in msg: 1096 self._sys_prefix = msg["prefix"] 1097 1098 if "exe_dirs" in msg: 1099 self._exe_dirs = msg["exe_dirs"] 1100 1101 if msg.get("executable"): 1102 self._reported_executable = msg["executable"] 1103 1104 def _publish_cwd(self, cwd): 1105 if self.uses_local_filesystem(): 1106 get_workbench().set_local_cwd(cwd) 1107 1108 def get_supported_features(self): 1109 return {"run"} 1110 1111 def get_site_packages(self): 1112 # NB! site.sitepackages may not be present in virtualenv 1113 for d in self._sys_path: 1114 if ("site-packages" in d or "dist-packages" in d) and path_startswith( 1115 d, self._sys_prefix 1116 ): 1117 return d 1118 1119 return None 1120 1121 def get_user_site_packages(self): 1122 return self._usersitepackages 1123 1124 def get_cwd(self): 1125 return self._cwd 1126 1127 def get_exe_dirs(self): 1128 return self._exe_dirs 1129 1130 def fetch_next_message(self): 1131 if not self._response_queue or len(self._response_queue) == 0: 1132 if self.is_terminated() and self._terminated_readers == 2: 1133 raise BackendTerminatedError(self._proc.returncode if self._proc else None) 1134 else: 1135 return None 1136 1137 msg = self._response_queue.popleft() 1138 self._store_state_info(msg) 1139 if msg.event_type == "ProgramOutput": 1140 # combine available small output messages to one single message, 1141 # in order to put less pressure on UI code 1142 1143 wait_time = 0.01 1144 total_wait_time = 0 1145 while True: 1146 if len(self._response_queue) == 0: 1147 if _ends_with_incomplete_ansi_code(msg["data"]) and total_wait_time < 0.1: 1148 # Allow reader to send the remaining part 1149 sleep(wait_time) 1150 total_wait_time += wait_time 1151 continue 1152 else: 1153 return msg 1154 else: 1155 next_msg = self._response_queue.popleft() 1156 if ( 1157 next_msg.event_type == "ProgramOutput" 1158 and next_msg["stream_name"] == msg["stream_name"] 1159 and ( 1160 len(msg["data"]) + len(next_msg["data"]) <= OUTPUT_MERGE_THRESHOLD 1161 and ("\n" not in msg["data"] or not io_animation_required) 1162 or _ends_with_incomplete_ansi_code(msg["data"]) 1163 ) 1164 ): 1165 msg["data"] += next_msg["data"] 1166 else: 1167 # not to be sent in the same block, put it back 1168 self._response_queue.appendleft(next_msg) 1169 return msg 1170 1171 else: 1172 return msg 1173 1174 1175def _ends_with_incomplete_ansi_code(data): 1176 pos = data.rfind("\033") 1177 if pos == -1: 1178 return False 1179 1180 # note ANSI_CODE_TERMINATOR also includes [ 1181 params_and_terminator = data[pos + 2 :] 1182 return not ANSI_CODE_TERMINATOR.search(params_and_terminator) 1183 1184 1185def is_bundled_python(executable): 1186 return os.path.exists(os.path.join(os.path.dirname(executable), "thonny_python.ini")) 1187 1188 1189def create_backend_python_process( 1190 args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT 1191): 1192 """Used for running helper commands (eg. pip) on CPython backend. 1193 Assumes current backend is CPython.""" 1194 1195 # TODO: if backend == frontend, then delegate to create_frontend_python_process 1196 1197 python_exe = get_runner().get_local_executable() 1198 1199 env = get_environment_for_python_subprocess(python_exe) 1200 env["PYTHONIOENCODING"] = "utf-8" 1201 env["PYTHONUNBUFFERED"] = "1" 1202 1203 # TODO: remove frontend python from path and add backend python to it 1204 1205 return _create_python_process(python_exe, args, stdin, stdout, stderr, env=env) 1206 1207 1208def create_frontend_python_process( 1209 args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT 1210): 1211 """Used for running helper commands (eg. for installing plug-ins on by the plug-ins)""" 1212 if _console_allocated: 1213 python_exe = get_interpreter_for_subprocess().replace("pythonw.exe", "python.exe") 1214 else: 1215 python_exe = get_interpreter_for_subprocess().replace("python.exe", "pythonw.exe") 1216 env = get_environment_for_python_subprocess(python_exe) 1217 env["PYTHONIOENCODING"] = "utf-8" 1218 env["PYTHONUNBUFFERED"] = "1" 1219 return _create_python_process(python_exe, args, stdin, stdout, stderr) 1220 1221 1222def _create_python_process( 1223 python_exe, 1224 args, 1225 stdin=None, 1226 stdout=subprocess.PIPE, 1227 stderr=subprocess.STDOUT, 1228 shell=False, 1229 env=None, 1230 universal_newlines=True, 1231): 1232 1233 cmd = [python_exe] + args 1234 1235 if running_on_windows(): 1236 creationflags = subprocess.CREATE_NEW_PROCESS_GROUP 1237 startupinfo = subprocess.STARTUPINFO() 1238 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW 1239 else: 1240 startupinfo = None 1241 creationflags = 0 1242 1243 proc = subprocess.Popen( 1244 cmd, 1245 stdin=stdin, 1246 stdout=stdout, 1247 stderr=stderr, 1248 shell=shell, 1249 env=env, 1250 universal_newlines=universal_newlines, 1251 startupinfo=startupinfo, 1252 creationflags=creationflags, 1253 ) 1254 1255 proc.cmd = cmd 1256 return proc 1257 1258 1259class BackendTerminatedError(Exception): 1260 def __init__(self, returncode=None): 1261 Exception.__init__(self) 1262 self.returncode = returncode 1263 1264 1265def is_venv_interpreter_of_current_interpreter(executable): 1266 for location in ["..", "."]: 1267 cfg_path = os.path.join(os.path.dirname(executable), location, "pyvenv.cfg") 1268 if os.path.isfile(cfg_path): 1269 with open(cfg_path) as fp: 1270 content = fp.read() 1271 for line in content.splitlines(): 1272 if line.replace(" ", "").startswith("home="): 1273 _, home = line.split("=", maxsplit=1) 1274 home = home.strip() 1275 if os.path.isdir(home) and ( 1276 is_same_path(home, sys.prefix) 1277 or is_same_path(home, os.path.join(sys.prefix, "bin")) 1278 or is_same_path(home, os.path.join(sys.prefix, "Scripts")) 1279 ): 1280 return True 1281 return False 1282 1283 1284def get_environment_for_python_subprocess(target_executable): 1285 overrides = get_environment_overrides_for_python_subprocess(target_executable) 1286 return get_environment_with_overrides(overrides) 1287 1288 1289def get_environment_with_overrides(overrides): 1290 env = os.environ.copy() 1291 for key in overrides: 1292 if overrides[key] is None and key in env: 1293 del env[key] 1294 else: 1295 assert isinstance(overrides[key], str) 1296 if key.upper() == "PATH": 1297 update_system_path(env, overrides[key]) 1298 else: 1299 env[key] = overrides[key] 1300 return env 1301 1302 1303def get_environment_overrides_for_python_subprocess(target_executable): 1304 """Take care of not not confusing different interpreter 1305 with variables meant for bundled interpreter""" 1306 1307 # At the moment I'm tweaking the environment only if current 1308 # exe is bundled for Thonny. 1309 # In remaining cases it is user's responsibility to avoid 1310 # calling Thonny with environment which may be confusing for 1311 # different Pythons called in a subprocess. 1312 1313 this_executable = sys.executable.replace("pythonw.exe", "python.exe") 1314 target_executable = target_executable.replace("pythonw.exe", "python.exe") 1315 1316 interpreter_specific_keys = [ 1317 "TCL_LIBRARY", 1318 "TK_LIBRARY", 1319 "LD_LIBRARY_PATH", 1320 "DYLD_LIBRARY_PATH", 1321 "SSL_CERT_DIR", 1322 "SSL_CERT_FILE", 1323 "PYTHONHOME", 1324 "PYTHONPATH", 1325 "PYTHONNOUSERSITE", 1326 "PYTHONUSERBASE", 1327 ] 1328 1329 result = {} 1330 1331 if os.path.samefile( 1332 target_executable, this_executable 1333 ) or is_venv_interpreter_of_current_interpreter(target_executable): 1334 # bring out some important variables so that they can 1335 # be explicitly set in macOS Terminal 1336 # (If they are set then it's most likely because current exe is in Thonny bundle) 1337 for key in interpreter_specific_keys: 1338 if key in os.environ: 1339 result[key] = os.environ[key] 1340 1341 # never pass some variables to different interpreter 1342 # (even if it's venv or symlink to current one) 1343 if not is_same_path(target_executable, this_executable): 1344 for key in ["PYTHONPATH", "PYTHONHOME", "PYTHONNOUSERSITE", "PYTHONUSERBASE"]: 1345 if key in os.environ: 1346 result[key] = None 1347 else: 1348 # interpreters are not related 1349 # interpreter specific keys most likely would confuse other interpreter 1350 for key in interpreter_specific_keys: 1351 if key in os.environ: 1352 result[key] = None 1353 1354 # some keys should be never passed 1355 for key in [ 1356 "PYTHONSTARTUP", 1357 "PYTHONBREAKPOINT", 1358 "PYTHONDEBUG", 1359 "PYTHONNOUSERSITE", 1360 "PYTHONASYNCIODEBUG", 1361 ]: 1362 if key in os.environ: 1363 result[key] = None 1364 1365 # venv may not find (correct) Tk without assistance (eg. in Ubuntu) 1366 if is_venv_interpreter_of_current_interpreter(target_executable): 1367 try: 1368 if "TCL_LIBRARY" not in os.environ or "TK_LIBRARY" not in os.environ: 1369 result["TCL_LIBRARY"] = get_workbench().tk.exprstring("$tcl_library") 1370 result["TK_LIBRARY"] = get_workbench().tk.exprstring("$tk_library") 1371 except Exception: 1372 logging.exception("Can't compute Tcl/Tk library location") 1373 1374 return result 1375 1376 1377def construct_cd_command(path) -> str: 1378 return construct_cmd_line(["%cd", path]) 1379 1380 1381_command_id_counter = 0 1382 1383 1384def generate_command_id(): 1385 global _command_id_counter 1386 _command_id_counter += 1 1387 return "cmd_" + str(_command_id_counter) 1388 1389 1390class InlineCommandDialog(WorkDialog): 1391 def __init__( 1392 self, 1393 master, 1394 cmd: Union[InlineCommand, Callable], 1395 title, 1396 instructions=None, 1397 output_prelude=None, 1398 autostart=True, 1399 ): 1400 self.response = None 1401 self._title = title 1402 self._instructions = instructions 1403 self._cmd = cmd 1404 self.returncode = None 1405 1406 get_shell().set_ignore_program_output(True) 1407 1408 get_workbench().bind("InlineResponse", self._on_response, True) 1409 get_workbench().bind("InlineProgress", self._on_progress, True) 1410 get_workbench().bind("ProgramOutput", self._on_output, True) 1411 1412 super().__init__(master, autostart=autostart) 1413 1414 if output_prelude: 1415 self.append_text(output_prelude) 1416 1417 def get_title(self): 1418 return self._title 1419 1420 def get_instructions(self) -> Optional[str]: 1421 return self._instructions or self._cmd.get("description", "Working...") 1422 1423 def _on_response(self, response): 1424 if response.get("command_id") == getattr(self._cmd, "id"): 1425 logger.debug("Dialog got response: %s", response) 1426 self.response = response 1427 self.returncode = response.get("returncode", None) 1428 success = ( 1429 not self.returncode and not response.get("error") and not response.get("errors") 1430 ) 1431 if success: 1432 self.set_action_text("Done!") 1433 else: 1434 self.set_action_text("Error") 1435 if response.get("error"): 1436 self.append_text("Error %s\n" % response["error"], stream_name="stderr") 1437 if response.get("errors"): 1438 self.append_text("Errors %s\n" % response["errors"], stream_name="stderr") 1439 if self.returncode: 1440 self.append_text( 1441 "Process returned with code %s\n" % self.returncode, stream_name="stderr" 1442 ) 1443 1444 self.report_done(success) 1445 1446 def _on_progress(self, msg): 1447 if msg.get("command_id") != getattr(self._cmd, "id"): 1448 return 1449 1450 if msg.get("value", None) is not None and msg.get("maximum", None) is not None: 1451 self.report_progress(msg["value"], msg["maximum"]) 1452 if msg.get("description"): 1453 self.set_action_text(msg["description"]) 1454 1455 def _on_output(self, msg): 1456 stream_name = msg.get("stream_name", "stdout") 1457 self.append_text(msg["data"], stream_name) 1458 self.set_action_text_smart(msg["data"]) 1459 1460 def start_work(self): 1461 self.send_command_to_backend() 1462 1463 def send_command_to_backend(self): 1464 if not isinstance(self._cmd, CommandToBackend): 1465 # it was a lazy definition 1466 try: 1467 self._cmd = self._cmd() 1468 except Exception as e: 1469 logger.error("Could not produce command for backend", self._cmd) 1470 self.set_action_text("Error!") 1471 self.append_text("Could not produce command for backend\n") 1472 self.append_text("".join(traceback.format_exc()) + "\n") 1473 self.report_done(False) 1474 return 1475 1476 logger.debug("Starting command in dialog: %s", self._cmd) 1477 get_runner().send_command(self._cmd) 1478 1479 def cancel_work(self): 1480 super(InlineCommandDialog, self).cancel_work() 1481 get_runner()._cmd_interrupt() 1482 1483 def close(self): 1484 get_workbench().unbind("InlineResponse", self._on_response) 1485 get_workbench().unbind("InlineProgress", self._on_progress) 1486 super(InlineCommandDialog, self).close() 1487 get_shell().set_ignore_program_output(False) 1488 1489 1490def get_frontend_python(): 1491 # TODO: deprecated (name can be misleading) 1492 warnings.warn("get_frontend_python is deprecated") 1493 return get_interpreter_for_subprocess(sys.executable) 1494 1495 1496def get_interpreter_for_subprocess(candidate=None): 1497 if candidate is None: 1498 candidate = sys.executable 1499 1500 pythonw = candidate.replace("python.exe", "pythonw.exe") 1501 if not _console_allocated and os.path.exists(pythonw): 1502 return pythonw 1503 else: 1504 return candidate.replace("pythonw.exe", "python.exe") 1505