1""" 2Each text will get its on SyntaxColorer. 3 4For performance reasons, coloring is updated in 2 phases: 5 1. recolor single-line tokens on the modified line(s) 6 2. recolor multi-line tokens (triple-quoted strings) in the whole text 7 8First phase may insert wrong tokens inside triple-quoted strings, but the 9priorities of triple-quoted-string tags are higher and therefore user 10doesn't see these wrong taggings. In some cases (eg. open strings) 11these wrong tags are removed later. 12 13In Shell only current command entry is colored 14 15Regexes are adapted from idlelib 16""" 17import logging 18import re 19 20import tkinter 21from thonny import get_workbench 22from thonny.codeview import CodeViewText 23from thonny.shell import ShellText 24 25logger = logging.getLogger(__name__) 26 27TODO = "COLOR_TODO" 28 29 30class SyntaxColorer: 31 def __init__(self, text: tkinter.Text): 32 self.text = text 33 self._compile_regexes() 34 self._config_tags() 35 self._update_scheduled = False 36 self._use_coloring = True 37 self._multiline_dirty = True 38 self._highlight_tabs = True 39 40 def _compile_regexes(self): 41 from thonny.token_utils import ( 42 BUILTIN, 43 COMMENT, 44 COMMENT_WITH_Q3DELIMITER, 45 KEYWORD, 46 MAGIC_COMMAND, 47 NUMBER, 48 STRING3, 49 STRING3_DELIMITER, 50 STRING_CLOSED, 51 STRING_OPEN, 52 TAB, 53 FUNCTION_CALL, 54 METHOD_CALL, 55 ) 56 57 self.uniline_regex = re.compile( 58 KEYWORD 59 + "|" 60 + BUILTIN 61 + "|" 62 + NUMBER 63 + "|" 64 + COMMENT 65 + "|" 66 + MAGIC_COMMAND 67 + "|" 68 + STRING3_DELIMITER # to avoid marking """ and ''' as single line string in uniline mode 69 + "|" 70 + STRING_CLOSED 71 + "|" 72 + STRING_OPEN 73 + "|" 74 + TAB 75 + "|" 76 + FUNCTION_CALL 77 + "|" 78 + METHOD_CALL, 79 re.S, # @UndefinedVariable 80 ) 81 82 # need to notice triple-quotes inside comments and magic commands 83 self.multiline_regex = re.compile( 84 "(" + STRING3 + ")|" + COMMENT_WITH_Q3DELIMITER + "|" + MAGIC_COMMAND, 85 re.S, # @UndefinedVariable 86 ) 87 88 self.id_regex = re.compile(r"\s+(\w+)", re.S) # @UndefinedVariable 89 90 def _config_tags(self): 91 self.uniline_tags = { 92 "comment", 93 "magic", 94 "string", 95 "open_string", 96 "keyword", 97 "number", 98 "builtin", 99 "definition", 100 "function_call", 101 "method_call", 102 "class_definition", 103 "function_definition", 104 } 105 self.multiline_tags = {"string3", "open_string3"} 106 self._raise_tags() 107 108 def _raise_tags(self): 109 self.text.tag_raise("string3") 110 # yes, unclosed_expression is another plugin's issue, 111 # but it must be higher than *string3 112 self.text.tag_raise("tab") 113 self.text.tag_raise("unclosed_expression") 114 self.text.tag_raise("open_string3") 115 self.text.tag_raise("open_string") 116 self.text.tag_raise("sel") 117 self.text.tag_raise("builtin", "function_call") 118 self.text.tag_raise("class_definition", "definition") 119 self.text.tag_raise("function_definition", "definition") 120 """ 121 tags = self.text.tag_names() 122 # take into account that without themes some tags may be undefined 123 if "string3" in tags: 124 self.text.tag_raise("string3") 125 if "open_string3" in tags: 126 self.text.tag_raise("open_string3") 127 """ 128 129 def mark_dirty(self, event=None): 130 start_index = "1.0" 131 end_index = "end" 132 133 if hasattr(event, "sequence"): 134 if event.sequence == "TextInsert": 135 index = self.text.index(event.index) 136 start_row = int(index.split(".")[0]) 137 end_row = start_row + event.text.count("\n") 138 start_index = "%d.%d" % (start_row, 0) 139 end_index = "%d.%d" % (end_row + 1, 0) 140 if not event.trivial_for_coloring: 141 self._multiline_dirty = True 142 143 elif event.sequence == "TextDelete": 144 index = self.text.index(event.index1) 145 start_row = int(index.split(".")[0]) 146 start_index = "%d.%d" % (start_row, 0) 147 end_index = "%d.%d" % (start_row + 1, 0) 148 if not event.trivial_for_coloring: 149 self._multiline_dirty = True 150 151 self.text.tag_add(TODO, start_index, end_index) 152 153 def schedule_update(self): 154 self._highlight_tabs = get_workbench().get_option("view.highlight_tabs") 155 self._use_coloring = ( 156 get_workbench().get_option("view.syntax_coloring") 157 and self.text.is_python_text() 158 or self.text.is_pythonlike_text() 159 ) 160 161 if not self._update_scheduled: 162 self._update_scheduled = True 163 self.text.after_idle(self.perform_update) 164 165 def perform_update(self): 166 try: 167 self._update_coloring() 168 finally: 169 self._update_scheduled = False 170 171 def _update_coloring(self): 172 raise NotImplementedError() 173 174 def _update_uniline_tokens(self, start, end): 175 chars = self.text.get(start, end) 176 177 # clear old tags 178 for tag in self.uniline_tags | {"tab"}: 179 self.text.tag_remove(tag, start, end) 180 181 if self._use_coloring: 182 for match in self.uniline_regex.finditer(chars): 183 for token_type, token_text in match.groupdict().items(): 184 if token_text and token_type in self.uniline_tags: 185 token_text = token_text.strip() 186 match_start, match_end = match.span(token_type) 187 188 self.text.tag_add( 189 token_type, start + "+%dc" % match_start, start + "+%dc" % match_end 190 ) 191 192 # Mark also the word following def or class 193 if token_text in ("def", "class"): 194 id_match = self.id_regex.match(chars, match_end) 195 if id_match: 196 id_match_start, id_match_end = id_match.span(1) 197 self.text.tag_add( 198 "definition", 199 start + "+%dc" % id_match_start, 200 start + "+%dc" % id_match_end, 201 ) 202 if token_text == "def": 203 tag_type = "function_definition" 204 else: 205 tag_type = "class_definition" 206 self.text.tag_add( 207 tag_type, 208 start + "+%dc" % id_match_start, 209 start + "+%dc" % id_match_end, 210 ) 211 212 if self._highlight_tabs: 213 self._update_tabs(start, end) 214 215 self.text.tag_remove(TODO, start, end) 216 217 def _update_multiline_tokens(self, start, end): 218 chars = self.text.get(start, end) 219 # clear old tags 220 for tag in self.multiline_tags: 221 self.text.tag_remove(tag, start, end) 222 223 if not self._use_coloring: 224 return 225 226 for match in self.multiline_regex.finditer(chars): 227 token_text = match.group(1) 228 if token_text is None: 229 # not string3 230 continue 231 232 match_start, match_end = match.span() 233 if ( 234 token_text.startswith('"""') 235 and not token_text.endswith('"""') 236 or token_text.startswith("'''") 237 and not token_text.endswith("'''") 238 or len(token_text) == 3 239 ): 240 token_type = "open_string3" 241 elif len(token_text) >= 4 and token_text[-4] == "\\": 242 token_type = "open_string3" 243 else: 244 token_type = "string3" 245 246 token_start = start + "+%dc" % match_start 247 token_end = start + "+%dc" % match_end 248 self.text.tag_add(token_type, token_start, token_end) 249 250 self._multiline_dirty = False 251 self._raise_tags() 252 253 def _update_tabs(self, start, end): 254 while True: 255 pos = self.text.search("\t", start, end) 256 if pos: 257 self.text.tag_add("tab", pos) 258 start = self.text.index("%s +1 c" % pos) 259 else: 260 break 261 262 263class CodeViewSyntaxColorer(SyntaxColorer): 264 def _update_coloring(self): 265 viewport_start = self.text.index("@0,0") 266 viewport_end = self.text.index( 267 "@%d,%d lineend" % (self.text.winfo_width(), self.text.winfo_height()) 268 ) 269 270 search_start = viewport_start 271 search_end = viewport_end 272 273 while True: 274 res = self.text.tag_nextrange(TODO, search_start, search_end) 275 if res: 276 update_start = res[0] 277 update_end = res[1] 278 else: 279 # maybe the range started earlier 280 res = self.text.tag_prevrange(TODO, search_start) 281 if res and self.text.compare(res[1], ">", search_end): 282 update_start = search_start 283 update_end = res[1] 284 else: 285 break 286 287 if self.text.compare(update_end, ">", search_end): 288 update_end = search_end 289 290 self._update_uniline_tokens(update_start, update_end) 291 292 if update_end == search_end: 293 break 294 else: 295 search_start = update_end 296 297 # Multiline tokens need to be searched from the whole source 298 if self._multiline_dirty: 299 self._update_multiline_tokens("1.0", "end") 300 301 # Get rid of wrong open string tags (https://github.com/thonny/thonny/issues/943) 302 search_start = viewport_start 303 while True: 304 tag_range = self.text.tag_nextrange("open_string", search_start, viewport_end) 305 if not tag_range: 306 break 307 308 if "string3" in self.text.tag_names(tag_range[0]): 309 self.text.tag_remove("open_string", tag_range[0], tag_range[1]) 310 311 search_start = tag_range[1] 312 313 314class ShellSyntaxColorer(SyntaxColorer): 315 def _update_coloring(self): 316 parts = self.text.tag_prevrange("command", "end") 317 318 if parts: 319 end_row, end_col = map(int, self.text.index(parts[1]).split(".")) 320 321 if end_col != 0: # if not just after the last linebreak 322 end_row += 1 # then extend the range to the beginning of next line 323 end_col = 0 # (otherwise open strings are not displayed correctly) 324 325 start_index = parts[0] 326 end_index = "%d.%d" % (end_row, end_col) 327 328 self._update_uniline_tokens(start_index, end_index) 329 self._update_multiline_tokens(start_index, end_index) 330 331 332def update_coloring_on_event(event): 333 if hasattr(event, "text_widget"): 334 text = event.text_widget 335 else: 336 text = event.widget 337 338 try: 339 update_coloring_on_text(text, event) 340 except Exception as e: 341 logger.error("Problem with coloring", exc_info=e) 342 343 344def update_coloring_on_text(text, event=None): 345 if not hasattr(text, "syntax_colorer"): 346 if isinstance(text, ShellText): 347 class_ = ShellSyntaxColorer 348 elif isinstance(text, CodeViewText): 349 class_ = CodeViewSyntaxColorer 350 else: 351 return 352 353 text.syntax_colorer = class_(text) 354 # mark whole text as unprocessed 355 text.syntax_colorer.mark_dirty() 356 else: 357 text.syntax_colorer.mark_dirty(event) 358 359 text.syntax_colorer.schedule_update() 360 361 362def load_plugin() -> None: 363 wb = get_workbench() 364 365 wb.set_default("view.syntax_coloring", True) 366 wb.set_default("view.highlight_tabs", True) 367 wb.bind("TextInsert", update_coloring_on_event, True) 368 wb.bind("TextDelete", update_coloring_on_event, True) 369 wb.bind_class("CodeViewText", "<<VerticalScroll>>", update_coloring_on_event, True) 370 wb.bind("<<UpdateAppearance>>", update_coloring_on_event, True) 371