1"""
2Each text will get its on SyntaxColorer.
3
4For performance reasons, coloring is updated in 2 phases:
5    1. recolor single-line tokens on the modified line(s)
6    2. recolor multi-line tokens (triple-quoted strings) in the whole text
7
8First phase may insert wrong tokens inside triple-quoted strings, but the
9priorities of triple-quoted-string tags are higher and therefore user
10doesn't see these wrong taggings. In some cases (eg. open strings)
11these wrong tags are removed later.
12
13In Shell only current command entry is colored
14
15Regexes are adapted from idlelib
16"""
17import logging
18import re
19
20import tkinter
21from thonny import get_workbench
22from thonny.codeview import CodeViewText
23from thonny.shell import ShellText
24
25logger = logging.getLogger(__name__)
26
27TODO = "COLOR_TODO"
28
29
30class SyntaxColorer:
31    def __init__(self, text: tkinter.Text):
32        self.text = text
33        self._compile_regexes()
34        self._config_tags()
35        self._update_scheduled = False
36        self._use_coloring = True
37        self._multiline_dirty = True
38        self._highlight_tabs = True
39
40    def _compile_regexes(self):
41        from thonny.token_utils import (
42            BUILTIN,
43            COMMENT,
44            COMMENT_WITH_Q3DELIMITER,
45            KEYWORD,
46            MAGIC_COMMAND,
47            NUMBER,
48            STRING3,
49            STRING3_DELIMITER,
50            STRING_CLOSED,
51            STRING_OPEN,
52            TAB,
53            FUNCTION_CALL,
54            METHOD_CALL,
55        )
56
57        self.uniline_regex = re.compile(
58            KEYWORD
59            + "|"
60            + BUILTIN
61            + "|"
62            + NUMBER
63            + "|"
64            + COMMENT
65            + "|"
66            + MAGIC_COMMAND
67            + "|"
68            + STRING3_DELIMITER  # to avoid marking """ and ''' as single line string in uniline mode
69            + "|"
70            + STRING_CLOSED
71            + "|"
72            + STRING_OPEN
73            + "|"
74            + TAB
75            + "|"
76            + FUNCTION_CALL
77            + "|"
78            + METHOD_CALL,
79            re.S,  # @UndefinedVariable
80        )
81
82        # need to notice triple-quotes inside comments and magic commands
83        self.multiline_regex = re.compile(
84            "(" + STRING3 + ")|" + COMMENT_WITH_Q3DELIMITER + "|" + MAGIC_COMMAND,
85            re.S,  # @UndefinedVariable
86        )
87
88        self.id_regex = re.compile(r"\s+(\w+)", re.S)  # @UndefinedVariable
89
90    def _config_tags(self):
91        self.uniline_tags = {
92            "comment",
93            "magic",
94            "string",
95            "open_string",
96            "keyword",
97            "number",
98            "builtin",
99            "definition",
100            "function_call",
101            "method_call",
102            "class_definition",
103            "function_definition",
104        }
105        self.multiline_tags = {"string3", "open_string3"}
106        self._raise_tags()
107
108    def _raise_tags(self):
109        self.text.tag_raise("string3")
110        # yes, unclosed_expression is another plugin's issue,
111        # but it must be higher than *string3
112        self.text.tag_raise("tab")
113        self.text.tag_raise("unclosed_expression")
114        self.text.tag_raise("open_string3")
115        self.text.tag_raise("open_string")
116        self.text.tag_raise("sel")
117        self.text.tag_raise("builtin", "function_call")
118        self.text.tag_raise("class_definition", "definition")
119        self.text.tag_raise("function_definition", "definition")
120        """
121        tags = self.text.tag_names()
122        # take into account that without themes some tags may be undefined
123        if "string3" in tags:
124            self.text.tag_raise("string3")
125        if "open_string3" in tags:
126            self.text.tag_raise("open_string3")
127        """
128
129    def mark_dirty(self, event=None):
130        start_index = "1.0"
131        end_index = "end"
132
133        if hasattr(event, "sequence"):
134            if event.sequence == "TextInsert":
135                index = self.text.index(event.index)
136                start_row = int(index.split(".")[0])
137                end_row = start_row + event.text.count("\n")
138                start_index = "%d.%d" % (start_row, 0)
139                end_index = "%d.%d" % (end_row + 1, 0)
140                if not event.trivial_for_coloring:
141                    self._multiline_dirty = True
142
143            elif event.sequence == "TextDelete":
144                index = self.text.index(event.index1)
145                start_row = int(index.split(".")[0])
146                start_index = "%d.%d" % (start_row, 0)
147                end_index = "%d.%d" % (start_row + 1, 0)
148                if not event.trivial_for_coloring:
149                    self._multiline_dirty = True
150
151        self.text.tag_add(TODO, start_index, end_index)
152
153    def schedule_update(self):
154        self._highlight_tabs = get_workbench().get_option("view.highlight_tabs")
155        self._use_coloring = (
156            get_workbench().get_option("view.syntax_coloring")
157            and self.text.is_python_text()
158            or self.text.is_pythonlike_text()
159        )
160
161        if not self._update_scheduled:
162            self._update_scheduled = True
163            self.text.after_idle(self.perform_update)
164
165    def perform_update(self):
166        try:
167            self._update_coloring()
168        finally:
169            self._update_scheduled = False
170
171    def _update_coloring(self):
172        raise NotImplementedError()
173
174    def _update_uniline_tokens(self, start, end):
175        chars = self.text.get(start, end)
176
177        # clear old tags
178        for tag in self.uniline_tags | {"tab"}:
179            self.text.tag_remove(tag, start, end)
180
181        if self._use_coloring:
182            for match in self.uniline_regex.finditer(chars):
183                for token_type, token_text in match.groupdict().items():
184                    if token_text and token_type in self.uniline_tags:
185                        token_text = token_text.strip()
186                        match_start, match_end = match.span(token_type)
187
188                        self.text.tag_add(
189                            token_type, start + "+%dc" % match_start, start + "+%dc" % match_end
190                        )
191
192                        # Mark also the word following def or class
193                        if token_text in ("def", "class"):
194                            id_match = self.id_regex.match(chars, match_end)
195                            if id_match:
196                                id_match_start, id_match_end = id_match.span(1)
197                                self.text.tag_add(
198                                    "definition",
199                                    start + "+%dc" % id_match_start,
200                                    start + "+%dc" % id_match_end,
201                                )
202                                if token_text == "def":
203                                    tag_type = "function_definition"
204                                else:
205                                    tag_type = "class_definition"
206                                self.text.tag_add(
207                                    tag_type,
208                                    start + "+%dc" % id_match_start,
209                                    start + "+%dc" % id_match_end,
210                                )
211
212        if self._highlight_tabs:
213            self._update_tabs(start, end)
214
215        self.text.tag_remove(TODO, start, end)
216
217    def _update_multiline_tokens(self, start, end):
218        chars = self.text.get(start, end)
219        # clear old tags
220        for tag in self.multiline_tags:
221            self.text.tag_remove(tag, start, end)
222
223        if not self._use_coloring:
224            return
225
226        for match in self.multiline_regex.finditer(chars):
227            token_text = match.group(1)
228            if token_text is None:
229                # not string3
230                continue
231
232            match_start, match_end = match.span()
233            if (
234                token_text.startswith('"""')
235                and not token_text.endswith('"""')
236                or token_text.startswith("'''")
237                and not token_text.endswith("'''")
238                or len(token_text) == 3
239            ):
240                token_type = "open_string3"
241            elif len(token_text) >= 4 and token_text[-4] == "\\":
242                token_type = "open_string3"
243            else:
244                token_type = "string3"
245
246            token_start = start + "+%dc" % match_start
247            token_end = start + "+%dc" % match_end
248            self.text.tag_add(token_type, token_start, token_end)
249
250        self._multiline_dirty = False
251        self._raise_tags()
252
253    def _update_tabs(self, start, end):
254        while True:
255            pos = self.text.search("\t", start, end)
256            if pos:
257                self.text.tag_add("tab", pos)
258                start = self.text.index("%s +1 c" % pos)
259            else:
260                break
261
262
263class CodeViewSyntaxColorer(SyntaxColorer):
264    def _update_coloring(self):
265        viewport_start = self.text.index("@0,0")
266        viewport_end = self.text.index(
267            "@%d,%d lineend" % (self.text.winfo_width(), self.text.winfo_height())
268        )
269
270        search_start = viewport_start
271        search_end = viewport_end
272
273        while True:
274            res = self.text.tag_nextrange(TODO, search_start, search_end)
275            if res:
276                update_start = res[0]
277                update_end = res[1]
278            else:
279                # maybe the range started earlier
280                res = self.text.tag_prevrange(TODO, search_start)
281                if res and self.text.compare(res[1], ">", search_end):
282                    update_start = search_start
283                    update_end = res[1]
284                else:
285                    break
286
287            if self.text.compare(update_end, ">", search_end):
288                update_end = search_end
289
290            self._update_uniline_tokens(update_start, update_end)
291
292            if update_end == search_end:
293                break
294            else:
295                search_start = update_end
296
297        # Multiline tokens need to be searched from the whole source
298        if self._multiline_dirty:
299            self._update_multiline_tokens("1.0", "end")
300
301        # Get rid of wrong open string tags (https://github.com/thonny/thonny/issues/943)
302        search_start = viewport_start
303        while True:
304            tag_range = self.text.tag_nextrange("open_string", search_start, viewport_end)
305            if not tag_range:
306                break
307
308            if "string3" in self.text.tag_names(tag_range[0]):
309                self.text.tag_remove("open_string", tag_range[0], tag_range[1])
310
311            search_start = tag_range[1]
312
313
314class ShellSyntaxColorer(SyntaxColorer):
315    def _update_coloring(self):
316        parts = self.text.tag_prevrange("command", "end")
317
318        if parts:
319            end_row, end_col = map(int, self.text.index(parts[1]).split("."))
320
321            if end_col != 0:  # if not just after the last linebreak
322                end_row += 1  # then extend the range to the beginning of next line
323                end_col = 0  # (otherwise open strings are not displayed correctly)
324
325            start_index = parts[0]
326            end_index = "%d.%d" % (end_row, end_col)
327
328            self._update_uniline_tokens(start_index, end_index)
329            self._update_multiline_tokens(start_index, end_index)
330
331
332def update_coloring_on_event(event):
333    if hasattr(event, "text_widget"):
334        text = event.text_widget
335    else:
336        text = event.widget
337
338    try:
339        update_coloring_on_text(text, event)
340    except Exception as e:
341        logger.error("Problem with coloring", exc_info=e)
342
343
344def update_coloring_on_text(text, event=None):
345    if not hasattr(text, "syntax_colorer"):
346        if isinstance(text, ShellText):
347            class_ = ShellSyntaxColorer
348        elif isinstance(text, CodeViewText):
349            class_ = CodeViewSyntaxColorer
350        else:
351            return
352
353        text.syntax_colorer = class_(text)
354        # mark whole text as unprocessed
355        text.syntax_colorer.mark_dirty()
356    else:
357        text.syntax_colorer.mark_dirty(event)
358
359    text.syntax_colorer.schedule_update()
360
361
362def load_plugin() -> None:
363    wb = get_workbench()
364
365    wb.set_default("view.syntax_coloring", True)
366    wb.set_default("view.highlight_tabs", True)
367    wb.bind("TextInsert", update_coloring_on_event, True)
368    wb.bind("TextDelete", update_coloring_on_event, True)
369    wb.bind_class("CodeViewText", "<<VerticalScroll>>", update_coloring_on_event, True)
370    wb.bind("<<UpdateAppearance>>", update_coloring_on_event, True)
371