1import builtins
2import keyword
3import re
4import time
5
6from idlelib.config import idleConf
7from idlelib.delegator import Delegator
8
9DEBUG = False
10
11
12def any(name, alternates):
13    "Return a named group pattern matching list of alternates."
14    return "(?P<%s>" % name + "|".join(alternates) + ")"
15
16
17def make_pat():
18    kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
19    match_softkw = (
20        r"^[ \t]*" +  # at beginning of line + possible indentation
21        r"(?P<MATCH_SOFTKW>match)\b" +
22        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
23            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
24                                 # pattern-matching statement
25            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
26        ]) +
27        r"))"
28    )
29    case_default = (
30        r"^[ \t]*" +  # at beginning of line + possible indentation
31        r"(?P<CASE_SOFTKW>case)" +
32        r"[ \t]+(?P<CASE_DEFAULT_UNDERSCORE>_\b)"
33    )
34    case_softkw_and_pattern = (
35        r"^[ \t]*" +  # at beginning of line + possible indentation
36        r"(?P<CASE_SOFTKW2>case)\b" +
37        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
38            r"_\b",  # a lone underscore
39            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
40                                 # pattern-matching case
41            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
42        ]) +
43        r"))"
44    )
45    builtinlist = [str(name) for name in dir(builtins)
46                   if not name.startswith('_') and
47                   name not in keyword.kwlist]
48    builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
49    comment = any("COMMENT", [r"#[^\n]*"])
50    stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?"
51    sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
52    dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
53    sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
54    dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
55    string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
56    prog = re.compile("|".join([
57                                builtin, comment, string, kw,
58                                match_softkw, case_default,
59                                case_softkw_and_pattern,
60                                any("SYNC", [r"\n"]),
61                               ]),
62                      re.DOTALL | re.MULTILINE)
63    return prog
64
65
66prog = make_pat()
67idprog = re.compile(r"\s+(\w+)")
68prog_group_name_to_tag = {
69    "MATCH_SOFTKW": "KEYWORD",
70    "CASE_SOFTKW": "KEYWORD",
71    "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
72    "CASE_SOFTKW2": "KEYWORD",
73}
74
75
76def matched_named_groups(re_match):
77    "Get only the non-empty named groups from an re.Match object."
78    return ((k, v) for (k, v) in re_match.groupdict().items() if v)
79
80
81def color_config(text):
82    """Set color options of Text widget.
83
84    If ColorDelegator is used, this should be called first.
85    """
86    # Called from htest, TextFrame, Editor, and Turtledemo.
87    # Not automatic because ColorDelegator does not know 'text'.
88    theme = idleConf.CurrentTheme()
89    normal_colors = idleConf.GetHighlight(theme, 'normal')
90    cursor_color = idleConf.GetHighlight(theme, 'cursor')['foreground']
91    select_colors = idleConf.GetHighlight(theme, 'hilite')
92    text.config(
93        foreground=normal_colors['foreground'],
94        background=normal_colors['background'],
95        insertbackground=cursor_color,
96        selectforeground=select_colors['foreground'],
97        selectbackground=select_colors['background'],
98        inactiveselectbackground=select_colors['background'],  # new in 8.5
99        )
100
101
102class ColorDelegator(Delegator):
103    """Delegator for syntax highlighting (text coloring).
104
105    Instance variables:
106        delegate: Delegator below this one in the stack, meaning the
107                one this one delegates to.
108
109        Used to track state:
110        after_id: Identifier for scheduled after event, which is a
111                timer for colorizing the text.
112        allow_colorizing: Boolean toggle for applying colorizing.
113        colorizing: Boolean flag when colorizing is in process.
114        stop_colorizing: Boolean flag to end an active colorizing
115                process.
116    """
117
118    def __init__(self):
119        Delegator.__init__(self)
120        self.init_state()
121        self.prog = prog
122        self.idprog = idprog
123        self.LoadTagDefs()
124
125    def init_state(self):
126        "Initialize variables that track colorizing state."
127        self.after_id = None
128        self.allow_colorizing = True
129        self.stop_colorizing = False
130        self.colorizing = False
131
132    def setdelegate(self, delegate):
133        """Set the delegate for this instance.
134
135        A delegate is an instance of a Delegator class and each
136        delegate points to the next delegator in the stack.  This
137        allows multiple delegators to be chained together for a
138        widget.  The bottom delegate for a colorizer is a Text
139        widget.
140
141        If there is a delegate, also start the colorizing process.
142        """
143        if self.delegate is not None:
144            self.unbind("<<toggle-auto-coloring>>")
145        Delegator.setdelegate(self, delegate)
146        if delegate is not None:
147            self.config_colors()
148            self.bind("<<toggle-auto-coloring>>", self.toggle_colorize_event)
149            self.notify_range("1.0", "end")
150        else:
151            # No delegate - stop any colorizing.
152            self.stop_colorizing = True
153            self.allow_colorizing = False
154
155    def config_colors(self):
156        "Configure text widget tags with colors from tagdefs."
157        for tag, cnf in self.tagdefs.items():
158            self.tag_configure(tag, **cnf)
159        self.tag_raise('sel')
160
161    def LoadTagDefs(self):
162        "Create dictionary of tag names to text colors."
163        theme = idleConf.CurrentTheme()
164        self.tagdefs = {
165            "COMMENT": idleConf.GetHighlight(theme, "comment"),
166            "KEYWORD": idleConf.GetHighlight(theme, "keyword"),
167            "BUILTIN": idleConf.GetHighlight(theme, "builtin"),
168            "STRING": idleConf.GetHighlight(theme, "string"),
169            "DEFINITION": idleConf.GetHighlight(theme, "definition"),
170            "SYNC": {'background': None, 'foreground': None},
171            "TODO": {'background': None, 'foreground': None},
172            "ERROR": idleConf.GetHighlight(theme, "error"),
173            # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but
174            # that currently isn't technically possible. This should be moved elsewhere in the future
175            # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a
176            # non-modal alternative.
177            "hit": idleConf.GetHighlight(theme, "hit"),
178            }
179        if DEBUG: print('tagdefs', self.tagdefs)
180
181    def insert(self, index, chars, tags=None):
182        "Insert chars into widget at index and mark for colorizing."
183        index = self.index(index)
184        self.delegate.insert(index, chars, tags)
185        self.notify_range(index, index + "+%dc" % len(chars))
186
187    def delete(self, index1, index2=None):
188        "Delete chars between indexes and mark for colorizing."
189        index1 = self.index(index1)
190        self.delegate.delete(index1, index2)
191        self.notify_range(index1)
192
193    def notify_range(self, index1, index2=None):
194        "Mark text changes for processing and restart colorizing, if active."
195        self.tag_add("TODO", index1, index2)
196        if self.after_id:
197            if DEBUG: print("colorizing already scheduled")
198            return
199        if self.colorizing:
200            self.stop_colorizing = True
201            if DEBUG: print("stop colorizing")
202        if self.allow_colorizing:
203            if DEBUG: print("schedule colorizing")
204            self.after_id = self.after(1, self.recolorize)
205        return
206
207    def close(self):
208        if self.after_id:
209            after_id = self.after_id
210            self.after_id = None
211            if DEBUG: print("cancel scheduled recolorizer")
212            self.after_cancel(after_id)
213        self.allow_colorizing = False
214        self.stop_colorizing = True
215
216    def toggle_colorize_event(self, event=None):
217        """Toggle colorizing on and off.
218
219        When toggling off, if colorizing is scheduled or is in
220        process, it will be cancelled and/or stopped.
221
222        When toggling on, colorizing will be scheduled.
223        """
224        if self.after_id:
225            after_id = self.after_id
226            self.after_id = None
227            if DEBUG: print("cancel scheduled recolorizer")
228            self.after_cancel(after_id)
229        if self.allow_colorizing and self.colorizing:
230            if DEBUG: print("stop colorizing")
231            self.stop_colorizing = True
232        self.allow_colorizing = not self.allow_colorizing
233        if self.allow_colorizing and not self.colorizing:
234            self.after_id = self.after(1, self.recolorize)
235        if DEBUG:
236            print("auto colorizing turned",
237                  "on" if self.allow_colorizing else "off")
238        return "break"
239
240    def recolorize(self):
241        """Timer event (every 1ms) to colorize text.
242
243        Colorizing is only attempted when the text widget exists,
244        when colorizing is toggled on, and when the colorizing
245        process is not already running.
246
247        After colorizing is complete, some cleanup is done to
248        make sure that all the text has been colorized.
249        """
250        self.after_id = None
251        if not self.delegate:
252            if DEBUG: print("no delegate")
253            return
254        if not self.allow_colorizing:
255            if DEBUG: print("auto colorizing is off")
256            return
257        if self.colorizing:
258            if DEBUG: print("already colorizing")
259            return
260        try:
261            self.stop_colorizing = False
262            self.colorizing = True
263            if DEBUG: print("colorizing...")
264            t0 = time.perf_counter()
265            self.recolorize_main()
266            t1 = time.perf_counter()
267            if DEBUG: print("%.3f seconds" % (t1-t0))
268        finally:
269            self.colorizing = False
270        if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
271            if DEBUG: print("reschedule colorizing")
272            self.after_id = self.after(1, self.recolorize)
273
274    def recolorize_main(self):
275        "Evaluate text and apply colorizing tags."
276        next = "1.0"
277        while todo_tag_range := self.tag_nextrange("TODO", next):
278            self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1])
279            sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0])
280            head = sync_tag_range[1] if sync_tag_range else "1.0"
281
282            chars = ""
283            next = head
284            lines_to_get = 1
285            ok = False
286            while not ok:
287                mark = next
288                next = self.index(mark + "+%d lines linestart" %
289                                         lines_to_get)
290                lines_to_get = min(lines_to_get * 2, 100)
291                ok = "SYNC" in self.tag_names(next + "-1c")
292                line = self.get(mark, next)
293                ##print head, "get", mark, next, "->", repr(line)
294                if not line:
295                    return
296                for tag in self.tagdefs:
297                    self.tag_remove(tag, mark, next)
298                chars += line
299                self._add_tags_in_section(chars, head)
300                if "SYNC" in self.tag_names(next + "-1c"):
301                    head = next
302                    chars = ""
303                else:
304                    ok = False
305                if not ok:
306                    # We're in an inconsistent state, and the call to
307                    # update may tell us to stop.  It may also change
308                    # the correct value for "next" (since this is a
309                    # line.col string, not a true mark).  So leave a
310                    # crumb telling the next invocation to resume here
311                    # in case update tells us to leave.
312                    self.tag_add("TODO", next)
313                self.update()
314                if self.stop_colorizing:
315                    if DEBUG: print("colorizing stopped")
316                    return
317
318    def _add_tag(self, start, end, head, matched_group_name):
319        """Add a tag to a given range in the text widget.
320
321        This is a utility function, receiving the range as `start` and
322        `end` positions, each of which is a number of characters
323        relative to the given `head` index in the text widget.
324
325        The tag to add is determined by `matched_group_name`, which is
326        the name of a regular expression "named group" as matched by
327        by the relevant highlighting regexps.
328        """
329        tag = prog_group_name_to_tag.get(matched_group_name,
330                                         matched_group_name)
331        self.tag_add(tag,
332                     f"{head}+{start:d}c",
333                     f"{head}+{end:d}c")
334
335    def _add_tags_in_section(self, chars, head):
336        """Parse and add highlighting tags to a given part of the text.
337
338        `chars` is a string with the text to parse and to which
339        highlighting is to be applied.
340
341            `head` is the index in the text widget where the text is found.
342        """
343        for m in self.prog.finditer(chars):
344            for name, matched_text in matched_named_groups(m):
345                a, b = m.span(name)
346                self._add_tag(a, b, head, name)
347                if matched_text in ("def", "class"):
348                    if m1 := self.idprog.match(chars, b):
349                        a, b = m1.span(1)
350                        self._add_tag(a, b, head, "DEFINITION")
351
352    def removecolors(self):
353        "Remove all colorizing tags."
354        for tag in self.tagdefs:
355            self.tag_remove(tag, "1.0", "end")
356
357
358def _color_delegator(parent):  # htest #
359    from tkinter import Toplevel, Text
360    from idlelib.idle_test.test_colorizer import source
361    from idlelib.percolator import Percolator
362
363    top = Toplevel(parent)
364    top.title("Test ColorDelegator")
365    x, y = map(int, parent.geometry().split('+')[1:])
366    top.geometry("700x550+%d+%d" % (x + 20, y + 175))
367
368    text = Text(top, background="white")
369    text.pack(expand=1, fill="both")
370    text.insert("insert", source)
371    text.focus_set()
372
373    color_config(text)
374    p = Percolator(text)
375    d = ColorDelegator()
376    p.insertfilter(d)
377
378
379if __name__ == "__main__":
380    from unittest import main
381    main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False)
382
383    from idlelib.idle_test.htest import run
384    run(_color_delegator)
385