1import builtins
2import keyword
3import re
4import time
5
6from idlelib.config import idleConf
7from idlelib.delegator import Delegator
8
9DEBUG = False
10
11
12def any(name, alternates):
13    "Return a named group pattern matching list of alternates."
14    return "(?P<%s>" % name + "|".join(alternates) + ")"
15
16
17def make_pat():
18    kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
19    builtinlist = [str(name) for name in dir(builtins)
20                   if not name.startswith('_') and
21                   name not in keyword.kwlist]
22    builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
23    comment = any("COMMENT", [r"#[^\n]*"])
24    stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?"
25    sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
26    dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
27    sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
28    dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
29    string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
30    return (kw + "|" + builtin + "|" + comment + "|" + string +
31            "|" + any("SYNC", [r"\n"]))
32
33
34prog = re.compile(make_pat(), re.S)
35idprog = re.compile(r"\s+(\w+)", re.S)
36
37
38def color_config(text):
39    """Set color options of Text widget.
40
41    If ColorDelegator is used, this should be called first.
42    """
43    # Called from htest, TextFrame, Editor, and Turtledemo.
44    # Not automatic because ColorDelegator does not know 'text'.
45    theme = idleConf.CurrentTheme()
46    normal_colors = idleConf.GetHighlight(theme, 'normal')
47    cursor_color = idleConf.GetHighlight(theme, 'cursor')['foreground']
48    select_colors = idleConf.GetHighlight(theme, 'hilite')
49    text.config(
50        foreground=normal_colors['foreground'],
51        background=normal_colors['background'],
52        insertbackground=cursor_color,
53        selectforeground=select_colors['foreground'],
54        selectbackground=select_colors['background'],
55        inactiveselectbackground=select_colors['background'],  # new in 8.5
56        )
57
58
59class ColorDelegator(Delegator):
60    """Delegator for syntax highlighting (text coloring).
61
62    Instance variables:
63        delegate: Delegator below this one in the stack, meaning the
64                one this one delegates to.
65
66        Used to track state:
67        after_id: Identifier for scheduled after event, which is a
68                timer for colorizing the text.
69        allow_colorizing: Boolean toggle for applying colorizing.
70        colorizing: Boolean flag when colorizing is in process.
71        stop_colorizing: Boolean flag to end an active colorizing
72                process.
73    """
74
75    def __init__(self):
76        Delegator.__init__(self)
77        self.init_state()
78        self.prog = prog
79        self.idprog = idprog
80        self.LoadTagDefs()
81
82    def init_state(self):
83        "Initialize variables that track colorizing state."
84        self.after_id = None
85        self.allow_colorizing = True
86        self.stop_colorizing = False
87        self.colorizing = False
88
89    def setdelegate(self, delegate):
90        """Set the delegate for this instance.
91
92        A delegate is an instance of a Delegator class and each
93        delegate points to the next delegator in the stack.  This
94        allows multiple delegators to be chained together for a
95        widget.  The bottom delegate for a colorizer is a Text
96        widget.
97
98        If there is a delegate, also start the colorizing process.
99        """
100        if self.delegate is not None:
101            self.unbind("<<toggle-auto-coloring>>")
102        Delegator.setdelegate(self, delegate)
103        if delegate is not None:
104            self.config_colors()
105            self.bind("<<toggle-auto-coloring>>", self.toggle_colorize_event)
106            self.notify_range("1.0", "end")
107        else:
108            # No delegate - stop any colorizing.
109            self.stop_colorizing = True
110            self.allow_colorizing = False
111
112    def config_colors(self):
113        "Configure text widget tags with colors from tagdefs."
114        for tag, cnf in self.tagdefs.items():
115            self.tag_configure(tag, **cnf)
116        self.tag_raise('sel')
117
118    def LoadTagDefs(self):
119        "Create dictionary of tag names to text colors."
120        theme = idleConf.CurrentTheme()
121        self.tagdefs = {
122            "COMMENT": idleConf.GetHighlight(theme, "comment"),
123            "KEYWORD": idleConf.GetHighlight(theme, "keyword"),
124            "BUILTIN": idleConf.GetHighlight(theme, "builtin"),
125            "STRING": idleConf.GetHighlight(theme, "string"),
126            "DEFINITION": idleConf.GetHighlight(theme, "definition"),
127            "SYNC": {'background': None, 'foreground': None},
128            "TODO": {'background': None, 'foreground': None},
129            "ERROR": idleConf.GetHighlight(theme, "error"),
130            # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but
131            # that currently isn't technically possible. This should be moved elsewhere in the future
132            # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a
133            # non-modal alternative.
134            "hit": idleConf.GetHighlight(theme, "hit"),
135            }
136
137        if DEBUG: print('tagdefs', self.tagdefs)
138
139    def insert(self, index, chars, tags=None):
140        "Insert chars into widget at index and mark for colorizing."
141        index = self.index(index)
142        self.delegate.insert(index, chars, tags)
143        self.notify_range(index, index + "+%dc" % len(chars))
144
145    def delete(self, index1, index2=None):
146        "Delete chars between indexes and mark for colorizing."
147        index1 = self.index(index1)
148        self.delegate.delete(index1, index2)
149        self.notify_range(index1)
150
151    def notify_range(self, index1, index2=None):
152        "Mark text changes for processing and restart colorizing, if active."
153        self.tag_add("TODO", index1, index2)
154        if self.after_id:
155            if DEBUG: print("colorizing already scheduled")
156            return
157        if self.colorizing:
158            self.stop_colorizing = True
159            if DEBUG: print("stop colorizing")
160        if self.allow_colorizing:
161            if DEBUG: print("schedule colorizing")
162            self.after_id = self.after(1, self.recolorize)
163        return
164
165    def close(self):
166        if self.after_id:
167            after_id = self.after_id
168            self.after_id = None
169            if DEBUG: print("cancel scheduled recolorizer")
170            self.after_cancel(after_id)
171        self.allow_colorizing = False
172        self.stop_colorizing = True
173
174    def toggle_colorize_event(self, event=None):
175        """Toggle colorizing on and off.
176
177        When toggling off, if colorizing is scheduled or is in
178        process, it will be cancelled and/or stopped.
179
180        When toggling on, colorizing will be scheduled.
181        """
182        if self.after_id:
183            after_id = self.after_id
184            self.after_id = None
185            if DEBUG: print("cancel scheduled recolorizer")
186            self.after_cancel(after_id)
187        if self.allow_colorizing and self.colorizing:
188            if DEBUG: print("stop colorizing")
189            self.stop_colorizing = True
190        self.allow_colorizing = not self.allow_colorizing
191        if self.allow_colorizing and not self.colorizing:
192            self.after_id = self.after(1, self.recolorize)
193        if DEBUG:
194            print("auto colorizing turned",
195                  "on" if self.allow_colorizing else "off")
196        return "break"
197
198    def recolorize(self):
199        """Timer event (every 1ms) to colorize text.
200
201        Colorizing is only attempted when the text widget exists,
202        when colorizing is toggled on, and when the colorizing
203        process is not already running.
204
205        After colorizing is complete, some cleanup is done to
206        make sure that all the text has been colorized.
207        """
208        self.after_id = None
209        if not self.delegate:
210            if DEBUG: print("no delegate")
211            return
212        if not self.allow_colorizing:
213            if DEBUG: print("auto colorizing is off")
214            return
215        if self.colorizing:
216            if DEBUG: print("already colorizing")
217            return
218        try:
219            self.stop_colorizing = False
220            self.colorizing = True
221            if DEBUG: print("colorizing...")
222            t0 = time.perf_counter()
223            self.recolorize_main()
224            t1 = time.perf_counter()
225            if DEBUG: print("%.3f seconds" % (t1-t0))
226        finally:
227            self.colorizing = False
228        if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
229            if DEBUG: print("reschedule colorizing")
230            self.after_id = self.after(1, self.recolorize)
231
232    def recolorize_main(self):
233        "Evaluate text and apply colorizing tags."
234        next = "1.0"
235        while True:
236            item = self.tag_nextrange("TODO", next)
237            if not item:
238                break
239            head, tail = item
240            self.tag_remove("SYNC", head, tail)
241            item = self.tag_prevrange("SYNC", head)
242            head = item[1] if item else "1.0"
243
244            chars = ""
245            next = head
246            lines_to_get = 1
247            ok = False
248            while not ok:
249                mark = next
250                next = self.index(mark + "+%d lines linestart" %
251                                         lines_to_get)
252                lines_to_get = min(lines_to_get * 2, 100)
253                ok = "SYNC" in self.tag_names(next + "-1c")
254                line = self.get(mark, next)
255                ##print head, "get", mark, next, "->", repr(line)
256                if not line:
257                    return
258                for tag in self.tagdefs:
259                    self.tag_remove(tag, mark, next)
260                chars = chars + line
261                m = self.prog.search(chars)
262                while m:
263                    for key, value in m.groupdict().items():
264                        if value:
265                            a, b = m.span(key)
266                            self.tag_add(key,
267                                         head + "+%dc" % a,
268                                         head + "+%dc" % b)
269                            if value in ("def", "class"):
270                                m1 = self.idprog.match(chars, b)
271                                if m1:
272                                    a, b = m1.span(1)
273                                    self.tag_add("DEFINITION",
274                                                 head + "+%dc" % a,
275                                                 head + "+%dc" % b)
276                    m = self.prog.search(chars, m.end())
277                if "SYNC" in self.tag_names(next + "-1c"):
278                    head = next
279                    chars = ""
280                else:
281                    ok = False
282                if not ok:
283                    # We're in an inconsistent state, and the call to
284                    # update may tell us to stop.  It may also change
285                    # the correct value for "next" (since this is a
286                    # line.col string, not a true mark).  So leave a
287                    # crumb telling the next invocation to resume here
288                    # in case update tells us to leave.
289                    self.tag_add("TODO", next)
290                self.update()
291                if self.stop_colorizing:
292                    if DEBUG: print("colorizing stopped")
293                    return
294
295    def removecolors(self):
296        "Remove all colorizing tags."
297        for tag in self.tagdefs:
298            self.tag_remove(tag, "1.0", "end")
299
300
301def _color_delegator(parent):  # htest #
302    from tkinter import Toplevel, Text
303    from idlelib.percolator import Percolator
304
305    top = Toplevel(parent)
306    top.title("Test ColorDelegator")
307    x, y = map(int, parent.geometry().split('+')[1:])
308    top.geometry("700x250+%d+%d" % (x + 20, y + 175))
309    source = (
310        "if True: int ('1') # keyword, builtin, string, comment\n"
311        "elif False: print(0)\n"
312        "else: float(None)\n"
313        "if iF + If + IF: 'keyword matching must respect case'\n"
314        "if'': x or''  # valid keyword-string no-space combinations\n"
315        "async def f(): await g()\n"
316        "# All valid prefixes for unicode and byte strings should be colored.\n"
317        "'x', '''x''', \"x\", \"\"\"x\"\"\"\n"
318        "r'x', u'x', R'x', U'x', f'x', F'x'\n"
319        "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n"
320        "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n"
321        "# Invalid combinations of legal characters should be half colored.\n"
322        "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n"
323        )
324    text = Text(top, background="white")
325    text.pack(expand=1, fill="both")
326    text.insert("insert", source)
327    text.focus_set()
328
329    color_config(text)
330    p = Percolator(text)
331    d = ColorDelegator()
332    p.insertfilter(d)
333
334
335if __name__ == "__main__":
336    from unittest import main
337    main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False)
338
339    from idlelib.idle_test.htest import run
340    run(_color_delegator)
341