1#!/usr/local/bin/python3.8
2# vim:fileencoding=utf-8
3
4
5__license__ = 'GPL v3'
6__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
7
8import numbers
9from functools import partial
10
11from qt.core import QTextBlockUserData
12from pygments.lexer import _TokenType, Error
13
14from calibre.gui2.tweak_book.editor.syntax.base import SyntaxHighlighter
15from calibre.gui2.tweak_book.editor.syntax.utils import format_for_pygments_token, NULL_FMT
16
17NORMAL = 0
18
19
20def create_lexer(base_class):
21    '''
22    Subclass the pygments RegexLexer to lex line by line instead of lexing full
23    text. The statestack at the end of each line is stored in the Qt block state.
24    '''
25
26    def get_tokens_unprocessed(self, text, statestack):
27        pos = 0
28        tokendefs = self._tokens
29        statetokens = tokendefs[statestack[-1]]
30        while True:
31            for rexmatch, action, new_state in statetokens:
32                m = rexmatch(text, pos)
33                if m is not None:
34                    if action is not None:
35                        if type(action) is _TokenType:
36                            yield pos, action, m.group()
37                        else:
38                            yield from action(self, m)
39                    pos = m.end()
40                    if new_state is not None:
41                        # state transition
42                        if isinstance(new_state, tuple):
43                            for state in new_state:
44                                if state == '#pop':
45                                    statestack.pop()
46                                elif state == '#push':
47                                    statestack.append(statestack[-1])
48                                else:
49                                    statestack.append(state)
50                        elif isinstance(new_state, numbers.Integral):
51                            # pop
52                            del statestack[new_state:]
53                        elif new_state == '#push':
54                            statestack.append(statestack[-1])
55                        else:
56                            assert False, "wrong state def: %r" % new_state
57                        statetokens = tokendefs[statestack[-1]]
58                    break
59            else:
60                try:
61                    if text[pos] == '\n':
62                        # at EOL, reset state to "root"
63                        statestack[:] = ['root']
64                        break
65                    yield pos, Error, text[pos]
66                    pos += 1
67                except IndexError:
68                    break
69
70    def lex_a_line(self, state, text, i, formats_map, user_data):
71        ' Get formats for a single block (line) '
72        statestack = list(state.pygments_stack) if state.pygments_stack is not None else ['root']
73
74        # Lex the text using Pygments
75        formats = []
76        if i > 0:
77            # This should never happen
78            state.pygments_stack = None
79            return [(len(text) - i, formats_map(Error))]
80        try:
81            # Pygments lexers expect newlines at the end of the line
82            for pos, token, txt in self.get_tokens_unprocessed(text + '\n', statestack):
83                if txt not in ('\n', ''):
84                    formats.append((len(txt), formats_map(token)))
85        except Exception:
86            import traceback
87            traceback.print_exc()
88            state.pygments_stack = None
89            return [(len(text) - i, formats_map(Error))]
90
91        state.pygments_stack = statestack
92        return formats
93
94    name_type = type(base_class.__name__)
95
96    return type(name_type('Qt'+base_class.__name__), (base_class,), {
97        'get_tokens_unprocessed': get_tokens_unprocessed,
98        'lex_a_line':lex_a_line,
99    })
100
101
102class State:
103
104    __slots__ = ('parse', 'pygments_stack')
105
106    def __init__(self):
107        self.parse = NORMAL
108        self.pygments_stack = None
109
110    def copy(self):
111        s = State()
112        s.pygments_stack = None if self.pygments_stack is None else list(self.pygments_stack)
113        return s
114
115    def __eq__(self, other):
116        return self.parse == getattr(other, 'parse', -1) and \
117            self.pygments_stack == getattr(other, 'pygments_stack', False)
118
119    def __ne__(self, other):
120        return not self.__eq__(other)
121
122    def __repr__(self):
123        return "PythonState(%r)" % self.pygments_stack
124    __str__ = __repr__
125
126
127class PygmentsUserData(QTextBlockUserData):
128
129    def __init__(self):
130        QTextBlockUserData.__init__(self)
131        self.state = State()
132        self.doc_name = None
133
134    def clear(self, state=None, doc_name=None):
135        self.state = State() if state is None else state
136        self.doc_name = doc_name
137
138
139def create_formats(highlighter):
140    cache = {}
141    theme = highlighter.theme.copy()
142    theme[None] = NULL_FMT
143    return partial(format_for_pygments_token, theme, cache)
144
145
146def create_highlighter(name, lexer_class):
147    name_type = type(lexer_class.__name__)
148    return type(name_type(name), (SyntaxHighlighter,), {
149        'state_map': {NORMAL:create_lexer(lexer_class)().lex_a_line},
150        'create_formats_func': create_formats,
151        'user_data_factory': PygmentsUserData,
152    })
153