1#!/usr/local/bin/python3.8 2# vim:fileencoding=utf-8 3 4 5__license__ = 'GPL v3' 6__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' 7 8import numbers 9from functools import partial 10 11from qt.core import QTextBlockUserData 12from pygments.lexer import _TokenType, Error 13 14from calibre.gui2.tweak_book.editor.syntax.base import SyntaxHighlighter 15from calibre.gui2.tweak_book.editor.syntax.utils import format_for_pygments_token, NULL_FMT 16 17NORMAL = 0 18 19 20def create_lexer(base_class): 21 ''' 22 Subclass the pygments RegexLexer to lex line by line instead of lexing full 23 text. The statestack at the end of each line is stored in the Qt block state. 24 ''' 25 26 def get_tokens_unprocessed(self, text, statestack): 27 pos = 0 28 tokendefs = self._tokens 29 statetokens = tokendefs[statestack[-1]] 30 while True: 31 for rexmatch, action, new_state in statetokens: 32 m = rexmatch(text, pos) 33 if m is not None: 34 if action is not None: 35 if type(action) is _TokenType: 36 yield pos, action, m.group() 37 else: 38 yield from action(self, m) 39 pos = m.end() 40 if new_state is not None: 41 # state transition 42 if isinstance(new_state, tuple): 43 for state in new_state: 44 if state == '#pop': 45 statestack.pop() 46 elif state == '#push': 47 statestack.append(statestack[-1]) 48 else: 49 statestack.append(state) 50 elif isinstance(new_state, numbers.Integral): 51 # pop 52 del statestack[new_state:] 53 elif new_state == '#push': 54 statestack.append(statestack[-1]) 55 else: 56 assert False, "wrong state def: %r" % new_state 57 statetokens = tokendefs[statestack[-1]] 58 break 59 else: 60 try: 61 if text[pos] == '\n': 62 # at EOL, reset state to "root" 63 statestack[:] = ['root'] 64 break 65 yield pos, Error, text[pos] 66 pos += 1 67 except IndexError: 68 break 69 70 def lex_a_line(self, state, text, i, formats_map, user_data): 71 ' Get formats for a single block (line) ' 72 statestack = list(state.pygments_stack) if state.pygments_stack is not None else ['root'] 73 74 # Lex the text using Pygments 75 formats = [] 76 if i > 0: 77 # This should never happen 78 state.pygments_stack = None 79 return [(len(text) - i, formats_map(Error))] 80 try: 81 # Pygments lexers expect newlines at the end of the line 82 for pos, token, txt in self.get_tokens_unprocessed(text + '\n', statestack): 83 if txt not in ('\n', ''): 84 formats.append((len(txt), formats_map(token))) 85 except Exception: 86 import traceback 87 traceback.print_exc() 88 state.pygments_stack = None 89 return [(len(text) - i, formats_map(Error))] 90 91 state.pygments_stack = statestack 92 return formats 93 94 name_type = type(base_class.__name__) 95 96 return type(name_type('Qt'+base_class.__name__), (base_class,), { 97 'get_tokens_unprocessed': get_tokens_unprocessed, 98 'lex_a_line':lex_a_line, 99 }) 100 101 102class State: 103 104 __slots__ = ('parse', 'pygments_stack') 105 106 def __init__(self): 107 self.parse = NORMAL 108 self.pygments_stack = None 109 110 def copy(self): 111 s = State() 112 s.pygments_stack = None if self.pygments_stack is None else list(self.pygments_stack) 113 return s 114 115 def __eq__(self, other): 116 return self.parse == getattr(other, 'parse', -1) and \ 117 self.pygments_stack == getattr(other, 'pygments_stack', False) 118 119 def __ne__(self, other): 120 return not self.__eq__(other) 121 122 def __repr__(self): 123 return "PythonState(%r)" % self.pygments_stack 124 __str__ = __repr__ 125 126 127class PygmentsUserData(QTextBlockUserData): 128 129 def __init__(self): 130 QTextBlockUserData.__init__(self) 131 self.state = State() 132 self.doc_name = None 133 134 def clear(self, state=None, doc_name=None): 135 self.state = State() if state is None else state 136 self.doc_name = doc_name 137 138 139def create_formats(highlighter): 140 cache = {} 141 theme = highlighter.theme.copy() 142 theme[None] = NULL_FMT 143 return partial(format_for_pygments_token, theme, cache) 144 145 146def create_highlighter(name, lexer_class): 147 name_type = type(lexer_class.__name__) 148 return type(name_type(name), (SyntaxHighlighter,), { 149 'state_map': {NORMAL:create_lexer(lexer_class)().lex_a_line}, 150 'create_formats_func': create_formats, 151 'user_data_factory': PygmentsUserData, 152 }) 153