1# -*- coding: utf-8 -*- 2""" 3Rather than trying to parse all the lines of code at once, this module implements methods 4for returning one line code at a time. 5""" 6 7from abc import abstractmethod, ABCMeta 8from chardet import detect 9 10 11class LineFeeder(metaclass=ABCMeta): 12 """An abstract representation for reading lines of characters, a 13 "feeder". The purpose of a feeder is to mediate the consumption of 14 characters between the tokeniser and the actual file being scaned, 15 as well to store messages regarding tokenization errors. 16 """ 17 18 def __init__(self, filename: str): 19 """ 20 :param filename: A string that describes the source of the feeder, i.e. 21 the filename that is being feed. 22 """ 23 self.messages = [] 24 self.lineno = 0 25 self.filename = filename 26 27 @abstractmethod 28 def feed(self): 29 """ 30 Consume and return next line of code. Each line should be followed by a 31 newline character. Returns '' after all lines are consumed. 32 """ 33 return "" 34 35 @abstractmethod 36 def empty(self): 37 """ 38 Return True once all lines have been consumed. 39 """ 40 return True 41 42 def message(self, sym: str, tag, *args): 43 """ 44 Append a generic message of type ``sym`` to the message queue. 45 """ 46 if sym == "Syntax": 47 message = self.syntax_message(sym, tag, *args) 48 else: 49 message = [sym, tag] + list(args) 50 self.messages.append(message) 51 52 def syntax_message(self, sym: str, tag, *args): 53 """ 54 Append a message concerning syntax errors to the message queue. 55 """ 56 if len(args) > 3: 57 raise ValueError("Too many args.") 58 message = [sym, tag] 59 for i in range(3): 60 if i < len(args): 61 message.append('"' + args[i] + '"') 62 else: 63 message.append('""') 64 message.append(self.lineno) 65 message.append('"' + self.filename + '"') 66 assert len(message) == 7 67 return message 68 69 # # TODO: Rethink this? 70 # def syntax_message(self, sym: str, tag, *args): 71 # for message in self.messages: 72 # evaluation.message(*message) 73 # self.messages = [] 74 75 76class MultiLineFeeder(LineFeeder): 77 "A feeder that feeds one line at a time." 78 79 def __init__(self, lines, filename=""): 80 """ 81 :param lines: The source of the feeder (a string). 82 :param filename: A string that describes the source of the feeder, i.e. 83 the filename that is being feed. 84 """ 85 super(MultiLineFeeder, self).__init__(filename) 86 self.lineno = 0 87 if isinstance(lines, str): 88 self.lines = lines.splitlines(True) 89 else: 90 self.lines = lines 91 92 def feed(self): 93 if self.lineno < len(self.lines): 94 result = self.lines[self.lineno] 95 self.lineno += 1 96 else: 97 result = "" 98 return result 99 100 def empty(self): 101 return self.lineno >= len(self.lines) 102 103 104class SingleLineFeeder(LineFeeder): 105 "A feeder that feeds all the code as a single line." 106 107 def __init__(self, code, filename=""): 108 """ 109 :param code: The source of the feeder (a string). 110 :param filename: A string that describes the source of the feeder, i.e. 111 the filename that is being feed. 112 """ 113 super().__init__(filename) 114 self.code = code 115 self._empty = False 116 117 def feed(self): 118 if self._empty: 119 return "" 120 self._empty = True 121 self.lineno += 1 122 return self.code 123 124 def empty(self): 125 return self._empty 126 127 128class FileLineFeeder(LineFeeder): 129 "A feeder that feeds lines from an open ``File`` object" 130 131 def __init__(self, fileobject, trace_fn=None): 132 """ 133 :param fileobject: The source of the feeder (a string). 134 :param filename: A string that describes the source of the feeder, 135 i.e. the filename that is being feed. 136 """ 137 super().__init__(fileobject.name) 138 self.fileobject = fileobject 139 self.lineno = 0 140 self.eof = False 141 self.trace_fn = trace_fn 142 143 def feed(self) -> str: 144 result = self.fileobject.readline() 145 while result == "\n": 146 result = self.fileobject.readline() 147 self.lineno += 1 148 if self.trace_fn: 149 self.trace_fn("%5d: %s" % (self.lineno, result), end="") 150 if result: 151 self.lineno += 1 152 else: 153 self.eof = True 154 return result 155 156 def empty(self) -> bool: 157 return self.eof 158