1def _isIdentifierCharacter(c): 2 return ((c == '_') or ('0' <= c <= '9') or ('A' <= c <= 'Z') or ('a' <= c <= 'z')) 3 4 5class Lexer(object): 6 def __init__(my, source): 7 my.source = source 8 my.i = 0 9 my.next() 10 11 def skipWhitespace(my): 12 while my.i < len(my.source): 13 c = my.source[my.i] 14 if ord(c) > 32: 15 break 16 my.i = my.i + 1 17 18 def skipQuotedString(my, quote): 19 my.i = my.i + 1 # skip leading quote 20 while (my.i < len(my.source)) and (my.source[my.i] != quote): 21 if my.source[my.i] == '\\': 22 my.i = my.i + 1 # skip backslash, then skip escaped character 23 my.i = my.i + 1 24 my.i = my.i + 1 # skip trailing quote 25 26 def next(my): 27 my.skipWhitespace() 28 if my.i >= len(my.source): 29 my.token = None 30 else: 31 my.token = my.source[my.i] 32 if my.token == '_': 33 my.lexIdentifier() 34 elif my.token == '"': 35 my.skipQuotedString('"') 36 my.next() 37 elif my.token == "'": 38 my.skipQuotedString("'") 39 my.next() 40 else: 41 my.i = my.i + 1 42 43 def lexIdentifier(my): 44 i = my.i 45 my.i = my.i + 1 46 while _isIdentifierCharacter(my.source[my.i]): 47 my.i = my.i + 1 48 my.token = my.source[i:my.i] 49 50