1package toml 2 3import ( 4 "fmt" 5 "unicode" 6) 7 8// Define tokens 9type tokenType int 10 11const ( 12 eof = -(iota + 1) 13) 14 15const ( 16 tokenError tokenType = iota 17 tokenEOF 18 tokenComment 19 tokenKey 20 tokenString 21 tokenInteger 22 tokenTrue 23 tokenFalse 24 tokenFloat 25 tokenInf 26 tokenNan 27 tokenEqual 28 tokenLeftBracket 29 tokenRightBracket 30 tokenLeftCurlyBrace 31 tokenRightCurlyBrace 32 tokenLeftParen 33 tokenRightParen 34 tokenDoubleLeftBracket 35 tokenDoubleRightBracket 36 tokenDate 37 tokenLocalDate 38 tokenKeyGroup 39 tokenKeyGroupArray 40 tokenComma 41 tokenColon 42 tokenDollar 43 tokenStar 44 tokenQuestion 45 tokenDot 46 tokenDotDot 47 tokenEOL 48) 49 50var tokenTypeNames = []string{ 51 "Error", 52 "EOF", 53 "Comment", 54 "Key", 55 "String", 56 "Integer", 57 "True", 58 "False", 59 "Float", 60 "Inf", 61 "NaN", 62 "=", 63 "[", 64 "]", 65 "{", 66 "}", 67 "(", 68 ")", 69 "]]", 70 "[[", 71 "LocalDate", 72 "LocalDate", 73 "KeyGroup", 74 "KeyGroupArray", 75 ",", 76 ":", 77 "$", 78 "*", 79 "?", 80 ".", 81 "..", 82 "EOL", 83} 84 85type token struct { 86 Position 87 typ tokenType 88 val string 89} 90 91func (tt tokenType) String() string { 92 idx := int(tt) 93 if idx < len(tokenTypeNames) { 94 return tokenTypeNames[idx] 95 } 96 return "Unknown" 97} 98 99func (t token) String() string { 100 switch t.typ { 101 case tokenEOF: 102 return "EOF" 103 case tokenError: 104 return t.val 105 } 106 107 return fmt.Sprintf("%q", t.val) 108} 109 110func isSpace(r rune) bool { 111 return r == ' ' || r == '\t' 112} 113 114func isAlphanumeric(r rune) bool { 115 return unicode.IsLetter(r) || r == '_' 116} 117 118func isKeyChar(r rune) bool { 119 // Keys start with the first character that isn't whitespace or [ and end 120 // with the last non-whitespace character before the equals sign. Keys 121 // cannot contain a # character." 122 return !(r == '\r' || r == '\n' || r == eof || r == '=') 123} 124 125func isKeyStartChar(r rune) bool { 126 return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[') 127} 128 129func isDigit(r rune) bool { 130 return unicode.IsNumber(r) 131} 132 133func isHexDigit(r rune) bool { 134 return isDigit(r) || 135 (r >= 'a' && r <= 'f') || 136 (r >= 'A' && r <= 'F') 137} 138