1package ini 2 3import ( 4 "bytes" 5 "io" 6 "io/ioutil" 7 8 "github.com/aws/aws-sdk-go/aws/awserr" 9) 10 11const ( 12 // ErrCodeUnableToReadFile is used when a file is failed to be 13 // opened or read from. 14 ErrCodeUnableToReadFile = "FailedRead" 15) 16 17// TokenType represents the various different tokens types 18type TokenType int 19 20func (t TokenType) String() string { 21 switch t { 22 case TokenNone: 23 return "none" 24 case TokenLit: 25 return "literal" 26 case TokenSep: 27 return "sep" 28 case TokenOp: 29 return "op" 30 case TokenWS: 31 return "ws" 32 case TokenNL: 33 return "newline" 34 case TokenComment: 35 return "comment" 36 case TokenComma: 37 return "comma" 38 default: 39 return "" 40 } 41} 42 43// TokenType enums 44const ( 45 TokenNone = TokenType(iota) 46 TokenLit 47 TokenSep 48 TokenComma 49 TokenOp 50 TokenWS 51 TokenNL 52 TokenComment 53) 54 55type iniLexer struct{} 56 57// Tokenize will return a list of tokens during lexical analysis of the 58// io.Reader. 59func (l *iniLexer) Tokenize(r io.Reader) ([]Token, error) { 60 b, err := ioutil.ReadAll(r) 61 if err != nil { 62 return nil, awserr.New(ErrCodeUnableToReadFile, "unable to read file", err) 63 } 64 65 return l.tokenize(b) 66} 67 68func (l *iniLexer) tokenize(b []byte) ([]Token, error) { 69 runes := bytes.Runes(b) 70 var err error 71 n := 0 72 tokenAmount := countTokens(runes) 73 tokens := make([]Token, tokenAmount) 74 count := 0 75 76 for len(runes) > 0 && count < tokenAmount { 77 switch { 78 case isWhitespace(runes[0]): 79 tokens[count], n, err = newWSToken(runes) 80 case isComma(runes[0]): 81 tokens[count], n = newCommaToken(), 1 82 case isComment(runes): 83 tokens[count], n, err = newCommentToken(runes) 84 case isNewline(runes): 85 tokens[count], n, err = newNewlineToken(runes) 86 case isSep(runes): 87 tokens[count], n, err = newSepToken(runes) 88 case isOp(runes): 89 tokens[count], n, err = newOpToken(runes) 90 default: 91 tokens[count], n, err = newLitToken(runes) 92 } 93 94 if err != nil { 95 return nil, err 96 } 97 98 count++ 99 100 runes = runes[n:] 101 } 102 103 return tokens[:count], nil 104} 105 106func countTokens(runes []rune) int { 107 count, n := 0, 0 108 var err error 109 110 for len(runes) > 0 { 111 switch { 112 case isWhitespace(runes[0]): 113 _, n, err = newWSToken(runes) 114 case isComma(runes[0]): 115 _, n = newCommaToken(), 1 116 case isComment(runes): 117 _, n, err = newCommentToken(runes) 118 case isNewline(runes): 119 _, n, err = newNewlineToken(runes) 120 case isSep(runes): 121 _, n, err = newSepToken(runes) 122 case isOp(runes): 123 _, n, err = newOpToken(runes) 124 default: 125 _, n, err = newLitToken(runes) 126 } 127 128 if err != nil { 129 return 0 130 } 131 132 count++ 133 runes = runes[n:] 134 } 135 136 return count + 1 137} 138 139// Token indicates a metadata about a given value. 140type Token struct { 141 t TokenType 142 ValueType ValueType 143 base int 144 raw []rune 145} 146 147var emptyValue = Value{} 148 149func newToken(t TokenType, raw []rune, v ValueType) Token { 150 return Token{ 151 t: t, 152 raw: raw, 153 ValueType: v, 154 } 155} 156 157// Raw return the raw runes that were consumed 158func (tok Token) Raw() []rune { 159 return tok.raw 160} 161 162// Type returns the token type 163func (tok Token) Type() TokenType { 164 return tok.t 165} 166