1// Package token defines constants representing the lexical tokens for HCL 2// (HashiCorp Configuration Language) 3package token 4 5import ( 6 "fmt" 7 "strconv" 8 "strings" 9 10 hclstrconv "github.com/hashicorp/hcl/hcl/strconv" 11) 12 13// Token defines a single HCL token which can be obtained via the Scanner 14type Token struct { 15 Type Type 16 Pos Pos 17 Text string 18 JSON bool 19} 20 21// Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language) 22type Type int 23 24const ( 25 // Special tokens 26 ILLEGAL Type = iota 27 EOF 28 COMMENT 29 30 identifier_beg 31 IDENT // literals 32 literal_beg 33 NUMBER // 12345 34 FLOAT // 123.45 35 BOOL // true,false 36 STRING // "abc" 37 HEREDOC // <<FOO\nbar\nFOO 38 literal_end 39 identifier_end 40 41 operator_beg 42 LBRACK // [ 43 LBRACE // { 44 COMMA // , 45 PERIOD // . 46 47 RBRACK // ] 48 RBRACE // } 49 50 ASSIGN // = 51 ADD // + 52 SUB // - 53 operator_end 54) 55 56var tokens = [...]string{ 57 ILLEGAL: "ILLEGAL", 58 59 EOF: "EOF", 60 COMMENT: "COMMENT", 61 62 IDENT: "IDENT", 63 NUMBER: "NUMBER", 64 FLOAT: "FLOAT", 65 BOOL: "BOOL", 66 STRING: "STRING", 67 68 LBRACK: "LBRACK", 69 LBRACE: "LBRACE", 70 COMMA: "COMMA", 71 PERIOD: "PERIOD", 72 HEREDOC: "HEREDOC", 73 74 RBRACK: "RBRACK", 75 RBRACE: "RBRACE", 76 77 ASSIGN: "ASSIGN", 78 ADD: "ADD", 79 SUB: "SUB", 80} 81 82// String returns the string corresponding to the token tok. 83func (t Type) String() string { 84 s := "" 85 if 0 <= t && t < Type(len(tokens)) { 86 s = tokens[t] 87 } 88 if s == "" { 89 s = "token(" + strconv.Itoa(int(t)) + ")" 90 } 91 return s 92} 93 94// IsIdentifier returns true for tokens corresponding to identifiers and basic 95// type literals; it returns false otherwise. 96func (t Type) IsIdentifier() bool { return identifier_beg < t && t < identifier_end } 97 98// IsLiteral returns true for tokens corresponding to basic type literals; it 99// returns false otherwise. 100func (t Type) IsLiteral() bool { return literal_beg < t && t < literal_end } 101 102// IsOperator returns true for tokens corresponding to operators and 103// delimiters; it returns false otherwise. 104func (t Type) IsOperator() bool { return operator_beg < t && t < operator_end } 105 106// String returns the token's literal text. Note that this is only 107// applicable for certain token types, such as token.IDENT, 108// token.STRING, etc.. 109func (t Token) String() string { 110 return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text) 111} 112 113// Value returns the properly typed value for this token. The type of 114// the returned interface{} is guaranteed based on the Type field. 115// 116// This can only be called for literal types. If it is called for any other 117// type, this will panic. 118func (t Token) Value() interface{} { 119 switch t.Type { 120 case BOOL: 121 if t.Text == "true" { 122 return true 123 } else if t.Text == "false" { 124 return false 125 } 126 127 panic("unknown bool value: " + t.Text) 128 case FLOAT: 129 v, err := strconv.ParseFloat(t.Text, 64) 130 if err != nil { 131 panic(err) 132 } 133 134 return float64(v) 135 case NUMBER: 136 v, err := strconv.ParseInt(t.Text, 0, 64) 137 if err != nil { 138 panic(err) 139 } 140 141 return int64(v) 142 case IDENT: 143 return t.Text 144 case HEREDOC: 145 return unindentHeredoc(t.Text) 146 case STRING: 147 // Determine the Unquote method to use. If it came from JSON, 148 // then we need to use the built-in unquote since we have to 149 // escape interpolations there. 150 f := hclstrconv.Unquote 151 if t.JSON { 152 f = strconv.Unquote 153 } 154 155 // This case occurs if json null is used 156 if t.Text == "" { 157 return "" 158 } 159 160 v, err := f(t.Text) 161 if err != nil { 162 panic(fmt.Sprintf("unquote %s err: %s", t.Text, err)) 163 } 164 165 return v 166 default: 167 panic(fmt.Sprintf("unimplemented Value for type: %s", t.Type)) 168 } 169} 170 171// unindentHeredoc returns the string content of a HEREDOC if it is started with << 172// and the content of a HEREDOC with the hanging indent removed if it is started with 173// a <<-, and the terminating line is at least as indented as the least indented line. 174func unindentHeredoc(heredoc string) string { 175 // We need to find the end of the marker 176 idx := strings.IndexByte(heredoc, '\n') 177 if idx == -1 { 178 panic("heredoc doesn't contain newline") 179 } 180 181 unindent := heredoc[2] == '-' 182 183 // We can optimize if the heredoc isn't marked for indentation 184 if !unindent { 185 return string(heredoc[idx+1 : len(heredoc)-idx+1]) 186 } 187 188 // We need to unindent each line based on the indentation level of the marker 189 lines := strings.Split(string(heredoc[idx+1:len(heredoc)-idx+2]), "\n") 190 whitespacePrefix := lines[len(lines)-1] 191 192 isIndented := true 193 for _, v := range lines { 194 if strings.HasPrefix(v, whitespacePrefix) { 195 continue 196 } 197 198 isIndented = false 199 break 200 } 201 202 // If all lines are not at least as indented as the terminating mark, return the 203 // heredoc as is, but trim the leading space from the marker on the final line. 204 if !isIndented { 205 return strings.TrimRight(string(heredoc[idx+1:len(heredoc)-idx+1]), " \t") 206 } 207 208 unindentedLines := make([]string, len(lines)) 209 for k, v := range lines { 210 if k == len(lines)-1 { 211 unindentedLines[k] = "" 212 break 213 } 214 215 unindentedLines[k] = strings.TrimPrefix(v, whitespacePrefix) 216 } 217 218 return strings.Join(unindentedLines, "\n") 219} 220