1package hclsyntax 2 3import ( 4 "bytes" 5 "fmt" 6 "path/filepath" 7 "runtime" 8 "strings" 9 10 "github.com/hashicorp/hcl2/hcl" 11) 12 13// This is set to true at init() time in tests, to enable more useful output 14// if a stack discipline error is detected. It should not be enabled in 15// normal mode since there is a performance penalty from accessing the 16// runtime stack to produce the traces, but could be temporarily set to 17// true for debugging if desired. 18var tracePeekerNewlinesStack = false 19 20type peeker struct { 21 Tokens Tokens 22 NextIndex int 23 24 IncludeComments bool 25 IncludeNewlinesStack []bool 26 27 // used only when tracePeekerNewlinesStack is set 28 newlineStackChanges []peekerNewlineStackChange 29} 30 31// for use in debugging the stack usage only 32type peekerNewlineStackChange struct { 33 Pushing bool // if false, then popping 34 Frame runtime.Frame 35 Include bool 36} 37 38func newPeeker(tokens Tokens, includeComments bool) *peeker { 39 return &peeker{ 40 Tokens: tokens, 41 IncludeComments: includeComments, 42 43 IncludeNewlinesStack: []bool{true}, 44 } 45} 46 47func (p *peeker) Peek() Token { 48 ret, _ := p.nextToken() 49 return ret 50} 51 52func (p *peeker) Read() Token { 53 ret, nextIdx := p.nextToken() 54 p.NextIndex = nextIdx 55 return ret 56} 57 58func (p *peeker) NextRange() hcl.Range { 59 return p.Peek().Range 60} 61 62func (p *peeker) PrevRange() hcl.Range { 63 if p.NextIndex == 0 { 64 return p.NextRange() 65 } 66 67 return p.Tokens[p.NextIndex-1].Range 68} 69 70func (p *peeker) nextToken() (Token, int) { 71 for i := p.NextIndex; i < len(p.Tokens); i++ { 72 tok := p.Tokens[i] 73 switch tok.Type { 74 case TokenComment: 75 if !p.IncludeComments { 76 // Single-line comment tokens, starting with # or //, absorb 77 // the trailing newline that terminates them as part of their 78 // bytes. When we're filtering out comments, we must as a 79 // special case transform these to newline tokens in order 80 // to properly parse newline-terminated block items. 81 82 if p.includingNewlines() { 83 if len(tok.Bytes) > 0 && tok.Bytes[len(tok.Bytes)-1] == '\n' { 84 fakeNewline := Token{ 85 Type: TokenNewline, 86 Bytes: tok.Bytes[len(tok.Bytes)-1 : len(tok.Bytes)], 87 88 // We use the whole token range as the newline 89 // range, even though that's a little... weird, 90 // because otherwise we'd need to go count 91 // characters again in order to figure out the 92 // column of the newline, and that complexity 93 // isn't justified when ranges of newlines are 94 // so rarely printed anyway. 95 Range: tok.Range, 96 } 97 return fakeNewline, i + 1 98 } 99 } 100 101 continue 102 } 103 case TokenNewline: 104 if !p.includingNewlines() { 105 continue 106 } 107 } 108 109 return tok, i + 1 110 } 111 112 // if we fall out here then we'll return the EOF token, and leave 113 // our index pointed off the end of the array so we'll keep 114 // returning EOF in future too. 115 return p.Tokens[len(p.Tokens)-1], len(p.Tokens) 116} 117 118func (p *peeker) includingNewlines() bool { 119 return p.IncludeNewlinesStack[len(p.IncludeNewlinesStack)-1] 120} 121 122func (p *peeker) PushIncludeNewlines(include bool) { 123 if tracePeekerNewlinesStack { 124 // Record who called us so that we can more easily track down any 125 // mismanagement of the stack in the parser. 126 callers := []uintptr{0} 127 runtime.Callers(2, callers) 128 frames := runtime.CallersFrames(callers) 129 frame, _ := frames.Next() 130 p.newlineStackChanges = append(p.newlineStackChanges, peekerNewlineStackChange{ 131 true, frame, include, 132 }) 133 } 134 135 p.IncludeNewlinesStack = append(p.IncludeNewlinesStack, include) 136} 137 138func (p *peeker) PopIncludeNewlines() bool { 139 stack := p.IncludeNewlinesStack 140 remain, ret := stack[:len(stack)-1], stack[len(stack)-1] 141 p.IncludeNewlinesStack = remain 142 143 if tracePeekerNewlinesStack { 144 // Record who called us so that we can more easily track down any 145 // mismanagement of the stack in the parser. 146 callers := []uintptr{0} 147 runtime.Callers(2, callers) 148 frames := runtime.CallersFrames(callers) 149 frame, _ := frames.Next() 150 p.newlineStackChanges = append(p.newlineStackChanges, peekerNewlineStackChange{ 151 false, frame, ret, 152 }) 153 } 154 155 return ret 156} 157 158// AssertEmptyNewlinesStack checks if the IncludeNewlinesStack is empty, doing 159// panicking if it is not. This can be used to catch stack mismanagement that 160// might otherwise just cause confusing downstream errors. 161// 162// This function is a no-op if the stack is empty when called. 163// 164// If newlines stack tracing is enabled by setting the global variable 165// tracePeekerNewlinesStack at init time, a full log of all of the push/pop 166// calls will be produced to help identify which caller in the parser is 167// misbehaving. 168func (p *peeker) AssertEmptyIncludeNewlinesStack() { 169 if len(p.IncludeNewlinesStack) != 1 { 170 // Should never happen; indicates mismanagement of the stack inside 171 // the parser. 172 if p.newlineStackChanges != nil { // only if traceNewlinesStack is enabled above 173 panic(fmt.Errorf( 174 "non-empty IncludeNewlinesStack after parse with %d calls unaccounted for:\n%s", 175 len(p.IncludeNewlinesStack)-1, 176 formatPeekerNewlineStackChanges(p.newlineStackChanges), 177 )) 178 } else { 179 panic(fmt.Errorf("non-empty IncludeNewlinesStack after parse: %#v", p.IncludeNewlinesStack)) 180 } 181 } 182} 183 184func formatPeekerNewlineStackChanges(changes []peekerNewlineStackChange) string { 185 indent := 0 186 var buf bytes.Buffer 187 for _, change := range changes { 188 funcName := change.Frame.Function 189 if idx := strings.LastIndexByte(funcName, '.'); idx != -1 { 190 funcName = funcName[idx+1:] 191 } 192 filename := change.Frame.File 193 if idx := strings.LastIndexByte(filename, filepath.Separator); idx != -1 { 194 filename = filename[idx+1:] 195 } 196 197 switch change.Pushing { 198 199 case true: 200 buf.WriteString(strings.Repeat(" ", indent)) 201 fmt.Fprintf(&buf, "PUSH %#v (%s at %s:%d)\n", change.Include, funcName, filename, change.Frame.Line) 202 indent++ 203 204 case false: 205 indent-- 206 buf.WriteString(strings.Repeat(" ", indent)) 207 fmt.Fprintf(&buf, "POP %#v (%s at %s:%d)\n", change.Include, funcName, filename, change.Frame.Line) 208 209 } 210 } 211 return buf.String() 212} 213