1 // Scintilla source code edit control
2 /** @file LexTCL.cxx
3  ** Lexer for TCL language.
4  **/
5 // Copyright 1998-2001 by Andre Arpin <arpin@kingston.net>
6 // The License.txt file describes the conditions under which this software may be distributed.
7 
8 #include <stdlib.h>
9 #include <string.h>
10 #include <stdio.h>
11 #include <stdarg.h>
12 #include <assert.h>
13 #include <ctype.h>
14 
15 #include "ILexer.h"
16 #include "Scintilla.h"
17 #include "SciLexer.h"
18 
19 #include "WordList.h"
20 #include "LexAccessor.h"
21 #include "Accessor.h"
22 #include "StyleContext.h"
23 #include "CharacterSet.h"
24 #include "LexerModule.h"
25 
26 using namespace Scintilla;
27 
28 // Extended to accept accented characters
IsAWordChar(int ch)29 static inline bool IsAWordChar(int ch) {
30 	return ch >= 0x80 ||
31 	       (isalnum(ch) || ch == '_' || ch ==':' || ch=='.'); // : name space separator
32 }
33 
IsAWordStart(int ch)34 static inline bool IsAWordStart(int ch) {
35 	return ch >= 0x80 || (ch ==':' || isalpha(ch) || ch == '_');
36 }
37 
IsANumberChar(int ch)38 static inline bool IsANumberChar(int ch) {
39 	// Not exactly following number definition (several dots are seen as OK, etc.)
40 	// but probably enough in most cases.
41 	return (ch < 0x80) &&
42 	       (IsADigit(ch, 0x10) || toupper(ch) == 'E' ||
43 	        ch == '.' || ch == '-' || ch == '+');
44 }
45 
ColouriseTCLDoc(Sci_PositionU startPos,Sci_Position length,int,WordList * keywordlists[],Accessor & styler)46 static void ColouriseTCLDoc(Sci_PositionU startPos, Sci_Position length, int , WordList *keywordlists[], Accessor &styler) {
47 #define  isComment(s) (s==SCE_TCL_COMMENT || s==SCE_TCL_COMMENTLINE || s==SCE_TCL_COMMENT_BOX || s==SCE_TCL_BLOCK_COMMENT)
48 	const bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
49 	const bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
50 	bool commentLevel = false;
51 	bool subBrace = false; // substitution begin with a brace ${.....}
52 	enum tLineState {LS_DEFAULT, LS_OPEN_COMMENT, LS_OPEN_DOUBLE_QUOTE, LS_COMMENT_BOX, LS_MASK_STATE = 0xf,
53 	                 LS_COMMAND_EXPECTED = 16, LS_BRACE_ONLY = 32
54 	                } lineState = LS_DEFAULT;
55 	bool prevSlash = false;
56 	int currentLevel = 0;
57 	bool expected = 0;
58 	bool subParen = 0;
59 
60 	Sci_Position currentLine = styler.GetLine(startPos);
61 	if (currentLine > 0)
62 		currentLine--;
63 	length += startPos - styler.LineStart(currentLine);
64 	// make sure lines overlap
65 	startPos = styler.LineStart(currentLine);
66 
67 	WordList &keywords = *keywordlists[0];
68 	WordList &keywords2 = *keywordlists[1];
69 	WordList &keywords3 = *keywordlists[2];
70 	WordList &keywords4 = *keywordlists[3];
71 	WordList &keywords5 = *keywordlists[4];
72 	WordList &keywords6 = *keywordlists[5];
73 	WordList &keywords7 = *keywordlists[6];
74 	WordList &keywords8 = *keywordlists[7];
75 	WordList &keywords9 = *keywordlists[8];
76 
77 	if (currentLine > 0) {
78 		int ls = styler.GetLineState(currentLine - 1);
79 		lineState = tLineState(ls & LS_MASK_STATE);
80 		expected = LS_COMMAND_EXPECTED == tLineState(ls & LS_COMMAND_EXPECTED);
81 		subBrace = LS_BRACE_ONLY == tLineState(ls & LS_BRACE_ONLY);
82 		currentLevel = styler.LevelAt(currentLine - 1) >> 17;
83 		commentLevel = (styler.LevelAt(currentLine - 1) >> 16) & 1;
84 	} else
85 		styler.SetLevel(0, SC_FOLDLEVELBASE | SC_FOLDLEVELHEADERFLAG);
86 	bool visibleChars = false;
87 
88 	int previousLevel = currentLevel;
89 	StyleContext sc(startPos, length, SCE_TCL_DEFAULT, styler);
90 	for (; ; sc.Forward()) {
91 next:
92 		if (sc.ch=='\r' && sc.chNext == '\n') // only ignore \r on PC process on the mac
93 			continue;
94 		bool atEnd = !sc.More();  // make sure we coloured the last word
95 		if (lineState != LS_DEFAULT) {
96 			sc.SetState(SCE_TCL_DEFAULT);
97 			if (lineState == LS_OPEN_COMMENT)
98 				sc.SetState(SCE_TCL_COMMENTLINE);
99 			else if (lineState == LS_OPEN_DOUBLE_QUOTE)
100 				sc.SetState(SCE_TCL_IN_QUOTE);
101 			else if (lineState == LS_COMMENT_BOX && (sc.ch == '#' || (sc.ch == ' ' && sc.chNext=='#')))
102 				sc.SetState(SCE_TCL_COMMENT_BOX);
103 			lineState = LS_DEFAULT;
104 		}
105 		if (subBrace) { // ${ overrides every thing even \ except }
106 			if (sc.ch == '}') {
107 				subBrace = false;
108 				sc.SetState(SCE_TCL_OPERATOR);
109 				sc.ForwardSetState(SCE_TCL_DEFAULT);
110 				goto next;
111 			} else
112 				sc.SetState(SCE_TCL_SUB_BRACE);
113 			if (!sc.atLineEnd)
114 				continue;
115 		} else if (sc.state == SCE_TCL_DEFAULT || sc.state ==SCE_TCL_OPERATOR) {
116 			expected &= isspacechar(static_cast<unsigned char>(sc.ch)) || IsAWordStart(sc.ch) || sc.ch =='#';
117 		} else if (sc.state == SCE_TCL_SUBSTITUTION) {
118 			switch (sc.ch) {
119 			case '(':
120 				subParen=true;
121 				sc.SetState(SCE_TCL_OPERATOR);
122 				sc.ForwardSetState(SCE_TCL_SUBSTITUTION);
123 				continue;
124 			case ')':
125 				sc.SetState(SCE_TCL_OPERATOR);
126 				subParen=false;
127 				continue;
128 			case '$':
129 				continue;
130 			case ',':
131 				sc.SetState(SCE_TCL_OPERATOR);
132 				if (subParen) {
133 					sc.ForwardSetState(SCE_TCL_SUBSTITUTION);
134 					goto next;	// Already forwarded so avoid loop's Forward()
135 				}
136 				continue;
137 			default :
138 				// maybe spaces should be allowed ???
139 				if (!IsAWordChar(sc.ch)) { // probably the code is wrong
140 					sc.SetState(SCE_TCL_DEFAULT);
141 					subParen = 0;
142 				}
143 				break;
144 			}
145 		} else if (isComment(sc.state)) {
146 		} else if (!IsAWordChar(sc.ch)) {
147 			if ((sc.state == SCE_TCL_IDENTIFIER && expected) ||  sc.state == SCE_TCL_MODIFIER) {
148 				char w[100];
149 				char *s=w;
150 				sc.GetCurrent(w, sizeof(w));
151 				if (w[strlen(w)-1]=='\r')
152 					w[strlen(w)-1]=0;
153 				while (*s == ':') // ignore leading : like in ::set a 10
154 					++s;
155 				bool quote = sc.state == SCE_TCL_IN_QUOTE;
156 				if (commentLevel  || expected) {
157 					if (keywords.InList(s)) {
158 						sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD);
159 					} else if (keywords2.InList(s)) {
160 						sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD2);
161 					} else if (keywords3.InList(s)) {
162 						sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD3);
163 					} else if (keywords4.InList(s)) {
164 						sc.ChangeState(quote ? SCE_TCL_WORD_IN_QUOTE : SCE_TCL_WORD4);
165 					} else if (sc.GetRelative(-static_cast<int>(strlen(s))-1) == '{' &&
166 					           keywords5.InList(s) && sc.ch == '}') { // {keyword} exactly no spaces
167 						sc.ChangeState(SCE_TCL_EXPAND);
168 					}
169 					if (keywords6.InList(s)) {
170 						sc.ChangeState(SCE_TCL_WORD5);
171 					} else if (keywords7.InList(s)) {
172 						sc.ChangeState(SCE_TCL_WORD6);
173 					} else if (keywords8.InList(s)) {
174 						sc.ChangeState(SCE_TCL_WORD7);
175 					} else if (keywords9.InList(s)) {
176 						sc.ChangeState(SCE_TCL_WORD8);
177 					}
178 				}
179 				expected = false;
180 				sc.SetState(quote ? SCE_TCL_IN_QUOTE : SCE_TCL_DEFAULT);
181 			} else if (sc.state == SCE_TCL_MODIFIER || sc.state == SCE_TCL_IDENTIFIER) {
182 				sc.SetState(SCE_TCL_DEFAULT);
183 			}
184 		}
185 		if (atEnd)
186 			break;
187 		if (sc.atLineEnd) {
188 			lineState = LS_DEFAULT;
189 			currentLine = styler.GetLine(sc.currentPos);
190 			if (foldComment && sc.state!=SCE_TCL_COMMENT && isComment(sc.state)) {
191 				if (currentLevel == 0) {
192 					++currentLevel;
193 					commentLevel = true;
194 				}
195 			} else {
196 				if (visibleChars && commentLevel) {
197 					--currentLevel;
198 					--previousLevel;
199 					commentLevel = false;
200 				}
201 			}
202 			int flag = 0;
203 			if (!visibleChars && foldCompact)
204 				flag = SC_FOLDLEVELWHITEFLAG;
205 			if (currentLevel > previousLevel)
206 				flag = SC_FOLDLEVELHEADERFLAG;
207 			styler.SetLevel(currentLine, flag + previousLevel + SC_FOLDLEVELBASE + (currentLevel << 17) + (commentLevel << 16));
208 
209 			// Update the line state, so it can be seen by next line
210 			if (sc.state == SCE_TCL_IN_QUOTE) {
211 				lineState = LS_OPEN_DOUBLE_QUOTE;
212 			} else {
213 				if (prevSlash) {
214 					if (isComment(sc.state))
215 						lineState = LS_OPEN_COMMENT;
216 				} else if (sc.state == SCE_TCL_COMMENT_BOX)
217 					lineState = LS_COMMENT_BOX;
218 			}
219 			styler.SetLineState(currentLine,
220 			                    (subBrace ? LS_BRACE_ONLY : 0) |
221 			                    (expected ? LS_COMMAND_EXPECTED : 0)  | lineState);
222 			if (lineState == LS_COMMENT_BOX)
223 				sc.ForwardSetState(SCE_TCL_COMMENT_BOX);
224 			else if (lineState == LS_OPEN_DOUBLE_QUOTE)
225 				sc.ForwardSetState(SCE_TCL_IN_QUOTE);
226 			else
227 				sc.ForwardSetState(SCE_TCL_DEFAULT);
228 			prevSlash = false;
229 			previousLevel = currentLevel;
230 			goto next;
231 		}
232 
233 		if (prevSlash) {
234 			prevSlash = false;
235 			if (sc.ch == '#' && IsANumberChar(sc.chNext))
236 				sc.ForwardSetState(SCE_TCL_NUMBER);
237 			continue;
238 		}
239 		prevSlash = sc.ch == '\\';
240 		if (isComment(sc.state))
241 			continue;
242 		if (sc.atLineStart) {
243 			visibleChars = false;
244 			if (sc.state!=SCE_TCL_IN_QUOTE && !isComment(sc.state))
245 			{
246 				sc.SetState(SCE_TCL_DEFAULT);
247 				expected = IsAWordStart(sc.ch)|| isspacechar(static_cast<unsigned char>(sc.ch));
248 			}
249 		}
250 
251 		switch (sc.state) {
252 		case SCE_TCL_NUMBER:
253 			if (!IsANumberChar(sc.ch))
254 				sc.SetState(SCE_TCL_DEFAULT);
255 			break;
256 		case SCE_TCL_IN_QUOTE:
257 			if (sc.ch == '"') {
258 				sc.ForwardSetState(SCE_TCL_DEFAULT);
259 				visibleChars = true; // necessary if a " is the first and only character on a line
260 				goto next;
261 			} else if (sc.ch == '[' || sc.ch == ']' || sc.ch == '$') {
262 				sc.SetState(SCE_TCL_OPERATOR);
263 				expected = sc.ch == '[';
264 				sc.ForwardSetState(SCE_TCL_IN_QUOTE);
265 				goto next;
266 			}
267 			continue;
268 		case SCE_TCL_OPERATOR:
269 			sc.SetState(SCE_TCL_DEFAULT);
270 			break;
271 		}
272 
273 		if (sc.ch == '#') {
274 			if (visibleChars) {
275 				if (sc.state != SCE_TCL_IN_QUOTE && expected)
276 					sc.SetState(SCE_TCL_COMMENT);
277 			} else {
278 				sc.SetState(SCE_TCL_COMMENTLINE);
279 				if (sc.chNext == '~')
280 					sc.SetState(SCE_TCL_BLOCK_COMMENT);
281 				if (sc.atLineStart && (sc.chNext == '#' || sc.chNext == '-'))
282 					sc.SetState(SCE_TCL_COMMENT_BOX);
283 			}
284 		}
285 
286 		if (!isspacechar(static_cast<unsigned char>(sc.ch))) {
287 			visibleChars = true;
288 		}
289 
290 		if (sc.ch == '\\') {
291 			prevSlash = true;
292 			continue;
293 		}
294 
295 		// Determine if a new state should be entered.
296 		if (sc.state == SCE_TCL_DEFAULT) {
297 			if (IsAWordStart(sc.ch)) {
298 				sc.SetState(SCE_TCL_IDENTIFIER);
299 			} else if (IsADigit(sc.ch) && !IsAWordChar(sc.chPrev)) {
300 				sc.SetState(SCE_TCL_NUMBER);
301 			} else {
302 				switch (sc.ch) {
303 				case '\"':
304 					sc.SetState(SCE_TCL_IN_QUOTE);
305 					break;
306 				case '{':
307 					sc.SetState(SCE_TCL_OPERATOR);
308 					expected = true;
309 					++currentLevel;
310 					break;
311 				case '}':
312 					sc.SetState(SCE_TCL_OPERATOR);
313 					expected = true;
314 					--currentLevel;
315 					break;
316 				case '[':
317 					expected = true;
318 					// Falls through.
319 				case ']':
320 				case '(':
321 				case ')':
322 					sc.SetState(SCE_TCL_OPERATOR);
323 					break;
324 				case ';':
325 					expected = true;
326 					break;
327 				case '$':
328 					subParen = 0;
329 					if (sc.chNext != '{') {
330 						sc.SetState(SCE_TCL_SUBSTITUTION);
331 					} else {
332 						sc.SetState(SCE_TCL_OPERATOR);  // $
333 						sc.Forward();  // {
334 						sc.ForwardSetState(SCE_TCL_SUB_BRACE);
335 						subBrace = true;
336 					}
337 					break;
338 				case '#':
339 					if ((isspacechar(static_cast<unsigned char>(sc.chPrev))||
340 					        isoperator(static_cast<char>(sc.chPrev))) && IsADigit(sc.chNext,0x10))
341 						sc.SetState(SCE_TCL_NUMBER);
342 					break;
343 				case '-':
344 					sc.SetState(IsADigit(sc.chNext)? SCE_TCL_NUMBER: SCE_TCL_MODIFIER);
345 					break;
346 				default:
347 					if (isoperator(static_cast<char>(sc.ch))) {
348 						sc.SetState(SCE_TCL_OPERATOR);
349 					}
350 				}
351 			}
352 		}
353 	}
354 	sc.Complete();
355 }
356 
357 static const char *const tclWordListDesc[] = {
358 	"TCL Keywords",
359 	"TK Keywords",
360 	"iTCL Keywords",
361 	"tkCommands",
362 	"expand",
363 	"user1",
364 	"user2",
365 	"user3",
366 	"user4",
367 	0
368 };
369 
370 // this code supports folding in the colourizer
371 LexerModule lmTCL(SCLEX_TCL, ColouriseTCLDoc, "tcl", 0, tclWordListDesc);
372