1 // Scintilla source code edit control
2 // Nimrod lexer
3 // (c) 2009 Andreas Rumpf
4 /** @file LexNimrod.cxx
5  ** Lexer for Nimrod.
6  **/
7 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
8 // The License.txt file describes the conditions under which this software may be distributed.
9 
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdio.h>
13 #include <stdarg.h>
14 #include <assert.h>
15 #include <ctype.h>
16 
17 #include "ILexer.h"
18 #include "Scintilla.h"
19 #include "SciLexer.h"
20 
21 #include "WordList.h"
22 #include "LexAccessor.h"
23 #include "Accessor.h"
24 #include "StyleContext.h"
25 #include "CharacterSet.h"
26 #include "LexerModule.h"
27 
28 using namespace Scintilla;
29 
IsAWordChar(int ch)30 static inline bool IsAWordChar(int ch) {
31 	return (ch >= 0x80) || isalnum(ch) || ch == '_';
32 }
33 
tillEndOfTripleQuote(Accessor & styler,Sci_Position pos,Sci_Position max)34 static Sci_Position tillEndOfTripleQuote(Accessor &styler, Sci_Position pos, Sci_Position max) {
35   /* search for """ */
36   for (;;) {
37     if (styler.SafeGetCharAt(pos, '\0') == '\0') return pos;
38     if (pos >= max) return pos;
39     if (styler.Match(pos, "\"\"\"")) {
40       return pos + 2;
41     }
42     pos++;
43   }
44 }
45 
46 #define CR 13 /* use both because Scite allows changing the line ending */
47 #define LF 10
48 
isNewLine(int ch)49 static bool inline isNewLine(int ch) {
50   return ch == CR || ch == LF;
51 }
52 
scanString(Accessor & styler,Sci_Position pos,Sci_Position max,bool rawMode)53 static Sci_Position scanString(Accessor &styler, Sci_Position pos, Sci_Position max, bool rawMode) {
54   for (;;) {
55     if (pos >= max) return pos;
56     char ch = styler.SafeGetCharAt(pos, '\0');
57     if (ch == CR || ch == LF || ch == '\0') return pos;
58     if (ch == '"') return pos;
59     if (ch == '\\' && !rawMode) {
60       pos += 2;
61     } else {
62       pos++;
63     }
64   }
65 }
66 
scanChar(Accessor & styler,Sci_Position pos,Sci_Position max)67 static Sci_Position scanChar(Accessor &styler, Sci_Position pos, Sci_Position max) {
68   for (;;) {
69     if (pos >= max) return pos;
70     char ch = styler.SafeGetCharAt(pos, '\0');
71     if (ch == CR || ch == LF || ch == '\0') return pos;
72     if (ch == '\'' && !isalnum(styler.SafeGetCharAt(pos+1, '\0')) )
73       return pos;
74     if (ch == '\\') {
75       pos += 2;
76     } else {
77       pos++;
78     }
79   }
80 }
81 
scanIdent(Accessor & styler,Sci_Position pos,WordList & keywords)82 static Sci_Position scanIdent(Accessor &styler, Sci_Position pos, WordList &keywords) {
83   char buf[100]; /* copy to lowercase and ignore underscores */
84   Sci_Position i = 0;
85 
86   for (;;) {
87     char ch = styler.SafeGetCharAt(pos, '\0');
88     if (!IsAWordChar(ch)) break;
89     if (ch != '_' && i < ((int)sizeof(buf))-1) {
90       buf[i] = static_cast<char>(tolower(ch));
91       i++;
92     }
93     pos++;
94   }
95   buf[i] = '\0';
96   /* look for keyword */
97   if (keywords.InList(buf)) {
98     styler.ColourTo(pos-1, SCE_P_WORD);
99   } else {
100     styler.ColourTo(pos-1, SCE_P_IDENTIFIER);
101   }
102   return pos;
103 }
104 
scanNumber(Accessor & styler,Sci_Position pos)105 static Sci_Position scanNumber(Accessor &styler, Sci_Position pos) {
106   char ch, ch2;
107   ch = styler.SafeGetCharAt(pos, '\0');
108   ch2 = styler.SafeGetCharAt(pos+1, '\0');
109   if (ch == '0' && (ch2 == 'b' || ch2 == 'B')) {
110     /* binary number: */
111     pos += 2;
112     for (;;) {
113       ch = styler.SafeGetCharAt(pos, '\0');
114       if (ch == '_' || (ch >= '0' && ch <= '1')) ++pos;
115       else break;
116     }
117   } else if (ch == '0' &&
118             (ch2 == 'o' || ch2 == 'O' || ch2 == 'c' || ch2 == 'C')) {
119     /* octal number: */
120     pos += 2;
121     for (;;) {
122       ch = styler.SafeGetCharAt(pos, '\0');
123       if (ch == '_' || (ch >= '0' && ch <= '7')) ++pos;
124       else break;
125     }
126   } else if (ch == '0' && (ch2 == 'x' || ch2 == 'X')) {
127     /* hexadecimal number: */
128     pos += 2;
129     for (;;) {
130       ch = styler.SafeGetCharAt(pos, '\0');
131       if (ch == '_' || (ch >= '0' && ch <= '9')
132           || (ch >= 'a' && ch <= 'f')
133           || (ch >= 'A' && ch <= 'F')) ++pos;
134       else break;
135     }
136   } else {
137     // skip decimal part:
138     for (;;) {
139       ch = styler.SafeGetCharAt(pos, '\0');
140       if (ch == '_' || (ch >= '0' && ch <= '9')) ++pos;
141       else break;
142     }
143     ch2 = styler.SafeGetCharAt(pos+1, '\0');
144     if (ch == '.' && ch2 >= '0' && ch2 <= '9') {
145       ++pos; // skip '.'
146       for (;;) {
147         ch = styler.SafeGetCharAt(pos, '\0');
148         if (ch == '_' || (ch >= '0' && ch <= '9')) ++pos;
149         else break;
150       }
151     }
152     if (ch == 'e' || ch == 'E') {
153       ++pos;
154       ch = styler.SafeGetCharAt(pos, '\0');
155       if (ch == '-' || ch == '+') ++pos;
156       for (;;) {
157         ch = styler.SafeGetCharAt(pos, '\0');
158         if (ch == '_' || (ch >= '0' && ch <= '9')) ++pos;
159         else break;
160       }
161     }
162   }
163   if (ch == '\'') {
164     /* a type suffix: */
165     pos++;
166     for (;;) {
167       ch = styler.SafeGetCharAt(pos);
168       if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z')
169          || (ch >= 'a' && ch <= 'z') || ch == '_') ++pos;
170       else break;
171     }
172   }
173   styler.ColourTo(pos-1, SCE_P_NUMBER);
174   return pos;
175 }
176 
177 /* rewritten from scratch, because I couldn't get rid of the bugs...
178    (A character based approach sucks!)
179 */
ColouriseNimrodDoc(Sci_PositionU startPos,Sci_Position length,int initStyle,WordList * keywordlists[],Accessor & styler)180 static void ColouriseNimrodDoc(Sci_PositionU startPos, Sci_Position length, int initStyle,
181                                 WordList *keywordlists[], Accessor &styler) {
182   Sci_Position pos = startPos;
183   Sci_Position max = startPos + length;
184   char ch;
185   WordList &keywords = *keywordlists[0];
186 
187   styler.StartAt(startPos);
188   styler.StartSegment(startPos);
189 
190   switch (initStyle) {
191     /* check where we are: */
192     case SCE_P_TRIPLEDOUBLE:
193       pos = tillEndOfTripleQuote(styler, pos, max);
194       styler.ColourTo(pos, SCE_P_TRIPLEDOUBLE);
195       pos++;
196     break;
197     default: /* nothing to do: */
198     break;
199   }
200   while (pos < max) {
201     ch = styler.SafeGetCharAt(pos, '\0');
202     switch (ch) {
203       case '\0': return;
204       case '#': {
205         bool doccomment = (styler.SafeGetCharAt(pos+1) == '#');
206         while (pos < max && !isNewLine(styler.SafeGetCharAt(pos, LF))) pos++;
207         if (doccomment)
208           styler.ColourTo(pos, SCE_C_COMMENTLINEDOC);
209         else
210           styler.ColourTo(pos, SCE_P_COMMENTLINE);
211       } break;
212       case 'r': case 'R': {
213         if (styler.SafeGetCharAt(pos+1) == '"') {
214           pos = scanString(styler, pos+2, max, true);
215           styler.ColourTo(pos, SCE_P_STRING);
216           pos++;
217         } else {
218           pos = scanIdent(styler, pos, keywords);
219         }
220       } break;
221       case '"':
222         if (styler.Match(pos+1, "\"\"")) {
223           pos = tillEndOfTripleQuote(styler, pos+3, max);
224           styler.ColourTo(pos, SCE_P_TRIPLEDOUBLE);
225         } else {
226           pos = scanString(styler, pos+1, max, false);
227           styler.ColourTo(pos, SCE_P_STRING);
228         }
229         pos++;
230       break;
231       case '\'':
232         pos = scanChar(styler, pos+1, max);
233         styler.ColourTo(pos, SCE_P_CHARACTER);
234         pos++;
235       break;
236       default: // identifers, numbers, operators, whitespace
237         if (ch >= '0' && ch <= '9') {
238           pos = scanNumber(styler, pos);
239         } else if (IsAWordChar(ch)) {
240           pos = scanIdent(styler, pos, keywords);
241         } else if (ch == '`') {
242           pos++;
243           while (pos < max) {
244             ch = styler.SafeGetCharAt(pos, LF);
245             if (ch == '`') {
246               ++pos;
247               break;
248             }
249             if (ch == CR || ch == LF) break;
250             ++pos;
251           }
252           styler.ColourTo(pos, SCE_P_IDENTIFIER);
253         } else if (strchr("()[]{}:=;-\\/&%$!+<>|^?,.*~@", ch)) {
254           styler.ColourTo(pos, SCE_P_OPERATOR);
255           pos++;
256         } else {
257           styler.ColourTo(pos, SCE_P_DEFAULT);
258           pos++;
259         }
260       break;
261     }
262   }
263 }
264 
IsCommentLine(Sci_Position line,Accessor & styler)265 static bool IsCommentLine(Sci_Position line, Accessor &styler) {
266 	Sci_Position pos = styler.LineStart(line);
267 	Sci_Position eol_pos = styler.LineStart(line + 1) - 1;
268 	for (Sci_Position i = pos; i < eol_pos; i++) {
269 		char ch = styler[i];
270 		if (ch == '#')
271 			return true;
272 		else if (ch != ' ' && ch != '\t')
273 			return false;
274 	}
275 	return false;
276 }
277 
IsQuoteLine(Sci_Position line,Accessor & styler)278 static bool IsQuoteLine(Sci_Position line, Accessor &styler) {
279 	int style = styler.StyleAt(styler.LineStart(line)) & 31;
280 	return ((style == SCE_P_TRIPLE) || (style == SCE_P_TRIPLEDOUBLE));
281 }
282 
283 
FoldNimrodDoc(Sci_PositionU startPos,Sci_Position length,int,WordList * [],Accessor & styler)284 static void FoldNimrodDoc(Sci_PositionU startPos, Sci_Position length,
285                           int /*initStyle - unused*/,
286                           WordList *[], Accessor &styler) {
287 	const Sci_Position maxPos = startPos + length;
288 	const Sci_Position maxLines = styler.GetLine(maxPos - 1); // Requested last line
289 	const Sci_Position docLines = styler.GetLine(styler.Length() - 1); // Available last line
290 	const bool foldComment = styler.GetPropertyInt("fold.comment.nimrod") != 0;
291 	const bool foldQuotes = styler.GetPropertyInt("fold.quotes.nimrod") != 0;
292 
293 	// Backtrack to previous non-blank line so we can determine indent level
294 	// for any white space lines (needed esp. within triple quoted strings)
295 	// and so we can fix any preceding fold level (which is why we go back
296 	// at least one line in all cases)
297 	int spaceFlags = 0;
298 	Sci_Position lineCurrent = styler.GetLine(startPos);
299 	int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
300 	while (lineCurrent > 0) {
301 		lineCurrent--;
302 		indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
303 		if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
304 		        (!IsCommentLine(lineCurrent, styler)) &&
305 		        (!IsQuoteLine(lineCurrent, styler)))
306 			break;
307 	}
308 	int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
309 
310 	// Set up initial loop state
311 	startPos = styler.LineStart(lineCurrent);
312 	int prev_state = SCE_P_DEFAULT & 31;
313 	if (lineCurrent >= 1)
314 		prev_state = styler.StyleAt(startPos - 1) & 31;
315 	int prevQuote = foldQuotes && ((prev_state == SCE_P_TRIPLE) ||
316 	                               (prev_state == SCE_P_TRIPLEDOUBLE));
317 	int prevComment = 0;
318 	if (lineCurrent >= 1)
319 		prevComment = foldComment && IsCommentLine(lineCurrent - 1, styler);
320 
321 	// Process all characters to end of requested range or end of any triple quote
322 	// or comment that hangs over the end of the range.  Cap processing in all cases
323 	// to end of document (in case of unclosed quote or comment at end).
324 	while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) ||
325 	                                      prevQuote || prevComment)) {
326 
327 		// Gather info
328 		int lev = indentCurrent;
329 		Sci_Position lineNext = lineCurrent + 1;
330 		int indentNext = indentCurrent;
331 		int quote = false;
332 		if (lineNext <= docLines) {
333 			// Information about next line is only available if not at end of document
334 			indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
335 			int style = styler.StyleAt(styler.LineStart(lineNext)) & 31;
336 			quote = foldQuotes && ((style == SCE_P_TRIPLE) || (style == SCE_P_TRIPLEDOUBLE));
337 		}
338 		const int quote_start = (quote && !prevQuote);
339 		const int quote_continue = (quote && prevQuote);
340 		const int comment = foldComment && IsCommentLine(lineCurrent, styler);
341 		const int comment_start = (comment && !prevComment && (lineNext <= docLines) &&
342 		                           IsCommentLine(lineNext, styler) &&
343 		                           (lev > SC_FOLDLEVELBASE));
344 		const int comment_continue = (comment && prevComment);
345 		if ((!quote || !prevQuote) && !comment)
346 			indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
347 		if (quote)
348 			indentNext = indentCurrentLevel;
349 		if (indentNext & SC_FOLDLEVELWHITEFLAG)
350 			indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
351 
352 		if (quote_start) {
353 			// Place fold point at start of triple quoted string
354 			lev |= SC_FOLDLEVELHEADERFLAG;
355 		} else if (quote_continue || prevQuote) {
356 			// Add level to rest of lines in the string
357 			lev = lev + 1;
358 		} else if (comment_start) {
359 			// Place fold point at start of a block of comments
360 			lev |= SC_FOLDLEVELHEADERFLAG;
361 		} else if (comment_continue) {
362 			// Add level to rest of lines in the block
363 			lev = lev + 1;
364 		}
365 
366 		// Skip past any blank lines for next indent level info; we skip also
367 		// comments (all comments, not just those starting in column 0)
368 		// which effectively folds them into surrounding code rather
369 		// than screwing up folding.
370 
371 		while (!quote &&
372 		        (lineNext < docLines) &&
373 		        ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
374 		         (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
375 
376 			lineNext++;
377 			indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
378 		}
379 
380 		const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
381 		const int levelBeforeComments =
382 		    Maximum(indentCurrentLevel,levelAfterComments);
383 
384 		// Now set all the indent levels on the lines we skipped
385 		// Do this from end to start.  Once we encounter one line
386 		// which is indented more than the line after the end of
387 		// the comment-block, use the level of the block before
388 
389 		Sci_Position skipLine = lineNext;
390 		int skipLevel = levelAfterComments;
391 
392 		while (--skipLine > lineCurrent) {
393 			int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
394 
395 			if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
396 				skipLevel = levelBeforeComments;
397 
398 			int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
399 
400 			styler.SetLevel(skipLine, skipLevel | whiteFlag);
401 		}
402 
403 		// Set fold header on non-quote/non-comment line
404 		if (!quote && !comment && !(indentCurrent & SC_FOLDLEVELWHITEFLAG) ) {
405 			if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) <
406 			     (indentNext & SC_FOLDLEVELNUMBERMASK))
407 				lev |= SC_FOLDLEVELHEADERFLAG;
408 		}
409 
410 		// Keep track of triple quote and block comment state of previous line
411 		prevQuote = quote;
412 		prevComment = comment_start || comment_continue;
413 
414 		// Set fold level for this line and move to next line
415 		styler.SetLevel(lineCurrent, lev);
416 		indentCurrent = indentNext;
417 		lineCurrent = lineNext;
418 	}
419 
420 	// NOTE: Cannot set level of last line here because indentCurrent doesn't have
421 	// header flag set; the loop above is crafted to take care of this case!
422 	//styler.SetLevel(lineCurrent, indentCurrent);
423 }
424 
425 static const char * const nimrodWordListDesc[] = {
426 	"Keywords",
427 	0
428 };
429 
430 LexerModule lmNimrod(SCLEX_NIMROD, ColouriseNimrodDoc, "nimrod", FoldNimrodDoc,
431 				     nimrodWordListDesc);
432