1 // Scintilla source code edit control
2 /** @file LexVerilog.cxx
3  ** Lexer for Verilog.
4  ** Written by Avi Yegudin, based on C++ lexer by Neil Hodgson
5  **/
6 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
7 // The License.txt file describes the conditions under which this software may be distributed.
8 
9 #include <stdlib.h>
10 #include <string.h>
11 #include <stdio.h>
12 #include <stdarg.h>
13 #include <assert.h>
14 #include <ctype.h>
15 
16 #include <string>
17 #include <vector>
18 #include <map>
19 #include <algorithm>
20 
21 #include "ILexer.h"
22 #include "Scintilla.h"
23 #include "SciLexer.h"
24 
25 #include "WordList.h"
26 #include "LexAccessor.h"
27 #include "Accessor.h"
28 #include "StyleContext.h"
29 #include "CharacterSet.h"
30 #include "LexerModule.h"
31 
32 #include "OptionSet.h"
33 #include "SubStyles.h"
34 #include "DefaultLexer.h"
35 
36 using namespace Scintilla;
37 
38 namespace {
39 	// Use an unnamed namespace to protect the functions and classes from name conflicts
40 
41 struct PPDefinition {
42 	Sci_Position line;
43 	std::string key;
44 	std::string value;
45 	bool isUndef;
46 	std::string arguments;
PPDefinition__anonf37426ac0111::PPDefinition47 	PPDefinition(Sci_Position line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_="") :
48 		line(line_), key(key_), value(value_), isUndef(isUndef_), arguments(arguments_) {
49 	}
50 };
51 
52 class LinePPState {
53 	int state;
54 	int ifTaken;
55 	int level;
ValidLevel() const56 	bool ValidLevel() const {
57 		return level >= 0 && level < 32;
58 	}
maskLevel() const59 	int maskLevel() const {
60 		if (level >= 0) {
61 			return 1 << level;
62 		} else {
63 			return 1;
64 		}
65 	}
66 public:
LinePPState()67 	LinePPState() : state(0), ifTaken(0), level(-1) {
68 	}
IsInactive() const69 	bool IsInactive() const {
70 		return state != 0;
71 	}
CurrentIfTaken() const72 	bool CurrentIfTaken() const {
73 		return (ifTaken & maskLevel()) != 0;
74 	}
StartSection(bool on)75 	void StartSection(bool on) {
76 		level++;
77 		if (ValidLevel()) {
78 			if (on) {
79 				state &= ~maskLevel();
80 				ifTaken |= maskLevel();
81 			} else {
82 				state |= maskLevel();
83 				ifTaken &= ~maskLevel();
84 			}
85 		}
86 	}
EndSection()87 	void EndSection() {
88 		if (ValidLevel()) {
89 			state &= ~maskLevel();
90 			ifTaken &= ~maskLevel();
91 		}
92 		level--;
93 	}
InvertCurrentLevel()94 	void InvertCurrentLevel() {
95 		if (ValidLevel()) {
96 			state ^= maskLevel();
97 			ifTaken |= maskLevel();
98 		}
99 	}
100 };
101 
102 // Hold the preprocessor state for each line seen.
103 // Currently one entry per line but could become sparse with just one entry per preprocessor line.
104 class PPStates {
105 	std::vector<LinePPState> vlls;
106 public:
ForLine(Sci_Position line) const107 	LinePPState ForLine(Sci_Position line) const {
108 		if ((line > 0) && (vlls.size() > static_cast<size_t>(line))) {
109 			return vlls[line];
110 		} else {
111 			return LinePPState();
112 		}
113 	}
Add(Sci_Position line,LinePPState lls)114 	void Add(Sci_Position line, LinePPState lls) {
115 		vlls.resize(line+1);
116 		vlls[line] = lls;
117 	}
118 };
119 
120 // Options used for LexerVerilog
121 struct OptionsVerilog {
122 	bool foldComment;
123 	bool foldPreprocessor;
124 	bool foldPreprocessorElse;
125 	bool foldCompact;
126 	bool foldAtElse;
127 	bool foldAtModule;
128 	bool trackPreprocessor;
129 	bool updatePreprocessor;
130 	bool portStyling;
131 	bool allUppercaseDocKeyword;
OptionsVerilog__anonf37426ac0111::OptionsVerilog132 	OptionsVerilog() {
133 		foldComment = false;
134 		foldPreprocessor = false;
135 		foldPreprocessorElse = false;
136 		foldCompact = false;
137 		foldAtElse = false;
138 		foldAtModule = false;
139 		// for backwards compatibility, preprocessor functionality is disabled by default
140 		trackPreprocessor = false;
141 		updatePreprocessor = false;
142 		// for backwards compatibility, treat input/output/inout as regular keywords
143 		portStyling = false;
144 		// for backwards compatibility, don't treat all uppercase identifiers as documentation keywords
145 		allUppercaseDocKeyword = false;
146 	}
147 };
148 
149 struct OptionSetVerilog : public OptionSet<OptionsVerilog> {
OptionSetVerilog__anonf37426ac0111::OptionSetVerilog150 	OptionSetVerilog() {
151 		DefineProperty("fold.comment", &OptionsVerilog::foldComment,
152 			"This option enables folding multi-line comments when using the Verilog lexer.");
153 		DefineProperty("fold.preprocessor", &OptionsVerilog::foldPreprocessor,
154 			"This option enables folding preprocessor directives when using the Verilog lexer.");
155 		DefineProperty("fold.compact", &OptionsVerilog::foldCompact);
156 		DefineProperty("fold.at.else", &OptionsVerilog::foldAtElse,
157 			"This option enables folding on the else line of an if statement.");
158 		DefineProperty("fold.verilog.flags", &OptionsVerilog::foldAtModule,
159 			"This option enables folding module definitions. Typically source files "
160 			"contain only one module definition so this option is somewhat useless.");
161 		DefineProperty("lexer.verilog.track.preprocessor", &OptionsVerilog::trackPreprocessor,
162 			"Set to 1 to interpret `if/`else/`endif to grey out code that is not active.");
163 		DefineProperty("lexer.verilog.update.preprocessor", &OptionsVerilog::updatePreprocessor,
164 			"Set to 1 to update preprocessor definitions when `define, `undef, or `undefineall found.");
165 		DefineProperty("lexer.verilog.portstyling", &OptionsVerilog::portStyling,
166 			"Set to 1 to style input, output, and inout ports differently from regular keywords.");
167 		DefineProperty("lexer.verilog.allupperkeywords", &OptionsVerilog::allUppercaseDocKeyword,
168 			"Set to 1 to style identifiers that are all uppercase as documentation keyword.");
169 		DefineProperty("lexer.verilog.fold.preprocessor.else", &OptionsVerilog::foldPreprocessorElse,
170 			"This option enables folding on `else and `elsif preprocessor directives.");
171 	}
172 };
173 
174 const char styleSubable[] = {0};
175 
176 }
177 
178 class LexerVerilog : public DefaultLexer {
179 	CharacterSet setWord;
180 	WordList keywords;
181 	WordList keywords2;
182 	WordList keywords3;
183 	WordList keywords4;
184 	WordList keywords5;
185 	WordList ppDefinitions;
186 	PPStates vlls;
187 	std::vector<PPDefinition> ppDefineHistory;
188 	struct SymbolValue {
189 		std::string value;
190 		std::string arguments;
SymbolValueLexerVerilog::SymbolValue191 		SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
192 		}
operator =LexerVerilog::SymbolValue193 		SymbolValue &operator = (const std::string &value_) {
194 			value = value_;
195 			arguments.clear();
196 			return *this;
197 		}
IsMacroLexerVerilog::SymbolValue198 		bool IsMacro() const {
199 			return !arguments.empty();
200 		}
201 	};
202 	typedef std::map<std::string, SymbolValue> SymbolTable;
203 	SymbolTable preprocessorDefinitionsStart;
204 	OptionsVerilog options;
205 	OptionSetVerilog osVerilog;
206 	enum { activeFlag = 0x40 };
207 	SubStyles subStyles;
208 
209 	// states at end of line (EOL) during fold operations:
210 	//		foldExternFlag: EOL while parsing an extern function/task declaration terminated by ';'
211 	//		foldWaitDisableFlag: EOL while parsing wait or disable statement, terminated by "fork" or '('
212 	//		typdefFlag: EOL while parsing typedef statement, terminated by ';'
213 	enum {foldExternFlag = 0x01, foldWaitDisableFlag = 0x02, typedefFlag = 0x04, protectedFlag = 0x08};
214 	// map using line number as key to store fold state information
215 	std::map<Sci_Position, int> foldState;
216 
217 public:
LexerVerilog()218 	LexerVerilog() :
219 		DefaultLexer("verilog", SCLEX_VERILOG),
220 		setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
221 		subStyles(styleSubable, 0x80, 0x40, activeFlag) {
222 		}
~LexerVerilog()223 	virtual ~LexerVerilog() {}
Version() const224 	int SCI_METHOD Version() const override {
225 		return lvRelease5;
226 	}
Release()227 	void SCI_METHOD Release() override {
228 		delete this;
229 	}
PropertyNames()230 	const char* SCI_METHOD PropertyNames() override {
231 		return osVerilog.PropertyNames();
232 	}
PropertyType(const char * name)233 	int SCI_METHOD PropertyType(const char* name) override {
234 		return osVerilog.PropertyType(name);
235 	}
DescribeProperty(const char * name)236 	const char* SCI_METHOD DescribeProperty(const char* name) override {
237 		return osVerilog.DescribeProperty(name);
238 	}
PropertySet(const char * key,const char * val)239 	Sci_Position SCI_METHOD PropertySet(const char* key, const char* val) override {
240 	    return osVerilog.PropertySet(&options, key, val);
241 	}
PropertyGet(const char * key)242 	const char * SCI_METHOD PropertyGet(const char *key) override {
243 		return osVerilog.PropertyGet(key);
244 	}
DescribeWordListSets()245 	const char* SCI_METHOD DescribeWordListSets() override {
246 		return osVerilog.DescribeWordListSets();
247 	}
248 	Sci_Position SCI_METHOD WordListSet(int n, const char* wl) override;
249 	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
250 	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
PrivateCall(int,void *)251 	void* SCI_METHOD PrivateCall(int, void*) override {
252 		return 0;
253 	}
LineEndTypesSupported()254 	int SCI_METHOD LineEndTypesSupported() override {
255 		return SC_LINE_END_TYPE_UNICODE;
256 	}
AllocateSubStyles(int styleBase,int numberStyles)257 	int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override {
258 		return subStyles.Allocate(styleBase, numberStyles);
259 	}
SubStylesStart(int styleBase)260 	int SCI_METHOD SubStylesStart(int styleBase) override {
261 		return subStyles.Start(styleBase);
262 	}
SubStylesLength(int styleBase)263 	int SCI_METHOD SubStylesLength(int styleBase) override {
264 		return subStyles.Length(styleBase);
265 	}
StyleFromSubStyle(int subStyle)266 	int SCI_METHOD StyleFromSubStyle(int subStyle) override {
267 		int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
268 		int active = subStyle & activeFlag;
269 		return styleBase | active;
270 	}
PrimaryStyleFromStyle(int style)271 	int SCI_METHOD PrimaryStyleFromStyle(int style) override {
272 		return MaskActive(style);
273  	}
FreeSubStyles()274 	void SCI_METHOD FreeSubStyles() override {
275 		subStyles.Free();
276 	}
SetIdentifiers(int style,const char * identifiers)277 	void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override {
278 		subStyles.SetIdentifiers(style, identifiers);
279 	}
DistanceToSecondaryStyles()280 	int SCI_METHOD DistanceToSecondaryStyles() override {
281 		return activeFlag;
282 	}
GetSubStyleBases()283 	const char * SCI_METHOD GetSubStyleBases() override {
284 		return styleSubable;
285 	}
LexerFactoryVerilog()286 	static ILexer5* LexerFactoryVerilog() {
287 		return new LexerVerilog();
288 	}
MaskActive(int style)289 	static int MaskActive(int style) {
290 		return style & ~activeFlag;
291 	}
292 	std::vector<std::string> Tokenize(const std::string &expr) const;
293 };
294 
WordListSet(int n,const char * wl)295 Sci_Position SCI_METHOD LexerVerilog::WordListSet(int n, const char *wl) {
296 	WordList *wordListN = 0;
297 	switch (n) {
298 	case 0:
299 		wordListN = &keywords;
300 		break;
301 	case 1:
302 		wordListN = &keywords2;
303 		break;
304 	case 2:
305 		wordListN = &keywords3;
306 		break;
307 	case 3:
308 		wordListN = &keywords4;
309 		break;
310 	case 4:
311 		wordListN = &keywords5;
312 		break;
313 	case 5:
314 		wordListN = &ppDefinitions;
315 		break;
316 	}
317 	Sci_Position firstModification = -1;
318 	if (wordListN) {
319 		WordList wlNew;
320 		wlNew.Set(wl);
321 		if (*wordListN != wlNew) {
322 			wordListN->Set(wl);
323 			firstModification = 0;
324 			if (n == 5) {
325 				// Rebuild preprocessorDefinitions
326 				preprocessorDefinitionsStart.clear();
327 				for (int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++) {
328 					const char *cpDefinition = ppDefinitions.WordAt(nDefinition);
329 					const char *cpEquals = strchr(cpDefinition, '=');
330 					if (cpEquals) {
331 						std::string name(cpDefinition, cpEquals - cpDefinition);
332 						std::string val(cpEquals+1);
333 						size_t bracket = name.find('(');
334 						size_t bracketEnd = name.find(')');
335 						if ((bracket != std::string::npos) && (bracketEnd != std::string::npos)) {
336 							// Macro
337 							std::string args = name.substr(bracket + 1, bracketEnd - bracket - 1);
338 							name = name.substr(0, bracket);
339 							preprocessorDefinitionsStart[name] = SymbolValue(val, args);
340 						} else {
341 							preprocessorDefinitionsStart[name] = val;
342 						}
343 					} else {
344 						std::string name(cpDefinition);
345 						std::string val("1");
346 						preprocessorDefinitionsStart[name] = val;
347 					}
348 				}
349 			}
350 		}
351 	}
352 	return firstModification;
353 }
354 
IsAWordChar(const int ch)355 static inline bool IsAWordChar(const int ch) {
356 	return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '\''|| ch == '$');
357 }
358 
IsAWordStart(const int ch)359 static inline bool IsAWordStart(const int ch) {
360 	return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '$');
361 }
362 
AllUpperCase(const char * a)363 static inline bool AllUpperCase(const char *a) {
364 	while (*a) {
365 		if (*a >= 'a' && *a <= 'z') return false;
366 		a++;
367 	}
368 	return true;
369 }
370 
371 // Functor used to truncate history
372 struct After {
373 	Sci_Position line;
AfterAfter374 	explicit After(Sci_Position line_) : line(line_) {}
operator ()After375 	bool operator()(PPDefinition &p) const {
376 		return p.line > line;
377 	}
378 };
379 
GetRestOfLine(LexAccessor & styler,Sci_Position start,bool allowSpace)380 static std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) {
381 	std::string restOfLine;
382 	Sci_Position i =0;
383 	char ch = styler.SafeGetCharAt(start, '\n');
384 	Sci_Position endLine = styler.LineEnd(styler.GetLine(start));
385 	while (((start+i) < endLine) && (ch != '\r')) {
386 		char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
387 		if (ch == '/' && (chNext == '/' || chNext == '*'))
388 			break;
389 		if (allowSpace || (ch != ' '))
390 			restOfLine += ch;
391 		i++;
392 		ch = chNext;
393 	}
394 	return restOfLine;
395 }
396 
IsSpaceOrTab(int ch)397 static bool IsSpaceOrTab(int ch) {
398 	return ch == ' ' || ch == '\t';
399 }
400 
Lex(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)401 void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
402 {
403 	LexAccessor styler(pAccess);
404 
405 	const int kwOther=0, kwDot=0x100, kwInput=0x200, kwOutput=0x300, kwInout=0x400, kwProtected=0x800;
406 	int lineState = kwOther;
407 	bool continuationLine = false;
408 
409 	Sci_Position curLine = styler.GetLine(startPos);
410 	if (curLine > 0) lineState = styler.GetLineState(curLine - 1);
411 
412 	// Do not leak onto next line
413 	if (initStyle == SCE_V_STRINGEOL)
414 		initStyle = SCE_V_DEFAULT;
415 
416 	if ((MaskActive(initStyle) == SCE_V_PREPROCESSOR) ||
417 			(MaskActive(initStyle) == SCE_V_COMMENTLINE) ||
418 			(MaskActive(initStyle) == SCE_V_COMMENTLINEBANG)) {
419 		// Set continuationLine if last character of previous line is '\'
420 		if (curLine > 0) {
421 			Sci_Position endLinePrevious = styler.LineEnd(curLine - 1);
422 			if (endLinePrevious > 0) {
423 				continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\\';
424 			}
425 		}
426 	}
427 
428 	StyleContext sc(startPos, length, initStyle, styler);
429 	LinePPState preproc = vlls.ForLine(curLine);
430 
431 	bool definitionsChanged = false;
432 
433 	// Truncate ppDefineHistory before current line
434 
435 	if (!options.updatePreprocessor)
436 		ppDefineHistory.clear();
437 
438 	std::vector<PPDefinition>::iterator itInvalid = std::find_if(ppDefineHistory.begin(), ppDefineHistory.end(), After(curLine-1));
439 	if (itInvalid != ppDefineHistory.end()) {
440 		ppDefineHistory.erase(itInvalid, ppDefineHistory.end());
441 		definitionsChanged = true;
442 	}
443 
444 	SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
445 	for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) {
446 		if (itDef->isUndef)
447 			preprocessorDefinitions.erase(itDef->key);
448 		else
449 			preprocessorDefinitions[itDef->key] = SymbolValue(itDef->value, itDef->arguments);
450 	}
451 
452 	int activitySet = preproc.IsInactive() ? activeFlag : 0;
453 	Sci_Position lineEndNext = styler.LineEnd(curLine);
454 	bool isEscapedId = false;    // true when parsing an escaped Identifier
455 	bool isProtected = (lineState&kwProtected) != 0;	// true when parsing a protected region
456 
457 	for (; sc.More(); sc.Forward()) {
458 		if (sc.atLineStart) {
459 			if (sc.state == SCE_V_STRING) {
460 				// Prevent SCE_V_STRINGEOL from leaking back to previous line
461 				sc.SetState(SCE_V_STRING);
462 			}
463 			if ((MaskActive(sc.state) == SCE_V_PREPROCESSOR) && (!continuationLine)) {
464 				sc.SetState(SCE_V_DEFAULT|activitySet);
465 			}
466 			if (preproc.IsInactive()) {
467 				activitySet = activeFlag;
468 				sc.SetState(sc.state | activitySet);
469 			}
470 		}
471 
472 		if (sc.atLineEnd) {
473 			curLine++;
474 			lineEndNext = styler.LineEnd(curLine);
475 			vlls.Add(curLine, preproc);
476 			// Update the line state, so it can be seen by next line
477 			styler.SetLineState(curLine, lineState);
478 			isEscapedId = false;    // EOL terminates an escaped Identifier
479 		}
480 
481 		// Handle line continuation generically.
482 		if (sc.ch == '\\') {
483 			if (static_cast<Sci_Position>((sc.currentPos+1)) >= lineEndNext) {
484 				curLine++;
485 				lineEndNext = styler.LineEnd(curLine);
486 				vlls.Add(curLine, preproc);
487 				// Update the line state, so it can be seen by next line
488 				styler.SetLineState(curLine, lineState);
489 				sc.Forward();
490 				if (sc.ch == '\r' && sc.chNext == '\n') {
491 					// Even in UTF-8, \r and \n are separate
492 					sc.Forward();
493 				}
494 				continuationLine = true;
495 				sc.Forward();
496 				continue;
497 			}
498 		}
499 
500 		// for comment keyword
501 		if (MaskActive(sc.state) == SCE_V_COMMENT_WORD && !IsAWordChar(sc.ch)) {
502 			char s[100];
503 			int state = lineState & 0xff;
504 			sc.GetCurrent(s, sizeof(s));
505 			if (keywords5.InList(s)) {
506 				sc.ChangeState(SCE_V_COMMENT_WORD|activitySet);
507 			} else {
508 				sc.ChangeState(state|activitySet);
509 			}
510 			sc.SetState(state|activitySet);
511 		}
512 
513 		const bool atLineEndBeforeSwitch = sc.atLineEnd;
514 
515 		// Determine if the current state should terminate.
516 		switch (MaskActive(sc.state)) {
517 			case SCE_V_OPERATOR:
518 				sc.SetState(SCE_V_DEFAULT|activitySet);
519 				break;
520 			case SCE_V_NUMBER:
521 				if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
522 					sc.SetState(SCE_V_DEFAULT|activitySet);
523 				}
524 				break;
525 			case SCE_V_IDENTIFIER:
526 				if (!isEscapedId &&(!IsAWordChar(sc.ch) || (sc.ch == '.'))) {
527 					char s[100];
528 					lineState &= 0xff00;
529 					sc.GetCurrent(s, sizeof(s));
530 					if (options.portStyling && (strcmp(s, "input") == 0)) {
531 						lineState = kwInput;
532 						sc.ChangeState(SCE_V_INPUT|activitySet);
533 					} else if (options.portStyling && (strcmp(s, "output") == 0)) {
534 						lineState = kwOutput;
535 						sc.ChangeState(SCE_V_OUTPUT|activitySet);
536 					} else if (options.portStyling && (strcmp(s, "inout") == 0)) {
537 						lineState = kwInout;
538 						sc.ChangeState(SCE_V_INOUT|activitySet);
539 					} else if (lineState == kwInput) {
540 						sc.ChangeState(SCE_V_INPUT|activitySet);
541 					} else if (lineState == kwOutput) {
542 						sc.ChangeState(SCE_V_OUTPUT|activitySet);
543 					} else if (lineState == kwInout) {
544 						sc.ChangeState(SCE_V_INOUT|activitySet);
545 					} else if (lineState == kwDot) {
546 						lineState = kwOther;
547 						if (options.portStyling)
548 							sc.ChangeState(SCE_V_PORT_CONNECT|activitySet);
549 					} else if (keywords.InList(s)) {
550 						sc.ChangeState(SCE_V_WORD|activitySet);
551 					} else if (keywords2.InList(s)) {
552 						sc.ChangeState(SCE_V_WORD2|activitySet);
553 					} else if (keywords3.InList(s)) {
554 						sc.ChangeState(SCE_V_WORD3|activitySet);
555 					} else if (keywords4.InList(s)) {
556 						sc.ChangeState(SCE_V_USER|activitySet);
557 					} else if (options.allUppercaseDocKeyword && AllUpperCase(s)) {
558 						sc.ChangeState(SCE_V_USER|activitySet);
559 					}
560 					sc.SetState(SCE_V_DEFAULT|activitySet);
561 				}
562 				break;
563 			case SCE_V_PREPROCESSOR:
564 				if (!IsAWordChar(sc.ch) || sc.atLineEnd) {
565 					sc.SetState(SCE_V_DEFAULT|activitySet);
566 				}
567 				break;
568 			case SCE_V_COMMENT:
569 				if (sc.Match('*', '/')) {
570 					sc.Forward();
571 					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
572 				} else if (IsAWordStart(sc.ch)) {
573 					lineState = sc.state | (lineState & 0xff00);
574 					sc.SetState(SCE_V_COMMENT_WORD|activitySet);
575 				}
576 				break;
577 			case SCE_V_COMMENTLINE:
578 			case SCE_V_COMMENTLINEBANG:
579 				if (sc.atLineStart) {
580 					sc.SetState(SCE_V_DEFAULT|activitySet);
581 				} else if (IsAWordStart(sc.ch)) {
582 					lineState = sc.state | (lineState & 0xff00);
583 					sc.SetState(SCE_V_COMMENT_WORD|activitySet);
584 				}
585 				break;
586 			case SCE_V_STRING:
587 				if (sc.ch == '\\') {
588 					if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
589 						sc.Forward();
590 					}
591 				} else if (sc.ch == '\"') {
592 					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
593 				} else if (sc.atLineEnd) {
594 					sc.ChangeState(SCE_V_STRINGEOL|activitySet);
595 					sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
596 				}
597 				break;
598 		}
599 
600 		if (sc.atLineEnd && !atLineEndBeforeSwitch) {
601 			// State exit processing consumed characters up to end of line.
602 			curLine++;
603 			lineEndNext = styler.LineEnd(curLine);
604 			vlls.Add(curLine, preproc);
605 			// Update the line state, so it can be seen by next line
606 			styler.SetLineState(curLine, lineState);
607 			isEscapedId = false;    // EOL terminates an escaped Identifier
608 		}
609 
610 		// Determine if a new state should be entered.
611 		if (MaskActive(sc.state) == SCE_V_DEFAULT) {
612 			if (sc.ch == '`') {
613 				sc.SetState(SCE_V_PREPROCESSOR|activitySet);
614 				// Skip whitespace between ` and preprocessor word
615 				do {
616 					sc.Forward();
617 				} while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
618 				if (sc.atLineEnd) {
619 					sc.SetState(SCE_V_DEFAULT|activitySet);
620 					styler.SetLineState(curLine, lineState);
621 				} else {
622 					if (sc.Match("protected")) {
623 						isProtected = true;
624 						lineState |= kwProtected;
625 						styler.SetLineState(curLine, lineState);
626 					} else if (sc.Match("endprotected")) {
627 						isProtected = false;
628 						lineState &= ~kwProtected;
629 						styler.SetLineState(curLine, lineState);
630 					} else if (!isProtected && options.trackPreprocessor) {
631 						if (sc.Match("ifdef") || sc.Match("ifndef")) {
632 							bool isIfDef = sc.Match("ifdef");
633 							int i = isIfDef ? 5 : 6;
634 							std::string restOfLine = GetRestOfLine(styler, sc.currentPos + i + 1, false);
635 							bool foundDef = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
636 							preproc.StartSection(isIfDef == foundDef);
637 						} else if (sc.Match("else")) {
638 							if (!preproc.CurrentIfTaken()) {
639 								preproc.InvertCurrentLevel();
640 								activitySet = preproc.IsInactive() ? activeFlag : 0;
641 								if (!activitySet) {
642 									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
643 								}
644 							} else if (!preproc.IsInactive()) {
645 								preproc.InvertCurrentLevel();
646 								activitySet = preproc.IsInactive() ? activeFlag : 0;
647 								if (!activitySet) {
648 									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
649 								}
650 							}
651 						} else if (sc.Match("elsif")) {
652 							// Ensure only one chosen out of `if .. `elsif .. `elsif .. `else .. `endif
653 							if (!preproc.CurrentIfTaken()) {
654 								// Similar to `ifdef
655 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
656 								bool ifGood = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
657 								if (ifGood) {
658 									preproc.InvertCurrentLevel();
659 									activitySet = preproc.IsInactive() ? activeFlag : 0;
660 									if (!activitySet)
661 										sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
662 								}
663 							} else if (!preproc.IsInactive()) {
664 								preproc.InvertCurrentLevel();
665 								activitySet = preproc.IsInactive() ? activeFlag : 0;
666 								if (!activitySet)
667 									sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
668 							}
669 						} else if (sc.Match("endif")) {
670 							preproc.EndSection();
671 							activitySet = preproc.IsInactive() ? activeFlag : 0;
672 							sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
673 						} else if (sc.Match("define")) {
674 							if (options.updatePreprocessor && !preproc.IsInactive()) {
675 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
676 								size_t startName = 0;
677 								while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
678 									startName++;
679 								size_t endName = startName;
680 								while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
681 									endName++;
682 								std::string key = restOfLine.substr(startName, endName-startName);
683 								if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
684 									// Macro
685 									size_t endArgs = endName;
686 									while ((endArgs < restOfLine.length()) && (restOfLine[endArgs] != ')'))
687 										endArgs++;
688 									std::string args = restOfLine.substr(endName + 1, endArgs - endName - 1);
689 									size_t startValue = endArgs+1;
690 									while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
691 										startValue++;
692 									std::string value;
693 									if (startValue < restOfLine.length())
694 										value = restOfLine.substr(startValue);
695 									preprocessorDefinitions[key] = SymbolValue(value, args);
696 									ppDefineHistory.push_back(PPDefinition(curLine, key, value, false, args));
697 									definitionsChanged = true;
698 								} else {
699 									// Value
700 									size_t startValue = endName;
701 									while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
702 										startValue++;
703 									std::string value = restOfLine.substr(startValue);
704 									preprocessorDefinitions[key] = value;
705 									ppDefineHistory.push_back(PPDefinition(curLine, key, value));
706 									definitionsChanged = true;
707 								}
708 							}
709 						} else if (sc.Match("undefineall")) {
710 							if (options.updatePreprocessor && !preproc.IsInactive()) {
711 								// remove all preprocessor definitions
712 								std::map<std::string, SymbolValue>::iterator itDef;
713 								for(itDef = preprocessorDefinitions.begin(); itDef != preprocessorDefinitions.end(); ++itDef) {
714 									ppDefineHistory.push_back(PPDefinition(curLine, itDef->first, "", true));
715 								}
716 								preprocessorDefinitions.clear();
717 								definitionsChanged = true;
718 							}
719 						} else if (sc.Match("undef")) {
720 							if (options.updatePreprocessor && !preproc.IsInactive()) {
721 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true);
722 								std::vector<std::string> tokens = Tokenize(restOfLine);
723 								std::string key;
724 								if (tokens.size() >= 1) {
725 									key = tokens[0];
726 									preprocessorDefinitions.erase(key);
727 									ppDefineHistory.push_back(PPDefinition(curLine, key, "", true));
728 									definitionsChanged = true;
729 								}
730 							}
731 						}
732 					}
733 				}
734 			} else if (!isProtected) {
735 				if (IsADigit(sc.ch) || (sc.ch == '\'') || (sc.ch == '.' && IsADigit(sc.chNext))) {
736 					sc.SetState(SCE_V_NUMBER|activitySet);
737 				} else if (IsAWordStart(sc.ch)) {
738 					sc.SetState(SCE_V_IDENTIFIER|activitySet);
739 				} else if (sc.Match('/', '*')) {
740 					sc.SetState(SCE_V_COMMENT|activitySet);
741 					sc.Forward();	// Eat the * so it isn't used for the end of the comment
742 				} else if (sc.Match('/', '/')) {
743 					if (sc.Match("//!"))	// Nice to have a different comment style
744 						sc.SetState(SCE_V_COMMENTLINEBANG|activitySet);
745 					else
746 						sc.SetState(SCE_V_COMMENTLINE|activitySet);
747 				} else if (sc.ch == '\"') {
748 					sc.SetState(SCE_V_STRING|activitySet);
749 				} else if (sc.ch == '\\') {
750 					// escaped identifier, everything is ok up to whitespace
751 					isEscapedId = true;
752 					sc.SetState(SCE_V_IDENTIFIER|activitySet);
753 				} else if (isoperator(static_cast<char>(sc.ch)) || sc.ch == '@' || sc.ch == '#') {
754 					sc.SetState(SCE_V_OPERATOR|activitySet);
755 					if (sc.ch == '.') lineState = kwDot;
756 					if (sc.ch == ';') lineState = kwOther;
757 				}
758 			}
759 		}
760 		if (isEscapedId && isspacechar(sc.ch)) {
761 			isEscapedId = false;
762 		}
763 	}
764 	if (definitionsChanged) {
765 		styler.ChangeLexerState(startPos, startPos + length);
766 	}
767 	sc.Complete();
768 }
769 
IsStreamCommentStyle(int style)770 static bool IsStreamCommentStyle(int style) {
771 	return style == SCE_V_COMMENT;
772 }
773 
IsCommentLine(Sci_Position line,LexAccessor & styler)774 static bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
775 	Sci_Position pos = styler.LineStart(line);
776 	Sci_Position eolPos = styler.LineStart(line + 1) - 1;
777 	for (Sci_Position i = pos; i < eolPos; i++) {
778 		char ch = styler[i];
779 		char chNext = styler.SafeGetCharAt(i + 1);
780 		int style = styler.StyleAt(i);
781 		if (ch == '/' && chNext == '/' &&
782 		   (style == SCE_V_COMMENTLINE || style == SCE_V_COMMENTLINEBANG)) {
783 			return true;
784 		} else if (!IsASpaceOrTab(ch)) {
785 			return false;
786 		}
787 	}
788 	return false;
789 }
790 
791 // Store both the current line's fold level and the next lines in the
792 // level store to make it easy to pick up with each increment
793 // and to make it possible to fiddle the current level for "} else {".
Fold(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)794 void SCI_METHOD LexerVerilog::Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
795 {
796 	LexAccessor styler(pAccess);
797 	bool foldAtBrace  = 1;
798 	bool foldAtParenthese  = 1;
799 
800 	Sci_Position lineCurrent = styler.GetLine(startPos);
801 	// Move back one line to be compatible with LexerModule::Fold behavior, fixes problem with foldComment behavior
802 	if (lineCurrent > 0) {
803 		lineCurrent--;
804 		Sci_Position newStartPos = styler.LineStart(lineCurrent);
805 		length += startPos - newStartPos;
806 		startPos = newStartPos;
807 		initStyle = 0;
808 		if (startPos > 0) {
809 			initStyle = styler.StyleAt(startPos - 1);
810 		}
811 	}
812 	Sci_PositionU endPos = startPos + length;
813 	int visibleChars = 0;
814 	int levelCurrent = SC_FOLDLEVELBASE;
815 	if (lineCurrent > 0)
816 		levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
817 	int levelMinCurrent = levelCurrent;
818 	int levelNext = levelCurrent;
819 	char chNext = styler[startPos];
820 	int styleNext = MaskActive(styler.StyleAt(startPos));
821 	int style = MaskActive(initStyle);
822 
823 	// restore fold state (if it exists) for prior line
824 	int stateCurrent = 0;
825 	std::map<Sci_Position,int>::iterator foldStateIterator = foldState.find(lineCurrent-1);
826 	if (foldStateIterator != foldState.end()) {
827 		stateCurrent = foldStateIterator->second;
828 	}
829 
830 	// remove all foldState entries after lineCurrent-1
831 	foldStateIterator = foldState.upper_bound(lineCurrent-1);
832 	if (foldStateIterator != foldState.end()) {
833 		foldState.erase(foldStateIterator, foldState.end());
834 	}
835 
836 	for (Sci_PositionU i = startPos; i < endPos; i++) {
837 		char ch = chNext;
838 		chNext = styler.SafeGetCharAt(i + 1);
839 		int stylePrev = style;
840 		style = styleNext;
841 		styleNext = MaskActive(styler.StyleAt(i + 1));
842 		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
843 		if (!(stateCurrent & protectedFlag)) {
844 			if (options.foldComment && IsStreamCommentStyle(style)) {
845 				if (!IsStreamCommentStyle(stylePrev)) {
846 					levelNext++;
847 				} else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
848 					// Comments don't end at end of line and the next character may be unstyled.
849 					levelNext--;
850 				}
851 			}
852 			if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
853 			{
854 				if (!IsCommentLine(lineCurrent - 1, styler)
855 					&& IsCommentLine(lineCurrent + 1, styler))
856 					levelNext++;
857 				else if (IsCommentLine(lineCurrent - 1, styler)
858 						 && !IsCommentLine(lineCurrent+1, styler))
859 					levelNext--;
860 			}
861 			if (options.foldComment && (style == SCE_V_COMMENTLINE)) {
862 				if ((ch == '/') && (chNext == '/')) {
863 					char chNext2 = styler.SafeGetCharAt(i + 2);
864 					if (chNext2 == '{') {
865 						levelNext++;
866 					} else if (chNext2 == '}') {
867 						levelNext--;
868 					}
869 				}
870 			}
871 		}
872 		if (ch == '`') {
873 			Sci_PositionU j = i + 1;
874 			while ((j < endPos) && IsASpaceOrTab(styler.SafeGetCharAt(j))) {
875 				j++;
876 			}
877 			if (styler.Match(j, "protected")) {
878 				stateCurrent |= protectedFlag;
879 				levelNext++;
880 			} else if (styler.Match(j, "endprotected")) {
881 				stateCurrent &= ~protectedFlag;
882 				levelNext--;
883 			} else if (!(stateCurrent & protectedFlag) && options.foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
884 				if (styler.Match(j, "if")) {
885 					if (options.foldPreprocessorElse) {
886 						// Measure the minimum before a begin to allow
887 						// folding on "end else begin"
888 						if (levelMinCurrent > levelNext) {
889 							levelMinCurrent = levelNext;
890 						}
891 					}
892 					levelNext++;
893 				} else if (options.foldPreprocessorElse && styler.Match(j, "else")) {
894 					levelNext--;
895 					if (levelMinCurrent > levelNext) {
896 						levelMinCurrent = levelNext;
897 					}
898 					levelNext++;
899 				} else if (options.foldPreprocessorElse && styler.Match(j, "elsif")) {
900 					levelNext--;
901 					// Measure the minimum before a begin to allow
902 					// folding on "end else begin"
903 					if (levelMinCurrent > levelNext) {
904 						levelMinCurrent = levelNext;
905 					}
906 					levelNext++;
907 				} else if (styler.Match(j, "endif")) {
908 					levelNext--;
909 				}
910 			}
911 		}
912 		if (style == SCE_V_OPERATOR) {
913 			if (foldAtParenthese) {
914 				if (ch == '(') {
915 					levelNext++;
916 				} else if (ch == ')') {
917 					levelNext--;
918 				}
919 			}
920 			// semicolons terminate external declarations
921 			if (ch == ';') {
922 				// extern and pure virtual declarations terminated by semicolon
923 				if (stateCurrent & foldExternFlag) {
924 					levelNext--;
925 					stateCurrent &= ~foldExternFlag;
926 				}
927 				// wait and disable statements terminated by semicolon
928 				if (stateCurrent & foldWaitDisableFlag) {
929 					stateCurrent &= ~foldWaitDisableFlag;
930 				}
931 				// typedef statements terminated by semicolon
932 				if (stateCurrent & typedefFlag) {
933 					stateCurrent &= ~typedefFlag;
934 				}
935 			}
936 			// wait and disable statements containing '(' will not contain "fork" keyword, special processing is not needed
937 			if (ch == '(') {
938 				if (stateCurrent & foldWaitDisableFlag) {
939 					stateCurrent &= ~foldWaitDisableFlag;
940 				}
941 			}
942 		}
943 		if (style == SCE_V_OPERATOR) {
944 			if (foldAtBrace) {
945 				if (ch == '{') {
946 					levelNext++;
947 				} else if (ch == '}') {
948 					levelNext--;
949 				}
950 			}
951 		}
952 		if (style == SCE_V_WORD && stylePrev != SCE_V_WORD) {
953 			Sci_PositionU j = i;
954 			if (styler.Match(j, "case") ||
955 				styler.Match(j, "casex") ||
956 				styler.Match(j, "casez") ||
957 				styler.Match(j, "covergroup") ||
958 				styler.Match(j, "function") ||
959 				styler.Match(j, "generate") ||
960 				styler.Match(j, "interface") ||
961 				styler.Match(j, "package") ||
962 				styler.Match(j, "primitive") ||
963 				styler.Match(j, "program") ||
964 				styler.Match(j, "sequence") ||
965 				styler.Match(j, "specify") ||
966 				styler.Match(j, "table") ||
967 				styler.Match(j, "task") ||
968 				(styler.Match(j, "module") && options.foldAtModule)) {
969 				levelNext++;
970 			} else if (styler.Match(j, "begin")) {
971 				// Measure the minimum before a begin to allow
972 				// folding on "end else begin"
973 				if (levelMinCurrent > levelNext) {
974 					levelMinCurrent = levelNext;
975 				}
976 				levelNext++;
977 			} else if (styler.Match(j, "class")) {
978 				// class does not introduce a block when used in a typedef statement
979 				if (!(stateCurrent & typedefFlag))
980 					levelNext++;
981 			} else if (styler.Match(j, "fork")) {
982 				// fork does not introduce a block when used in a wait or disable statement
983 				if (stateCurrent & foldWaitDisableFlag) {
984 					stateCurrent &= ~foldWaitDisableFlag;
985 				} else
986 					levelNext++;
987 			} else if (styler.Match(j, "endcase") ||
988 				styler.Match(j, "endclass") ||
989 				styler.Match(j, "endfunction") ||
990 				styler.Match(j, "endgenerate") ||
991 				styler.Match(j, "endgroup") ||
992 				styler.Match(j, "endinterface") ||
993 				styler.Match(j, "endpackage") ||
994 				styler.Match(j, "endprimitive") ||
995 				styler.Match(j, "endprogram") ||
996 				styler.Match(j, "endsequence") ||
997 				styler.Match(j, "endspecify") ||
998 				styler.Match(j, "endtable") ||
999 				styler.Match(j, "endtask") ||
1000 				styler.Match(j, "join") ||
1001 				styler.Match(j, "join_any") ||
1002 				styler.Match(j, "join_none") ||
1003 				(styler.Match(j, "endmodule") && options.foldAtModule) ||
1004 				(styler.Match(j, "end") && !IsAWordChar(styler.SafeGetCharAt(j + 3)))) {
1005 				levelNext--;
1006 			} else if (styler.Match(j, "extern") ||
1007 				styler.Match(j, "pure")) {
1008 				// extern and pure virtual functions/tasks are terminated by ';' not endfunction/endtask
1009 				stateCurrent |= foldExternFlag;
1010 			} else if (styler.Match(j, "disable") ||
1011 				styler.Match(j, "wait")) {
1012 				// fork does not introduce a block when used in a wait or disable statement
1013 				stateCurrent |= foldWaitDisableFlag;
1014 			} else if (styler.Match(j, "typedef")) {
1015 				stateCurrent |= typedefFlag;
1016 			}
1017 		}
1018 		if (atEOL) {
1019 			int levelUse = levelCurrent;
1020 			if (options.foldAtElse||options.foldPreprocessorElse) {
1021 				levelUse = levelMinCurrent;
1022 			}
1023 			int lev = levelUse | levelNext << 16;
1024 			if (visibleChars == 0 && options.foldCompact)
1025 				lev |= SC_FOLDLEVELWHITEFLAG;
1026 			if (levelUse < levelNext)
1027 				lev |= SC_FOLDLEVELHEADERFLAG;
1028 			if (stateCurrent) {
1029 				foldState[lineCurrent] = stateCurrent;
1030 			}
1031 			if (lev != styler.LevelAt(lineCurrent)) {
1032 				styler.SetLevel(lineCurrent, lev);
1033 			}
1034 			lineCurrent++;
1035 			levelCurrent = levelNext;
1036 			levelMinCurrent = levelCurrent;
1037 			visibleChars = 0;
1038 		}
1039 		if (!isspacechar(ch))
1040 			visibleChars++;
1041 	}
1042 }
1043 
Tokenize(const std::string & expr) const1044 std::vector<std::string> LexerVerilog::Tokenize(const std::string &expr) const {
1045 	// Break into tokens
1046 	std::vector<std::string> tokens;
1047 	const char *cp = expr.c_str();
1048 	while (*cp) {
1049 		std::string word;
1050 		if (setWord.Contains(static_cast<unsigned char>(*cp))) {
1051 			// Identifiers and numbers
1052 			while (setWord.Contains(static_cast<unsigned char>(*cp))) {
1053 				word += *cp;
1054 				cp++;
1055 			}
1056 		} else if (IsSpaceOrTab(*cp)) {
1057 			while (IsSpaceOrTab(*cp)) {
1058 				cp++;
1059 			}
1060 			continue;
1061 		} else {
1062 			// Should handle strings, characters, and comments here
1063 			word += *cp;
1064 			cp++;
1065 		}
1066 		tokens.push_back(word);
1067 	}
1068 	return tokens;
1069 }
1070 
1071 static const char * const verilogWordLists[] = {
1072             "Primary keywords and identifiers",
1073             "Secondary keywords and identifiers",
1074             "System Tasks",
1075             "User defined tasks and identifiers",
1076             "Documentation comment keywords",
1077             "Preprocessor definitions",
1078             0,
1079         };
1080 
1081 LexerModule lmVerilog(SCLEX_VERILOG, LexerVerilog::LexerFactoryVerilog, "verilog", verilogWordLists);
1082