1 // Scintilla source code edit control
2 /** @file LexVerilog.cxx
3 ** Lexer for Verilog.
4 ** Written by Avi Yegudin, based on C++ lexer by Neil Hodgson
5 **/
6 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 #include <stdlib.h>
10 #include <string.h>
11 #include <stdio.h>
12 #include <stdarg.h>
13 #include <assert.h>
14 #include <ctype.h>
15
16 #include <string>
17 #include <vector>
18 #include <map>
19 #include <algorithm>
20
21 #include "ILexer.h"
22 #include "Scintilla.h"
23 #include "SciLexer.h"
24
25 #include "WordList.h"
26 #include "LexAccessor.h"
27 #include "Accessor.h"
28 #include "StyleContext.h"
29 #include "CharacterSet.h"
30 #include "LexerModule.h"
31
32 #include "OptionSet.h"
33 #include "SubStyles.h"
34 #include "DefaultLexer.h"
35
36 using namespace Scintilla;
37
38 namespace {
39 // Use an unnamed namespace to protect the functions and classes from name conflicts
40
41 struct PPDefinition {
42 Sci_Position line;
43 std::string key;
44 std::string value;
45 bool isUndef;
46 std::string arguments;
PPDefinition__anonfbcfa2e40111::PPDefinition47 PPDefinition(Sci_Position line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_="") :
48 line(line_), key(key_), value(value_), isUndef(isUndef_), arguments(arguments_) {
49 }
50 };
51
52 class LinePPState {
53 int state;
54 int ifTaken;
55 int level;
ValidLevel() const56 bool ValidLevel() const {
57 return level >= 0 && level < 32;
58 }
maskLevel() const59 int maskLevel() const {
60 if (level >= 0) {
61 return 1 << level;
62 } else {
63 return 1;
64 }
65 }
66 public:
LinePPState()67 LinePPState() : state(0), ifTaken(0), level(-1) {
68 }
IsInactive() const69 bool IsInactive() const {
70 return state != 0;
71 }
CurrentIfTaken() const72 bool CurrentIfTaken() const {
73 return (ifTaken & maskLevel()) != 0;
74 }
StartSection(bool on)75 void StartSection(bool on) {
76 level++;
77 if (ValidLevel()) {
78 if (on) {
79 state &= ~maskLevel();
80 ifTaken |= maskLevel();
81 } else {
82 state |= maskLevel();
83 ifTaken &= ~maskLevel();
84 }
85 }
86 }
EndSection()87 void EndSection() {
88 if (ValidLevel()) {
89 state &= ~maskLevel();
90 ifTaken &= ~maskLevel();
91 }
92 level--;
93 }
InvertCurrentLevel()94 void InvertCurrentLevel() {
95 if (ValidLevel()) {
96 state ^= maskLevel();
97 ifTaken |= maskLevel();
98 }
99 }
100 };
101
102 // Hold the preprocessor state for each line seen.
103 // Currently one entry per line but could become sparse with just one entry per preprocessor line.
104 class PPStates {
105 std::vector<LinePPState> vlls;
106 public:
ForLine(Sci_Position line) const107 LinePPState ForLine(Sci_Position line) const {
108 if ((line > 0) && (vlls.size() > static_cast<size_t>(line))) {
109 return vlls[line];
110 } else {
111 return LinePPState();
112 }
113 }
Add(Sci_Position line,LinePPState lls)114 void Add(Sci_Position line, LinePPState lls) {
115 vlls.resize(line+1);
116 vlls[line] = lls;
117 }
118 };
119
120 // Options used for LexerVerilog
121 struct OptionsVerilog {
122 bool foldComment;
123 bool foldPreprocessor;
124 bool foldPreprocessorElse;
125 bool foldCompact;
126 bool foldAtElse;
127 bool foldAtModule;
128 bool trackPreprocessor;
129 bool updatePreprocessor;
130 bool portStyling;
131 bool allUppercaseDocKeyword;
OptionsVerilog__anonfbcfa2e40111::OptionsVerilog132 OptionsVerilog() {
133 foldComment = false;
134 foldPreprocessor = false;
135 foldPreprocessorElse = false;
136 foldCompact = false;
137 foldAtElse = false;
138 foldAtModule = false;
139 // for backwards compatibility, preprocessor functionality is disabled by default
140 trackPreprocessor = false;
141 updatePreprocessor = false;
142 // for backwards compatibility, treat input/output/inout as regular keywords
143 portStyling = false;
144 // for backwards compatibility, don't treat all uppercase identifiers as documentation keywords
145 allUppercaseDocKeyword = false;
146 }
147 };
148
149 struct OptionSetVerilog : public OptionSet<OptionsVerilog> {
OptionSetVerilog__anonfbcfa2e40111::OptionSetVerilog150 OptionSetVerilog() {
151 DefineProperty("fold.comment", &OptionsVerilog::foldComment,
152 "This option enables folding multi-line comments when using the Verilog lexer.");
153 DefineProperty("fold.preprocessor", &OptionsVerilog::foldPreprocessor,
154 "This option enables folding preprocessor directives when using the Verilog lexer.");
155 DefineProperty("fold.compact", &OptionsVerilog::foldCompact);
156 DefineProperty("fold.at.else", &OptionsVerilog::foldAtElse,
157 "This option enables folding on the else line of an if statement.");
158 DefineProperty("fold.verilog.flags", &OptionsVerilog::foldAtModule,
159 "This option enables folding module definitions. Typically source files "
160 "contain only one module definition so this option is somewhat useless.");
161 DefineProperty("lexer.verilog.track.preprocessor", &OptionsVerilog::trackPreprocessor,
162 "Set to 1 to interpret `if/`else/`endif to grey out code that is not active.");
163 DefineProperty("lexer.verilog.update.preprocessor", &OptionsVerilog::updatePreprocessor,
164 "Set to 1 to update preprocessor definitions when `define, `undef, or `undefineall found.");
165 DefineProperty("lexer.verilog.portstyling", &OptionsVerilog::portStyling,
166 "Set to 1 to style input, output, and inout ports differently from regular keywords.");
167 DefineProperty("lexer.verilog.allupperkeywords", &OptionsVerilog::allUppercaseDocKeyword,
168 "Set to 1 to style identifiers that are all uppercase as documentation keyword.");
169 DefineProperty("lexer.verilog.fold.preprocessor.else", &OptionsVerilog::foldPreprocessorElse,
170 "This option enables folding on `else and `elsif preprocessor directives.");
171 }
172 };
173
174 const char styleSubable[] = {0};
175
176 }
177
178 class LexerVerilog : public DefaultLexer {
179 CharacterSet setWord;
180 WordList keywords;
181 WordList keywords2;
182 WordList keywords3;
183 WordList keywords4;
184 WordList keywords5;
185 WordList ppDefinitions;
186 PPStates vlls;
187 std::vector<PPDefinition> ppDefineHistory;
188 struct SymbolValue {
189 std::string value;
190 std::string arguments;
SymbolValueLexerVerilog::SymbolValue191 SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
192 }
operator =LexerVerilog::SymbolValue193 SymbolValue &operator = (const std::string &value_) {
194 value = value_;
195 arguments.clear();
196 return *this;
197 }
IsMacroLexerVerilog::SymbolValue198 bool IsMacro() const {
199 return !arguments.empty();
200 }
201 };
202 typedef std::map<std::string, SymbolValue> SymbolTable;
203 SymbolTable preprocessorDefinitionsStart;
204 OptionsVerilog options;
205 OptionSetVerilog osVerilog;
206 enum { activeFlag = 0x40 };
207 SubStyles subStyles;
208
209 // states at end of line (EOL) during fold operations:
210 // foldExternFlag: EOL while parsing an extern function/task declaration terminated by ';'
211 // foldWaitDisableFlag: EOL while parsing wait or disable statement, terminated by "fork" or '('
212 // typdefFlag: EOL while parsing typedef statement, terminated by ';'
213 enum {foldExternFlag = 0x01, foldWaitDisableFlag = 0x02, typedefFlag = 0x04, protectedFlag = 0x08};
214 // map using line number as key to store fold state information
215 std::map<Sci_Position, int> foldState;
216
217 public:
LexerVerilog()218 LexerVerilog() :
219 setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
220 subStyles(styleSubable, 0x80, 0x40, activeFlag) {
221 }
~LexerVerilog()222 virtual ~LexerVerilog() {}
Version() const223 int SCI_METHOD Version() const override {
224 return lvSubStyles;
225 }
Release()226 void SCI_METHOD Release() override {
227 delete this;
228 }
PropertyNames()229 const char* SCI_METHOD PropertyNames() override {
230 return osVerilog.PropertyNames();
231 }
PropertyType(const char * name)232 int SCI_METHOD PropertyType(const char* name) override {
233 return osVerilog.PropertyType(name);
234 }
DescribeProperty(const char * name)235 const char* SCI_METHOD DescribeProperty(const char* name) override {
236 return osVerilog.DescribeProperty(name);
237 }
PropertySet(const char * key,const char * val)238 Sci_Position SCI_METHOD PropertySet(const char* key, const char* val) override {
239 return osVerilog.PropertySet(&options, key, val);
240 }
DescribeWordListSets()241 const char* SCI_METHOD DescribeWordListSets() override {
242 return osVerilog.DescribeWordListSets();
243 }
244 Sci_Position SCI_METHOD WordListSet(int n, const char* wl) override;
245 void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
246 void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
PrivateCall(int,void *)247 void* SCI_METHOD PrivateCall(int, void*) override {
248 return 0;
249 }
LineEndTypesSupported()250 int SCI_METHOD LineEndTypesSupported() override {
251 return SC_LINE_END_TYPE_UNICODE;
252 }
AllocateSubStyles(int styleBase,int numberStyles)253 int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override {
254 return subStyles.Allocate(styleBase, numberStyles);
255 }
SubStylesStart(int styleBase)256 int SCI_METHOD SubStylesStart(int styleBase) override {
257 return subStyles.Start(styleBase);
258 }
SubStylesLength(int styleBase)259 int SCI_METHOD SubStylesLength(int styleBase) override {
260 return subStyles.Length(styleBase);
261 }
StyleFromSubStyle(int subStyle)262 int SCI_METHOD StyleFromSubStyle(int subStyle) override {
263 int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
264 int active = subStyle & activeFlag;
265 return styleBase | active;
266 }
PrimaryStyleFromStyle(int style)267 int SCI_METHOD PrimaryStyleFromStyle(int style) override {
268 return MaskActive(style);
269 }
FreeSubStyles()270 void SCI_METHOD FreeSubStyles() override {
271 subStyles.Free();
272 }
SetIdentifiers(int style,const char * identifiers)273 void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override {
274 subStyles.SetIdentifiers(style, identifiers);
275 }
DistanceToSecondaryStyles()276 int SCI_METHOD DistanceToSecondaryStyles() override {
277 return activeFlag;
278 }
GetSubStyleBases()279 const char * SCI_METHOD GetSubStyleBases() override {
280 return styleSubable;
281 }
LexerFactoryVerilog()282 static ILexer* LexerFactoryVerilog() {
283 return new LexerVerilog();
284 }
MaskActive(int style)285 static int MaskActive(int style) {
286 return style & ~activeFlag;
287 }
288 std::vector<std::string> Tokenize(const std::string &expr) const;
289 };
290
WordListSet(int n,const char * wl)291 Sci_Position SCI_METHOD LexerVerilog::WordListSet(int n, const char *wl) {
292 WordList *wordListN = 0;
293 switch (n) {
294 case 0:
295 wordListN = &keywords;
296 break;
297 case 1:
298 wordListN = &keywords2;
299 break;
300 case 2:
301 wordListN = &keywords3;
302 break;
303 case 3:
304 wordListN = &keywords4;
305 break;
306 case 4:
307 wordListN = &keywords5;
308 break;
309 case 5:
310 wordListN = &ppDefinitions;
311 break;
312 }
313 Sci_Position firstModification = -1;
314 if (wordListN) {
315 WordList wlNew;
316 wlNew.Set(wl);
317 if (*wordListN != wlNew) {
318 wordListN->Set(wl);
319 firstModification = 0;
320 if (n == 5) {
321 // Rebuild preprocessorDefinitions
322 preprocessorDefinitionsStart.clear();
323 for (int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++) {
324 const char *cpDefinition = ppDefinitions.WordAt(nDefinition);
325 const char *cpEquals = strchr(cpDefinition, '=');
326 if (cpEquals) {
327 std::string name(cpDefinition, cpEquals - cpDefinition);
328 std::string val(cpEquals+1);
329 size_t bracket = name.find('(');
330 size_t bracketEnd = name.find(')');
331 if ((bracket != std::string::npos) && (bracketEnd != std::string::npos)) {
332 // Macro
333 std::string args = name.substr(bracket + 1, bracketEnd - bracket - 1);
334 name = name.substr(0, bracket);
335 preprocessorDefinitionsStart[name] = SymbolValue(val, args);
336 } else {
337 preprocessorDefinitionsStart[name] = val;
338 }
339 } else {
340 std::string name(cpDefinition);
341 std::string val("1");
342 preprocessorDefinitionsStart[name] = val;
343 }
344 }
345 }
346 }
347 }
348 return firstModification;
349 }
350
IsAWordChar(const int ch)351 static inline bool IsAWordChar(const int ch) {
352 return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '\''|| ch == '$');
353 }
354
IsAWordStart(const int ch)355 static inline bool IsAWordStart(const int ch) {
356 return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '$');
357 }
358
AllUpperCase(const char * a)359 static inline bool AllUpperCase(const char *a) {
360 while (*a) {
361 if (*a >= 'a' && *a <= 'z') return false;
362 a++;
363 }
364 return true;
365 }
366
367 // Functor used to truncate history
368 struct After {
369 Sci_Position line;
AfterAfter370 explicit After(Sci_Position line_) : line(line_) {}
operator ()After371 bool operator()(PPDefinition &p) const {
372 return p.line > line;
373 }
374 };
375
GetRestOfLine(LexAccessor & styler,Sci_Position start,bool allowSpace)376 static std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) {
377 std::string restOfLine;
378 Sci_Position i =0;
379 char ch = styler.SafeGetCharAt(start, '\n');
380 Sci_Position endLine = styler.LineEnd(styler.GetLine(start));
381 while (((start+i) < endLine) && (ch != '\r')) {
382 char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
383 if (ch == '/' && (chNext == '/' || chNext == '*'))
384 break;
385 if (allowSpace || (ch != ' '))
386 restOfLine += ch;
387 i++;
388 ch = chNext;
389 }
390 return restOfLine;
391 }
392
IsSpaceOrTab(int ch)393 static bool IsSpaceOrTab(int ch) {
394 return ch == ' ' || ch == '\t';
395 }
396
Lex(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)397 void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
398 {
399 LexAccessor styler(pAccess);
400
401 const int kwOther=0, kwDot=0x100, kwInput=0x200, kwOutput=0x300, kwInout=0x400, kwProtected=0x800;
402 int lineState = kwOther;
403 bool continuationLine = false;
404
405 Sci_Position curLine = styler.GetLine(startPos);
406 if (curLine > 0) lineState = styler.GetLineState(curLine - 1);
407
408 // Do not leak onto next line
409 if (initStyle == SCE_V_STRINGEOL)
410 initStyle = SCE_V_DEFAULT;
411
412 if ((MaskActive(initStyle) == SCE_V_PREPROCESSOR) ||
413 (MaskActive(initStyle) == SCE_V_COMMENTLINE) ||
414 (MaskActive(initStyle) == SCE_V_COMMENTLINEBANG)) {
415 // Set continuationLine if last character of previous line is '\'
416 if (curLine > 0) {
417 Sci_Position endLinePrevious = styler.LineEnd(curLine - 1);
418 if (endLinePrevious > 0) {
419 continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\\';
420 }
421 }
422 }
423
424 StyleContext sc(startPos, length, initStyle, styler);
425 LinePPState preproc = vlls.ForLine(curLine);
426
427 bool definitionsChanged = false;
428
429 // Truncate ppDefineHistory before current line
430
431 if (!options.updatePreprocessor)
432 ppDefineHistory.clear();
433
434 std::vector<PPDefinition>::iterator itInvalid = std::find_if(ppDefineHistory.begin(), ppDefineHistory.end(), After(curLine-1));
435 if (itInvalid != ppDefineHistory.end()) {
436 ppDefineHistory.erase(itInvalid, ppDefineHistory.end());
437 definitionsChanged = true;
438 }
439
440 SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
441 for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) {
442 if (itDef->isUndef)
443 preprocessorDefinitions.erase(itDef->key);
444 else
445 preprocessorDefinitions[itDef->key] = SymbolValue(itDef->value, itDef->arguments);
446 }
447
448 int activitySet = preproc.IsInactive() ? activeFlag : 0;
449 Sci_Position lineEndNext = styler.LineEnd(curLine);
450 bool isEscapedId = false; // true when parsing an escaped Identifier
451 bool isProtected = (lineState&kwProtected) != 0; // true when parsing a protected region
452
453 for (; sc.More(); sc.Forward()) {
454 if (sc.atLineStart) {
455 if (sc.state == SCE_V_STRING) {
456 // Prevent SCE_V_STRINGEOL from leaking back to previous line
457 sc.SetState(SCE_V_STRING);
458 }
459 if ((MaskActive(sc.state) == SCE_V_PREPROCESSOR) && (!continuationLine)) {
460 sc.SetState(SCE_V_DEFAULT|activitySet);
461 }
462 if (preproc.IsInactive()) {
463 activitySet = activeFlag;
464 sc.SetState(sc.state | activitySet);
465 }
466 }
467
468 if (sc.atLineEnd) {
469 curLine++;
470 lineEndNext = styler.LineEnd(curLine);
471 vlls.Add(curLine, preproc);
472 // Update the line state, so it can be seen by next line
473 styler.SetLineState(curLine, lineState);
474 isEscapedId = false; // EOL terminates an escaped Identifier
475 }
476
477 // Handle line continuation generically.
478 if (sc.ch == '\\') {
479 if (static_cast<Sci_Position>((sc.currentPos+1)) >= lineEndNext) {
480 curLine++;
481 lineEndNext = styler.LineEnd(curLine);
482 vlls.Add(curLine, preproc);
483 // Update the line state, so it can be seen by next line
484 styler.SetLineState(curLine, lineState);
485 sc.Forward();
486 if (sc.ch == '\r' && sc.chNext == '\n') {
487 // Even in UTF-8, \r and \n are separate
488 sc.Forward();
489 }
490 continuationLine = true;
491 sc.Forward();
492 continue;
493 }
494 }
495
496 // for comment keyword
497 if (MaskActive(sc.state) == SCE_V_COMMENT_WORD && !IsAWordChar(sc.ch)) {
498 char s[100];
499 int state = lineState & 0xff;
500 sc.GetCurrent(s, sizeof(s));
501 if (keywords5.InList(s)) {
502 sc.ChangeState(SCE_V_COMMENT_WORD|activitySet);
503 } else {
504 sc.ChangeState(state|activitySet);
505 }
506 sc.SetState(state|activitySet);
507 }
508
509 const bool atLineEndBeforeSwitch = sc.atLineEnd;
510
511 // Determine if the current state should terminate.
512 switch (MaskActive(sc.state)) {
513 case SCE_V_OPERATOR:
514 sc.SetState(SCE_V_DEFAULT|activitySet);
515 break;
516 case SCE_V_NUMBER:
517 if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
518 sc.SetState(SCE_V_DEFAULT|activitySet);
519 }
520 break;
521 case SCE_V_IDENTIFIER:
522 if (!isEscapedId &&(!IsAWordChar(sc.ch) || (sc.ch == '.'))) {
523 char s[100];
524 lineState &= 0xff00;
525 sc.GetCurrent(s, sizeof(s));
526 if (options.portStyling && (strcmp(s, "input") == 0)) {
527 lineState = kwInput;
528 sc.ChangeState(SCE_V_INPUT|activitySet);
529 } else if (options.portStyling && (strcmp(s, "output") == 0)) {
530 lineState = kwOutput;
531 sc.ChangeState(SCE_V_OUTPUT|activitySet);
532 } else if (options.portStyling && (strcmp(s, "inout") == 0)) {
533 lineState = kwInout;
534 sc.ChangeState(SCE_V_INOUT|activitySet);
535 } else if (lineState == kwInput) {
536 sc.ChangeState(SCE_V_INPUT|activitySet);
537 } else if (lineState == kwOutput) {
538 sc.ChangeState(SCE_V_OUTPUT|activitySet);
539 } else if (lineState == kwInout) {
540 sc.ChangeState(SCE_V_INOUT|activitySet);
541 } else if (lineState == kwDot) {
542 lineState = kwOther;
543 if (options.portStyling)
544 sc.ChangeState(SCE_V_PORT_CONNECT|activitySet);
545 } else if (keywords.InList(s)) {
546 sc.ChangeState(SCE_V_WORD|activitySet);
547 } else if (keywords2.InList(s)) {
548 sc.ChangeState(SCE_V_WORD2|activitySet);
549 } else if (keywords3.InList(s)) {
550 sc.ChangeState(SCE_V_WORD3|activitySet);
551 } else if (keywords4.InList(s)) {
552 sc.ChangeState(SCE_V_USER|activitySet);
553 } else if (options.allUppercaseDocKeyword && AllUpperCase(s)) {
554 sc.ChangeState(SCE_V_USER|activitySet);
555 }
556 sc.SetState(SCE_V_DEFAULT|activitySet);
557 }
558 break;
559 case SCE_V_PREPROCESSOR:
560 if (!IsAWordChar(sc.ch) || sc.atLineEnd) {
561 sc.SetState(SCE_V_DEFAULT|activitySet);
562 }
563 break;
564 case SCE_V_COMMENT:
565 if (sc.Match('*', '/')) {
566 sc.Forward();
567 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
568 } else if (IsAWordStart(sc.ch)) {
569 lineState = sc.state | (lineState & 0xff00);
570 sc.SetState(SCE_V_COMMENT_WORD|activitySet);
571 }
572 break;
573 case SCE_V_COMMENTLINE:
574 case SCE_V_COMMENTLINEBANG:
575 if (sc.atLineStart) {
576 sc.SetState(SCE_V_DEFAULT|activitySet);
577 } else if (IsAWordStart(sc.ch)) {
578 lineState = sc.state | (lineState & 0xff00);
579 sc.SetState(SCE_V_COMMENT_WORD|activitySet);
580 }
581 break;
582 case SCE_V_STRING:
583 if (sc.ch == '\\') {
584 if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
585 sc.Forward();
586 }
587 } else if (sc.ch == '\"') {
588 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
589 } else if (sc.atLineEnd) {
590 sc.ChangeState(SCE_V_STRINGEOL|activitySet);
591 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
592 }
593 break;
594 }
595
596 if (sc.atLineEnd && !atLineEndBeforeSwitch) {
597 // State exit processing consumed characters up to end of line.
598 curLine++;
599 lineEndNext = styler.LineEnd(curLine);
600 vlls.Add(curLine, preproc);
601 // Update the line state, so it can be seen by next line
602 styler.SetLineState(curLine, lineState);
603 isEscapedId = false; // EOL terminates an escaped Identifier
604 }
605
606 // Determine if a new state should be entered.
607 if (MaskActive(sc.state) == SCE_V_DEFAULT) {
608 if (sc.ch == '`') {
609 sc.SetState(SCE_V_PREPROCESSOR|activitySet);
610 // Skip whitespace between ` and preprocessor word
611 do {
612 sc.Forward();
613 } while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
614 if (sc.atLineEnd) {
615 sc.SetState(SCE_V_DEFAULT|activitySet);
616 styler.SetLineState(curLine, lineState);
617 } else {
618 if (sc.Match("protected")) {
619 isProtected = true;
620 lineState |= kwProtected;
621 styler.SetLineState(curLine, lineState);
622 } else if (sc.Match("endprotected")) {
623 isProtected = false;
624 lineState &= ~kwProtected;
625 styler.SetLineState(curLine, lineState);
626 } else if (!isProtected && options.trackPreprocessor) {
627 if (sc.Match("ifdef") || sc.Match("ifndef")) {
628 bool isIfDef = sc.Match("ifdef");
629 int i = isIfDef ? 5 : 6;
630 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + i + 1, false);
631 bool foundDef = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
632 preproc.StartSection(isIfDef == foundDef);
633 } else if (sc.Match("else")) {
634 if (!preproc.CurrentIfTaken()) {
635 preproc.InvertCurrentLevel();
636 activitySet = preproc.IsInactive() ? activeFlag : 0;
637 if (!activitySet) {
638 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
639 }
640 } else if (!preproc.IsInactive()) {
641 preproc.InvertCurrentLevel();
642 activitySet = preproc.IsInactive() ? activeFlag : 0;
643 if (!activitySet) {
644 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
645 }
646 }
647 } else if (sc.Match("elsif")) {
648 // Ensure only one chosen out of `if .. `elsif .. `elsif .. `else .. `endif
649 if (!preproc.CurrentIfTaken()) {
650 // Similar to `ifdef
651 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
652 bool ifGood = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
653 if (ifGood) {
654 preproc.InvertCurrentLevel();
655 activitySet = preproc.IsInactive() ? activeFlag : 0;
656 if (!activitySet)
657 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
658 }
659 } else if (!preproc.IsInactive()) {
660 preproc.InvertCurrentLevel();
661 activitySet = preproc.IsInactive() ? activeFlag : 0;
662 if (!activitySet)
663 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
664 }
665 } else if (sc.Match("endif")) {
666 preproc.EndSection();
667 activitySet = preproc.IsInactive() ? activeFlag : 0;
668 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
669 } else if (sc.Match("define")) {
670 if (options.updatePreprocessor && !preproc.IsInactive()) {
671 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
672 size_t startName = 0;
673 while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
674 startName++;
675 size_t endName = startName;
676 while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
677 endName++;
678 std::string key = restOfLine.substr(startName, endName-startName);
679 if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
680 // Macro
681 size_t endArgs = endName;
682 while ((endArgs < restOfLine.length()) && (restOfLine[endArgs] != ')'))
683 endArgs++;
684 std::string args = restOfLine.substr(endName + 1, endArgs - endName - 1);
685 size_t startValue = endArgs+1;
686 while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
687 startValue++;
688 std::string value;
689 if (startValue < restOfLine.length())
690 value = restOfLine.substr(startValue);
691 preprocessorDefinitions[key] = SymbolValue(value, args);
692 ppDefineHistory.push_back(PPDefinition(curLine, key, value, false, args));
693 definitionsChanged = true;
694 } else {
695 // Value
696 size_t startValue = endName;
697 while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
698 startValue++;
699 std::string value = restOfLine.substr(startValue);
700 preprocessorDefinitions[key] = value;
701 ppDefineHistory.push_back(PPDefinition(curLine, key, value));
702 definitionsChanged = true;
703 }
704 }
705 } else if (sc.Match("undefineall")) {
706 if (options.updatePreprocessor && !preproc.IsInactive()) {
707 // remove all preprocessor definitions
708 std::map<std::string, SymbolValue>::iterator itDef;
709 for(itDef = preprocessorDefinitions.begin(); itDef != preprocessorDefinitions.end(); ++itDef) {
710 ppDefineHistory.push_back(PPDefinition(curLine, itDef->first, "", true));
711 }
712 preprocessorDefinitions.clear();
713 definitionsChanged = true;
714 }
715 } else if (sc.Match("undef")) {
716 if (options.updatePreprocessor && !preproc.IsInactive()) {
717 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true);
718 std::vector<std::string> tokens = Tokenize(restOfLine);
719 std::string key;
720 if (tokens.size() >= 1) {
721 key = tokens[0];
722 preprocessorDefinitions.erase(key);
723 ppDefineHistory.push_back(PPDefinition(curLine, key, "", true));
724 definitionsChanged = true;
725 }
726 }
727 }
728 }
729 }
730 } else if (!isProtected) {
731 if (IsADigit(sc.ch) || (sc.ch == '\'') || (sc.ch == '.' && IsADigit(sc.chNext))) {
732 sc.SetState(SCE_V_NUMBER|activitySet);
733 } else if (IsAWordStart(sc.ch)) {
734 sc.SetState(SCE_V_IDENTIFIER|activitySet);
735 } else if (sc.Match('/', '*')) {
736 sc.SetState(SCE_V_COMMENT|activitySet);
737 sc.Forward(); // Eat the * so it isn't used for the end of the comment
738 } else if (sc.Match('/', '/')) {
739 if (sc.Match("//!")) // Nice to have a different comment style
740 sc.SetState(SCE_V_COMMENTLINEBANG|activitySet);
741 else
742 sc.SetState(SCE_V_COMMENTLINE|activitySet);
743 } else if (sc.ch == '\"') {
744 sc.SetState(SCE_V_STRING|activitySet);
745 } else if (sc.ch == '\\') {
746 // escaped identifier, everything is ok up to whitespace
747 isEscapedId = true;
748 sc.SetState(SCE_V_IDENTIFIER|activitySet);
749 } else if (isoperator(static_cast<char>(sc.ch)) || sc.ch == '@' || sc.ch == '#') {
750 sc.SetState(SCE_V_OPERATOR|activitySet);
751 if (sc.ch == '.') lineState = kwDot;
752 if (sc.ch == ';') lineState = kwOther;
753 }
754 }
755 }
756 if (isEscapedId && isspacechar(sc.ch)) {
757 isEscapedId = false;
758 }
759 }
760 if (definitionsChanged) {
761 styler.ChangeLexerState(startPos, startPos + length);
762 }
763 sc.Complete();
764 }
765
IsStreamCommentStyle(int style)766 static bool IsStreamCommentStyle(int style) {
767 return style == SCE_V_COMMENT;
768 }
769
IsCommentLine(Sci_Position line,LexAccessor & styler)770 static bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
771 Sci_Position pos = styler.LineStart(line);
772 Sci_Position eolPos = styler.LineStart(line + 1) - 1;
773 for (Sci_Position i = pos; i < eolPos; i++) {
774 char ch = styler[i];
775 char chNext = styler.SafeGetCharAt(i + 1);
776 int style = styler.StyleAt(i);
777 if (ch == '/' && chNext == '/' &&
778 (style == SCE_V_COMMENTLINE || style == SCE_V_COMMENTLINEBANG)) {
779 return true;
780 } else if (!IsASpaceOrTab(ch)) {
781 return false;
782 }
783 }
784 return false;
785 }
786
787 // Store both the current line's fold level and the next lines in the
788 // level store to make it easy to pick up with each increment
789 // and to make it possible to fiddle the current level for "} else {".
Fold(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)790 void SCI_METHOD LexerVerilog::Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
791 {
792 LexAccessor styler(pAccess);
793 bool foldAtBrace = 1;
794 bool foldAtParenthese = 1;
795
796 Sci_Position lineCurrent = styler.GetLine(startPos);
797 // Move back one line to be compatible with LexerModule::Fold behavior, fixes problem with foldComment behavior
798 if (lineCurrent > 0) {
799 lineCurrent--;
800 Sci_Position newStartPos = styler.LineStart(lineCurrent);
801 length += startPos - newStartPos;
802 startPos = newStartPos;
803 initStyle = 0;
804 if (startPos > 0) {
805 initStyle = styler.StyleAt(startPos - 1);
806 }
807 }
808 Sci_PositionU endPos = startPos + length;
809 int visibleChars = 0;
810 int levelCurrent = SC_FOLDLEVELBASE;
811 if (lineCurrent > 0)
812 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
813 int levelMinCurrent = levelCurrent;
814 int levelNext = levelCurrent;
815 char chNext = styler[startPos];
816 int styleNext = MaskActive(styler.StyleAt(startPos));
817 int style = MaskActive(initStyle);
818
819 // restore fold state (if it exists) for prior line
820 int stateCurrent = 0;
821 std::map<Sci_Position,int>::iterator foldStateIterator = foldState.find(lineCurrent-1);
822 if (foldStateIterator != foldState.end()) {
823 stateCurrent = foldStateIterator->second;
824 }
825
826 // remove all foldState entries after lineCurrent-1
827 foldStateIterator = foldState.upper_bound(lineCurrent-1);
828 if (foldStateIterator != foldState.end()) {
829 foldState.erase(foldStateIterator, foldState.end());
830 }
831
832 for (Sci_PositionU i = startPos; i < endPos; i++) {
833 char ch = chNext;
834 chNext = styler.SafeGetCharAt(i + 1);
835 int stylePrev = style;
836 style = styleNext;
837 styleNext = MaskActive(styler.StyleAt(i + 1));
838 bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
839 if (!(stateCurrent & protectedFlag)) {
840 if (options.foldComment && IsStreamCommentStyle(style)) {
841 if (!IsStreamCommentStyle(stylePrev)) {
842 levelNext++;
843 } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
844 // Comments don't end at end of line and the next character may be unstyled.
845 levelNext--;
846 }
847 }
848 if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
849 {
850 if (!IsCommentLine(lineCurrent - 1, styler)
851 && IsCommentLine(lineCurrent + 1, styler))
852 levelNext++;
853 else if (IsCommentLine(lineCurrent - 1, styler)
854 && !IsCommentLine(lineCurrent+1, styler))
855 levelNext--;
856 }
857 if (options.foldComment && (style == SCE_V_COMMENTLINE)) {
858 if ((ch == '/') && (chNext == '/')) {
859 char chNext2 = styler.SafeGetCharAt(i + 2);
860 if (chNext2 == '{') {
861 levelNext++;
862 } else if (chNext2 == '}') {
863 levelNext--;
864 }
865 }
866 }
867 }
868 if (ch == '`') {
869 Sci_PositionU j = i + 1;
870 while ((j < endPos) && IsASpaceOrTab(styler.SafeGetCharAt(j))) {
871 j++;
872 }
873 if (styler.Match(j, "protected")) {
874 stateCurrent |= protectedFlag;
875 levelNext++;
876 } else if (styler.Match(j, "endprotected")) {
877 stateCurrent &= ~protectedFlag;
878 levelNext--;
879 } else if (!(stateCurrent & protectedFlag) && options.foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
880 if (styler.Match(j, "if")) {
881 if (options.foldPreprocessorElse) {
882 // Measure the minimum before a begin to allow
883 // folding on "end else begin"
884 if (levelMinCurrent > levelNext) {
885 levelMinCurrent = levelNext;
886 }
887 }
888 levelNext++;
889 } else if (options.foldPreprocessorElse && styler.Match(j, "else")) {
890 levelNext--;
891 if (levelMinCurrent > levelNext) {
892 levelMinCurrent = levelNext;
893 }
894 levelNext++;
895 } else if (options.foldPreprocessorElse && styler.Match(j, "elsif")) {
896 levelNext--;
897 // Measure the minimum before a begin to allow
898 // folding on "end else begin"
899 if (levelMinCurrent > levelNext) {
900 levelMinCurrent = levelNext;
901 }
902 levelNext++;
903 } else if (styler.Match(j, "endif")) {
904 levelNext--;
905 }
906 }
907 }
908 if (style == SCE_V_OPERATOR) {
909 if (foldAtParenthese) {
910 if (ch == '(') {
911 levelNext++;
912 } else if (ch == ')') {
913 levelNext--;
914 }
915 }
916 // semicolons terminate external declarations
917 if (ch == ';') {
918 // extern and pure virtual declarations terminated by semicolon
919 if (stateCurrent & foldExternFlag) {
920 levelNext--;
921 stateCurrent &= ~foldExternFlag;
922 }
923 // wait and disable statements terminated by semicolon
924 if (stateCurrent & foldWaitDisableFlag) {
925 stateCurrent &= ~foldWaitDisableFlag;
926 }
927 // typedef statements terminated by semicolon
928 if (stateCurrent & typedefFlag) {
929 stateCurrent &= ~typedefFlag;
930 }
931 }
932 // wait and disable statements containing '(' will not contain "fork" keyword, special processing is not needed
933 if (ch == '(') {
934 if (stateCurrent & foldWaitDisableFlag) {
935 stateCurrent &= ~foldWaitDisableFlag;
936 }
937 }
938 }
939 if (style == SCE_V_OPERATOR) {
940 if (foldAtBrace) {
941 if (ch == '{') {
942 levelNext++;
943 } else if (ch == '}') {
944 levelNext--;
945 }
946 }
947 }
948 if (style == SCE_V_WORD && stylePrev != SCE_V_WORD) {
949 Sci_PositionU j = i;
950 if (styler.Match(j, "case") ||
951 styler.Match(j, "casex") ||
952 styler.Match(j, "casez") ||
953 styler.Match(j, "covergroup") ||
954 styler.Match(j, "function") ||
955 styler.Match(j, "generate") ||
956 styler.Match(j, "interface") ||
957 styler.Match(j, "package") ||
958 styler.Match(j, "primitive") ||
959 styler.Match(j, "program") ||
960 styler.Match(j, "sequence") ||
961 styler.Match(j, "specify") ||
962 styler.Match(j, "table") ||
963 styler.Match(j, "task") ||
964 (styler.Match(j, "module") && options.foldAtModule)) {
965 levelNext++;
966 } else if (styler.Match(j, "begin")) {
967 // Measure the minimum before a begin to allow
968 // folding on "end else begin"
969 if (levelMinCurrent > levelNext) {
970 levelMinCurrent = levelNext;
971 }
972 levelNext++;
973 } else if (styler.Match(j, "class")) {
974 // class does not introduce a block when used in a typedef statement
975 if (!(stateCurrent & typedefFlag))
976 levelNext++;
977 } else if (styler.Match(j, "fork")) {
978 // fork does not introduce a block when used in a wait or disable statement
979 if (stateCurrent & foldWaitDisableFlag) {
980 stateCurrent &= ~foldWaitDisableFlag;
981 } else
982 levelNext++;
983 } else if (styler.Match(j, "endcase") ||
984 styler.Match(j, "endclass") ||
985 styler.Match(j, "endfunction") ||
986 styler.Match(j, "endgenerate") ||
987 styler.Match(j, "endgroup") ||
988 styler.Match(j, "endinterface") ||
989 styler.Match(j, "endpackage") ||
990 styler.Match(j, "endprimitive") ||
991 styler.Match(j, "endprogram") ||
992 styler.Match(j, "endsequence") ||
993 styler.Match(j, "endspecify") ||
994 styler.Match(j, "endtable") ||
995 styler.Match(j, "endtask") ||
996 styler.Match(j, "join") ||
997 styler.Match(j, "join_any") ||
998 styler.Match(j, "join_none") ||
999 (styler.Match(j, "endmodule") && options.foldAtModule) ||
1000 (styler.Match(j, "end") && !IsAWordChar(styler.SafeGetCharAt(j + 3)))) {
1001 levelNext--;
1002 } else if (styler.Match(j, "extern") ||
1003 styler.Match(j, "pure")) {
1004 // extern and pure virtual functions/tasks are terminated by ';' not endfunction/endtask
1005 stateCurrent |= foldExternFlag;
1006 } else if (styler.Match(j, "disable") ||
1007 styler.Match(j, "wait")) {
1008 // fork does not introduce a block when used in a wait or disable statement
1009 stateCurrent |= foldWaitDisableFlag;
1010 } else if (styler.Match(j, "typedef")) {
1011 stateCurrent |= typedefFlag;
1012 }
1013 }
1014 if (atEOL) {
1015 int levelUse = levelCurrent;
1016 if (options.foldAtElse||options.foldPreprocessorElse) {
1017 levelUse = levelMinCurrent;
1018 }
1019 int lev = levelUse | levelNext << 16;
1020 if (visibleChars == 0 && options.foldCompact)
1021 lev |= SC_FOLDLEVELWHITEFLAG;
1022 if (levelUse < levelNext)
1023 lev |= SC_FOLDLEVELHEADERFLAG;
1024 if (stateCurrent) {
1025 foldState[lineCurrent] = stateCurrent;
1026 }
1027 if (lev != styler.LevelAt(lineCurrent)) {
1028 styler.SetLevel(lineCurrent, lev);
1029 }
1030 lineCurrent++;
1031 levelCurrent = levelNext;
1032 levelMinCurrent = levelCurrent;
1033 visibleChars = 0;
1034 }
1035 if (!isspacechar(ch))
1036 visibleChars++;
1037 }
1038 }
1039
Tokenize(const std::string & expr) const1040 std::vector<std::string> LexerVerilog::Tokenize(const std::string &expr) const {
1041 // Break into tokens
1042 std::vector<std::string> tokens;
1043 const char *cp = expr.c_str();
1044 while (*cp) {
1045 std::string word;
1046 if (setWord.Contains(static_cast<unsigned char>(*cp))) {
1047 // Identifiers and numbers
1048 while (setWord.Contains(static_cast<unsigned char>(*cp))) {
1049 word += *cp;
1050 cp++;
1051 }
1052 } else if (IsSpaceOrTab(*cp)) {
1053 while (IsSpaceOrTab(*cp)) {
1054 cp++;
1055 }
1056 continue;
1057 } else {
1058 // Should handle strings, characters, and comments here
1059 word += *cp;
1060 cp++;
1061 }
1062 tokens.push_back(word);
1063 }
1064 return tokens;
1065 }
1066
1067 static const char * const verilogWordLists[] = {
1068 "Primary keywords and identifiers",
1069 "Secondary keywords and identifiers",
1070 "System Tasks",
1071 "User defined tasks and identifiers",
1072 "Documentation comment keywords",
1073 "Preprocessor definitions",
1074 0,
1075 };
1076
1077 LexerModule lmVerilog(SCLEX_VERILOG, LexerVerilog::LexerFactoryVerilog, "verilog", verilogWordLists);
1078