1 // Scintilla source code edit control
2 /** @file LexVerilog.cxx
3 ** Lexer for Verilog.
4 ** Written by Avi Yegudin, based on C++ lexer by Neil Hodgson
5 **/
6 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
7 // The License.txt file describes the conditions under which this software may be distributed.
8
9 #include <stdlib.h>
10 #include <string.h>
11 #include <stdio.h>
12 #include <stdarg.h>
13 #include <assert.h>
14 #include <ctype.h>
15
16 #include <string>
17 #include <vector>
18 #include <map>
19 #include <algorithm>
20
21 #include "ILexer.h"
22 #include "Scintilla.h"
23 #include "SciLexer.h"
24
25 #include "WordList.h"
26 #include "LexAccessor.h"
27 #include "Accessor.h"
28 #include "StyleContext.h"
29 #include "CharacterSet.h"
30 #include "LexerModule.h"
31
32 #include "OptionSet.h"
33 #include "SubStyles.h"
34 #include "DefaultLexer.h"
35
36 using namespace Scintilla;
37
38 namespace {
39 // Use an unnamed namespace to protect the functions and classes from name conflicts
40
41 struct PPDefinition {
42 Sci_Position line;
43 std::string key;
44 std::string value;
45 bool isUndef;
46 std::string arguments;
PPDefinition__anonf37426ac0111::PPDefinition47 PPDefinition(Sci_Position line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_="") :
48 line(line_), key(key_), value(value_), isUndef(isUndef_), arguments(arguments_) {
49 }
50 };
51
52 class LinePPState {
53 int state;
54 int ifTaken;
55 int level;
ValidLevel() const56 bool ValidLevel() const {
57 return level >= 0 && level < 32;
58 }
maskLevel() const59 int maskLevel() const {
60 if (level >= 0) {
61 return 1 << level;
62 } else {
63 return 1;
64 }
65 }
66 public:
LinePPState()67 LinePPState() : state(0), ifTaken(0), level(-1) {
68 }
IsInactive() const69 bool IsInactive() const {
70 return state != 0;
71 }
CurrentIfTaken() const72 bool CurrentIfTaken() const {
73 return (ifTaken & maskLevel()) != 0;
74 }
StartSection(bool on)75 void StartSection(bool on) {
76 level++;
77 if (ValidLevel()) {
78 if (on) {
79 state &= ~maskLevel();
80 ifTaken |= maskLevel();
81 } else {
82 state |= maskLevel();
83 ifTaken &= ~maskLevel();
84 }
85 }
86 }
EndSection()87 void EndSection() {
88 if (ValidLevel()) {
89 state &= ~maskLevel();
90 ifTaken &= ~maskLevel();
91 }
92 level--;
93 }
InvertCurrentLevel()94 void InvertCurrentLevel() {
95 if (ValidLevel()) {
96 state ^= maskLevel();
97 ifTaken |= maskLevel();
98 }
99 }
100 };
101
102 // Hold the preprocessor state for each line seen.
103 // Currently one entry per line but could become sparse with just one entry per preprocessor line.
104 class PPStates {
105 std::vector<LinePPState> vlls;
106 public:
ForLine(Sci_Position line) const107 LinePPState ForLine(Sci_Position line) const {
108 if ((line > 0) && (vlls.size() > static_cast<size_t>(line))) {
109 return vlls[line];
110 } else {
111 return LinePPState();
112 }
113 }
Add(Sci_Position line,LinePPState lls)114 void Add(Sci_Position line, LinePPState lls) {
115 vlls.resize(line+1);
116 vlls[line] = lls;
117 }
118 };
119
120 // Options used for LexerVerilog
121 struct OptionsVerilog {
122 bool foldComment;
123 bool foldPreprocessor;
124 bool foldPreprocessorElse;
125 bool foldCompact;
126 bool foldAtElse;
127 bool foldAtModule;
128 bool trackPreprocessor;
129 bool updatePreprocessor;
130 bool portStyling;
131 bool allUppercaseDocKeyword;
OptionsVerilog__anonf37426ac0111::OptionsVerilog132 OptionsVerilog() {
133 foldComment = false;
134 foldPreprocessor = false;
135 foldPreprocessorElse = false;
136 foldCompact = false;
137 foldAtElse = false;
138 foldAtModule = false;
139 // for backwards compatibility, preprocessor functionality is disabled by default
140 trackPreprocessor = false;
141 updatePreprocessor = false;
142 // for backwards compatibility, treat input/output/inout as regular keywords
143 portStyling = false;
144 // for backwards compatibility, don't treat all uppercase identifiers as documentation keywords
145 allUppercaseDocKeyword = false;
146 }
147 };
148
149 struct OptionSetVerilog : public OptionSet<OptionsVerilog> {
OptionSetVerilog__anonf37426ac0111::OptionSetVerilog150 OptionSetVerilog() {
151 DefineProperty("fold.comment", &OptionsVerilog::foldComment,
152 "This option enables folding multi-line comments when using the Verilog lexer.");
153 DefineProperty("fold.preprocessor", &OptionsVerilog::foldPreprocessor,
154 "This option enables folding preprocessor directives when using the Verilog lexer.");
155 DefineProperty("fold.compact", &OptionsVerilog::foldCompact);
156 DefineProperty("fold.at.else", &OptionsVerilog::foldAtElse,
157 "This option enables folding on the else line of an if statement.");
158 DefineProperty("fold.verilog.flags", &OptionsVerilog::foldAtModule,
159 "This option enables folding module definitions. Typically source files "
160 "contain only one module definition so this option is somewhat useless.");
161 DefineProperty("lexer.verilog.track.preprocessor", &OptionsVerilog::trackPreprocessor,
162 "Set to 1 to interpret `if/`else/`endif to grey out code that is not active.");
163 DefineProperty("lexer.verilog.update.preprocessor", &OptionsVerilog::updatePreprocessor,
164 "Set to 1 to update preprocessor definitions when `define, `undef, or `undefineall found.");
165 DefineProperty("lexer.verilog.portstyling", &OptionsVerilog::portStyling,
166 "Set to 1 to style input, output, and inout ports differently from regular keywords.");
167 DefineProperty("lexer.verilog.allupperkeywords", &OptionsVerilog::allUppercaseDocKeyword,
168 "Set to 1 to style identifiers that are all uppercase as documentation keyword.");
169 DefineProperty("lexer.verilog.fold.preprocessor.else", &OptionsVerilog::foldPreprocessorElse,
170 "This option enables folding on `else and `elsif preprocessor directives.");
171 }
172 };
173
174 const char styleSubable[] = {0};
175
176 }
177
178 class LexerVerilog : public DefaultLexer {
179 CharacterSet setWord;
180 WordList keywords;
181 WordList keywords2;
182 WordList keywords3;
183 WordList keywords4;
184 WordList keywords5;
185 WordList ppDefinitions;
186 PPStates vlls;
187 std::vector<PPDefinition> ppDefineHistory;
188 struct SymbolValue {
189 std::string value;
190 std::string arguments;
SymbolValueLexerVerilog::SymbolValue191 SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
192 }
operator =LexerVerilog::SymbolValue193 SymbolValue &operator = (const std::string &value_) {
194 value = value_;
195 arguments.clear();
196 return *this;
197 }
IsMacroLexerVerilog::SymbolValue198 bool IsMacro() const {
199 return !arguments.empty();
200 }
201 };
202 typedef std::map<std::string, SymbolValue> SymbolTable;
203 SymbolTable preprocessorDefinitionsStart;
204 OptionsVerilog options;
205 OptionSetVerilog osVerilog;
206 enum { activeFlag = 0x40 };
207 SubStyles subStyles;
208
209 // states at end of line (EOL) during fold operations:
210 // foldExternFlag: EOL while parsing an extern function/task declaration terminated by ';'
211 // foldWaitDisableFlag: EOL while parsing wait or disable statement, terminated by "fork" or '('
212 // typdefFlag: EOL while parsing typedef statement, terminated by ';'
213 enum {foldExternFlag = 0x01, foldWaitDisableFlag = 0x02, typedefFlag = 0x04, protectedFlag = 0x08};
214 // map using line number as key to store fold state information
215 std::map<Sci_Position, int> foldState;
216
217 public:
LexerVerilog()218 LexerVerilog() :
219 DefaultLexer("verilog", SCLEX_VERILOG),
220 setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
221 subStyles(styleSubable, 0x80, 0x40, activeFlag) {
222 }
~LexerVerilog()223 virtual ~LexerVerilog() {}
Version() const224 int SCI_METHOD Version() const override {
225 return lvRelease5;
226 }
Release()227 void SCI_METHOD Release() override {
228 delete this;
229 }
PropertyNames()230 const char* SCI_METHOD PropertyNames() override {
231 return osVerilog.PropertyNames();
232 }
PropertyType(const char * name)233 int SCI_METHOD PropertyType(const char* name) override {
234 return osVerilog.PropertyType(name);
235 }
DescribeProperty(const char * name)236 const char* SCI_METHOD DescribeProperty(const char* name) override {
237 return osVerilog.DescribeProperty(name);
238 }
PropertySet(const char * key,const char * val)239 Sci_Position SCI_METHOD PropertySet(const char* key, const char* val) override {
240 return osVerilog.PropertySet(&options, key, val);
241 }
PropertyGet(const char * key)242 const char * SCI_METHOD PropertyGet(const char *key) override {
243 return osVerilog.PropertyGet(key);
244 }
DescribeWordListSets()245 const char* SCI_METHOD DescribeWordListSets() override {
246 return osVerilog.DescribeWordListSets();
247 }
248 Sci_Position SCI_METHOD WordListSet(int n, const char* wl) override;
249 void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
250 void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
PrivateCall(int,void *)251 void* SCI_METHOD PrivateCall(int, void*) override {
252 return 0;
253 }
LineEndTypesSupported()254 int SCI_METHOD LineEndTypesSupported() override {
255 return SC_LINE_END_TYPE_UNICODE;
256 }
AllocateSubStyles(int styleBase,int numberStyles)257 int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override {
258 return subStyles.Allocate(styleBase, numberStyles);
259 }
SubStylesStart(int styleBase)260 int SCI_METHOD SubStylesStart(int styleBase) override {
261 return subStyles.Start(styleBase);
262 }
SubStylesLength(int styleBase)263 int SCI_METHOD SubStylesLength(int styleBase) override {
264 return subStyles.Length(styleBase);
265 }
StyleFromSubStyle(int subStyle)266 int SCI_METHOD StyleFromSubStyle(int subStyle) override {
267 int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
268 int active = subStyle & activeFlag;
269 return styleBase | active;
270 }
PrimaryStyleFromStyle(int style)271 int SCI_METHOD PrimaryStyleFromStyle(int style) override {
272 return MaskActive(style);
273 }
FreeSubStyles()274 void SCI_METHOD FreeSubStyles() override {
275 subStyles.Free();
276 }
SetIdentifiers(int style,const char * identifiers)277 void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override {
278 subStyles.SetIdentifiers(style, identifiers);
279 }
DistanceToSecondaryStyles()280 int SCI_METHOD DistanceToSecondaryStyles() override {
281 return activeFlag;
282 }
GetSubStyleBases()283 const char * SCI_METHOD GetSubStyleBases() override {
284 return styleSubable;
285 }
LexerFactoryVerilog()286 static ILexer5* LexerFactoryVerilog() {
287 return new LexerVerilog();
288 }
MaskActive(int style)289 static int MaskActive(int style) {
290 return style & ~activeFlag;
291 }
292 std::vector<std::string> Tokenize(const std::string &expr) const;
293 };
294
WordListSet(int n,const char * wl)295 Sci_Position SCI_METHOD LexerVerilog::WordListSet(int n, const char *wl) {
296 WordList *wordListN = 0;
297 switch (n) {
298 case 0:
299 wordListN = &keywords;
300 break;
301 case 1:
302 wordListN = &keywords2;
303 break;
304 case 2:
305 wordListN = &keywords3;
306 break;
307 case 3:
308 wordListN = &keywords4;
309 break;
310 case 4:
311 wordListN = &keywords5;
312 break;
313 case 5:
314 wordListN = &ppDefinitions;
315 break;
316 }
317 Sci_Position firstModification = -1;
318 if (wordListN) {
319 WordList wlNew;
320 wlNew.Set(wl);
321 if (*wordListN != wlNew) {
322 wordListN->Set(wl);
323 firstModification = 0;
324 if (n == 5) {
325 // Rebuild preprocessorDefinitions
326 preprocessorDefinitionsStart.clear();
327 for (int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++) {
328 const char *cpDefinition = ppDefinitions.WordAt(nDefinition);
329 const char *cpEquals = strchr(cpDefinition, '=');
330 if (cpEquals) {
331 std::string name(cpDefinition, cpEquals - cpDefinition);
332 std::string val(cpEquals+1);
333 size_t bracket = name.find('(');
334 size_t bracketEnd = name.find(')');
335 if ((bracket != std::string::npos) && (bracketEnd != std::string::npos)) {
336 // Macro
337 std::string args = name.substr(bracket + 1, bracketEnd - bracket - 1);
338 name = name.substr(0, bracket);
339 preprocessorDefinitionsStart[name] = SymbolValue(val, args);
340 } else {
341 preprocessorDefinitionsStart[name] = val;
342 }
343 } else {
344 std::string name(cpDefinition);
345 std::string val("1");
346 preprocessorDefinitionsStart[name] = val;
347 }
348 }
349 }
350 }
351 }
352 return firstModification;
353 }
354
IsAWordChar(const int ch)355 static inline bool IsAWordChar(const int ch) {
356 return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '\''|| ch == '$');
357 }
358
IsAWordStart(const int ch)359 static inline bool IsAWordStart(const int ch) {
360 return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '$');
361 }
362
AllUpperCase(const char * a)363 static inline bool AllUpperCase(const char *a) {
364 while (*a) {
365 if (*a >= 'a' && *a <= 'z') return false;
366 a++;
367 }
368 return true;
369 }
370
371 // Functor used to truncate history
372 struct After {
373 Sci_Position line;
AfterAfter374 explicit After(Sci_Position line_) : line(line_) {}
operator ()After375 bool operator()(PPDefinition &p) const {
376 return p.line > line;
377 }
378 };
379
GetRestOfLine(LexAccessor & styler,Sci_Position start,bool allowSpace)380 static std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) {
381 std::string restOfLine;
382 Sci_Position i =0;
383 char ch = styler.SafeGetCharAt(start, '\n');
384 Sci_Position endLine = styler.LineEnd(styler.GetLine(start));
385 while (((start+i) < endLine) && (ch != '\r')) {
386 char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
387 if (ch == '/' && (chNext == '/' || chNext == '*'))
388 break;
389 if (allowSpace || (ch != ' '))
390 restOfLine += ch;
391 i++;
392 ch = chNext;
393 }
394 return restOfLine;
395 }
396
IsSpaceOrTab(int ch)397 static bool IsSpaceOrTab(int ch) {
398 return ch == ' ' || ch == '\t';
399 }
400
Lex(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)401 void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
402 {
403 LexAccessor styler(pAccess);
404
405 const int kwOther=0, kwDot=0x100, kwInput=0x200, kwOutput=0x300, kwInout=0x400, kwProtected=0x800;
406 int lineState = kwOther;
407 bool continuationLine = false;
408
409 Sci_Position curLine = styler.GetLine(startPos);
410 if (curLine > 0) lineState = styler.GetLineState(curLine - 1);
411
412 // Do not leak onto next line
413 if (initStyle == SCE_V_STRINGEOL)
414 initStyle = SCE_V_DEFAULT;
415
416 if ((MaskActive(initStyle) == SCE_V_PREPROCESSOR) ||
417 (MaskActive(initStyle) == SCE_V_COMMENTLINE) ||
418 (MaskActive(initStyle) == SCE_V_COMMENTLINEBANG)) {
419 // Set continuationLine if last character of previous line is '\'
420 if (curLine > 0) {
421 Sci_Position endLinePrevious = styler.LineEnd(curLine - 1);
422 if (endLinePrevious > 0) {
423 continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\\';
424 }
425 }
426 }
427
428 StyleContext sc(startPos, length, initStyle, styler);
429 LinePPState preproc = vlls.ForLine(curLine);
430
431 bool definitionsChanged = false;
432
433 // Truncate ppDefineHistory before current line
434
435 if (!options.updatePreprocessor)
436 ppDefineHistory.clear();
437
438 std::vector<PPDefinition>::iterator itInvalid = std::find_if(ppDefineHistory.begin(), ppDefineHistory.end(), After(curLine-1));
439 if (itInvalid != ppDefineHistory.end()) {
440 ppDefineHistory.erase(itInvalid, ppDefineHistory.end());
441 definitionsChanged = true;
442 }
443
444 SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
445 for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) {
446 if (itDef->isUndef)
447 preprocessorDefinitions.erase(itDef->key);
448 else
449 preprocessorDefinitions[itDef->key] = SymbolValue(itDef->value, itDef->arguments);
450 }
451
452 int activitySet = preproc.IsInactive() ? activeFlag : 0;
453 Sci_Position lineEndNext = styler.LineEnd(curLine);
454 bool isEscapedId = false; // true when parsing an escaped Identifier
455 bool isProtected = (lineState&kwProtected) != 0; // true when parsing a protected region
456
457 for (; sc.More(); sc.Forward()) {
458 if (sc.atLineStart) {
459 if (sc.state == SCE_V_STRING) {
460 // Prevent SCE_V_STRINGEOL from leaking back to previous line
461 sc.SetState(SCE_V_STRING);
462 }
463 if ((MaskActive(sc.state) == SCE_V_PREPROCESSOR) && (!continuationLine)) {
464 sc.SetState(SCE_V_DEFAULT|activitySet);
465 }
466 if (preproc.IsInactive()) {
467 activitySet = activeFlag;
468 sc.SetState(sc.state | activitySet);
469 }
470 }
471
472 if (sc.atLineEnd) {
473 curLine++;
474 lineEndNext = styler.LineEnd(curLine);
475 vlls.Add(curLine, preproc);
476 // Update the line state, so it can be seen by next line
477 styler.SetLineState(curLine, lineState);
478 isEscapedId = false; // EOL terminates an escaped Identifier
479 }
480
481 // Handle line continuation generically.
482 if (sc.ch == '\\') {
483 if (static_cast<Sci_Position>((sc.currentPos+1)) >= lineEndNext) {
484 curLine++;
485 lineEndNext = styler.LineEnd(curLine);
486 vlls.Add(curLine, preproc);
487 // Update the line state, so it can be seen by next line
488 styler.SetLineState(curLine, lineState);
489 sc.Forward();
490 if (sc.ch == '\r' && sc.chNext == '\n') {
491 // Even in UTF-8, \r and \n are separate
492 sc.Forward();
493 }
494 continuationLine = true;
495 sc.Forward();
496 continue;
497 }
498 }
499
500 // for comment keyword
501 if (MaskActive(sc.state) == SCE_V_COMMENT_WORD && !IsAWordChar(sc.ch)) {
502 char s[100];
503 int state = lineState & 0xff;
504 sc.GetCurrent(s, sizeof(s));
505 if (keywords5.InList(s)) {
506 sc.ChangeState(SCE_V_COMMENT_WORD|activitySet);
507 } else {
508 sc.ChangeState(state|activitySet);
509 }
510 sc.SetState(state|activitySet);
511 }
512
513 const bool atLineEndBeforeSwitch = sc.atLineEnd;
514
515 // Determine if the current state should terminate.
516 switch (MaskActive(sc.state)) {
517 case SCE_V_OPERATOR:
518 sc.SetState(SCE_V_DEFAULT|activitySet);
519 break;
520 case SCE_V_NUMBER:
521 if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
522 sc.SetState(SCE_V_DEFAULT|activitySet);
523 }
524 break;
525 case SCE_V_IDENTIFIER:
526 if (!isEscapedId &&(!IsAWordChar(sc.ch) || (sc.ch == '.'))) {
527 char s[100];
528 lineState &= 0xff00;
529 sc.GetCurrent(s, sizeof(s));
530 if (options.portStyling && (strcmp(s, "input") == 0)) {
531 lineState = kwInput;
532 sc.ChangeState(SCE_V_INPUT|activitySet);
533 } else if (options.portStyling && (strcmp(s, "output") == 0)) {
534 lineState = kwOutput;
535 sc.ChangeState(SCE_V_OUTPUT|activitySet);
536 } else if (options.portStyling && (strcmp(s, "inout") == 0)) {
537 lineState = kwInout;
538 sc.ChangeState(SCE_V_INOUT|activitySet);
539 } else if (lineState == kwInput) {
540 sc.ChangeState(SCE_V_INPUT|activitySet);
541 } else if (lineState == kwOutput) {
542 sc.ChangeState(SCE_V_OUTPUT|activitySet);
543 } else if (lineState == kwInout) {
544 sc.ChangeState(SCE_V_INOUT|activitySet);
545 } else if (lineState == kwDot) {
546 lineState = kwOther;
547 if (options.portStyling)
548 sc.ChangeState(SCE_V_PORT_CONNECT|activitySet);
549 } else if (keywords.InList(s)) {
550 sc.ChangeState(SCE_V_WORD|activitySet);
551 } else if (keywords2.InList(s)) {
552 sc.ChangeState(SCE_V_WORD2|activitySet);
553 } else if (keywords3.InList(s)) {
554 sc.ChangeState(SCE_V_WORD3|activitySet);
555 } else if (keywords4.InList(s)) {
556 sc.ChangeState(SCE_V_USER|activitySet);
557 } else if (options.allUppercaseDocKeyword && AllUpperCase(s)) {
558 sc.ChangeState(SCE_V_USER|activitySet);
559 }
560 sc.SetState(SCE_V_DEFAULT|activitySet);
561 }
562 break;
563 case SCE_V_PREPROCESSOR:
564 if (!IsAWordChar(sc.ch) || sc.atLineEnd) {
565 sc.SetState(SCE_V_DEFAULT|activitySet);
566 }
567 break;
568 case SCE_V_COMMENT:
569 if (sc.Match('*', '/')) {
570 sc.Forward();
571 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
572 } else if (IsAWordStart(sc.ch)) {
573 lineState = sc.state | (lineState & 0xff00);
574 sc.SetState(SCE_V_COMMENT_WORD|activitySet);
575 }
576 break;
577 case SCE_V_COMMENTLINE:
578 case SCE_V_COMMENTLINEBANG:
579 if (sc.atLineStart) {
580 sc.SetState(SCE_V_DEFAULT|activitySet);
581 } else if (IsAWordStart(sc.ch)) {
582 lineState = sc.state | (lineState & 0xff00);
583 sc.SetState(SCE_V_COMMENT_WORD|activitySet);
584 }
585 break;
586 case SCE_V_STRING:
587 if (sc.ch == '\\') {
588 if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
589 sc.Forward();
590 }
591 } else if (sc.ch == '\"') {
592 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
593 } else if (sc.atLineEnd) {
594 sc.ChangeState(SCE_V_STRINGEOL|activitySet);
595 sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
596 }
597 break;
598 }
599
600 if (sc.atLineEnd && !atLineEndBeforeSwitch) {
601 // State exit processing consumed characters up to end of line.
602 curLine++;
603 lineEndNext = styler.LineEnd(curLine);
604 vlls.Add(curLine, preproc);
605 // Update the line state, so it can be seen by next line
606 styler.SetLineState(curLine, lineState);
607 isEscapedId = false; // EOL terminates an escaped Identifier
608 }
609
610 // Determine if a new state should be entered.
611 if (MaskActive(sc.state) == SCE_V_DEFAULT) {
612 if (sc.ch == '`') {
613 sc.SetState(SCE_V_PREPROCESSOR|activitySet);
614 // Skip whitespace between ` and preprocessor word
615 do {
616 sc.Forward();
617 } while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
618 if (sc.atLineEnd) {
619 sc.SetState(SCE_V_DEFAULT|activitySet);
620 styler.SetLineState(curLine, lineState);
621 } else {
622 if (sc.Match("protected")) {
623 isProtected = true;
624 lineState |= kwProtected;
625 styler.SetLineState(curLine, lineState);
626 } else if (sc.Match("endprotected")) {
627 isProtected = false;
628 lineState &= ~kwProtected;
629 styler.SetLineState(curLine, lineState);
630 } else if (!isProtected && options.trackPreprocessor) {
631 if (sc.Match("ifdef") || sc.Match("ifndef")) {
632 bool isIfDef = sc.Match("ifdef");
633 int i = isIfDef ? 5 : 6;
634 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + i + 1, false);
635 bool foundDef = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
636 preproc.StartSection(isIfDef == foundDef);
637 } else if (sc.Match("else")) {
638 if (!preproc.CurrentIfTaken()) {
639 preproc.InvertCurrentLevel();
640 activitySet = preproc.IsInactive() ? activeFlag : 0;
641 if (!activitySet) {
642 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
643 }
644 } else if (!preproc.IsInactive()) {
645 preproc.InvertCurrentLevel();
646 activitySet = preproc.IsInactive() ? activeFlag : 0;
647 if (!activitySet) {
648 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
649 }
650 }
651 } else if (sc.Match("elsif")) {
652 // Ensure only one chosen out of `if .. `elsif .. `elsif .. `else .. `endif
653 if (!preproc.CurrentIfTaken()) {
654 // Similar to `ifdef
655 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
656 bool ifGood = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
657 if (ifGood) {
658 preproc.InvertCurrentLevel();
659 activitySet = preproc.IsInactive() ? activeFlag : 0;
660 if (!activitySet)
661 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
662 }
663 } else if (!preproc.IsInactive()) {
664 preproc.InvertCurrentLevel();
665 activitySet = preproc.IsInactive() ? activeFlag : 0;
666 if (!activitySet)
667 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
668 }
669 } else if (sc.Match("endif")) {
670 preproc.EndSection();
671 activitySet = preproc.IsInactive() ? activeFlag : 0;
672 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
673 } else if (sc.Match("define")) {
674 if (options.updatePreprocessor && !preproc.IsInactive()) {
675 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
676 size_t startName = 0;
677 while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
678 startName++;
679 size_t endName = startName;
680 while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
681 endName++;
682 std::string key = restOfLine.substr(startName, endName-startName);
683 if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
684 // Macro
685 size_t endArgs = endName;
686 while ((endArgs < restOfLine.length()) && (restOfLine[endArgs] != ')'))
687 endArgs++;
688 std::string args = restOfLine.substr(endName + 1, endArgs - endName - 1);
689 size_t startValue = endArgs+1;
690 while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
691 startValue++;
692 std::string value;
693 if (startValue < restOfLine.length())
694 value = restOfLine.substr(startValue);
695 preprocessorDefinitions[key] = SymbolValue(value, args);
696 ppDefineHistory.push_back(PPDefinition(curLine, key, value, false, args));
697 definitionsChanged = true;
698 } else {
699 // Value
700 size_t startValue = endName;
701 while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
702 startValue++;
703 std::string value = restOfLine.substr(startValue);
704 preprocessorDefinitions[key] = value;
705 ppDefineHistory.push_back(PPDefinition(curLine, key, value));
706 definitionsChanged = true;
707 }
708 }
709 } else if (sc.Match("undefineall")) {
710 if (options.updatePreprocessor && !preproc.IsInactive()) {
711 // remove all preprocessor definitions
712 std::map<std::string, SymbolValue>::iterator itDef;
713 for(itDef = preprocessorDefinitions.begin(); itDef != preprocessorDefinitions.end(); ++itDef) {
714 ppDefineHistory.push_back(PPDefinition(curLine, itDef->first, "", true));
715 }
716 preprocessorDefinitions.clear();
717 definitionsChanged = true;
718 }
719 } else if (sc.Match("undef")) {
720 if (options.updatePreprocessor && !preproc.IsInactive()) {
721 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true);
722 std::vector<std::string> tokens = Tokenize(restOfLine);
723 std::string key;
724 if (tokens.size() >= 1) {
725 key = tokens[0];
726 preprocessorDefinitions.erase(key);
727 ppDefineHistory.push_back(PPDefinition(curLine, key, "", true));
728 definitionsChanged = true;
729 }
730 }
731 }
732 }
733 }
734 } else if (!isProtected) {
735 if (IsADigit(sc.ch) || (sc.ch == '\'') || (sc.ch == '.' && IsADigit(sc.chNext))) {
736 sc.SetState(SCE_V_NUMBER|activitySet);
737 } else if (IsAWordStart(sc.ch)) {
738 sc.SetState(SCE_V_IDENTIFIER|activitySet);
739 } else if (sc.Match('/', '*')) {
740 sc.SetState(SCE_V_COMMENT|activitySet);
741 sc.Forward(); // Eat the * so it isn't used for the end of the comment
742 } else if (sc.Match('/', '/')) {
743 if (sc.Match("//!")) // Nice to have a different comment style
744 sc.SetState(SCE_V_COMMENTLINEBANG|activitySet);
745 else
746 sc.SetState(SCE_V_COMMENTLINE|activitySet);
747 } else if (sc.ch == '\"') {
748 sc.SetState(SCE_V_STRING|activitySet);
749 } else if (sc.ch == '\\') {
750 // escaped identifier, everything is ok up to whitespace
751 isEscapedId = true;
752 sc.SetState(SCE_V_IDENTIFIER|activitySet);
753 } else if (isoperator(static_cast<char>(sc.ch)) || sc.ch == '@' || sc.ch == '#') {
754 sc.SetState(SCE_V_OPERATOR|activitySet);
755 if (sc.ch == '.') lineState = kwDot;
756 if (sc.ch == ';') lineState = kwOther;
757 }
758 }
759 }
760 if (isEscapedId && isspacechar(sc.ch)) {
761 isEscapedId = false;
762 }
763 }
764 if (definitionsChanged) {
765 styler.ChangeLexerState(startPos, startPos + length);
766 }
767 sc.Complete();
768 }
769
IsStreamCommentStyle(int style)770 static bool IsStreamCommentStyle(int style) {
771 return style == SCE_V_COMMENT;
772 }
773
IsCommentLine(Sci_Position line,LexAccessor & styler)774 static bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
775 Sci_Position pos = styler.LineStart(line);
776 Sci_Position eolPos = styler.LineStart(line + 1) - 1;
777 for (Sci_Position i = pos; i < eolPos; i++) {
778 char ch = styler[i];
779 char chNext = styler.SafeGetCharAt(i + 1);
780 int style = styler.StyleAt(i);
781 if (ch == '/' && chNext == '/' &&
782 (style == SCE_V_COMMENTLINE || style == SCE_V_COMMENTLINEBANG)) {
783 return true;
784 } else if (!IsASpaceOrTab(ch)) {
785 return false;
786 }
787 }
788 return false;
789 }
790
791 // Store both the current line's fold level and the next lines in the
792 // level store to make it easy to pick up with each increment
793 // and to make it possible to fiddle the current level for "} else {".
Fold(Sci_PositionU startPos,Sci_Position length,int initStyle,IDocument * pAccess)794 void SCI_METHOD LexerVerilog::Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
795 {
796 LexAccessor styler(pAccess);
797 bool foldAtBrace = 1;
798 bool foldAtParenthese = 1;
799
800 Sci_Position lineCurrent = styler.GetLine(startPos);
801 // Move back one line to be compatible with LexerModule::Fold behavior, fixes problem with foldComment behavior
802 if (lineCurrent > 0) {
803 lineCurrent--;
804 Sci_Position newStartPos = styler.LineStart(lineCurrent);
805 length += startPos - newStartPos;
806 startPos = newStartPos;
807 initStyle = 0;
808 if (startPos > 0) {
809 initStyle = styler.StyleAt(startPos - 1);
810 }
811 }
812 Sci_PositionU endPos = startPos + length;
813 int visibleChars = 0;
814 int levelCurrent = SC_FOLDLEVELBASE;
815 if (lineCurrent > 0)
816 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
817 int levelMinCurrent = levelCurrent;
818 int levelNext = levelCurrent;
819 char chNext = styler[startPos];
820 int styleNext = MaskActive(styler.StyleAt(startPos));
821 int style = MaskActive(initStyle);
822
823 // restore fold state (if it exists) for prior line
824 int stateCurrent = 0;
825 std::map<Sci_Position,int>::iterator foldStateIterator = foldState.find(lineCurrent-1);
826 if (foldStateIterator != foldState.end()) {
827 stateCurrent = foldStateIterator->second;
828 }
829
830 // remove all foldState entries after lineCurrent-1
831 foldStateIterator = foldState.upper_bound(lineCurrent-1);
832 if (foldStateIterator != foldState.end()) {
833 foldState.erase(foldStateIterator, foldState.end());
834 }
835
836 for (Sci_PositionU i = startPos; i < endPos; i++) {
837 char ch = chNext;
838 chNext = styler.SafeGetCharAt(i + 1);
839 int stylePrev = style;
840 style = styleNext;
841 styleNext = MaskActive(styler.StyleAt(i + 1));
842 bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
843 if (!(stateCurrent & protectedFlag)) {
844 if (options.foldComment && IsStreamCommentStyle(style)) {
845 if (!IsStreamCommentStyle(stylePrev)) {
846 levelNext++;
847 } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
848 // Comments don't end at end of line and the next character may be unstyled.
849 levelNext--;
850 }
851 }
852 if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
853 {
854 if (!IsCommentLine(lineCurrent - 1, styler)
855 && IsCommentLine(lineCurrent + 1, styler))
856 levelNext++;
857 else if (IsCommentLine(lineCurrent - 1, styler)
858 && !IsCommentLine(lineCurrent+1, styler))
859 levelNext--;
860 }
861 if (options.foldComment && (style == SCE_V_COMMENTLINE)) {
862 if ((ch == '/') && (chNext == '/')) {
863 char chNext2 = styler.SafeGetCharAt(i + 2);
864 if (chNext2 == '{') {
865 levelNext++;
866 } else if (chNext2 == '}') {
867 levelNext--;
868 }
869 }
870 }
871 }
872 if (ch == '`') {
873 Sci_PositionU j = i + 1;
874 while ((j < endPos) && IsASpaceOrTab(styler.SafeGetCharAt(j))) {
875 j++;
876 }
877 if (styler.Match(j, "protected")) {
878 stateCurrent |= protectedFlag;
879 levelNext++;
880 } else if (styler.Match(j, "endprotected")) {
881 stateCurrent &= ~protectedFlag;
882 levelNext--;
883 } else if (!(stateCurrent & protectedFlag) && options.foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
884 if (styler.Match(j, "if")) {
885 if (options.foldPreprocessorElse) {
886 // Measure the minimum before a begin to allow
887 // folding on "end else begin"
888 if (levelMinCurrent > levelNext) {
889 levelMinCurrent = levelNext;
890 }
891 }
892 levelNext++;
893 } else if (options.foldPreprocessorElse && styler.Match(j, "else")) {
894 levelNext--;
895 if (levelMinCurrent > levelNext) {
896 levelMinCurrent = levelNext;
897 }
898 levelNext++;
899 } else if (options.foldPreprocessorElse && styler.Match(j, "elsif")) {
900 levelNext--;
901 // Measure the minimum before a begin to allow
902 // folding on "end else begin"
903 if (levelMinCurrent > levelNext) {
904 levelMinCurrent = levelNext;
905 }
906 levelNext++;
907 } else if (styler.Match(j, "endif")) {
908 levelNext--;
909 }
910 }
911 }
912 if (style == SCE_V_OPERATOR) {
913 if (foldAtParenthese) {
914 if (ch == '(') {
915 levelNext++;
916 } else if (ch == ')') {
917 levelNext--;
918 }
919 }
920 // semicolons terminate external declarations
921 if (ch == ';') {
922 // extern and pure virtual declarations terminated by semicolon
923 if (stateCurrent & foldExternFlag) {
924 levelNext--;
925 stateCurrent &= ~foldExternFlag;
926 }
927 // wait and disable statements terminated by semicolon
928 if (stateCurrent & foldWaitDisableFlag) {
929 stateCurrent &= ~foldWaitDisableFlag;
930 }
931 // typedef statements terminated by semicolon
932 if (stateCurrent & typedefFlag) {
933 stateCurrent &= ~typedefFlag;
934 }
935 }
936 // wait and disable statements containing '(' will not contain "fork" keyword, special processing is not needed
937 if (ch == '(') {
938 if (stateCurrent & foldWaitDisableFlag) {
939 stateCurrent &= ~foldWaitDisableFlag;
940 }
941 }
942 }
943 if (style == SCE_V_OPERATOR) {
944 if (foldAtBrace) {
945 if (ch == '{') {
946 levelNext++;
947 } else if (ch == '}') {
948 levelNext--;
949 }
950 }
951 }
952 if (style == SCE_V_WORD && stylePrev != SCE_V_WORD) {
953 Sci_PositionU j = i;
954 if (styler.Match(j, "case") ||
955 styler.Match(j, "casex") ||
956 styler.Match(j, "casez") ||
957 styler.Match(j, "covergroup") ||
958 styler.Match(j, "function") ||
959 styler.Match(j, "generate") ||
960 styler.Match(j, "interface") ||
961 styler.Match(j, "package") ||
962 styler.Match(j, "primitive") ||
963 styler.Match(j, "program") ||
964 styler.Match(j, "sequence") ||
965 styler.Match(j, "specify") ||
966 styler.Match(j, "table") ||
967 styler.Match(j, "task") ||
968 (styler.Match(j, "module") && options.foldAtModule)) {
969 levelNext++;
970 } else if (styler.Match(j, "begin")) {
971 // Measure the minimum before a begin to allow
972 // folding on "end else begin"
973 if (levelMinCurrent > levelNext) {
974 levelMinCurrent = levelNext;
975 }
976 levelNext++;
977 } else if (styler.Match(j, "class")) {
978 // class does not introduce a block when used in a typedef statement
979 if (!(stateCurrent & typedefFlag))
980 levelNext++;
981 } else if (styler.Match(j, "fork")) {
982 // fork does not introduce a block when used in a wait or disable statement
983 if (stateCurrent & foldWaitDisableFlag) {
984 stateCurrent &= ~foldWaitDisableFlag;
985 } else
986 levelNext++;
987 } else if (styler.Match(j, "endcase") ||
988 styler.Match(j, "endclass") ||
989 styler.Match(j, "endfunction") ||
990 styler.Match(j, "endgenerate") ||
991 styler.Match(j, "endgroup") ||
992 styler.Match(j, "endinterface") ||
993 styler.Match(j, "endpackage") ||
994 styler.Match(j, "endprimitive") ||
995 styler.Match(j, "endprogram") ||
996 styler.Match(j, "endsequence") ||
997 styler.Match(j, "endspecify") ||
998 styler.Match(j, "endtable") ||
999 styler.Match(j, "endtask") ||
1000 styler.Match(j, "join") ||
1001 styler.Match(j, "join_any") ||
1002 styler.Match(j, "join_none") ||
1003 (styler.Match(j, "endmodule") && options.foldAtModule) ||
1004 (styler.Match(j, "end") && !IsAWordChar(styler.SafeGetCharAt(j + 3)))) {
1005 levelNext--;
1006 } else if (styler.Match(j, "extern") ||
1007 styler.Match(j, "pure")) {
1008 // extern and pure virtual functions/tasks are terminated by ';' not endfunction/endtask
1009 stateCurrent |= foldExternFlag;
1010 } else if (styler.Match(j, "disable") ||
1011 styler.Match(j, "wait")) {
1012 // fork does not introduce a block when used in a wait or disable statement
1013 stateCurrent |= foldWaitDisableFlag;
1014 } else if (styler.Match(j, "typedef")) {
1015 stateCurrent |= typedefFlag;
1016 }
1017 }
1018 if (atEOL) {
1019 int levelUse = levelCurrent;
1020 if (options.foldAtElse||options.foldPreprocessorElse) {
1021 levelUse = levelMinCurrent;
1022 }
1023 int lev = levelUse | levelNext << 16;
1024 if (visibleChars == 0 && options.foldCompact)
1025 lev |= SC_FOLDLEVELWHITEFLAG;
1026 if (levelUse < levelNext)
1027 lev |= SC_FOLDLEVELHEADERFLAG;
1028 if (stateCurrent) {
1029 foldState[lineCurrent] = stateCurrent;
1030 }
1031 if (lev != styler.LevelAt(lineCurrent)) {
1032 styler.SetLevel(lineCurrent, lev);
1033 }
1034 lineCurrent++;
1035 levelCurrent = levelNext;
1036 levelMinCurrent = levelCurrent;
1037 visibleChars = 0;
1038 }
1039 if (!isspacechar(ch))
1040 visibleChars++;
1041 }
1042 }
1043
Tokenize(const std::string & expr) const1044 std::vector<std::string> LexerVerilog::Tokenize(const std::string &expr) const {
1045 // Break into tokens
1046 std::vector<std::string> tokens;
1047 const char *cp = expr.c_str();
1048 while (*cp) {
1049 std::string word;
1050 if (setWord.Contains(static_cast<unsigned char>(*cp))) {
1051 // Identifiers and numbers
1052 while (setWord.Contains(static_cast<unsigned char>(*cp))) {
1053 word += *cp;
1054 cp++;
1055 }
1056 } else if (IsSpaceOrTab(*cp)) {
1057 while (IsSpaceOrTab(*cp)) {
1058 cp++;
1059 }
1060 continue;
1061 } else {
1062 // Should handle strings, characters, and comments here
1063 word += *cp;
1064 cp++;
1065 }
1066 tokens.push_back(word);
1067 }
1068 return tokens;
1069 }
1070
1071 static const char * const verilogWordLists[] = {
1072 "Primary keywords and identifiers",
1073 "Secondary keywords and identifiers",
1074 "System Tasks",
1075 "User defined tasks and identifiers",
1076 "Documentation comment keywords",
1077 "Preprocessor definitions",
1078 0,
1079 };
1080
1081 LexerModule lmVerilog(SCLEX_VERILOG, LexerVerilog::LexerFactoryVerilog, "verilog", verilogWordLists);
1082