1 /*
2 * This file is part of the Code::Blocks IDE and licensed under the GNU General Public License, version 3
3 * http://www.gnu.org/licenses/gpl-3.0.html
4 *
5 * $Revision: 11572 $
6 * $Id: parserthread.cpp 11572 2019-02-16 06:52:30Z ollydbg $
7 * $HeadURL: svn://svn.code.sf.net/p/codeblocks/code/branches/release-20.xx/src/plugins/codecompletion/parser/parserthread.cpp $
8 */
9
10 #include <sdk.h>
11
12 #ifndef CB_PRECOMP
13 #include <cctype>
14 #include <queue>
15
16 #include <wx/app.h>
17 #include <wx/msgdlg.h>
18
19 #include <cbexception.h>
20 #include <globals.h>
21 #include <logmanager.h>
22 #include <manager.h>
23 #endif
24
25 #include <wx/tokenzr.h>
26
27 #include "parserthread.h"
28 #include "parser.h"
29 #include "expression.h" // used to calculate the enumerator value
30
31 #define CC_PARSERTHREAD_DEBUG_OUTPUT 0
32
33 #if defined(CC_GLOBAL_DEBUG_OUTPUT)
34 #if CC_GLOBAL_DEBUG_OUTPUT == 1
35 #undef CC_PARSERTHREAD_DEBUG_OUTPUT
36 #define CC_PARSERTHREAD_DEBUG_OUTPUT 1
37 #elif CC_GLOBAL_DEBUG_OUTPUT == 2
38 #undef CC_PARSERTHREAD_DEBUG_OUTPUT
39 #define CC_PARSERTHREAD_DEBUG_OUTPUT 2
40 #endif
41 #endif
42
43 #ifdef CC_PARSER_TEST
44 #define ADDTOKEN(format, args...) \
45 CCLogger::Get()->AddToken(F(format, ##args))
46 #define TRACE(format, args...) \
47 CCLogger::Get()->DebugLog(F(format, ##args))
48 #define TRACE2(format, args...) \
49 CCLogger::Get()->DebugLog(F(format, ##args))
50 #else
51 #if CC_PARSERTHREAD_DEBUG_OUTPUT == 1
52 #define ADDTOKEN(format, args...) \
53 CCLogger::Get()->AddToken(F(format, ##args))
54 #define TRACE(format, args...) \
55 CCLogger::Get()->DebugLog(F(format, ##args))
56 #define TRACE2(format, args...)
57 #elif CC_PARSERTHREAD_DEBUG_OUTPUT == 2
58 #define ADDTOKEN(format, args...) \
59 CCLogger::Get()->AddToken(F(format, ##args))
60 #define TRACE(format, args...) \
61 do \
62 { \
63 if (g_EnableDebugTrace) \
64 CCLogger::Get()->DebugLog(F(format, ##args)); \
65 } \
66 while (false)
67 #define TRACE2(format, args...) \
68 CCLogger::Get()->DebugLog(F(format, ##args))
69 #else
70 #define ADDTOKEN(format, args...)
71 #define TRACE(format, args...)
72 #define TRACE2(format, args...)
73 #endif
74 #endif
75
76 #define CC_PARSERTHREAD_TESTDESTROY 0
77
78 #if CC_PARSERTHREAD_TESTDESTROY
79 #define IS_ALIVE IsStillAlive(wxString(__PRETTY_FUNCTION__, wxConvUTF8))
80 #else
81 #define IS_ALIVE !TestDestroy()
82 #endif
83
84 const wxString g_UnnamedSymbol = _T("__Unnamed");
85
86 namespace ParserConsts
87 {
88 // length: 0
89 const wxString empty (_T(""));
90 const wxChar null (_T('\0'));
91 // length: 1
92 const wxChar eol_chr (_T('\n'));
93 const wxString space (_T(" "));
94 const wxChar space_chr (_T(' '));
95 const wxChar tab_chr (_T('\t'));
96 const wxString equals (_T("="));
97 const wxChar equals_chr (_T('='));
98 const wxString hash (_T("#"));
99 const wxChar hash_chr (_T('#'));
100 const wxString plus (_T("+"));
101 const wxChar plus_chr (_T('+'));
102 const wxString dash (_T("-"));
103 const wxChar dash_chr (_T('-'));
104 const wxString ptr (_T("*"));
105 const wxChar ptr_chr (_T('*'));
106 const wxString ref (_T("&"));
107 const wxChar ref_chr (_T('&'));
108 const wxString comma (_T(","));
109 const wxChar comma_chr (_T(','));
110 const wxString dot (_T("."));
111 const wxChar dot_chr (_T('.'));
112 const wxString colon (_T(":"));
113 const wxChar colon_chr (_T(':'));
114 const wxString semicolon (_T(";"));
115 const wxChar semicolon_chr (_T(';'));
116 const wxChar opbracket_chr (_T('('));
117 const wxChar clbracket_chr (_T(')'));
118 const wxString opbracket (_T("("));
119 const wxString clbracket (_T(")"));
120 const wxString opbrace (_T("{"));
121 const wxChar opbrace_chr (_T('{'));
122 const wxString clbrace (_T("}"));
123 const wxChar clbrace_chr (_T('}'));
124 const wxString oparray (_T("["));
125 const wxChar oparray_chr (_T('['));
126 const wxString clarray (_T("]"));
127 const wxChar clarray_chr (_T(']'));
128 const wxString tilde (_T("~"));
129 const wxString lt (_T("<"));
130 const wxChar lt_chr (_T('<'));
131 const wxString gt (_T(">"));
132 const wxChar gt_chr (_T('>'));
133 const wxChar underscore_chr (_T('_'));
134 const wxChar question_chr (_T('?'));
135 // length: 2
136 const wxString dcolon (_T("::"));
137 const wxString opbracesemicolon(_T("{;"));
138 const wxString commaclbrace (_T(",}"));
139 const wxString semicolonopbrace(_T(";{"));
140 const wxString semicolonclbrace(_T(";}"));
141 const wxString gtsemicolon (_T(">;"));
142 const wxString quot (_T("\""));
143 const wxString kw_do (_T("do"));
144 const wxString kw_if (_T("if"));
145 // length: 3
146 const wxString spaced_colon (_T(" : "));
147 const wxString kw__C_ (_T("\"C\""));
148 const wxString kw_for (_T("for"));
149 const wxString kw_try (_T("try"));
150 const wxString commasemicolonopbrace(_T(",;{"));
151 // length: 4
152 const wxString kw___at (_T("__at"));
153 const wxString kw_else (_T("else"));
154 const wxString kw_enum (_T("enum"));
155 const wxString kw_elif (_T("elif"));
156 const wxString kw_case (_T("case"));
157 // length: 5
158 const wxString kw__CPP_ (_T("\"C++\""));
159 const wxString kw___asm (_T("__asm"));
160 const wxString kw_catch (_T("catch"));
161 const wxString kw_class (_T("class"));
162 const wxString kw_const (_T("const"));
163 const wxString kw_union (_T("union"));
164 const wxString kw_using (_T("using"));
165 const wxString kw_throw (_T("throw"));
166 const wxString kw_while (_T("while"));
167 // length: 6
168 const wxString kw_delete (_T("delete"));
169 const wxString kw_extern (_T("extern"));
170 const wxString kw_friend (_T("friend"));
171 const wxString kw_inline (_T("inline"));
172 const wxString kw_public (_T("public"));
173 const wxString kw_return (_T("return"));
174 const wxString kw_static (_T("static"));
175 const wxString kw_struct (_T("struct"));
176 const wxString kw_switch (_T("switch"));
177 // length: 7
178 const wxString kw_include (_T("include"));
179 const wxString kw_private (_T("private"));
180 const wxString kw_typedef (_T("typedef"));
181 const wxString kw_virtual (_T("virtual"));
182 // length: 8
183 const wxString kw_noexcept (_T("noexcept"));
184 const wxString kw_operator (_T("operator"));
185 const wxString kw_template (_T("template"));
186 const wxString kw_typename (_T("typename"));
187 const wxString kw_volatile (_T("volatile"));
188 // length: 9
189 const wxString kw_namespace (_T("namespace"));
190 const wxString kw_protected (_T("protected"));
191 // length: 10
192 const wxString kw_declspec (_T("__declspec"));
193 // length: 13
194 const wxString kw_attribute (_T("__attribute__"));
195 }
196
ParserThread(ParserBase * parent,const wxString & bufferOrFilename,bool isLocal,ParserThreadOptions & parserThreadOptions,TokenTree * tokenTree)197 ParserThread::ParserThread(ParserBase* parent,
198 const wxString& bufferOrFilename,
199 bool isLocal,
200 ParserThreadOptions& parserThreadOptions,
201 TokenTree* tokenTree) :
202 m_Tokenizer(tokenTree),
203 m_Parent(parent),
204 m_TokenTree(tokenTree),
205 m_LastParent(0),
206 m_LastScope(tsUndefined),
207 m_FileSize(0),
208 m_FileIdx(0),
209 m_IsLocal(isLocal),
210 m_Options(parserThreadOptions),
211 m_ParsingTypedef(false),
212 m_Buffer(bufferOrFilename),
213 m_StructUnionUnnamedCount(0),
214 m_EnumUnnamedCount(0)
215 {
216 m_Tokenizer.SetTokenizerOption(parserThreadOptions.wantPreprocessor,
217 parserThreadOptions.storeDocumentation);
218 if (!m_TokenTree)
219 cbThrow(_T("m_TokenTree is a nullptr?!"));
220 }
221
~ParserThread()222 ParserThread::~ParserThread()
223 {
224 // wait for file loader object to complete (can't abort it)
225 if (m_Options.loader)
226 {
227 m_Options.loader->Sync();
228 delete m_Options.loader;
229 }
230 }
231
SkipToOneOfChars(const wxString & chars,bool supportNesting,bool singleCharToken)232 wxChar ParserThread::SkipToOneOfChars(const wxString& chars, bool supportNesting, bool singleCharToken)
233 {
234 unsigned int level = m_Tokenizer.GetNestingLevel();
235 while (IS_ALIVE)
236 {
237 wxString token = m_Tokenizer.GetToken(); // grab next token...
238 if (token.IsEmpty())
239 return ParserConsts::null; // eof
240
241 // if supportNesting==true, we only do a match in the same brace/nesting level,
242 // thus we preserve the brace level when the function returned. But if
243 // supportNesting==false, we do not consider the brace level on matching.
244 if (!supportNesting || m_Tokenizer.GetNestingLevel() == level)
245 {
246 // only consider tokens of length one, if requested
247 if (singleCharToken && token.length() > 1)
248 continue;
249
250 wxChar ch = token.GetChar(0);
251 if (chars.Find(ch) != wxNOT_FOUND) // match one char
252 return ch;
253 }
254 }
255
256 return ParserConsts::null; // not found
257 }
258
SkipBlock()259 void ParserThread::SkipBlock()
260 {
261 // need to force the tokenizer _not_ skip anything
262 // or else default values for template params would cause us to miss everything (because of the '=' symbol)
263 TokenizerState oldState = m_Tokenizer.GetState();
264 m_Tokenizer.SetState(tsNormal);
265
266 // skip tokens until we reach }
267 // block nesting is taken into consideration too ;)
268
269 // this is the nesting level we start at
270 // we subtract 1 because we 're already inside the block
271 // (since we 've read the {)
272 unsigned int level = m_Tokenizer.GetNestingLevel() - 1;
273 while (IS_ALIVE)
274 {
275 wxString token = m_Tokenizer.GetToken();
276 if (token.IsEmpty())
277 break; // eof
278
279 // if we reach the initial nesting level, we are done
280 if (level == m_Tokenizer.GetNestingLevel())
281 break;
282 }
283
284 // reset tokenizer's functionality
285 m_Tokenizer.SetState(oldState);
286 }
287
SkipAngleBraces()288 void ParserThread::SkipAngleBraces()
289 {
290 // need to force the tokenizer _not_ skip anything
291 // or else default values for template params would cause us to miss everything (because of the '=' symbol)
292 TokenizerState oldState = m_Tokenizer.GetState();
293 m_Tokenizer.SetState(tsNormal);
294
295 int nestLvl = 0;
296 // NOTE: only exit this loop with 'break' so the tokenizer's state can
297 // be reset afterwards (i.e. don't use 'return')
298 while (IS_ALIVE)
299 {
300 wxString tmp = m_Tokenizer.GetToken();
301 if (tmp==ParserConsts::lt)
302 ++nestLvl;
303 else if (tmp==ParserConsts::gt)
304 --nestLvl;
305 else if (tmp==ParserConsts::semicolon)
306 {
307 // unget token - leave ; on the stack
308 m_Tokenizer.UngetToken();
309 break;
310 }
311 else if (tmp.IsEmpty())
312 break;
313 if (nestLvl <= 0)
314 break;
315 }
316
317 // reset tokenizer's functionality
318 m_Tokenizer.SetState(oldState);
319 }
320
ParseBufferForNamespaces(const wxString & buffer,NameSpaceVec & result)321 bool ParserThread::ParseBufferForNamespaces(const wxString& buffer, NameSpaceVec& result)
322 {
323 m_Tokenizer.InitFromBuffer(buffer);
324 if (!m_Tokenizer.IsOK())
325 return false;
326
327 result.clear();
328
329 wxArrayString nsStack;
330 m_Tokenizer.SetState(tsNormal);
331 m_ParsingTypedef = false;
332
333 while (m_Tokenizer.NotEOF() && IS_ALIVE)
334 {
335 wxString token = m_Tokenizer.GetToken();
336 if (token.IsEmpty())
337 continue;
338
339 if (token == ParserConsts::kw_using)
340 SkipToOneOfChars(ParserConsts::semicolonclbrace);
341 else if (token == ParserConsts::opbrace)
342 SkipBlock();
343 else if (token == ParserConsts::kw_namespace)
344 {
345 wxString name = m_Tokenizer.GetToken();
346 if (name == ParserConsts::opbrace)
347 name = wxEmptyString; // anonymous namespace
348 else
349 {
350 wxString next = m_Tokenizer.PeekToken();
351 if (next == ParserConsts::equals)
352 {
353 SkipToOneOfChars(ParserConsts::semicolonclbrace);
354 continue;
355 }
356 else if (next == ParserConsts::opbrace)
357 {
358 m_Tokenizer.GetToken();
359 name += ParserConsts::dcolon;
360 }
361 }
362
363 nsStack.Add(name);
364 NameSpace ns;
365 for (size_t i = 0; i < nsStack.Count(); ++i)
366 ns.Name << nsStack[i];
367 ns.StartLine = m_Tokenizer.GetLineNumber() - 1;
368 ns.EndLine = -1;
369
370 result.push_back(ns);
371 }
372 else if (token == ParserConsts::clbrace)
373 {
374 NameSpaceVec::reverse_iterator it;
375 for (it = result.rbegin(); it != result.rend(); ++it)
376 {
377 NameSpace& ns = *it;
378 if (ns.EndLine == -1)
379 {
380 ns.EndLine = m_Tokenizer.GetLineNumber() - 1;
381 break;
382 }
383 }
384
385 if (!nsStack.IsEmpty())
386 nsStack.RemoveAt(nsStack.GetCount() - 1);
387 }
388 }
389 return true;
390 }
391
ParseBufferForUsingNamespace(const wxString & buffer,wxArrayString & result)392 bool ParserThread::ParseBufferForUsingNamespace(const wxString& buffer, wxArrayString& result)
393 {
394 m_Tokenizer.InitFromBuffer(buffer);
395 if (!m_Tokenizer.IsOK())
396 return false;
397
398 result.Clear();
399 m_Str.Clear();
400 m_LastUnnamedTokenName.Clear();
401 m_ParsingTypedef = false;
402
403 // Notice: clears the queue "m_EncounteredTypeNamespaces"
404 while (!m_EncounteredTypeNamespaces.empty())
405 m_EncounteredTypeNamespaces.pop();
406
407 // Notice: clears the queue "m_EncounteredNamespaces"
408 while (!m_EncounteredNamespaces.empty())
409 m_EncounteredNamespaces.pop();
410
411 while (m_Tokenizer.NotEOF() && IS_ALIVE)
412 {
413 wxString token = m_Tokenizer.GetToken();
414 if (token.IsEmpty())
415 continue;
416
417 if (token==ParserConsts::kw_namespace)
418 {
419 // need this too
420 token = m_Tokenizer.GetToken();
421 SkipToOneOfChars(ParserConsts::opbrace);
422
423 if (!token.IsEmpty())
424 result.Add(token);
425 }
426 else if (token==ParserConsts::opbrace && m_Options.bufferSkipBlocks)
427 {
428 SkipBlock();
429 }
430 else if (token==ParserConsts::kw_using)
431 {
432 // there are some kinds of using keyword usage
433 // (1) using namespace A;
434 // (2) using namespace A::B; // where B is a namespace
435 // (3) using A::B; // where B is NOT a namespace
436 // (4) using A = B; // doesn't import anything, so we don't handle this here
437 token = m_Tokenizer.GetToken();
438 wxString peek = m_Tokenizer.PeekToken();
439 if (token == ParserConsts::kw_namespace || peek == ParserConsts::dcolon)
440 {
441 if (peek == ParserConsts::dcolon) // using declaration, such as case (3)
442 m_Str << token; // push the A to the m_Str(type stack)
443 else //handling the case (1) and (2)
444 {
445 // using directive
446 while (IS_ALIVE) // support full namespaces
447 {
448 m_Str << m_Tokenizer.GetToken();
449 if (m_Tokenizer.PeekToken() == ParserConsts::dcolon)
450 m_Str << m_Tokenizer.GetToken();
451 else
452 break;
453 }
454 }
455 // m_Str must end with a namespace for CC to work
456 // now, m_Str contains "A" in case (1) and (3), and "A::B" in case (2)
457 if (!m_Str.IsEmpty())
458 result.Add(m_Str);
459 m_Str.Clear();
460 }
461 else
462 SkipToOneOfChars(ParserConsts::semicolonclbrace);
463 }
464 }
465 return true;
466 }
467
InitTokenizer()468 bool ParserThread::InitTokenizer()
469 {
470 if (!m_Buffer.IsEmpty())
471 {
472 if (!m_Options.useBuffer)
473 {
474 if (wxFileExists(m_Buffer))
475 {
476 wxFile file(m_Buffer);
477 if (file.IsOpened())
478 {
479 m_Filename = m_Buffer;
480 m_FileSize = file.Length();
481
482 TRACE(_T("InitTokenizer() : m_Filename='%s', m_FileSize=%u."), m_Filename.wx_str(), m_FileSize);
483
484 bool ret = m_Tokenizer.Init(m_Filename, m_Options.loader);
485 // must delete the loader, since it was allocated by SDK's Load() function
486 Delete(m_Options.loader);
487
488 if (!ret) { TRACE(_T("InitTokenizer() : Could not initialise tokenizer for file '%s'."), m_Filename.wx_str()); }
489 return ret;
490 }
491 }
492
493 TRACE(_T("InitTokenizer() : Could not open file: '%s'."), m_Buffer.wx_str());
494 return false;
495 }
496 else
497 {
498 // record filename for buffer parsing
499 m_Filename = m_Options.fileOfBuffer;
500 m_FileIdx = m_TokenTree->InsertFileOrGetIndex(m_Filename);
501
502 return m_Tokenizer.InitFromBuffer(m_Buffer, m_Filename, m_Options.initLineOfBuffer);
503 }
504 }
505
506 TRACE(_T("InitTokenizer() : Buffer is empty."));
507 return false;
508 }
509
Parse()510 bool ParserThread::Parse()
511 {
512 if (!IS_ALIVE || !InitTokenizer())
513 return false;
514
515 TRACE(_T("Parse() : Parsing '%s'"), m_Filename.wx_str());
516
517 bool result = false;
518 m_ParsingTypedef = false;
519
520 do
521 {
522 if (!m_TokenTree || !m_Tokenizer.IsOK())
523 break;
524
525 if (!m_Options.useBuffer) // Parse a file
526 {
527 // the second parameter of ReserveFileForParsing() is false, so set it to fpsBeingParsed
528 m_FileIdx = m_TokenTree->ReserveFileForParsing(m_Filename);
529 if (!m_FileIdx)
530 break;
531 }
532
533 DoParse();
534
535 if (!m_Options.useBuffer) // Parsing a file
536 m_TokenTree->FlagFileAsParsed(m_Filename);
537
538 result = true;
539 }
540 while (false);
541
542 return result;
543 }
544
DoParse()545 void ParserThread::DoParse()
546 {
547 // need to reset tokenizer's behaviour
548 // don't forget to reset that if you add any early exit condition!
549 TokenizerState oldState = m_Tokenizer.GetState();
550 m_Tokenizer.SetState(tsNormal);
551
552 m_Str.Clear();
553 m_LastToken.Clear();
554 m_LastUnnamedTokenName.Clear();
555
556 // Notice: clears the queue "m_EncounteredTypeNamespaces"
557 while (!m_EncounteredTypeNamespaces.empty())
558 m_EncounteredTypeNamespaces.pop();
559
560 // Notice: clears the queue "m_EncounteredNamespaces"
561 while (!m_EncounteredNamespaces.empty())
562 m_EncounteredNamespaces.pop();
563
564 while (m_Tokenizer.NotEOF() && IS_ALIVE)
565 {
566 wxString token = m_Tokenizer.GetToken();
567 if (token.IsEmpty())
568 continue;
569
570 TRACE(_T("DoParse() : Loop:m_Str='%s', token='%s'"), m_Str.wx_str(), token.wx_str());
571
572 bool switchHandled = true;
573 switch (token.Length())
574 {
575 // ---------------------------------------------------------------
576 // token length of 1
577 // ---------------------------------------------------------------
578 case 1:
579 switch (static_cast<wxChar>(token[0]))
580 {
581 case ParserConsts::semicolon_chr:
582 {
583 m_Str.Clear();
584 m_PointerOrRef.Clear();
585 // Notice: clears the queue "m_EncounteredTypeNamespaces"
586 while (!m_EncounteredTypeNamespaces.empty())
587 m_EncounteredTypeNamespaces.pop();
588 m_TemplateArgument.Clear();
589 }
590 break;
591
592 case ParserConsts::dot_chr:
593 {
594 m_Str.Clear();
595 SkipToOneOfChars(ParserConsts::semicolonclbrace);
596 }
597 break;
598
599 case ParserConsts::gt_chr:
600 {
601 if (m_LastToken == ParserConsts::dash)
602 {
603 m_Str.Clear();
604 SkipToOneOfChars(ParserConsts::semicolonclbrace);
605 }
606 else
607 switchHandled = false;
608 }
609 break;
610
611 case ParserConsts::opbrace_chr:
612 {
613 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
614 SkipBlock();
615 m_Str.Clear();
616 }
617 break;
618
619 case ParserConsts::clbrace_chr:
620 {
621 m_LastParent = 0L;
622 m_LastScope = tsUndefined;
623 m_Str.Clear();
624 // the only time we get to find a } is when recursively called by e.g. HandleClass
625 // we have to return now...
626 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
627 {
628 m_Tokenizer.SetState(oldState); // This uses the top-level oldState (renamed shadowed versions below)
629 return;
630 }
631 }
632 break;
633
634 case ParserConsts::colon_chr:
635 {
636 if (m_LastToken == ParserConsts::kw_public)
637 m_LastScope = tsPublic;
638 else if (m_LastToken == ParserConsts::kw_protected)
639 m_LastScope = tsProtected;
640 else if (m_LastToken == ParserConsts::kw_private)
641 m_LastScope = tsPrivate;
642 m_Str.Clear();
643 }
644 break;
645
646 case ParserConsts::hash_chr:
647 {
648 token = m_Tokenizer.GetToken();
649 // only the ptOthers kinds of preprocessor directives will be passed here
650 // see details in: Tokenizer::SkipPreprocessorBranch()
651 // those could be: "#include" or "#warning" or "#xxx" and more
652 if (token == ParserConsts::kw_include)
653 HandleIncludes();
654 else // handle "#warning" or "#xxx" and more, just skip them
655 m_Tokenizer.SkipToEOL();
656
657 m_Str.Clear();
658 }
659 break;
660
661 case ParserConsts::ptr_chr:
662 case ParserConsts::ref_chr:
663 {
664 m_PointerOrRef << token;
665 }
666 break;
667
668 case ParserConsts::equals_chr:
669 {
670 // pattern int a = 3;
671 // m_Str.Clear();
672 SkipToOneOfChars(ParserConsts::commasemicolonopbrace, true);
673 m_Tokenizer.UngetToken();
674 }
675 break;
676
677 case ParserConsts::question_chr:
678 {
679 m_Str.Clear();
680 SkipToOneOfChars(ParserConsts::semicolonopbrace, true);
681 }
682 break;
683
684 case ParserConsts::plus_chr:
685 {
686 m_Str.Clear();
687 SkipToOneOfChars(ParserConsts::semicolonclbrace);
688 }
689 break;
690
691 case ParserConsts::dash_chr:
692 {
693 if (m_LastToken == ParserConsts::dash)
694 {
695 m_Str.Clear();
696 SkipToOneOfChars(ParserConsts::semicolonclbrace);
697 }
698 }
699 break;
700
701 case ParserConsts::oparray_chr:
702 {
703 SkipToOneOfChars(ParserConsts::clarray);
704 }
705 break;
706
707 case ParserConsts::comma_chr:
708 break;
709
710 default:
711 switchHandled = false;
712 break;
713 }
714 break;
715
716 // ---------------------------------------------------------------
717 // token length of 2
718 // ---------------------------------------------------------------
719 case 2:
720 if (token == ParserConsts::kw_if || token == ParserConsts::kw_do)
721 {
722 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
723 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
724 else
725 HandleConditionalArguments();
726
727 m_Str.Clear();
728 }
729 else
730 switchHandled = false;
731 break;
732
733 // ---------------------------------------------------------------
734 // token length of 3
735 // ---------------------------------------------------------------
736 case 3:
737 if (token == ParserConsts::kw_for)
738 {
739 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
740 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
741 else
742 HandleForLoopArguments();
743
744 m_Str.Clear();
745 }
746 else
747 switchHandled = false;
748 break;
749
750 // ---------------------------------------------------------------
751 // token length of 4
752 // ---------------------------------------------------------------
753 case 4:
754 if (token == ParserConsts::kw_else)
755 {
756 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
757 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
758 m_Str.Clear();
759 }
760 else if (token == ParserConsts::kw_enum)
761 {
762 m_Str.Clear();
763 if (m_Options.handleEnums)
764 HandleEnum();
765 else
766 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
767 }
768 else if (token == ParserConsts::kw_case)
769 {
770 m_Str.Clear();
771 SkipToOneOfChars(ParserConsts::colon, true, true);
772 }
773 else if (token == ParserConsts::kw___at)
774 {
775 m_Tokenizer.GetToken(); // skip arguments
776 }
777 else
778 switchHandled = false;
779 break;
780
781 // ---------------------------------------------------------------
782 // token length of 5
783 // ---------------------------------------------------------------
784 case 5:
785 if (token == ParserConsts::kw_while || token == ParserConsts::kw_catch)
786 {
787 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
788 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
789 else
790 HandleConditionalArguments();
791
792 m_Str.Clear();
793 }
794 else if (token == ParserConsts::kw_const)
795 {
796 m_Str << token << _T(" ");
797 }
798 else if (token==ParserConsts::kw_using)
799 {
800 // there are some kinds of using keyword usage
801 // (1) using namespace A;
802 // (2) using namespace A::B;
803 // (3) using A::B;
804 // (4) using A = B;
805 token = m_Tokenizer.GetToken();
806 wxString peek = m_Tokenizer.PeekToken();
807 if (peek == ParserConsts::kw_namespace)
808 {
809 while (true) // support full namespaces
810 {
811 m_Str << m_Tokenizer.GetToken();
812 if (m_Tokenizer.PeekToken() == ParserConsts::dcolon)
813 m_Str << m_Tokenizer.GetToken();
814 else
815 break;
816 }
817 if ( !m_Str.IsEmpty()
818 && m_LastParent != 0L
819 && m_LastParent->m_Index != -1
820 && m_LastParent->m_TokenKind == tkNamespace )
821 {
822 if (m_LastParent->m_AncestorsString.IsEmpty())
823 m_LastParent->m_AncestorsString << m_Str;
824 else
825 m_LastParent->m_AncestorsString << ParserConsts::comma_chr << m_Str;
826 }
827 else if ( !m_Str.IsEmpty()
828 && (m_LastParent == 0 || m_LastParent->m_Index == -1) )
829 {
830 // using namespace in global scope
831 // "using namespace first::second::third;"
832
833 Token* foundNsToken = nullptr;
834 wxStringTokenizer tokenizer(m_Str, ParserConsts::dcolon);
835 while (tokenizer.HasMoreTokens())
836 {
837 std::queue<wxString> nsQuqe;
838 nsQuqe.push(tokenizer.GetNextToken());
839 foundNsToken = FindTokenFromQueue(nsQuqe, foundNsToken, true, foundNsToken);
840 foundNsToken->m_TokenKind = tkNamespace;
841 }
842 m_UsedNamespacesIds.insert(foundNsToken->m_Index);
843 }
844 }
845 else if (peek == ParserConsts::equals)
846 {
847 // Type alias pattern: using AAA = BBB::CCC;
848 // Handle same as a typedef
849 wxString args;
850 size_t lineNr = m_Tokenizer.GetLineNumber();
851 Token* tdef = DoAddToken(tkTypedef, token, lineNr, 0, 0, args);
852
853 m_Tokenizer.GetToken(); // eat equals
854 wxString type;
855
856 while (IS_ALIVE) // support full namespaces
857 {
858 type << m_Tokenizer.GetToken();
859 if (m_Tokenizer.PeekToken() == ParserConsts::dcolon)
860 type << m_Tokenizer.GetToken();
861 else
862 break;
863 }
864
865 if (tdef)
866 {
867 tdef->m_FullType = type;
868 tdef->m_BaseType = type;
869 if (tdef->IsValidAncestor(type))
870 tdef->m_AncestorsString = type;
871 }
872 }
873 else
874 SkipToOneOfChars(ParserConsts::semicolonclbrace);
875
876 m_Str.Clear();
877 }
878 else if (token == ParserConsts::kw_class)
879 {
880 m_Str.Clear();
881 if (m_Options.handleClasses)
882 HandleClass(ctClass);
883 else
884 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
885 }
886 else if (token == ParserConsts::kw_union)
887 {
888 m_Str.Clear();
889 if (m_Options.handleClasses)
890 HandleClass(ctUnion);
891 else
892 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
893 }
894 else
895 switchHandled = false;
896 break;
897
898 // ---------------------------------------------------------------
899 // token length of 6
900 // ---------------------------------------------------------------
901 case 6:
902 if (token == ParserConsts::kw_delete)
903 {
904 m_Str.Clear();
905 SkipToOneOfChars(ParserConsts::semicolonclbrace);
906 }
907 else if (token == ParserConsts::kw_switch)
908 {
909 if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
910 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
911 else
912 m_Tokenizer.GetToken(); // skip arguments
913 m_Str.Clear();
914 }
915 else if (token == ParserConsts::kw_return)
916 {
917 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
918 m_Str.Clear();
919 }
920 else if (token == ParserConsts::kw_extern)
921 {
922 // check for "C", "C++"
923 m_Str = m_Tokenizer.GetToken();
924 if (m_Str == ParserConsts::kw__C_ || m_Str == ParserConsts::kw__CPP_)
925 {
926 if (m_Tokenizer.PeekToken() == ParserConsts::opbrace)
927 {
928 m_Tokenizer.GetToken(); // "eat" {
929 DoParse(); // time for recursion ;)
930 }
931 }
932 else
933 {
934 // do nothing, just skip keyword "extern", otherwise uncomment:
935 //SkipToOneOfChars(ParserConsts::semicolon); // skip externs
936 m_Tokenizer.UngetToken();
937 }
938 m_Str.Clear();
939 }
940 else if ( token == ParserConsts::kw_static
941 || token == ParserConsts::kw_inline )
942 {
943 // do nothing, just skip keyword "static" / "inline"
944 }
945 else if (token == ParserConsts::kw_friend)
946 {
947 // friend methods can be either the decl only or an inline implementation
948 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
949 m_Str.Clear();
950 }
951 else if (token == ParserConsts::kw_struct)
952 {
953 m_Str.Clear();
954 if (m_Options.handleClasses)
955 HandleClass(ctStructure);
956 else
957 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
958 }
959 else
960 switchHandled = false;
961 break;
962
963 // ---------------------------------------------------------------
964 // token length of 7
965 // ---------------------------------------------------------------
966 case 7:
967 if (token == ParserConsts::kw_typedef)
968 {
969 if (m_Options.handleTypedefs)
970 HandleTypedef();
971 else
972 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
973 m_Str.Clear();
974 }
975 else if (token == ParserConsts::kw_virtual)
976 {
977 // do nothing, just skip keyword "virtual"
978 }
979 else
980 switchHandled = false;
981 break;
982
983 // ---------------------------------------------------------------
984 // token length of 8
985 // ---------------------------------------------------------------
986 case 8:
987 if (token == ParserConsts::kw_template)
988 {
989 // There are some template definitions that are not working like
990 // within gcc headers (NB: This syntax is a GNU extension):
991 // extern template
992 // const codecvt<char, char, mbstate_t>&
993 // use_facet<codecvt<char, char, mbstate_t> >(const locale&);
994 // read <> as a whole token
995 m_TemplateArgument = ReadAngleBrackets();
996 TRACE(_T("DoParse() : Template argument='%s'"), m_TemplateArgument.wx_str());
997 m_Str.Clear();
998 if (m_Tokenizer.PeekToken() != ParserConsts::kw_class)
999 m_TemplateArgument.clear();
1000 }
1001 else if (token == ParserConsts::kw_noexcept)
1002 {
1003 m_Str << token << _T(" ");
1004 }
1005 else if (token == ParserConsts::kw_operator)
1006 {
1007 wxString func = token;
1008 while (IS_ALIVE)
1009 {
1010 token = m_Tokenizer.GetToken();
1011 if (!token.IsEmpty())
1012 {
1013 if (token.GetChar(0) == ParserConsts::opbracket_chr)
1014 {
1015 // check for operator()()
1016 wxString peek = m_Tokenizer.PeekToken();
1017 if ( !peek.IsEmpty()
1018 && peek.GetChar(0) != ParserConsts::opbracket_chr)
1019 m_Tokenizer.UngetToken();
1020 else
1021 func << token;
1022 break;
1023 }
1024 else
1025 func << token;
1026 }
1027 else
1028 break;
1029 }
1030 HandleFunction(func, true);
1031 }
1032 else
1033 switchHandled = false;
1034 break;
1035
1036 // ---------------------------------------------------------------
1037 // token length of 9
1038 // ---------------------------------------------------------------
1039 case 9:
1040 if (token == ParserConsts::kw_namespace)
1041 {
1042 m_Str.Clear();
1043 HandleNamespace();
1044 }
1045 else
1046 switchHandled = false;
1047 break;
1048
1049 // ---------------------------------------------------------------
1050 // token length of 10
1051 // ---------------------------------------------------------------
1052 case 10:
1053 if (token == ParserConsts::kw_declspec)
1054 {
1055 // Handle stuff like: int __declspec ((whatever)) fun();
1056 // __declspec already be eat
1057 m_Tokenizer.GetToken(); // eat (( whatever ))
1058 }
1059 else
1060 switchHandled = false;
1061 break;
1062
1063 // ---------------------------------------------------------------
1064 // token length of 13
1065 // ---------------------------------------------------------------
1066 case 13:
1067 if (token == ParserConsts::kw_attribute)
1068 {
1069 // Handle stuff like: int __attribute__((whatever)) fun();
1070 // __attribute__ already be eat
1071 m_Tokenizer.GetToken(); // eat (( whatever ))
1072 }
1073 else
1074 switchHandled = false;
1075 break;
1076
1077 // token length of other than 1 .. 13
1078 default:
1079 switchHandled = false;
1080 break;
1081 }
1082
1083 if (token.StartsWith(ParserConsts::kw___asm))
1084 {
1085 // Handle: __asm assembly-instruction [ ; ] OR
1086 // __asm { assembly-instruction-list } [ ; ] OR
1087 // __asm __volatile("assembly-instruction-list");
1088 // TODO: You can also put __asm in front of each assembly instruction:
1089 // __asm mov al, 2
1090 // __asm mov dx, 0xD007
1091 // OR: __asm mov al, 2 __asm mov dx, 0xD007
1092 SkipToOneOfChars(ParserConsts::semicolon, true, true);
1093 }
1094 else if (!switchHandled)
1095 {
1096 // since we can't recognize the pattern by token, then the token
1097 // is normally an identifier style lexeme, now we try to peek the next token
1098 wxString peek = m_Tokenizer.PeekToken();
1099 if (!peek.IsEmpty())
1100 {
1101
1102 if ( (peek.GetChar(0) == ParserConsts::opbracket_chr)
1103 && m_Options.handleFunctions )
1104 {
1105 if ( m_Str.IsEmpty()
1106 && m_EncounteredNamespaces.empty()
1107 && m_EncounteredTypeNamespaces.empty()
1108 && (!m_LastParent || m_LastParent->m_Name != token) ) // if func has same name as current scope (class)
1109 {
1110 // see what is inside the (...)
1111 wxString arg = m_Tokenizer.GetToken(); // eat args ()
1112 // try to see whether the peek pattern is (* BBB)
1113 int pos = peek.find(ParserConsts::ptr);
1114 if (pos != wxNOT_FOUND)
1115 {
1116 peek = m_Tokenizer.PeekToken();
1117 // see whether there is a (...) after (* BBB)
1118 if (peek.GetChar(0) == ParserConsts::opbracket_chr)
1119 {
1120 // pattern: AAA (* BBB) (...)
1121 // where peek is (...) and arg is (* BBB)
1122
1123 // NOTE: support func ptr in local block, show return type.
1124 // if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
1125 // HandleFunction(arg); // function
1126 // AAA now becomes the last element of stacked type string
1127 // which is the return type of function ptr
1128 m_Str << token << ParserConsts::space_chr;
1129 // BBB is now the function ptr's name
1130 HandleFunction(/*function name*/ arg,
1131 /*isOperator*/ false,
1132 /*isPointer*/ true);
1133 }
1134 }
1135 else // wxString arg = m_Tokenizer.GetToken(); // eat args ()
1136 m_Str = token + arg;
1137 }
1138 // NOTE: support some more cases..., such as m_Str is not empty
1139 // if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
1140 // HandleFunction(token); // function
1141 // else
1142 // m_Tokenizer.GetToken(); // eat args when parsing block
1143
1144 // list of function ptrs
1145 // eg: void (*fun1)(void), (*fun2)(size_t size);
1146 // where, m_Str=void, token=(*fun2), peek=(size_t size)
1147
1148 // function ptr with pointer return type
1149 // eg: void *(*Alloc)(void *p, size_t size);
1150 // where, m_Str=void, token=(*Alloc), peek=(void *p, size_t size)
1151 else if (token.GetChar(0) == ParserConsts::opbracket_chr)
1152 {
1153 int pos = token.find(ParserConsts::ptr);
1154 if (pos != wxNOT_FOUND)
1155 {
1156 wxString arg = token;
1157 HandleFunction(/*function name*/ arg,
1158 /*isOperator*/ false,
1159 /*isPointer*/ true);
1160 }
1161 }
1162 else if ( m_Options.useBuffer
1163 && m_Str.GetChar(0) == ParserConsts::opbracket_chr
1164 && m_Str.GetChar(m_Str.Len() - 2) == ParserConsts::clbracket_chr)
1165 {
1166 // pattern: (void) fun (...)
1167 // m_Str="(void) " token="fun" peek="(...)"
1168 // this is a return value cast, we should reset the m_Str with fun
1169 m_Str = token;
1170 }
1171 else
1172 {
1173 // pattern unsigned int (*getClientLibVersion)(char** result);
1174 // currently, m_Str = unsigned, token = int, peek = (*getClientLibVersion)
1175 // this may be a function pointer declaration, we can guess without
1176 // reading the next token, if "peek" has a ptr char and only 1 argument
1177 // in it.
1178
1179 // see what is inside the (...)
1180 // try to see whether the peek pattern is (* BBB)
1181 if (peek.GetChar(1) == ParserConsts::ptr)
1182 {
1183 wxString arg = peek;
1184 m_Str << token;
1185 token = m_Tokenizer.GetToken(); //consume the peek
1186 // BBB is now the function ptr's name
1187 HandleFunction(/*function name*/ arg,
1188 /*isOperator*/ false,
1189 /*isPointer*/ true);
1190 }
1191 else if (!m_Options.useBuffer || m_Options.bufferSkipBlocks)
1192 {
1193 // pattern AAA BBB (...) in global namespace (not in local block)
1194 // so, this is mostly like a function declaration, but in-fact this
1195 // can also be a global variable initialized with ctor, but for
1196 // simplicity, we drop the later case
1197 HandleFunction(token); // function
1198 }
1199 else
1200 {
1201 // local variables initialized with ctor
1202 if (!m_Str.IsEmpty() && m_Options.handleVars)
1203 {
1204 Token* newToken = DoAddToken(tkVariable, token, m_Tokenizer.GetLineNumber());
1205 if (newToken && !m_TemplateArgument.IsEmpty())
1206 ResolveTemplateArgs(newToken);
1207 }
1208 m_Tokenizer.GetToken(); // eat args when parsing block
1209 }
1210 }
1211 }
1212 else if ( (peek == ParserConsts::colon)
1213 && (token != ParserConsts::kw_private)
1214 && (token != ParserConsts::kw_protected)
1215 && (token != ParserConsts::kw_public) )
1216 {
1217 // example decl to encounter a colon is when defining a bitfield: int x:1,y:1,z:1;
1218 // token should hold the var (x/y/z)
1219 // m_Str should hold the type (int)
1220 if (m_Options.handleVars)
1221 DoAddToken(tkVariable, token, m_Tokenizer.GetLineNumber());
1222
1223 m_Tokenizer.GetToken(); // skip colon
1224 m_Tokenizer.GetToken(); // skip bitfield
1225 }
1226 else if (peek==ParserConsts::comma)
1227 {
1228 // example decl to encounter a comma: int x,y,z;
1229 // token should hold the var (x/y/z)
1230 // m_Str should hold the type (int)
1231 if (!m_Str.IsEmpty() && m_Options.handleVars)
1232 {
1233 Token* newToken = DoAddToken(tkVariable, token, m_Tokenizer.GetLineNumber());
1234 if (newToken && !m_TemplateArgument.IsEmpty())
1235 ResolveTemplateArgs(newToken);
1236 }
1237
1238 // else it's a syntax error; let's hope we can recover from this...
1239 // skip comma (we had peeked it)
1240 m_Tokenizer.GetToken();
1241 }
1242 else if (peek==ParserConsts::lt)
1243 {
1244 // a template, e.g. someclass<void>::memberfunc
1245 // we have to skip <>, so we 're left with someclass::memberfunc
1246 // about 'const' handle, e.g.
1247 /* template<typename T> class A{};
1248 const A<int> var; */
1249 if (m_Str.IsEmpty() || m_Str.StartsWith(ParserConsts::kw_const))
1250 GetTemplateArgs();
1251 else
1252 SkipAngleBraces();
1253 peek = m_Tokenizer.PeekToken();
1254 if (peek==ParserConsts::dcolon)
1255 {
1256 TRACE(_T("DoParse() : peek='::', token='") + token + _T("', m_LastToken='") + m_LastToken + _T("', m_Str='") + m_Str + _T("'"));
1257 if (m_Str.IsEmpty())
1258 m_EncounteredTypeNamespaces.push(token); // it's a type's namespace
1259 else
1260 m_EncounteredNamespaces.push(token);
1261 m_Tokenizer.GetToken(); // eat ::
1262 }
1263 else // case like, std::map<int, int> somevar;
1264 m_Str << token << ParserConsts::space_chr;
1265 }
1266 else if (peek==ParserConsts::dcolon)
1267 {
1268 wxString str_stripped(m_Str); str_stripped.Trim(true).Trim(false);
1269 if ( str_stripped.IsEmpty()
1270 || str_stripped.IsSameAs(ParserConsts::kw_const)
1271 || str_stripped.IsSameAs(ParserConsts::kw_volatile) ) // what else?!
1272 m_EncounteredTypeNamespaces.push(token); // it's a type's namespace
1273 else
1274 m_EncounteredNamespaces.push(token);
1275 m_Tokenizer.GetToken(); // eat ::
1276 }
1277 // NOTE: opbracket_chr already handled above
1278 else if ( peek==ParserConsts::semicolon
1279 || peek==ParserConsts::oparray_chr
1280 || peek==ParserConsts::equals_chr)
1281 {
1282 if ( !m_Str.IsEmpty()
1283 && ( wxIsalpha(token.GetChar(0))
1284 || (token.GetChar(0) == ParserConsts::underscore_chr) ) )
1285 {
1286 // pattern: m_Str AAA;
1287 // pattern: m_Str AAA[X][Y];
1288 // pattern: m_Str AAA = BBB;
1289 // where AAA is the variable name, m_Str contains type string
1290 if (m_Options.handleVars)
1291 {
1292 Token* newToken = DoAddToken(tkVariable, token, m_Tokenizer.GetLineNumber());
1293 if (newToken && !m_TemplateArgument.IsEmpty())
1294 ResolveTemplateArgs(newToken);
1295 }
1296 else
1297 SkipToOneOfChars(ParserConsts::semicolonclbrace, true, true);
1298 }
1299
1300 if (peek==ParserConsts::oparray_chr)
1301 SkipToOneOfChars(ParserConsts::clarray);
1302 else if (peek==ParserConsts::equals_chr)
1303 {
1304 SkipToOneOfChars(ParserConsts::commasemicolonopbrace, true);
1305 m_Tokenizer.UngetToken();
1306 }
1307 }
1308 else if (!m_EncounteredNamespaces.empty())
1309 {
1310 // Notice: clears the queue "m_EncounteredNamespaces", too
1311 while (!m_EncounteredNamespaces.empty())
1312 {
1313 m_EncounteredTypeNamespaces.push( m_EncounteredNamespaces.front() );
1314 m_EncounteredNamespaces.pop();
1315 }
1316 m_Str = token;
1317 }
1318 else
1319 m_Str << token << ParserConsts::space_chr;
1320 }
1321 }
1322
1323 m_LastToken = token;
1324 }
1325
1326 // reset tokenizer behaviour
1327 m_Tokenizer.SetState(oldState);
1328 }
1329
TokenExists(const wxString & name,const Token * parent,short int kindMask)1330 Token* ParserThread::TokenExists(const wxString& name, const Token* parent, short int kindMask)
1331 {
1332 // no critical section needed here:
1333 // all functions that call this, already entered a critical section.
1334
1335 // Lookup in local parent or in global scope
1336 int foundIdx = m_TokenTree->TokenExists(name, parent ? parent->m_Index : -1, kindMask);
1337 if (foundIdx != wxNOT_FOUND)
1338 return m_TokenTree->at(foundIdx);
1339
1340 // Lookup in included namespaces
1341 foundIdx = m_TokenTree->TokenExists(name, m_UsedNamespacesIds, kindMask);
1342 return m_TokenTree->at(foundIdx);
1343 }
1344
TokenExists(const wxString & name,const wxString & baseArgs,const Token * parent,TokenKind kind)1345 Token* ParserThread::TokenExists(const wxString& name, const wxString& baseArgs, const Token* parent, TokenKind kind)
1346 {
1347 // no critical section needed here:
1348 // all functions that call this, already entered a critical section.
1349
1350 // Lookup in local parent or in global scope
1351 int foundIdx = m_TokenTree->TokenExists(name, baseArgs, parent ? parent->m_Index : -1, kind);
1352 if(foundIdx != wxNOT_FOUND)
1353 return m_TokenTree->at(foundIdx);
1354
1355 // Lookup in included namespaces
1356 foundIdx = m_TokenTree->TokenExists(name, baseArgs, m_UsedNamespacesIds, kind);
1357 return m_TokenTree->at(foundIdx);
1358 }
1359
GetTokenBaseType()1360 wxString ParserThread::GetTokenBaseType()
1361 {
1362 TRACE(_T("GetTokenBaseType() : Searching within m_Str='%s'"), m_Str.wx_str());
1363
1364 // Compensate for spaces between namespaces (e.g. NAMESPACE :: SomeType)
1365 // which is valid C++ construct.
1366 // Also, spaces that follow a semicolon are removed.
1367 int pos = 0;
1368 while (pos < static_cast<int>(m_Str.Length()))
1369 {
1370 if ( wxIsspace(m_Str.GetChar(pos))
1371 && ( ( (pos > 0)
1372 && (m_Str.GetChar(pos - 1) == ParserConsts::colon_chr) )
1373 || ( (pos < static_cast<int>(m_Str.Length()) - 1)
1374 && (m_Str.GetChar(pos + 1) == ParserConsts::colon_chr) ) ) )
1375 {
1376 m_Str.Remove(pos, 1);
1377 }
1378 else
1379 ++pos;
1380 }
1381
1382 TRACE(_T("GetTokenBaseType() : Compensated m_Str='%s'"), m_Str.wx_str());
1383
1384 // TODO (Morten#5#): Handle stuff like the following gracefully:
1385 // int __cdecl __MINGW_NOTHROW vscanf (const char * __restrict__, __VALIST);
1386
1387 // m_Str contains the full text before the token's declaration
1388 // an example, for a variable Token: "const wxString& s;"
1389 // m_Str would be: const wxString&
1390 // what we do here is locate the actual return value (wxString in this example)
1391 // it will be needed by code completion code ;)
1392 // Note that generally the returned type string is the identifier like token near the variable
1393 // name, there may be some exceptions. E.g. "wxString const &s;", here, "const" should not be
1394 // returned as a type name.
1395
1396 pos = m_Str.Length() - 1; // search start at the end of m_Str
1397
1398 while (pos >= 0)
1399 {
1400 // we walk m_Str backwards until we find a non-space character which also is
1401 // not * or &
1402 // const wxString&
1403 // in this example, we would stop here ^
1404 while ( (pos >= 0)
1405 && ( wxIsspace(m_Str.GetChar(pos))
1406 || (m_Str.GetChar(pos) == ParserConsts::ptr_chr)
1407 || (m_Str.GetChar(pos) == ParserConsts::ref_chr)) )
1408 {
1409 --pos;
1410 }
1411
1412 if (pos >= 0)
1413 {
1414 // we have the end of the word we're interested in
1415 int end = pos;
1416
1417 // continue walking backwards until we find the start of the word
1418 // const wxString&
1419 // in this example, we would stop here ^
1420 while ( (pos >= 0)
1421 && ( wxIsalnum(m_Str.GetChar(pos))
1422 || (m_Str.GetChar(pos) == ParserConsts::underscore_chr)
1423 || (m_Str.GetChar(pos) == ParserConsts::colon_chr)) )
1424 {
1425 --pos;
1426 }
1427 wxString typeCandidate = m_Str.Mid(pos + 1, end - pos);
1428 // "const" should not be returned as a type name, so we try next candidate.
1429 if (typeCandidate.IsSameAs(ParserConsts::kw_const))
1430 continue;
1431
1432 TRACE(_T("GetTokenBaseType() : Found '%s'"), typeCandidate.wx_str());
1433 return typeCandidate;
1434 }
1435 }
1436
1437 TRACE(_T("GetTokenBaseType() : Returning '%s'"), m_Str.wx_str());
1438 return m_Str; // token ends at start of phrase
1439 }
1440
FindTokenFromQueue(std::queue<wxString> & q,Token * parent,bool createIfNotExist,Token * parentIfCreated)1441 Token* ParserThread::FindTokenFromQueue(std::queue<wxString>& q, Token* parent, bool createIfNotExist,
1442 Token* parentIfCreated)
1443 {
1444 if (q.empty())
1445 return 0;
1446
1447 wxString ns = q.front();
1448 q.pop();
1449
1450 Token* result = TokenExists(ns, parent, tkNamespace | tkClass);
1451
1452 // if we can't find one in global namespace, then we check the local parent
1453 if (!result && parent == 0)
1454 {
1455 result = TokenExists(ns, parentIfCreated, tkNamespace | tkClass);
1456 }
1457
1458 if (!result && createIfNotExist)
1459 {
1460 result = new Token(ns, m_FileIdx, 0, ++m_TokenTree->m_TokenTicketCount);
1461 result->m_TokenKind = q.empty() ? tkClass : tkNamespace;
1462 result->m_IsLocal = m_IsLocal;
1463 result->m_ParentIndex = parentIfCreated ? parentIfCreated->m_Index : -1;
1464 int newidx = m_TokenTree->insert(result);
1465 if (parentIfCreated)
1466 parentIfCreated->AddChild(newidx);
1467
1468 TRACE(_T("FindTokenFromQueue() : Created unknown class/namespace %s (%d) under %s (%d)"),
1469 ns.wx_str(),
1470 newidx,
1471 parent ? parent->m_Name.wx_str() : _T("<globals>"),
1472 parent ? parent->m_Index : -1);
1473 }
1474
1475 if (q.empty())
1476 return result;
1477
1478 if (result)
1479 result = FindTokenFromQueue(q, result, createIfNotExist, parentIfCreated);
1480
1481 return result;
1482 }
1483
DoAddToken(TokenKind kind,const wxString & name,int line,int implLineStart,int implLineEnd,const wxString & args,bool isOperator,bool isImpl)1484 Token* ParserThread::DoAddToken(TokenKind kind,
1485 const wxString& name,
1486 int line,
1487 int implLineStart,
1488 int implLineEnd,
1489 const wxString& args,
1490 bool isOperator,
1491 bool isImpl)
1492 {
1493 if (name.IsEmpty())
1494 {
1495 TRACE(_T("DoAddToken() : Token name is empty!"));
1496 return 0; // oops!
1497 }
1498
1499 Token* newToken = 0;
1500 wxString newname(name);
1501 m_Str.Trim(true).Trim(false);
1502 if (kind == tkDestructor)
1503 {
1504 // special class destructors case
1505 newname.Prepend(ParserConsts::tilde);
1506 m_Str.Clear();
1507 }
1508
1509 wxString baseArgs;
1510 if (kind & tkAnyFunction)
1511 {
1512 if ( !GetBaseArgs(args, baseArgs) )
1513 kind = tkVariable;
1514 }
1515
1516 Token* localParent = 0;
1517
1518 // preserve m_EncounteredTypeNamespaces; needed further down this function
1519 std::queue<wxString> q = m_EncounteredTypeNamespaces;
1520 if ((kind == tkDestructor || kind == tkConstructor) && !q.empty())
1521 {
1522 // look in m_EncounteredTypeNamespaces
1523 localParent = FindTokenFromQueue(q, 0, true, m_LastParent);
1524 if (localParent)
1525 newToken = TokenExists(newname, baseArgs, localParent, kind);
1526 if (newToken)
1527 { TRACE(_T("DoAddToken() : Found token (ctor/dtor).")); }
1528 }
1529
1530 // check for implementation member function
1531 if (!newToken && !m_EncounteredNamespaces.empty())
1532 {
1533 localParent = FindTokenFromQueue(m_EncounteredNamespaces, 0, true, m_LastParent);
1534 if (localParent)
1535 newToken = TokenExists(newname, baseArgs, localParent, kind);
1536 if (newToken)
1537 {
1538 TRACE(_T("DoAddToken() : Found token (member function)."));
1539 // Special handling function implementation here, a function declaration and its
1540 // function implementation share one Token. But the function implementation's arguments
1541 // should take precedence, as they will be used for code-completion.
1542 if (isImpl && (kind & tkAnyFunction))
1543 newToken->m_Args = args;
1544 }
1545 }
1546
1547 // none of the above; check for token under parent
1548 if (!newToken)
1549 {
1550 newToken = TokenExists(newname, baseArgs, m_LastParent, kind);
1551 if (newToken)
1552 {
1553 TRACE(_T("DoAddToken() : Found token (parent)."));
1554 // Special handling function implementation, see comments above
1555 if (isImpl && (kind & tkAnyFunction))
1556 newToken->m_Args = args;
1557 }
1558 }
1559
1560 // need to check if the current token already exists in the tokenTree
1561 // if newToken is valid (non zero), it points to a Token with same kind and same name, so there
1562 // is a chance we can update the existing Token and not create a new one. This usually happens
1563 // we are reparsing a header file, but some class definition Token is shared with an implementation
1564 // file, so the Token can be updated.
1565 // In some special cases, the a new Token instance is need. E.g. token's template argument is
1566 // checked to support template specialization
1567 // eg: template<typename T> class A {...} and template<> class A<int> {...}
1568 // we record them as different tokens
1569 if ( newToken
1570 && (newToken->m_TemplateArgument == m_TemplateArgument)
1571 && ( kind & tkAnyFunction
1572 || newToken->m_Args == args
1573 || kind & tkAnyContainer ) )
1574 {
1575 ; // nothing to do
1576 }
1577 else
1578 {
1579 newToken = new Token(newname, m_FileIdx, line, ++m_TokenTree->m_TokenTicketCount);
1580 TRACE(_T("DoAddToken() : Created token='%s', file_idx=%u, line=%d, ticket=%lu"), newname.wx_str(),
1581 m_FileIdx, line, static_cast<unsigned long>(m_TokenTree->m_TokenTicketCount));
1582
1583 Token* finalParent = localParent ? localParent : m_LastParent;
1584 if (kind == tkVariable && m_Options.parentIdxOfBuffer != -1)
1585 finalParent = m_TokenTree->at(m_Options.parentIdxOfBuffer);
1586
1587 newToken->m_ParentIndex = finalParent ? finalParent->m_Index : -1;
1588 newToken->m_TokenKind = kind;
1589 newToken->m_Scope = m_LastScope;
1590 newToken->m_BaseArgs = baseArgs;
1591
1592 if (newToken->m_TokenKind == tkClass)
1593 newToken->m_BaseArgs = args; // save template args
1594 else
1595 newToken->m_Args = args;
1596
1597 int newidx = m_TokenTree->insert(newToken);
1598
1599 if (finalParent)
1600 finalParent->AddChild(newidx);
1601 }
1602
1603 if (!(kind & (tkConstructor | tkDestructor)))
1604 {
1605 wxString tokenFullType = m_Str;
1606 if (!m_PointerOrRef.IsEmpty())
1607 {
1608 tokenFullType << m_PointerOrRef;
1609 m_PointerOrRef.Clear();
1610 }
1611 wxString tokenBaseType = GetTokenBaseType();
1612 if (tokenBaseType.Find(ParserConsts::space_chr) == wxNOT_FOUND)
1613 {
1614 // token type must contain all namespaces
1615 wxString prepend;
1616
1617 // Notice: clears the queue "m_EncounteredTypeNamespaces", too
1618 while (!m_EncounteredTypeNamespaces.empty())
1619 {
1620 prepend << m_EncounteredTypeNamespaces.front() << ParserConsts::dcolon;
1621 m_EncounteredTypeNamespaces.pop();
1622 }
1623
1624 TRACE(_T("DoAddToken() : Prepending '%s'"), prepend.wx_str());
1625 tokenBaseType.Prepend(prepend);
1626 }
1627 newToken->m_FullType = tokenFullType;
1628 newToken->m_BaseType = tokenBaseType;
1629 }
1630
1631 newToken->m_IsLocal = m_IsLocal;
1632 newToken->m_IsTemp = m_Options.isTemp;
1633 newToken->m_IsOperator = isOperator;
1634
1635 if (!isImpl)
1636 {
1637 newToken->m_FileIdx = m_FileIdx;
1638 newToken->m_Line = line;
1639 }
1640 else
1641 {
1642 newToken->m_ImplFileIdx = m_FileIdx;
1643 newToken->m_ImplLine = line;
1644 newToken->m_ImplLineStart = implLineStart;
1645 newToken->m_ImplLineEnd = implLineEnd;
1646 m_TokenTree->InsertTokenBelongToFile(newToken->m_ImplFileIdx, newToken->m_Index);
1647 }
1648
1649 // this will append the doxygen style comments to the Token
1650 m_Tokenizer.SetLastTokenIdx(newToken->m_Index);
1651
1652 TRACE(_T("DoAddToken() : Added/updated token '%s' (%d), kind '%s', type '%s', actual '%s'. Parent is %s (%d)"),
1653 name.wx_str(), newToken->m_Index, newToken->GetTokenKindString().wx_str(), newToken->m_FullType.wx_str(),
1654 newToken->m_BaseType.wx_str(), m_TokenTree->at(newToken->m_ParentIndex) ?
1655 m_TokenTree->at(newToken->m_ParentIndex)->m_Name.wx_str() : wxEmptyString,
1656 newToken->m_ParentIndex);
1657 ADDTOKEN(_T("Token: Index %7d Line %7d: Type: %s: -> '%s'"),
1658 newToken->m_Index, line, newToken->GetTokenKindString().wx_str(), name.wx_str());
1659
1660 // Notice: clears the queue "m_EncounteredTypeNamespaces"
1661 while (!m_EncounteredTypeNamespaces.empty())
1662 m_EncounteredTypeNamespaces.pop();
1663
1664 // Notice: clears the queue "m_EncounteredNamespaces"
1665 while (!m_EncounteredNamespaces.empty())
1666 m_EncounteredNamespaces.pop();
1667
1668 return newToken;
1669 }
1670
HandleIncludes()1671 void ParserThread::HandleIncludes()
1672 {
1673 wxString filename;
1674 bool isGlobal = !m_IsLocal;
1675 wxString token = m_Tokenizer.GetToken();
1676
1677 // now token holds something like:
1678 // "someheader.h"
1679 // < and will follow iostream.h, >
1680 if (!token.IsEmpty())
1681 {
1682 if (token.GetChar(0) == '"')
1683 {
1684 // "someheader.h"
1685 // don't use wxString::Replace(); it's too costly
1686 size_t pos = 0;
1687 while (pos < token.Length())
1688 {
1689 wxChar c = token.GetChar(pos);
1690 if (c != _T('"'))
1691 filename << c;
1692 ++pos;
1693 }
1694 }
1695 else if (token.GetChar(0) == ParserConsts::lt_chr)
1696 {
1697 isGlobal = true;
1698 // next token is filename, next is '.' (dot), next is extension
1699 // basically we'll loop until '>' (gt)
1700 while (IS_ALIVE)
1701 {
1702 token = m_Tokenizer.GetToken();
1703 if (token.IsEmpty())
1704 break;
1705 if (token.GetChar(0) != ParserConsts::gt_chr)
1706 filename << token;
1707 else
1708 break;
1709 }
1710 }
1711 }
1712
1713 if (ParserCommon::FileType(filename) == ParserCommon::ftOther)
1714 return;
1715
1716 if (!filename.IsEmpty())
1717 {
1718 TRACE(_T("HandleIncludes() : Found include file '%s'"), filename.wx_str());
1719 do
1720 {
1721 // setting all #includes as global
1722 // it's amazing how many projects use #include "..." for global headers (MSVC mainly - booh)
1723 isGlobal = true;
1724
1725 if (!(isGlobal ? m_Options.followGlobalIncludes : m_Options.followLocalIncludes))
1726 {
1727 TRACE(_T("HandleIncludes() : File '%s' not requested to parse after checking options, skipping"), filename.wx_str());
1728 break; // Nothing to do!
1729 }
1730
1731 wxString real_filename = m_Parent->GetFullFileName(m_Filename, filename, isGlobal);
1732 // Parser::GetFullFileName is thread-safe :)
1733
1734 if (real_filename.IsEmpty())
1735 {
1736 TRACE(_T("HandleIncludes() : File '%s' not found, skipping"), filename.wx_str());
1737 break; // File not found, do nothing.
1738 }
1739
1740 if (m_TokenTree->IsFileParsed(real_filename))
1741 {
1742 TRACE(_T("HandleIncludes() : File '%s' is already being parsed, skipping"), real_filename.wx_str());
1743 break; // Already being parsed elsewhere
1744 }
1745
1746 TRACE(_T("HandleIncludes() : Adding include file '%s'"), real_filename.wx_str());
1747 m_Parent->ParseFile(real_filename, isGlobal, true);
1748 }
1749 while (false);
1750 }
1751 }
1752
HandleNamespace()1753 void ParserThread::HandleNamespace()
1754 {
1755 wxString ns = m_Tokenizer.GetToken();
1756 int line = m_Tokenizer.GetLineNumber();
1757
1758 if (ns == ParserConsts::opbrace)
1759 {
1760 // parse inside anonymous namespace
1761 Token* lastParent = m_LastParent;
1762 TokenScope lastScope = m_LastScope;
1763
1764 DoParse();
1765
1766 m_LastParent = lastParent;
1767 m_LastScope = lastScope;
1768 }
1769 else
1770 {
1771
1772 while (true)
1773 {
1774 // for namespace aliases to be parsed, we need to tell the tokenizer
1775 // not to skip the usually unwanted tokens. One of those tokens is the
1776 // "assignment" (=).
1777 // we just have to remember to revert this setting below, or else problems will follow
1778 m_Tokenizer.SetState(tsNormal);
1779
1780 wxString next = m_Tokenizer.PeekToken(); // named namespace
1781 if (next==ParserConsts::opbrace)
1782 {
1783 m_Tokenizer.SetState(tsNormal);
1784
1785 // use the existing copy (if any)
1786 Token* newToken = TokenExists(ns, m_LastParent, tkNamespace);
1787 if (!newToken)
1788 newToken = DoAddToken(tkNamespace, ns, line);
1789 if (!newToken)
1790 {
1791 TRACE(_T("HandleNamespace() : Unable to create/add new token: ") + ns);
1792 return;
1793 }
1794
1795 m_Tokenizer.GetToken(); // eat {
1796 int lineStart = m_Tokenizer.GetLineNumber();
1797
1798 Token* lastParent = m_LastParent; // save status, will restore after DoParse()
1799 TokenScope lastScope = m_LastScope;
1800
1801 m_LastParent = newToken;
1802 // default scope is: public for namespaces (actually no, but emulate it)
1803 m_LastScope = tsPublic;
1804
1805 DoParse();
1806
1807 m_LastParent = lastParent;
1808 m_LastScope = lastScope;
1809
1810 // update implementation file and lines of namespace.
1811 // this doesn't make much sense because namespaces are all over the place,
1812 // but do it anyway so that buffer-based parsing returns the correct values.
1813 newToken->m_ImplFileIdx = m_FileIdx;
1814 newToken->m_ImplLine = line;
1815 newToken->m_ImplLineStart = lineStart;
1816 newToken->m_ImplLineEnd = m_Tokenizer.GetLineNumber();
1817
1818 // the namespace body is correctly parsed
1819 break;
1820 }
1821 else if (next==ParserConsts::equals)
1822 {
1823 // namespace alias; example from cxxabi.h:
1824 //
1825 // namespace __cxxabiv1
1826 // {
1827 // ...
1828 // }
1829 // namespace abi = __cxxabiv1; <-- we 're in this case now
1830
1831 m_Tokenizer.GetToken(); // eat '='
1832 m_Tokenizer.SetState(tsNormal);
1833
1834 Token* lastParent = m_LastParent;
1835 Token* aliasToken = NULL;
1836
1837 while (IS_ALIVE)
1838 {
1839 wxString aliasStr = m_Tokenizer.GetToken();
1840
1841 // use the existing copy (if any)
1842 aliasToken = TokenExists(aliasStr, m_LastParent, tkNamespace);
1843 if (!aliasToken)
1844 aliasToken = DoAddToken(tkNamespace, aliasStr, line);
1845 if (!aliasToken)
1846 return;
1847
1848 if (m_Tokenizer.PeekToken() == ParserConsts::dcolon)
1849 {
1850 m_Tokenizer.GetToken();
1851 m_LastParent = aliasToken;
1852 }
1853 else
1854 break;
1855 }
1856
1857 aliasToken->m_Aliases.Add(ns);
1858 m_LastParent = lastParent;
1859
1860 // the namespace alias statement is correctly parsed
1861 break;
1862 }
1863 else
1864 {
1865 m_Tokenizer.SetState(tsNormal);
1866 // probably some kind of error in code ?
1867 SkipToOneOfChars(ParserConsts::semicolonopbrace);
1868
1869 // in case of the code:
1870 //
1871 // # define _GLIBCXX_VISIBILITY(V) _GLIBCXX_PSEUDO_VISIBILITY(V)
1872 // namespace std _GLIBCXX_VISIBILITY(default)
1873 // {
1874 // class vector
1875 // {
1876 // size_t size();
1877 // }
1878 // }
1879 // we still want to parse the body of the namespace, but skip the tokens before "{"
1880 m_Tokenizer.UngetToken();
1881 wxString peek = m_Tokenizer.PeekToken();
1882 if(peek == ParserConsts::opbrace)
1883 continue;
1884 else
1885 break;
1886 }
1887 } // while(true)
1888 }
1889 }
1890
HandleClass(EClassType ct)1891 void ParserThread::HandleClass(EClassType ct)
1892 {
1893 // need to force the tokenizer _not_ skip anything
1894 // as we 're manually parsing class decls
1895 // don't forget to reset that if you add any early exit condition!
1896 TokenizerState oldState = m_Tokenizer.GetState();
1897 m_Tokenizer.SetState(tsNormal);
1898
1899 int lineNr = m_Tokenizer.GetLineNumber();
1900 wxString ancestors;
1901 wxString lastCurrent;
1902 while (IS_ALIVE)
1903 {
1904 wxString current = m_Tokenizer.GetToken(); // class name
1905 wxString next = m_Tokenizer.PeekToken();
1906
1907 // remove __attribute__ or __declspec qualifiers
1908 if (current == ParserConsts::kw_attribute || current == ParserConsts::kw_declspec)
1909 {
1910 TRACE(_T("HandleClass() : Skip __attribute__ or __declspec"));
1911
1912 // Handle stuff like: __attribute__(( whatever ))
1913 m_Tokenizer.GetToken(); // eat __attribute__
1914 current = m_Tokenizer.GetToken(); // eat (( whatever ))
1915 next = m_Tokenizer.PeekToken(); // peek again
1916 }
1917
1918 TRACE(_T("HandleClass() : Found class '%s', next='%s'"), current.wx_str(), next.wx_str());
1919
1920 if (current.IsEmpty() || next.IsEmpty())
1921 break;
1922
1923 // -------------------------------------------------------------------
1924 if (next == ParserConsts::lt) // template specialization
1925 // -------------------------------------------------------------------
1926 {
1927 // eg: template<> class A<int> {...}, then we update template argument with "<int>"
1928 GetTemplateArgs();
1929 next = m_Tokenizer.PeekToken();
1930 }
1931
1932 // -------------------------------------------------------------------
1933 if (next == ParserConsts::colon) // has ancestor(s)
1934 // -------------------------------------------------------------------
1935 {
1936 TRACE(_T("HandleClass() : Class '%s' has ancestors"), current.wx_str());
1937 m_Tokenizer.GetToken(); // eat ":"
1938 while (IS_ALIVE)
1939 {
1940 wxString tmp = m_Tokenizer.GetToken();
1941 next = m_Tokenizer.PeekToken();
1942 // -----------------------------------------------------------
1943 if ( tmp == ParserConsts::kw_public
1944 || tmp == ParserConsts::kw_protected
1945 || tmp == ParserConsts::kw_private )
1946 // -----------------------------------------------------------
1947 {
1948 continue;
1949 }
1950
1951 // -----------------------------------------------------------
1952 if (!(tmp == ParserConsts::comma || tmp == ParserConsts::gt))
1953 // -----------------------------------------------------------
1954 {
1955 // fix for namespace usage in ancestors
1956 if (tmp == ParserConsts::dcolon || next == ParserConsts::dcolon)
1957 ancestors << tmp;
1958 else
1959 ancestors << tmp << ParserConsts::comma_chr;
1960 TRACE(_T("HandleClass() : Adding ancestor ") + tmp);
1961 }
1962
1963 // -----------------------------------------------------------
1964 if ( next.IsEmpty()
1965 || next == ParserConsts::opbrace
1966 || next == ParserConsts::semicolon )
1967 // -----------------------------------------------------------
1968 {
1969 break;
1970 }
1971 // -----------------------------------------------------------
1972 else if (next == ParserConsts::lt)
1973 // -----------------------------------------------------------
1974 {
1975 // template class
1976 //m_Tokenizer.GetToken(); // reach "<"
1977 // must not "eat" the token,
1978 // SkipAngleBraces() will do it to see what it must match
1979 SkipAngleBraces();
1980 // also need to 'unget' the last token (>)
1981 // so next iteration will see the { or ; in 'next'
1982 m_Tokenizer.UngetToken();
1983 }
1984 }
1985 TRACE(_T("HandleClass() : Ancestors: ") + ancestors);
1986 }
1987
1988 // -------------------------------------------------------------------
1989 if (current == ParserConsts::opbrace) // unnamed class/struct/union
1990 // -------------------------------------------------------------------
1991 {
1992 wxString unnamedTmp;
1993 unnamedTmp.Printf(_T("%s%s%u_%lu"),
1994 g_UnnamedSymbol.wx_str(),
1995 (ct == ctClass ? _T("Class") : (ct == ctUnion ? _T("Union") : _T("Struct"))),
1996 m_FileIdx, static_cast<unsigned long>(m_StructUnionUnnamedCount++));
1997 Token* newToken = DoAddToken(tkClass, unnamedTmp, lineNr);
1998 // Maybe it is a bug here. I just fixed it.
1999 if (!newToken)
2000 {
2001 TRACE(_T("HandleClass() : Unable to create/add new token: ") + unnamedTmp);
2002
2003 // restore tokenizer's functionality
2004 m_Tokenizer.SetState(oldState);
2005 return;
2006 }
2007
2008 newToken->m_TemplateArgument = m_TemplateArgument;
2009 wxArrayString formals;
2010 SplitTemplateFormalParameters(m_TemplateArgument, formals);
2011 #ifdef CC_PARSER_TEST
2012 for (size_t i = 0; i < formals.GetCount(); ++i)
2013 TRACE(_T("The template formal arguments are '%s'."), formals[i].wx_str());
2014 #endif
2015 newToken->m_TemplateType = formals;
2016 m_TemplateArgument.Clear();
2017
2018 Token* lastParent = m_LastParent;
2019 TokenScope lastScope = m_LastScope;
2020 bool parsingTypedef = m_ParsingTypedef;
2021
2022 m_LastParent = newToken;
2023 // default scope is: private for classes, public for structs, public for unions
2024 m_LastScope = ct == ctClass ? tsPrivate : tsPublic;
2025 m_ParsingTypedef = false;
2026
2027 newToken->m_ImplLine = lineNr;
2028 newToken->m_ImplLineStart = m_Tokenizer.GetLineNumber();
2029
2030 newToken->m_IsAnonymous = true;
2031
2032 DoParse(); // recursion
2033
2034 m_LastParent = lastParent;
2035 m_LastScope = lastScope;
2036 m_ParsingTypedef = parsingTypedef;
2037
2038 m_LastUnnamedTokenName = unnamedTmp; // used for typedef'ing anonymous class/struct/union
2039
2040 // we should now be right after the closing brace
2041 // no vars are defined on a typedef, only types
2042 // In the former example, aa is not part of the typedef.
2043 if (m_ParsingTypedef)
2044 {
2045 m_Str.Clear();
2046 TRACE(_T("HandleClass() : Unable to create/add new token: ") + current);
2047 if ( !ReadClsNames(newToken->m_Name) )
2048 { TRACE(_T("HandleClass() : ReadClsNames returned false [1].")); }
2049 break;
2050 }
2051 else
2052 {
2053 m_Str = newToken->m_Name;
2054 if ( !ReadVarNames() )
2055 { TRACE(_T("HandleClass() : ReadVarNames returned false [1].")); }
2056 m_Str.Clear();
2057 break;
2058 }
2059 }
2060 // -------------------------------------------------------------------
2061 else if (next == ParserConsts::opbrace)
2062 // -------------------------------------------------------------------
2063 {
2064 // for a template class definition like
2065 // template <typename x, typename y>class AAA : public BBB, CCC {;}
2066 // we would like to show its ancestors and template formal parameters on the tooltip,
2067 // so we re-used the m_Args member to store those informations, the tooltip shows like:
2068 // class AAA<x,y> : BBB, CCC {...} instead of class AAA {...}
2069 wxStringTokenizer tkz(ancestors, ParserConsts::comma);
2070 wxString args;
2071 while (tkz.HasMoreTokens())
2072 {
2073 const wxString& ancestor = tkz.GetNextToken();
2074 if (ancestor.IsEmpty())
2075 continue;
2076 if (args.IsEmpty())
2077 args += ParserConsts::space + ParserConsts::colon;
2078 else
2079 args += ParserConsts::comma;
2080 args += ParserConsts::space + ancestor;
2081 }
2082 wxArrayString formals;
2083 SplitTemplateFormalParameters(m_TemplateArgument, formals);
2084 if (!formals.IsEmpty())
2085 args.Prepend(ParserConsts::lt + GetStringFromArray(formals, ParserConsts::comma, false) + ParserConsts::gt);
2086
2087 Token* newToken = DoAddToken(tkClass, current, lineNr, 0, 0, args);
2088 if (!newToken)
2089 {
2090 TRACE(_T("HandleClass() : Unable to create/add new token: ") + current);
2091
2092 // restore tokenizer's functionality
2093 m_Tokenizer.SetState(oldState);
2094 return;
2095 }
2096 newToken->m_AncestorsString = ancestors;
2097
2098 m_Tokenizer.GetToken(); // eat {
2099
2100 Token* lastParent = m_LastParent; // save status, and will restore after DoParse()
2101 TokenScope lastScope = m_LastScope;
2102 bool parsingTypedef = m_ParsingTypedef;
2103
2104 m_LastParent = newToken;
2105 // default scope is: private for classes, public for structs, public for unions
2106 m_LastScope = ct == ctClass ? tsPrivate : tsPublic;
2107 m_ParsingTypedef = false;
2108
2109 newToken->m_ImplLine = lineNr;
2110 newToken->m_ImplLineStart = m_Tokenizer.GetLineNumber();
2111
2112 newToken->m_TemplateArgument = m_TemplateArgument;
2113
2114 #ifdef CC_PARSER_TEST
2115 for (size_t i = 0; i < formals.GetCount(); ++i)
2116 TRACE(_T("The template formal arguments are '%s'."), formals[i].wx_str());
2117 #endif
2118 newToken->m_TemplateType = formals;
2119 m_TemplateArgument.Clear();
2120
2121 DoParse();
2122
2123 newToken->m_ImplLineEnd = m_Tokenizer.GetLineNumber();
2124
2125 m_ParsingTypedef = parsingTypedef;
2126 m_LastParent = lastParent;
2127 m_LastScope = lastScope;
2128
2129 // we should now be right after the closing brace
2130 // no vars are defined on a typedef, only types
2131 // In the former example, aa is not part of the typedef.
2132 if (m_ParsingTypedef)
2133 {
2134 m_Str.Clear();
2135 if ( !ReadClsNames(newToken->m_Name) )
2136 { TRACE(_T("HandleClass() : ReadClsNames returned false [2].")); }
2137 break;
2138 }
2139 else
2140 {
2141 m_Str = newToken->m_Name; // pattern: class A{} b; b is a variable
2142 if ( !ReadVarNames() )
2143 { TRACE(_T("HandleClass() : ReadVarNames returned false [2].")); }
2144 m_Str.Clear();
2145 break;
2146 }
2147 }
2148 // -------------------------------------------------------------------
2149 else if (next == ParserConsts::semicolon)
2150 // -------------------------------------------------------------------
2151 {
2152 // e.g. struct A {}; struct B { struct A a; };
2153 if ( m_LastParent
2154 && m_LastParent->m_TokenKind == tkClass
2155 && !lastCurrent.IsEmpty() )
2156 {
2157 m_Str << lastCurrent << ParserConsts::space_chr;
2158 DoAddToken(tkVariable, current, m_Tokenizer.GetLineNumber());
2159 break;
2160 }
2161 else
2162 break; // forward decl; we don't care
2163 }
2164 // -------------------------------------------------------------------
2165 else if (next.GetChar(0) == ParserConsts::opbracket_chr) // function: struct xyz& DoSomething()...
2166 // -------------------------------------------------------------------
2167 {
2168 HandleFunction(current);
2169 break;
2170 }
2171 // -------------------------------------------------------------------
2172 else if ( (next.GetChar(0) == ParserConsts::ptr_chr)
2173 || (next.GetChar(0) == ParserConsts::ref_chr) )
2174 // -------------------------------------------------------------------
2175 {
2176 // e.g. typedef struct A * a;
2177 if (next.GetChar(0) == ParserConsts::ptr_chr && m_ParsingTypedef)
2178 {
2179 wxString args;
2180
2181 Token* newToken = DoAddToken(tkClass, current, lineNr, 0, 0, args);
2182
2183 if (!newToken)
2184 {
2185 TRACE(_T("HandleClass() : Unable to create/add new token: ") + current);
2186
2187 // restore tokenizer's functionality
2188 m_Tokenizer.SetState(oldState);
2189 return;
2190 }
2191 newToken->m_AncestorsString = ancestors;
2192
2193 m_PointerOrRef << m_Tokenizer.GetToken();
2194
2195 m_Str.Clear();
2196 if ( !ReadClsNames(newToken->m_Name) )
2197 {
2198 TRACE(_T("HandleClass() : ReadClsNames returned false [2]."));
2199 }
2200
2201 break;
2202 }
2203 else
2204 m_Str << current;
2205 break;
2206 }
2207 // -------------------------------------------------------------------
2208 else if(next == ParserConsts::equals)
2209 // -------------------------------------------------------------------
2210 {
2211 // some patterns like: struct AAA a = {.x = 1, .y=2};
2212 // In (ANSI) C99, you can use a designated initializer to initialize a structure
2213 if (!lastCurrent.IsEmpty() )
2214 {
2215 m_Str << lastCurrent << ParserConsts::space_chr;
2216 DoAddToken(tkVariable, current, m_Tokenizer.GetLineNumber());
2217 }
2218 // so we have to eat the brace pair
2219 SkipToOneOfChars(ParserConsts::semicolon, /* supportNesting*/ true, /*singleCharToken*/ true);
2220 break;
2221 }
2222 // -------------------------------------------------------------------
2223 else
2224 // -------------------------------------------------------------------
2225 {
2226 // might be instantiation, see the following
2227 // e.g. struct HiddenStruct { int val; }; struct HiddenStruct yy;
2228 if (m_ParsingTypedef)
2229 {
2230 m_Tokenizer.UngetToken();
2231 break;
2232 }
2233 if (TokenExists(current, m_LastParent, tkClass))
2234 {
2235 if (!TokenExists(next, m_LastParent, tkVariable))
2236 {
2237 wxString farnext;
2238
2239 m_Tokenizer.GetToken(); // go ahead of identifier
2240 farnext = m_Tokenizer.PeekToken();
2241 // struct Point p1, p2;
2242 // current="Point", next="p1"
2243 if (farnext == ParserConsts::semicolon || farnext == ParserConsts::comma)
2244 {
2245 while (m_Options.handleVars
2246 && ( farnext == ParserConsts::semicolon
2247 || farnext == ParserConsts::comma ))
2248 {
2249 if (m_Str.IsEmpty())
2250 m_Str = current;
2251 DoAddToken(tkVariable, next, m_Tokenizer.GetLineNumber());
2252
2253 if (farnext == ParserConsts::comma)
2254 {
2255 m_Tokenizer.GetToken(); // eat the ","
2256 next = m_Tokenizer.GetToken(); // next = "p2"
2257 farnext = m_Tokenizer.PeekToken(); // farnext = "," or the final ";"
2258 continue;
2259 }
2260 else // we meet a ";", so break the loop
2261 {
2262 m_Str.Clear();
2263 break;
2264 }
2265 }
2266
2267 m_Tokenizer.GetToken(); // eat semi-colon
2268 break;
2269 }
2270 else
2271 m_Tokenizer.UngetToken(); // restore the identifier
2272 }
2273 }
2274 }
2275 lastCurrent = current;
2276 }
2277
2278 // restore tokenizer's functionality
2279 m_Tokenizer.SetState(oldState);
2280 }
2281
HandleFunction(wxString & name,bool isOperator,bool isPointer)2282 void ParserThread::HandleFunction(wxString& name, bool isOperator, bool isPointer)
2283 {
2284 TRACE(_T("HandleFunction() : Adding function '")+name+_T("': m_Str='")+m_Str+_T("'"));
2285 int lineNr = m_Tokenizer.GetLineNumber();
2286 wxString args = m_Tokenizer.GetToken();
2287 wxString peek = m_Tokenizer.PeekToken();
2288 TRACE(_T("HandleFunction() : name='")+name+_T("', args='")+args+_T("', peek='")+peek+_T("'"));
2289
2290 // NOTE: Avoid using return, because m_Str needs to be cleared
2291 // at the end of this function.
2292
2293 // special case for function pointers
2294 if (isPointer)
2295 {
2296 int pos = name.find(ParserConsts::ptr);
2297
2298 // pattern: m_Str AAA (*BBB) (...);
2299 // pattern: m_Str AAA (*BBB) (...) = some_function;
2300 if (pos != wxNOT_FOUND && ( peek == ParserConsts::semicolon
2301 || peek == ParserConsts::equals
2302 || peek == ParserConsts::comma))
2303 {
2304 name.RemoveLast(); // remove ")"
2305 name.Remove(0, pos+1).Trim(false); // remove "(* "
2306
2307 // pattern: m_Str AAA (*BBB[X][Y]) (...);
2308 // Trim(true) for safety, in case the name contains a trailing space
2309 pos = name.find(ParserConsts::oparray_chr);
2310 if (pos != wxNOT_FOUND)
2311 name.Remove(pos).Trim(true);
2312
2313 TRACE(_T("HandleFunction() : Add token name='")+name+_T("', args='")+args+_T("', return type='") + m_Str+ _T("'"));
2314 Token* newToken = DoAddToken(tkFunction, name, lineNr, 0, 0, args);
2315 if (newToken)
2316 {
2317 newToken->m_IsConst = false;
2318 newToken->m_TemplateArgument = m_TemplateArgument;
2319 if (!m_TemplateArgument.IsEmpty() && newToken->m_TemplateMap.empty())
2320 ResolveTemplateArgs(newToken);
2321 }
2322 else
2323 {
2324 TRACE(_T("HandleFunction() : Unable to create/add new token: ") + name);
2325 }
2326 m_TemplateArgument.Clear();
2327 }
2328 }
2329 else if (!m_Str.StartsWith(ParserConsts::kw_friend))
2330 {
2331 int lineStart = 0;
2332 int lineEnd = 0;
2333 bool isCtor = m_Str.IsEmpty();
2334 bool isDtor = m_Str.StartsWith(ParserConsts::tilde);
2335 Token* localParent = 0;
2336
2337 if ((isCtor || isDtor) && !m_EncounteredTypeNamespaces.empty())
2338 {
2339 // probably a ctor/dtor
2340 std::queue<wxString> q = m_EncounteredTypeNamespaces; // preserve m_EncounteredTypeNamespaces; needed in DoAddToken()
2341 localParent = FindTokenFromQueue(q, m_LastParent);
2342
2343 TRACE(_T("HandleFunction() : Ctor/Dtor '%s', m_Str='%s', localParent='%s'"),
2344 name.wx_str(),
2345 m_Str.wx_str(),
2346 localParent ? localParent->m_Name.wx_str() : _T("<none>"));
2347 }
2348 else
2349 {
2350 std::queue<wxString> q = m_EncounteredNamespaces; // preserve m_EncounteredNamespaces; needed in DoAddToken()
2351 localParent = FindTokenFromQueue(q, m_LastParent);
2352
2353 TRACE(_T("HandleFunction() : !(Ctor/Dtor) '%s', m_Str='%s', localParent='%s'"),
2354 name.wx_str(),
2355 m_Str.wx_str(),
2356 localParent ? localParent->m_Name.wx_str() : _T("<none>"));
2357 }
2358
2359 bool isCtorOrDtor = m_LastParent && name == m_LastParent->m_Name;
2360
2361 if (!isCtorOrDtor)
2362 isCtorOrDtor = localParent && name == localParent->m_Name;
2363
2364 if (!isCtorOrDtor && m_Options.useBuffer)
2365 isCtorOrDtor = isCtor || isDtor;
2366
2367 TRACE(_T("HandleFunction() : Adding function '%s', ': m_Str='%s', enc_ns='%s'."),
2368 name.wx_str(),
2369 m_Str.wx_str(),
2370 m_EncounteredNamespaces.size() ? m_EncounteredNamespaces.front().wx_str() : wxT("nil"));
2371
2372 bool isImpl = false;
2373 bool isConst = false;
2374 bool isNoExcept = false;
2375 while (!peek.IsEmpty()) // !eof
2376 {
2377 if (peek == ParserConsts::colon) // probably a ctor with member initializers
2378 {
2379 SkipToOneOfChars(ParserConsts::opbrace);
2380 m_Tokenizer.UngetToken(); // leave brace there
2381 peek = m_Tokenizer.PeekToken();
2382 continue;
2383 }
2384 else if (peek == ParserConsts::opbrace)// function implementation
2385 {
2386 isImpl = true;
2387 m_Tokenizer.GetToken(); // eat {
2388 lineStart = m_Tokenizer.GetLineNumber();
2389 SkipBlock(); // skip to matching }
2390 lineEnd = m_Tokenizer.GetLineNumber();
2391 break;
2392 }
2393 else if ( peek == ParserConsts::clbrace
2394 || peek == ParserConsts::semicolon
2395 || peek == ParserConsts::comma)
2396 break; // function decl
2397 else if (peek == ParserConsts::kw_const)
2398 isConst = true;
2399 else if (peek == ParserConsts::kw_noexcept)
2400 isNoExcept = true;
2401 else if (peek == ParserConsts::kw_throw)
2402 {
2403 // Handle something like: std::string MyClass::MyMethod() throw(std::exception)
2404 wxString arg = m_Tokenizer.GetToken(); // eat args ()
2405 }
2406 else if (peek == ParserConsts::kw_try)
2407 {
2408 // function-try-block pattern: AAA(...)try{}catch{}
2409 m_Tokenizer.GetToken(); // eat the try keyword
2410
2411 if (m_Tokenizer.PeekToken() == ParserConsts::colon)
2412 {
2413 // skip ctor initialization list
2414 SkipToOneOfChars(ParserConsts::opbrace);
2415 m_Tokenizer.UngetToken(); // leave brace there
2416 }
2417 if (m_Tokenizer.PeekToken() == ParserConsts::opbrace)
2418 {
2419 isImpl = true;
2420 m_Tokenizer.GetToken(); // eat {
2421 lineStart = m_Tokenizer.GetLineNumber();
2422 SkipBlock(); // skip to matching }
2423
2424 while (m_Tokenizer.PeekToken() == ParserConsts::kw_catch)
2425 {
2426 m_Tokenizer.GetToken(); // eat catch
2427 m_Tokenizer.GetToken(); // eat catch args
2428
2429 if (m_Tokenizer.PeekToken() == ParserConsts::opbrace)
2430 {
2431 m_Tokenizer.GetToken(); // eat {
2432 SkipBlock(); // skip to matching }
2433 }
2434 }
2435
2436 lineEnd = m_Tokenizer.GetLineNumber();
2437 break;
2438 }
2439 }
2440 else
2441 {
2442 TRACE(_T("HandleFunction() : Possible macro '%s' in function '%s' (file name='%s', line numer %d)."),
2443 peek.wx_str(), name.wx_str(), m_Filename.wx_str(), m_Tokenizer.GetLineNumber());
2444 break; // darned macros that do not end with a semicolon :/
2445 }
2446
2447 // if we reached here, eat the token so peek gets a new value
2448 m_Tokenizer.GetToken();
2449 peek = m_Tokenizer.PeekToken();
2450 } // while
2451
2452 TRACE(_T("HandleFunction() : Add token name='")+name+_T("', args='")+args+_T("', return type='") + m_Str+ _T("'"));
2453 TokenKind tokenKind = !isCtorOrDtor ? tkFunction : (isDtor ? tkDestructor : tkConstructor);
2454 Token* newToken = DoAddToken(tokenKind, name, lineNr, lineStart, lineEnd, args, isOperator, isImpl);
2455 if (newToken)
2456 {
2457 newToken->m_IsConst = isConst;
2458 newToken->m_IsNoExcept = isNoExcept;
2459 newToken->m_TemplateArgument = m_TemplateArgument;
2460 if (!m_TemplateArgument.IsEmpty() && newToken->m_TemplateMap.empty())
2461 ResolveTemplateArgs(newToken);
2462 }
2463 else
2464 {
2465 TRACE(_T("HandleFunction() : Unable to create/add new token: ") + name);
2466 }
2467 m_TemplateArgument.Clear();
2468 }
2469
2470 // NOTE: If we peek an equals or comma, this could be a list of function
2471 // declarations. In that case, don't clear return type (m_Str).
2472 peek = m_Tokenizer.PeekToken();
2473 if (peek != ParserConsts::equals && peek != ParserConsts::comma)
2474 m_Str.Clear();
2475 }
2476
HandleConditionalArguments()2477 void ParserThread::HandleConditionalArguments()
2478 {
2479 // if these aren't empty at this point, we have a syntax error
2480 if (!m_Str.empty())
2481 return;
2482
2483 if (!m_PointerOrRef.empty())
2484 return;
2485
2486 if (!m_TemplateArgument.empty())
2487 return;
2488
2489 // conditional arguments can look like this:
2490 // (int i = 12)
2491 // (Foo *bar = getFooBar())
2492 // (var <= 12 && (getType() != 23))
2493 wxString args = m_Tokenizer.GetToken();
2494
2495 // remove braces
2496 if (args.StartsWith(_T("(")))
2497 args = args.Mid(1, args.length() - 1);
2498
2499 if (args.EndsWith(_T(")")))
2500 args = args.Mid(0, args.length() - 1);
2501
2502 // parse small tokens inside for loop head
2503 TokenTree tree;
2504 wxString fileName = m_Tokenizer.GetFilename();
2505 Tokenizer smallTokenizer(&tree);
2506
2507 smallTokenizer.InitFromBuffer(args, fileName, m_Tokenizer.GetLineNumber());
2508
2509 while (IS_ALIVE)
2510 {
2511 wxString token = smallTokenizer.GetToken();
2512 if (token.empty())
2513 break;
2514
2515 wxString peek = smallTokenizer.PeekToken();
2516
2517 if (peek.empty())
2518 {
2519 if (!m_Str.empty())
2520 {
2521 // remove template argument if there is one
2522 wxString varType, templateArgs;
2523 RemoveTemplateArgs(m_Str, varType, templateArgs);
2524
2525 m_Str = varType;
2526 m_TemplateArgument = templateArgs;
2527
2528 Token *newToken = DoAddToken(tkVariable, token, smallTokenizer.GetLineNumber());
2529 if (newToken && !m_TemplateArgument.IsEmpty())
2530 ResolveTemplateArgs(newToken);
2531 else
2532 { TRACE(_T("HandleConditionalArguments() : Unable to create/add new token: ") + token); }
2533
2534 }
2535
2536 break;
2537 }
2538 else
2539 {
2540 if (token == ParserConsts::ref_chr || token == ParserConsts::ptr_chr)
2541 m_PointerOrRef << token;
2542 else
2543 {
2544 if (!m_Str.empty())
2545 m_Str << _T(" ");
2546
2547 m_Str << token;
2548 }
2549 }
2550 }
2551
2552 m_Str.clear();
2553 m_PointerOrRef.clear();
2554 m_TemplateArgument.clear();
2555 }
2556
HandleForLoopArguments()2557 void ParserThread::HandleForLoopArguments()
2558 {
2559 // if these aren't empty at this point, we have a syntax error
2560 if (!m_Str.empty())
2561 return;
2562
2563 if (!m_PointerOrRef.empty())
2564 return;
2565
2566 if (!m_TemplateArgument.empty())
2567 return;
2568
2569 // for loop heads look like this:
2570 // ([init1 [, init2 ...] ] ; [cond1 [, cond2 ..]]; [mod1 [, mod2 ..]])
2571 wxString args = m_Tokenizer.GetToken();
2572
2573 // remove braces
2574 if (args.StartsWith(_T("(")))
2575 args = args.Mid(1, args.length() - 1);
2576 if (args.EndsWith(_T(")")))
2577 args = args.Mid(0, args.length() - 1);
2578
2579 // parse small tokens inside for loop head
2580 TokenTree tree;
2581 wxString fileName = m_Tokenizer.GetFilename();
2582 Tokenizer smallTokenizer(&tree);
2583
2584 smallTokenizer.InitFromBuffer(args, fileName, m_Tokenizer.GetLineNumber());
2585
2586 while (IS_ALIVE)
2587 {
2588 wxString token = smallTokenizer.GetToken();
2589 if (token.empty())
2590 break;
2591
2592 // pattern for (; ...)
2593 // the first token is a ';'
2594 if (token == ParserConsts::semicolon)
2595 break;
2596
2597 wxString peek = smallTokenizer.PeekToken();
2598
2599 bool createNewToken = false;
2600 bool finished = false;
2601
2602 // pattern for(int i = 5; ...)
2603 // there is a "=" after the token "i"
2604 if (peek == ParserConsts::equals)
2605 {
2606 // skip to ',' or ';'
2607 while (IS_ALIVE)
2608 {
2609 smallTokenizer.GetToken();
2610
2611 peek = smallTokenizer.PeekToken();
2612 if (peek == ParserConsts::comma
2613 || peek == ParserConsts::semicolon
2614 || peek.empty())
2615 break;
2616 }
2617 }
2618
2619 if (peek == ParserConsts::comma)
2620 {
2621 smallTokenizer.GetToken(); // eat comma
2622 createNewToken = true;
2623 }
2624 else if (peek == ParserConsts::colon
2625 || peek == ParserConsts::semicolon
2626 || peek.empty())
2627 {
2628 createNewToken = true;
2629 finished = true; // after this point there will be no further declarations
2630 }
2631 else
2632 {
2633 if (token == ParserConsts::ref_chr || token == ParserConsts::ptr_chr)
2634 m_PointerOrRef << token;
2635 else
2636 {
2637 if (!m_Str.empty())
2638 m_Str << _T(" ");
2639
2640 m_Str << token;
2641 }
2642 }
2643
2644 if (createNewToken && !m_Str.empty())
2645 {
2646 // remove template argument if there is one
2647 wxString name, templateArgs;
2648 RemoveTemplateArgs(m_Str, name, templateArgs);
2649
2650 m_Str = name;
2651 m_TemplateArgument = templateArgs;
2652
2653 Token *newToken = DoAddToken(tkVariable, token, smallTokenizer.GetLineNumber());
2654 if (newToken && !m_TemplateArgument.IsEmpty())
2655 ResolveTemplateArgs(newToken);
2656 else
2657 { TRACE(_T("HandleForLoopArguments() : Unable to create/add new token: ") + token); }
2658
2659 }
2660
2661 if (finished)
2662 break;
2663 }
2664
2665 m_Str.clear();
2666 m_PointerOrRef.clear();
2667 m_TemplateArgument.clear();
2668 }
2669
HandleEnum()2670 void ParserThread::HandleEnum()
2671 {
2672 // enums have the following rough definition:
2673 // enum [xxx] { type1 name1 [= 1][, [type2 name2 [= 2]]] };
2674 bool isUnnamed = false;
2675 bool isEnumClass = false;
2676 int lineNr = m_Tokenizer.GetLineNumber();
2677 wxString token = m_Tokenizer.GetToken();
2678
2679 // C++11 has some enhanced enumeration declaration
2680 // see: http://en.cppreference.com/w/cpp/language/enum
2681 if (token == ParserConsts::kw_class)
2682 {
2683 token = m_Tokenizer.GetToken();
2684 isEnumClass = true;
2685 }
2686 else if (token == ParserConsts::colon)
2687 {
2688 // enum : int {...}
2689 SkipToOneOfChars(ParserConsts::semicolonopbrace); // jump to the "{" or ";"
2690 // note in this case, the "{" or ";" is already eaten, so we need to go back one step
2691 m_Tokenizer.UngetToken();
2692 token = m_Tokenizer.PeekToken();
2693 }
2694
2695 if (token.IsEmpty())
2696 return;
2697 else if (token==ParserConsts::opbrace)
2698 {
2699 // we have an un-named enum
2700 if (m_ParsingTypedef)
2701 {
2702 token.Printf(_T("%sEnum%u_%lu"), g_UnnamedSymbol.wx_str(), m_FileIdx, static_cast<unsigned long>(m_EnumUnnamedCount++));
2703 m_LastUnnamedTokenName = token;
2704 }
2705 else
2706 token = g_UnnamedSymbol;
2707 m_Tokenizer.UngetToken(); // return '{' back
2708 isUnnamed = true;
2709 }
2710
2711 // the token is now the expected enum name
2712 Token* newEnum = 0L;
2713 unsigned int level = 0;
2714 if ( wxIsalpha(token.GetChar(0))
2715 || (token.GetChar(0) == ParserConsts::underscore_chr) )
2716 {
2717 // we have such pattern: enum name {
2718 // ^^^^
2719 // token peek
2720 wxString peek = m_Tokenizer.PeekToken();
2721 if (peek == ParserConsts::colon) // enum name : type {
2722 {
2723 m_Tokenizer.GetToken(); // eat the ":"
2724 SkipToOneOfChars(ParserConsts::semicolonopbrace); // jump to the "{" or ";"
2725 // note in this case, the "{" or ";" is already eaten, so we need to go back one step
2726 m_Tokenizer.UngetToken();
2727 peek = m_Tokenizer.PeekToken();
2728 }
2729
2730 if (peek.GetChar(0) != ParserConsts::opbrace_chr)
2731 {
2732 // pattern: enum E var;
2733 // now peek=var, so we try to see it is a variable definition
2734 if (TokenExists(token, m_LastParent, tkEnum))
2735 {
2736 if (!TokenExists(m_Tokenizer.PeekToken(), m_LastParent, tkVariable) )
2737 {
2738 wxString ident = m_Tokenizer.GetToken(); // go ahead of identifier
2739
2740 if (m_Tokenizer.PeekToken()==ParserConsts::semicolon)
2741 {
2742 if (m_Options.handleEnums)
2743 {
2744 m_Str = token;
2745 DoAddToken(tkVariable, ident, m_Tokenizer.GetLineNumber());
2746 m_Str.Clear();
2747 }
2748
2749 m_Tokenizer.GetToken(); // eat semi-colon
2750 }
2751 else
2752 { // peek is not ";", mostly it is some pattern like:
2753 // enum E fun (..) ;
2754 // enum E fun (..) {...};
2755 // so we just push the "E" to the m_Str, and return
2756 // this make the just like:
2757 // E fun (..) ;
2758 // E fun (..) {...};
2759 m_Str = token;
2760 m_Tokenizer.UngetToken(); // restore the identifier
2761 }
2762 }
2763 }
2764 return;
2765 }
2766
2767 if (isUnnamed && !m_ParsingTypedef)
2768 {
2769 // for unnamed enums, look if we already have "Unnamed", so we don't
2770 // add a new one for every unnamed enum we encounter, in this scope...
2771 newEnum = TokenExists(token, m_LastParent, tkEnum);
2772 }
2773
2774 if (!newEnum) // either named or first unnamed enum
2775 {
2776 newEnum = DoAddToken(tkEnum, token, lineNr);
2777 newEnum->m_IsAnonymous = true;
2778 }
2779
2780 level = m_Tokenizer.GetNestingLevel();
2781 m_Tokenizer.GetToken(); // skip {
2782 }
2783 else
2784 {
2785 if (token.GetChar(0) != ParserConsts::opbrace_chr)
2786 return;
2787 level = m_Tokenizer.GetNestingLevel() - 1; // we 've already entered the { block
2788 }
2789
2790 int lineStart = m_Tokenizer.GetLineNumber();
2791 //Implementation for showing Enum values: resolves expressions, preprocessor and enum tokens.
2792 int enumValue = 0;
2793 bool updateValue = true;
2794
2795 const TokenizerState oldState = m_Tokenizer.GetState();
2796 m_Tokenizer.SetState(tsNormal);
2797
2798 while (IS_ALIVE)
2799 {
2800 // process enumerators
2801 token = m_Tokenizer.GetToken();
2802 wxString peek = m_Tokenizer.PeekToken();
2803 if (token.IsEmpty() || peek.IsEmpty())
2804 return; //eof
2805 if (token==ParserConsts::clbrace && level == m_Tokenizer.GetNestingLevel())
2806 break;
2807 // assignments (=xxx) are ignored by the tokenizer,
2808 // so we don't have to worry about them here,
2809 // if (peek==ParserConsts::comma || peek==ParserConsts::clbrace || peek==ParserConsts::colon)
2810 if (peek==ParserConsts::colon)
2811 {
2812 peek = SkipToOneOfChars(ParserConsts::equals + ParserConsts::commaclbrace);
2813 }
2814 if (peek == ParserConsts::equals)
2815 {
2816 m_Tokenizer.GetToken(); //eat '='
2817 long result;
2818 updateValue = false;
2819 if (CalcEnumExpression(newEnum, result, peek))
2820 {
2821 enumValue = result;
2822 updateValue = true;
2823 }
2824 }
2825 if (peek == ParserConsts::comma || peek == ParserConsts::clbrace)
2826 {
2827 // this "if", avoids non-valid enumerators
2828 // like a comma (if no enumerators follow)
2829 if ( wxIsalpha(token.GetChar(0))
2830 || (token.GetChar(0) == ParserConsts::underscore_chr) )
2831 {
2832 wxString args;
2833 if (updateValue)
2834 args << enumValue++;
2835
2836 Token* lastParent = m_LastParent;
2837 m_LastParent = newEnum;
2838 Token* enumerator = DoAddToken(tkEnumerator, token, m_Tokenizer.GetLineNumber(), 0, 0, args);
2839 enumerator->m_Scope = isEnumClass ? tsPrivate : tsPublic;
2840 m_LastParent = lastParent;
2841 }
2842 }
2843 }
2844
2845 m_Tokenizer.SetState(oldState);
2846
2847 newEnum->m_ImplLine = lineNr;
2848 newEnum->m_ImplLineStart = lineStart;
2849 newEnum->m_ImplLineEnd = m_Tokenizer.GetLineNumber();
2850
2851 // // skip to ;
2852 // SkipToOneOfChars(ParserConsts::semicolon);
2853 }
2854
CalcEnumExpression(Token * tokenParent,long & result,wxString & peek)2855 bool ParserThread::CalcEnumExpression(Token* tokenParent, long& result, wxString& peek)
2856 {
2857 // need to force the tokenizer skip raw expression
2858 const TokenizerState oldState = m_Tokenizer.GetState();
2859 // expand macros, but don't read a single parentheses
2860 m_Tokenizer.SetState(tsRawExpression);
2861
2862 Expression exp;
2863 wxString token, next;
2864
2865 while (IS_ALIVE)
2866 {
2867 token = m_Tokenizer.GetToken();
2868 if (token.IsEmpty())
2869 return false;
2870 if (token == _T("\\"))
2871 continue;
2872 if (token == ParserConsts::comma || token == ParserConsts::clbrace)
2873 {
2874 m_Tokenizer.UngetToken();
2875 peek = token;
2876 break;
2877 }
2878 if (token == ParserConsts::dcolon)
2879 {
2880 peek = SkipToOneOfChars(ParserConsts::commaclbrace);
2881 m_Tokenizer.UngetToken();
2882 exp.Clear();
2883 break;
2884 }
2885
2886 if (wxIsalpha(token[0]) || token[0] == ParserConsts::underscore_chr) // handle enum or macro
2887 {
2888 const Token* tk = m_TokenTree->at(m_TokenTree->TokenExists(token, tokenParent->m_Index, tkEnumerator));
2889
2890 if (tk) // the enumerator token
2891 {
2892 if (!tk->m_Args.IsEmpty() && wxIsdigit(tk->m_Args[0]))
2893 token = tk->m_Args; // add the value to exp
2894 }
2895 else
2896 {
2897 peek = SkipToOneOfChars(ParserConsts::commaclbrace);
2898 m_Tokenizer.UngetToken();
2899 exp.Clear();
2900 break;
2901 }
2902 }
2903
2904 // only remaining number now
2905 if (!token.StartsWith(_T("0x")))
2906 exp.AddToInfixExpression(token);
2907 else
2908 {
2909 long value;
2910 if (token.ToLong(&value, 16))
2911 exp.AddToInfixExpression(wxString::Format(_T("%ld"), value));
2912 else
2913 {
2914 peek = SkipToOneOfChars(ParserConsts::commaclbrace);
2915 exp.Clear();
2916 break;
2917 }
2918 }
2919 }
2920
2921 // reset tokenizer's functionality
2922 m_Tokenizer.SetState(oldState);
2923
2924 exp.ConvertInfixToPostfix();
2925 if (exp.CalcPostfix() && exp.GetStatus())
2926 {
2927 result = exp.GetResult();
2928 return true;
2929 }
2930
2931 return false;
2932 }
2933
HandleTypedef()2934 void ParserThread::HandleTypedef()
2935 {
2936 // typedefs are handled as tkClass and we put the typedef'd type as the
2937 // class's ancestor. This way, it will work through inheritance.
2938 // Function pointers are a different beast and are handled differently.
2939
2940 // this is going to be tough :(
2941 // let's see some examples:
2942 //
2943 // relatively easy:
2944 // typedef unsigned int uint32;
2945 // typedef std::map<String, StringVector> AnimableDictionaryMap;
2946 // typedef class|struct|enum [name] {...} type;
2947 //
2948 // special case of above:
2949 // typedef struct __attribute__((packed)) _PSTRUCT {...} PSTRUCT;
2950 //
2951 // harder:
2952 // typedef void dMessageFunction (int errnum, const char *msg, va_list ap);
2953 //
2954 // even harder:
2955 // typedef void (*dMessageFunction)(int errnum, const char *msg, va_list ap);
2956 // or
2957 // typedef void (MyClass::*Function)(int);
2958
2959 size_t lineNr = m_Tokenizer.GetLineNumber();
2960 bool is_function_pointer = false;
2961 wxString typ;
2962 std::queue<wxString> components;
2963 // get everything on the same line
2964
2965 TRACE(_T("HandleTypedef() : Typedef start"));
2966 wxString args;
2967 wxString token;
2968 wxString peek;
2969 m_ParsingTypedef = true;
2970
2971 while (IS_ALIVE)
2972 {
2973 token = m_Tokenizer.GetToken();
2974 peek = m_Tokenizer.PeekToken();
2975
2976 TRACE(_T("HandleTypedef() : token=%s, peek=%s"), token.wx_str(), peek.wx_str());
2977 if (token.IsEmpty() || token == ParserConsts::semicolon)
2978 {
2979 m_Tokenizer.UngetToken(); // NOTE: preserve ';' for the next GetToken();
2980 break;
2981 }
2982
2983 if (token == ParserConsts::kw_const)
2984 continue;
2985
2986 if ( token == ParserConsts::kw_class
2987 || token == ParserConsts::kw_struct
2988 || token == ParserConsts::kw_union)
2989 {
2990 // "typedef struct|class|union"
2991 TRACE(_("HandleTypedef() : Before HandleClass m_LastUnnamedTokenName='%s'"), m_LastUnnamedTokenName.wx_str());
2992 HandleClass(token == ParserConsts::kw_class ? ctClass :
2993 token == ParserConsts::kw_union ? ctUnion :
2994 ctStructure);
2995 token = m_LastUnnamedTokenName;
2996 TRACE(_("HandleTypedef() : After HandleClass m_LastUnnamedTokenName='%s'"), m_LastUnnamedTokenName.wx_str());
2997 }
2998 else if (token == ParserConsts::ptr || token == ParserConsts::ref)
2999 {
3000 m_PointerOrRef << token;
3001 continue;
3002 }
3003 else if (peek == ParserConsts::comma)
3004 {
3005 m_Tokenizer.UngetToken();
3006 if (components.size() != 0)
3007 {
3008 wxString ancestor;
3009 while (components.size() > 0)
3010 {
3011 wxString tempToken = components.front();
3012 components.pop();
3013
3014 if (!ancestor.IsEmpty())
3015 ancestor << ParserConsts::space_chr;
3016 ancestor << tempToken;
3017 }
3018 if ( !ReadClsNames(ancestor) )
3019 {
3020 TRACE(_T("HandleTypedef() : ReadClsNames returned false."));
3021 m_Tokenizer.GetToken(); // eat it
3022 }
3023 }
3024 }
3025 else if (token == ParserConsts::kw_enum)
3026 {
3027 // "typedef enum"
3028 HandleEnum();
3029 token = m_LastUnnamedTokenName;
3030 }
3031
3032 // keep namespaces together
3033 while (peek == ParserConsts::dcolon)
3034 {
3035 token << peek;
3036 m_Tokenizer.GetToken(); // eat it
3037 token << m_Tokenizer.GetToken(); // get what's next
3038 peek = m_Tokenizer.PeekToken();
3039 }
3040
3041 if (token.GetChar(0) == ParserConsts::opbracket_chr)
3042 {
3043 // function pointer (probably)
3044 is_function_pointer = true;
3045 if (peek.GetChar(0) == ParserConsts::opbracket_chr)
3046 {
3047 // typedef void (*dMessageFunction)(int errnum, const char *msg, va_list ap);
3048 // typedef void (MyClass::*Function)(int);
3049
3050 // remove parentheses and keep everything after the dereferencing symbol
3051 token.RemoveLast();
3052 int pos = token.Find(ParserConsts::ptr_chr, true);
3053 if (pos != wxNOT_FOUND)
3054 {
3055 typ << ParserConsts::opbracket_chr
3056 << token.Mid(1, pos)
3057 << ParserConsts::clbracket_chr;
3058 token.Remove(0, pos + 1);
3059 }
3060 else
3061 {
3062 typ = _T("(*)");
3063 token.Remove(0, 1); // remove opening parenthesis
3064 }
3065 args = peek;
3066 m_Tokenizer.GetToken(); // eat args
3067
3068 TRACE(_("HandleTypedef() : Pushing component='%s' (typedef args='%s')"), token.Trim(true).Trim(false).wx_str(), args.wx_str());
3069 components.push(token.Trim(true).Trim(false));
3070 }
3071 else
3072 {
3073 // typedef void dMessageFunction (int errnum, const char *msg, va_list ap);
3074
3075 // last component is already the name and this is the args
3076 args = token;
3077 TRACE(_("HandleTypedef() : Typedef args='%s'"), args.wx_str());
3078 }
3079 break;
3080 }
3081
3082 TRACE(_("HandleTypedef() : Pushing component='%s', typedef args='%s'"), token.Trim(true).Trim(false).wx_str(), args.wx_str());
3083 components.push(token.Trim(true).Trim(false));
3084
3085 // skip templates <>
3086 if (peek == ParserConsts::lt)
3087 {
3088 GetTemplateArgs();
3089 continue;
3090 }
3091
3092 TRACE(_T(" + '%s'"), token.wx_str());
3093 }
3094 TRACE(_T("HandleTypedef() : Typedef done"));
3095 m_ParsingTypedef = false;
3096
3097 if (components.empty())
3098 return; // invalid typedef
3099
3100 if (!is_function_pointer && components.size() <= 1)
3101 return; // invalid typedef
3102
3103 // now get the type
3104 wxString ancestor;
3105 wxString alias;
3106
3107 // handle the special cases below, a template type parameter is used in typedef
3108 // template<typename _Tp>
3109 // class c2
3110 // {
3111 // public:
3112 // typedef _Tp alise;
3113 //
3114 // };
3115 if ( (components.size() == 2)
3116 && m_LastParent
3117 && m_LastParent->m_TokenKind == tkClass
3118 && (!m_LastParent->m_TemplateType.IsEmpty())
3119 && m_LastParent->m_TemplateType.Index(components.front()) != wxNOT_FOUND )
3120 {
3121 wxArrayString templateType = m_LastParent->m_TemplateType;
3122 alias = components.front();
3123 components.pop();
3124 ancestor = components.front();
3125 }
3126 else
3127 {
3128 while (components.size() > 1)
3129 {
3130 token = components.front();
3131 components.pop();
3132
3133 if (!ancestor.IsEmpty())
3134 ancestor << ParserConsts::space_chr;
3135 ancestor << token;
3136 }
3137 }
3138
3139 // no return type
3140 m_Str.Clear();
3141
3142 TRACE(_("HandleTypedef() : Adding typedef: name='%s', ancestor='%s', args='%s'"), components.front().wx_str(), ancestor.wx_str(), args.wx_str());
3143 Token* tdef = DoAddToken(tkTypedef /*tkClass*/, components.front(), lineNr, 0, 0, args);
3144 if (tdef)
3145 {
3146 wxString actualAncestor = ancestor.BeforeFirst(ParserConsts::lt_chr).Trim();
3147 TRACE(_("HandleTypedef() : Ancestor='%s', actual ancestor='%s'"), ancestor.wx_str(), actualAncestor.wx_str());
3148
3149 if (is_function_pointer)
3150 {
3151 tdef->m_FullType = ancestor + typ; // + args;
3152 tdef->m_BaseType = actualAncestor;
3153 if (tdef->IsValidAncestor(ancestor))
3154 tdef->m_AncestorsString = ancestor;
3155 }
3156 else
3157 {
3158 tdef->m_FullType = ancestor;
3159 tdef->m_BaseType = actualAncestor;
3160 tdef->m_TemplateAlias = alias;
3161 TRACE(_T("The typedef alias is %s."), tdef->m_TemplateAlias.wx_str());
3162
3163 if (tdef->IsValidAncestor(ancestor))
3164 tdef->m_AncestorsString = ancestor;
3165 if (!m_TemplateArgument.IsEmpty())
3166 ResolveTemplateArgs(tdef);
3167 }
3168 }
3169 }
3170
ReadVarNames()3171 bool ParserThread::ReadVarNames()
3172 {
3173 bool success = true; // optimistic start value
3174
3175 while (IS_ALIVE)
3176 {
3177 wxString token = m_Tokenizer.GetToken();
3178
3179 if (token.IsEmpty()) // end of file / tokens
3180 break;
3181
3182 if (token==ParserConsts::comma) // another variable name
3183 continue;
3184 else if (token==ParserConsts::semicolon) // end of variable name(s)
3185 {
3186 m_PointerOrRef.Clear();
3187 break;
3188 }
3189 else if (token == ParserConsts::oparray)
3190 {
3191 SkipToOneOfChars(ParserConsts::clarray);
3192 }
3193 else if (token == ParserConsts::ptr) // variable is a pointer
3194 m_PointerOrRef << token;
3195 else if ( wxIsalpha(token.GetChar(0))
3196 || (token.GetChar(0) == ParserConsts::underscore_chr) )
3197 {
3198 TRACE(_T("ReadVarNames() : Adding variable '%s' as '%s' to '%s'"),
3199 token.wx_str(), m_Str.wx_str(),
3200 (m_LastParent ? m_LastParent->m_Name.wx_str() : _T("<no-parent>")));
3201
3202 // Detects anonymous ancestor and gives him a name based on the first found alias.
3203 if (m_Str.StartsWith(g_UnnamedSymbol))
3204 RefineAnonymousTypeToken(tkUndefined, token);
3205
3206 Token* newToken = DoAddToken(tkVariable, token, m_Tokenizer.GetLineNumber());
3207 if (!newToken)
3208 {
3209 TRACE(_T("ReadVarNames() : Unable to create/add new token: ") + token);
3210 break;
3211 }
3212 }
3213 else // unexpected
3214 {
3215 TRACE(F(_T("ReadVarNames() : Unexpected token '%s' for '%s', file '%s', line %d."),
3216 token.wx_str(), m_Str.wx_str(), m_Tokenizer.GetFilename().wx_str(), m_Tokenizer.GetLineNumber()));
3217 CCLogger::Get()->DebugLog(F(_T("ReadVarNames() : Unexpected token '%s' for '%s', file '%s', line %d."),
3218 token.wx_str(), m_Str.wx_str(), m_Tokenizer.GetFilename().wx_str(), m_Tokenizer.GetLineNumber()));
3219 success = false;
3220 break;
3221 }
3222 }
3223 return success;
3224 }
3225
ReadClsNames(wxString & ancestor)3226 bool ParserThread::ReadClsNames(wxString& ancestor)
3227 {
3228 bool success = true; // optimistic start value
3229
3230 while (IS_ALIVE)
3231 {
3232 wxString token = m_Tokenizer.GetToken();
3233
3234 if (token.IsEmpty()) // end of file / tokens
3235 break;
3236
3237 if (token==ParserConsts::comma) // another class name
3238 continue;
3239 else if (token==ParserConsts::kw_attribute)
3240 {
3241 m_Tokenizer.GetToken(); // eat (( whatever ))
3242 continue;
3243 }
3244 else if (token==ParserConsts::semicolon) // end of class name(s)
3245 {
3246 m_Tokenizer.UngetToken();
3247 m_PointerOrRef.Clear();
3248 break;
3249 }
3250 else if (token == ParserConsts::ptr) // variable is a pointer
3251 m_PointerOrRef << token;
3252 else if ( wxIsalpha(token.GetChar(0))
3253 || (token.GetChar(0) == ParserConsts::underscore_chr) )
3254 {
3255 TRACE(_T("ReadClsNames() : Adding variable '%s' as '%s' to '%s'"),
3256 token.wx_str(),
3257 m_Str.wx_str(),
3258 (m_LastParent ? m_LastParent->m_Name.wx_str() : _T("<no-parent>")));
3259
3260 m_Str.clear();
3261 m_Str = ancestor;
3262
3263 // Detects anonymous ancestor and gives him a name based on the first found alias.
3264 if (m_Str.StartsWith(g_UnnamedSymbol))
3265 {
3266 RefineAnonymousTypeToken(tkTypedef | tkClass, token);
3267 ancestor = m_Str;
3268 }
3269
3270 Token* newToken = DoAddToken(tkTypedef, token, m_Tokenizer.GetLineNumber());
3271 if (!newToken)
3272 {
3273 TRACE(_T("ReadClsNames() : Unable to create/add new token: ") + token);
3274 break;
3275 }
3276 else
3277 newToken->m_AncestorsString = ancestor;
3278 }
3279 else // unexpected
3280 {
3281 TRACE(F(_T("ReadClsNames() : Unexpected token '%s' for '%s', file '%s', line %d."),
3282 token.wx_str(), m_Str.wx_str(), m_Tokenizer.GetFilename().wx_str(), m_Tokenizer.GetLineNumber()));
3283 CCLogger::Get()->DebugLog(F(_T("ReadClsNames() : Unexpected token '%s' for '%s', file '%s', line %d."),
3284 token.wx_str(), m_Str.wx_str(), m_Tokenizer.GetFilename().wx_str(), m_Tokenizer.GetLineNumber()));
3285 // The following code snippet freezes CC here:
3286 // typedef std::enable_if<N > 1, get_type_N<N-1, Tail...>> type;
3287 m_Tokenizer.UngetToken();
3288 // Note: Do NOT remove m_Tokenizer.UngetToken();, otherwise it freezes somewhere else
3289 success = false;
3290 break;
3291 }
3292 }
3293 return success;
3294 }
3295
GetBaseArgs(const wxString & args,wxString & baseArgs)3296 bool ParserThread::GetBaseArgs(const wxString& args, wxString& baseArgs)
3297 {
3298 const wxChar* ptr = args.wx_str(); // pointer to current char in args string
3299 wxString word; // compiled word of last arg
3300 bool skip = false; // skip the next char (do not add to stripped args)
3301 bool sym = false; // current char symbol
3302 bool one = true; // only one argument
3303 // ( int abc = 5 , float * def )
3304 // ^
3305 // ptr point to the next char of "int"
3306 // word = "int"
3307 // sym = true means ptr is point to an identifier like token
3308 // here, if we find an identifier like token which is "int", we just skip the next token
3309 // until we meet a "," or ")".
3310
3311
3312 TRACE(_T("GetBaseArgs() : args='%s'."), args.wx_str());
3313 baseArgs.Alloc(args.Len() + 1);
3314
3315 // Verify ptr is valid (still within the range of the string)
3316 while (*ptr != ParserConsts::null)
3317 {
3318 switch (*ptr)
3319 {
3320 case ParserConsts::eol_chr:
3321 // skip the "\r\n"
3322 while (*ptr != ParserConsts::null && *ptr <= ParserConsts::space_chr)
3323 ++ptr;
3324 break;
3325 case ParserConsts::space_chr:
3326 // take care of args like:
3327 // - enum my_enum the_enum_my_enum
3328 // - const int the_const_int
3329 // - volatile long the_volatile_long
3330 if ( (word == ParserConsts::kw_enum)
3331 || (word == ParserConsts::kw_const)
3332 || (word == ParserConsts::kw_volatile) )
3333 skip = false; // don't skip this (it's part of the stripped arg)
3334 else
3335 skip = true; // safely skip this as it is the args name
3336 word = _T(""); // reset
3337 sym = false;
3338 break;
3339 case ParserConsts::ptr_chr: // handle pointer args
3340 // handle multiple pointer like in: main (int argc, void** argv)
3341 // or ((int *, char ***))
3342 while (*(ptr+1) != ParserConsts::null && *(ptr+1) == ParserConsts::ptr_chr)
3343 {
3344 baseArgs << *ptr; // append one more '*' to baseArgs
3345 ptr++; // next char
3346 }
3347 // ...and fall through:
3348 case ParserConsts::ref_chr: // handle references
3349 word = _T(""); // reset
3350 skip = true;
3351 sym = true;
3352
3353 // TODO (Morten#5#): Do comment the following even more. It's still not exactly clear to me...
3354 // verify completeness of last stripped argument (handle nested brackets correctly)
3355 {
3356 // extract last stripped argument from baseArgs
3357 wxString lastStrippedArg;
3358 int lastArgComma = baseArgs.Find(ParserConsts::comma_chr, true);
3359 if (lastArgComma)
3360 lastStrippedArg = baseArgs.Mid(1);
3361 else
3362 lastStrippedArg = baseArgs.Mid(lastArgComma);
3363
3364
3365 // input: (float a = 0.0, int* f1(char x, char y), void z)
3366 // output: (float, int*, z)
3367 // the internal "(char x, char y)" should be removed
3368 // No opening brackets in last stripped arg?
3369 // this means if we have function pointer in the argument
3370 // input: void foo(double (*fn)(double))
3371 // we should not skip the content after '*', since the '(' before '*' is already
3372 // pushed to the lastStrippedArg.
3373 if ( lastStrippedArg.Find(ParserConsts::opbracket_chr) == wxNOT_FOUND )
3374 {
3375 baseArgs << *ptr; // append to baseArgs
3376
3377 // find end
3378 int brackets = 0;
3379 ptr++; // next char
3380
3381 while (*ptr != ParserConsts::null)
3382 {
3383 if (*ptr == ParserConsts::opbracket_chr)
3384 brackets++;
3385 else if (*ptr == ParserConsts::clbracket_chr)
3386 {
3387 if (brackets == 0)
3388 break;
3389 brackets--;
3390 }
3391 else if (*ptr == ParserConsts::comma_chr && brackets == 0)
3392 {
3393 // don't stop at the inner comma char, such as '^' pointed below
3394 // (float a = 0.0, int* f1(char x, char y), void z)
3395 // ^
3396 skip = false;
3397 break;
3398 }
3399 ptr++; // next char
3400 }
3401 }
3402 }
3403 break;
3404 case ParserConsts::colon_chr: // namespace handling like for 'std::vector'
3405 skip = false;
3406 sym = true;
3407 break;
3408 case ParserConsts::oparray_chr: // array handling like for 'int[20]'
3409 // [ 128 ] -> [128]
3410 // space between the [] is stripped
3411 while ( *ptr != ParserConsts::null
3412 && *ptr != ParserConsts::clarray_chr )
3413 {
3414 if (*ptr != ParserConsts::space_chr)
3415 baseArgs << *ptr; // append to baseArgs, skipping spaces
3416 ptr++; // next char
3417 }
3418 skip = true;
3419 sym = true;
3420 break;
3421 case ParserConsts::lt_chr: // template arg handling like for 'vector<int>'
3422 // < int > -> <int>
3423 // space between the <> is stripped
3424 // note that embeded <> such as vector<vector<int>> is not handled here
3425 while ( *ptr != ParserConsts::null
3426 && *ptr != ParserConsts::gt_chr )
3427 {
3428 if (*ptr != ParserConsts::space_chr)
3429 baseArgs << *ptr; // append to baseArgs, skipping spaces
3430 ptr++; // next char
3431 }
3432 skip = true;
3433 sym = true;
3434 break;
3435 case ParserConsts::comma_chr: // fall through
3436 case ParserConsts::clbracket_chr: // fall through
3437 case ParserConsts::opbracket_chr:
3438 // ( int abc, .....)
3439 // we have just skip the "abc", and now, we see the ","
3440 if (skip && *ptr == ParserConsts::comma_chr)
3441 one = false; // see a comma, which means we have at least two parameter!
3442
3443 // try to remove the __attribute__(xxx) decoration in the parameter
3444 // such as: int f(__attribute__(xxx) wxCommandEvent & event);
3445 // should be convert to : int f(wxCommandEvent & event);
3446 if(*ptr == ParserConsts::opbracket_chr && word == ParserConsts::kw_attribute)
3447 {
3448 // remove the "__attribute__" keywords from the baseArgs
3449 // the length of "__attribute__" is 13
3450 baseArgs = baseArgs.Mid(0, baseArgs.Len()-13);
3451
3452 // skip the next "(xxx)
3453 int brackets = 1; // skip the first "(" already
3454 ptr++; // next char
3455
3456 while (*ptr != ParserConsts::null)
3457 {
3458 if (*ptr == ParserConsts::opbracket_chr)
3459 brackets++;
3460 else if (*ptr == ParserConsts::clbracket_chr)
3461 {
3462 brackets--;
3463 if (brackets == 0)
3464 {
3465 ptr++;
3466 break;
3467 }
3468
3469 }
3470 ptr++; // next char
3471 }
3472 // skip the spaces after the "__attribute__(xxx)"
3473 while ( *ptr != ParserConsts::null
3474 && *(ptr) == ParserConsts::space_chr )
3475 {
3476 ++ptr; // next char
3477 }
3478 word = _T(""); // reset
3479 sym = false; // no symbol is added
3480 skip = false; // don't skip the next token
3481 break;
3482 }
3483 word = _T(""); // reset
3484 sym = true;
3485 skip = false;
3486 break;
3487 default:
3488 sym = false;
3489 }// switch (*ptr)
3490
3491 // Now handle the char processed in this loop:
3492 if (!skip || sym)
3493 {
3494 // append to stripped argument and save the last word
3495 // (it's probably a type specifier like 'const' or alike)
3496 if (*ptr != ParserConsts::null)
3497 {
3498 baseArgs << *ptr; // append to baseArgs
3499 if (wxIsalnum(*ptr) || *ptr == ParserConsts::underscore_chr)
3500 word << *ptr; // append to word
3501 }
3502 }
3503
3504 if (!skip && sym)
3505 {
3506 // skip white spaces and increase pointer
3507 while ( *ptr != ParserConsts::null
3508 && *(ptr+1) == ParserConsts::space_chr )
3509 {
3510 ++ptr; // next char
3511 }
3512 }
3513
3514 if (*ptr != ParserConsts::null)
3515 {
3516 ++ptr; // next char
3517 }
3518 }
3519
3520 if (one && baseArgs.Len() > 2)
3521 {
3522 const wxChar ch = baseArgs[1];
3523 if ( (ch <= _T('9') && ch >= _T('0')) // number, 0 ~ 9
3524 || baseArgs.Find(_T('"')) != wxNOT_FOUND // string
3525 || baseArgs.Find(_T('\'')) != wxNOT_FOUND ) // character
3526 {
3527 return false; // not function, it should be variable
3528 }
3529
3530 if (baseArgs == _T("(void)"))
3531 baseArgs = _T("()");
3532 }
3533
3534 TRACE(_T("GetBaseArgs() : baseArgs='%s'."), baseArgs.wx_str());
3535 return true;
3536 }
3537
GetTemplateArgs()3538 void ParserThread::GetTemplateArgs()
3539 {
3540 // need to force the tokenizer _not_ skip anything
3541 // otherwise default values for template params would cause us to miss everything (because of the '=' symbol)
3542 TokenizerState oldState = m_Tokenizer.GetState();
3543 m_Tokenizer.SetState(tsNormal);
3544 m_TemplateArgument.clear();
3545 int nestLvl = 0;
3546 // NOTE: only exit this loop with 'break' so the tokenizer's state can
3547 // be reset afterwards (i.e. don't use 'return')
3548 while (IS_ALIVE)
3549 {
3550 wxString tmp = m_Tokenizer.GetToken();
3551
3552 if (tmp==ParserConsts::lt)
3553 {
3554 ++nestLvl;
3555 m_TemplateArgument << tmp;
3556
3557 }
3558 else if (tmp==ParserConsts::gt)
3559 {
3560 --nestLvl;
3561 m_TemplateArgument << tmp;
3562 }
3563 else if (tmp==ParserConsts::semicolon)
3564 {
3565 // unget token - leave ; on the stack
3566 m_Tokenizer.UngetToken();
3567 m_TemplateArgument.clear();
3568 break;
3569 }
3570 else if (tmp.IsEmpty())
3571 break;
3572 else
3573 m_TemplateArgument << tmp;
3574 if (nestLvl <= 0)
3575 break;
3576 }
3577
3578 // reset tokenizer's functionality
3579 m_Tokenizer.SetState(oldState);
3580 }
3581
ResolveTemplateArgs(Token * newToken)3582 void ParserThread::ResolveTemplateArgs(Token* newToken)
3583 {
3584 TRACE(_T("The variable template arguments are '%s'."), m_TemplateArgument.wx_str());
3585 newToken->m_TemplateArgument = m_TemplateArgument;
3586 wxArrayString actuals;
3587 SplitTemplateActualParameters(m_TemplateArgument, actuals);
3588 for (size_t i=0; i<actuals.GetCount(); ++i)
3589 TRACE(_T("The template actual arguments are '%s'."), actuals[i].wx_str());
3590
3591 newToken->m_TemplateType = actuals;
3592 // now resolve the template normal and actual map
3593 // wxString parentType = m_Str;
3594 std::map<wxString, wxString> templateMap;
3595 ResolveTemplateMap(newToken->m_FullType, actuals, templateMap);
3596 newToken->m_TemplateMap = templateMap;
3597 }
3598
GetTemplateArgArray(const wxString & templateArgs,bool remove_gt_lt,bool add_last)3599 wxArrayString ParserThread::GetTemplateArgArray(const wxString& templateArgs, bool remove_gt_lt, bool add_last)
3600 {
3601 wxString word;
3602 wxString args = templateArgs;
3603 args.Trim(true).Trim(false);
3604 if (remove_gt_lt)
3605 {
3606 args.Remove(0, 1);
3607 args.RemoveLast();
3608 }
3609
3610 wxArrayString container;
3611 for (size_t i = 0; i < args.Len(); ++i)
3612 {
3613 wxChar arg = args.GetChar(i);
3614 switch (arg)
3615 {
3616 case ParserConsts::space_chr:
3617 container.Add(word);
3618 word.clear();
3619 continue;
3620 case ParserConsts::lt_chr:
3621 case ParserConsts::gt_chr:
3622 case ParserConsts::comma_chr:
3623 container.Add(word);
3624 word.clear();
3625 container.Add(args[i]);
3626 continue;
3627 default:
3628 word << args[i];
3629 }
3630 }
3631
3632 if (add_last && !word.IsEmpty())
3633 container.Add(word);
3634
3635 return container;
3636 }
3637
SplitTemplateFormalParameters(const wxString & templateArgs,wxArrayString & formals)3638 void ParserThread::SplitTemplateFormalParameters(const wxString& templateArgs, wxArrayString& formals)
3639 {
3640 wxArrayString container = GetTemplateArgArray(templateArgs, false, false);
3641 size_t n = container.GetCount();
3642 for (size_t j = 0; j < n; ++j)
3643 {
3644 if ( (container[j] == ParserConsts::kw_typename)
3645 || (container[j] == ParserConsts::kw_class) )
3646 {
3647 if ( (j+1) < n )
3648 {
3649 formals.Add(container[j+1]);
3650 ++j; // skip
3651 }
3652 }
3653 }
3654
3655 }
3656
SplitTemplateActualParameters(const wxString & templateArgs,wxArrayString & actuals)3657 void ParserThread::SplitTemplateActualParameters(const wxString& templateArgs, wxArrayString& actuals)
3658 {
3659 wxArrayString container = GetTemplateArgArray(templateArgs, true, true);
3660 size_t n = container.GetCount();
3661
3662 // for debug purposes only
3663 for (size_t j = 0; j < n; ++j)
3664 TRACE(_T("The container elements are '%s'."), container[j].wx_str());
3665
3666 int level = 0;
3667 for (size_t j = 0; j < n; ++j)
3668 {
3669 if (container[j] == ParserConsts::lt)
3670 {
3671 ++level;
3672 while (level > 0 && (j+1) < n)
3673 {
3674 if (container[j] == ParserConsts::gt)
3675 --level;
3676 ++j; // skip
3677 }
3678
3679 }
3680 else if (container[j] == ParserConsts::comma)
3681 {
3682 ++j; // skip
3683 continue;
3684 }
3685 else
3686 actuals.Add(container[j]);
3687 ++j; // skip
3688 }
3689 }
3690
ResolveTemplateMap(const wxString & typeStr,const wxArrayString & actuals,std::map<wxString,wxString> & results)3691 bool ParserThread::ResolveTemplateMap(const wxString& typeStr, const wxArrayString& actuals,
3692 std::map<wxString, wxString>& results)
3693 {
3694 // Check if type is an alias template. If it is, then we use the actual type's template map.
3695 // For example, given:
3696 // template <class T> using AAA = BBB<T>;
3697 // AAA<MyClass> obj;
3698 // When handling obj, typeStr would equal AAA, but we would use BBB's template map.
3699 wxString tokenFullType = typeStr;
3700 TokenIdxSet fullTypeMatches;
3701 size_t matchesCount = m_TokenTree->FindMatches(tokenFullType, fullTypeMatches, true, false, tkTypedef);
3702 if (matchesCount > 0)
3703 {
3704 for (TokenIdxSet::const_iterator it= fullTypeMatches.begin(); it!= fullTypeMatches.end(); ++it)
3705 {
3706 int id = (*it);
3707 Token* token = m_TokenTree->at(id);
3708
3709 if (token->m_TokenKind == tkTypedef)
3710 {
3711 tokenFullType = token->m_FullType;
3712 // we are only interested in the type name, so remove the scope qualifiers
3713 if (tokenFullType.Find(_T("::")) != wxNOT_FOUND)
3714 tokenFullType = tokenFullType.substr(tokenFullType.Find(_T("::"))+2);
3715 break;
3716 }
3717 }
3718 }
3719
3720 wxString parentType = tokenFullType;
3721 parentType.Trim(true).Trim(false);
3722 // Note that we only search by the type name, and we don't care about the scope qualifiers
3723 // I add this for temporary support of templates under std, I will write better code later.
3724 TokenIdxSet parentResult;
3725 size_t tokenCounts = m_TokenTree->FindMatches(parentType, parentResult, true, false, tkClass);
3726 if (tokenCounts > 0)
3727 {
3728 for (TokenIdxSet::const_iterator it=parentResult.begin(); it!=parentResult.end(); ++it)
3729 {
3730 int id = (*it);
3731 Token* normalToken = m_TokenTree->at(id);
3732 if (normalToken)
3733 {
3734 // Get the formal template argument lists
3735 wxArrayString formals = normalToken->m_TemplateType;
3736 for (size_t i=0; i<formals.GetCount(); ++i)
3737 TRACE(_T("ResolveTemplateMap get the formal template arguments are '%s'."), formals[i].wx_str());
3738
3739 size_t n = formals.GetCount() < actuals.GetCount() ? formals.GetCount() : actuals.GetCount();
3740 for (size_t i=0; i<n; ++i)
3741 {
3742 results[formals[i]] = actuals[i];
3743 TRACE(_T("In ResolveTemplateMap function the normal is '%s',the actual is '%s'."), formals[i].wx_str(), actuals[i].wx_str());
3744 }
3745 }
3746 }
3747 return (results.size()>0) ? true : false;
3748 }
3749 else
3750 return false;
3751 }
3752
RemoveTemplateArgs(const wxString & exp,wxString & expNoArgs,wxString & templateArgs)3753 void ParserThread::RemoveTemplateArgs(const wxString &exp, wxString &expNoArgs, wxString &templateArgs)
3754 {
3755 expNoArgs.clear();
3756 templateArgs.clear();
3757
3758 int nestLvl = 0;
3759 for (unsigned int i = 0; i < exp.length(); i++)
3760 {
3761 wxChar c = exp[i];
3762
3763 if (c == ParserConsts::lt_chr)
3764 {
3765 nestLvl++;
3766 templateArgs << c;
3767 continue;
3768 }
3769
3770 if (c == ParserConsts::gt_chr)
3771 {
3772 nestLvl--;
3773 templateArgs << c;
3774 continue;
3775 }
3776
3777 if (nestLvl == 0)
3778 expNoArgs << c;
3779 else
3780 {
3781 bool wanted = true;
3782
3783 // don't add unwanted whitespaces, i.e. ws around '<' and '>'
3784 if(c == ParserConsts::space)
3785 {
3786 wxChar last = 0;
3787 wxChar next = 0;
3788
3789 if (i > 0) last = exp[i - 1];
3790 if (i < exp.length() - 1) next = exp[i + 1];
3791
3792 if (last == ParserConsts::gt || last == ParserConsts::lt)
3793 wanted = false;
3794
3795 if (next == ParserConsts::gt || next == ParserConsts::lt)
3796 wanted = false;
3797 }
3798
3799 if (wanted == true)
3800 templateArgs << c;
3801 }
3802 }
3803 }
3804
IsStillAlive(cb_unused const wxString & funcInfo)3805 bool ParserThread::IsStillAlive(cb_unused const wxString& funcInfo)
3806 {
3807 const bool alive = !TestDestroy();
3808 if (!alive)
3809 {
3810 TRACE(_T("IsStillAlive() : %s "), funcInfo.wx_str());
3811 free(0);
3812 }
3813 return alive;
3814 }
3815
RefineAnonymousTypeToken(short int typeMask,wxString alias)3816 void ParserThread::RefineAnonymousTypeToken(short int typeMask, wxString alias)
3817 {
3818 // we expect the m_Str are storing the unnamed type token, like UnnamedClassAA_BBB
3819 // AA is the file index, BBB is the unnamed token index
3820 // now, we are going to rename its name to classAA_CCC, CCC is the alias name
3821 Token* unnamedAncestor = TokenExists(m_Str, m_LastParent, typeMask);
3822 if (unnamedAncestor && unnamedAncestor->m_IsAnonymous) // Unnamed ancestor found - rename it to something useful.
3823 {
3824 if (m_Str.Contains(_T("Union")))
3825 m_Str = _T("union");
3826 else if (m_Str.Contains(_T("Struct")))
3827 m_Str = _T("struct");
3828 else
3829 m_Str = _T("tag");
3830 m_Str << m_FileIdx << _T("_") << alias;
3831 m_TokenTree->RenameToken(unnamedAncestor, m_Str);
3832 }
3833 }
3834
ReadAngleBrackets()3835 wxString ParserThread::ReadAngleBrackets()
3836 {
3837 wxString str = m_Tokenizer.GetToken();
3838 if (str != wxT("<"))
3839 return wxEmptyString;
3840
3841 int level = 1; // brace level of '<' and '>'
3842
3843 while (m_Tokenizer.NotEOF())
3844 {
3845 wxString token = m_Tokenizer.GetToken();
3846 if (token == _T("<"))
3847 {
3848 ++level;
3849 str << token;
3850 }
3851 else if (token == _T(">"))
3852 {
3853 --level;
3854 str << token;
3855 if (level == 0)
3856 break;
3857
3858 }
3859 else if (token == _T("*") || token == _T("&") || token == _T(","))
3860 {
3861 str << token;
3862 }
3863 else
3864 {
3865 if (str.Last() == _T('<')) // there is no space between '(' and the following token
3866 str << token;
3867 else // otherwise, a space is needed
3868 str << _T(" ") << token;
3869 }
3870
3871 if (level == 0)
3872 break;
3873 }//while (NotEOF())
3874
3875 return str;
3876 }
3877