1 /******************************************************************************
2  *
3  * $Id: $
4  *
5  *
6  * Copyright (C) 1997-2015 by Dimitri van Heesch.
7  *
8  * Permission to use, copy, modify, and distribute this software and its
9  * documentation under the terms of the GNU General Public License is hereby
10  * granted. No representations are made about the suitability of this software
11  * for any purpose. It is provided "as is" without express or implied warranty.
12  * See the GNU General Public License for more details.
13  *
14  * Documents produced by Doxygen are derivative works derived from the
15  * input used in their production; they are not affected by this license.
16  *
17  */
18 
19 %option never-interactive
20 %option prefix="doctokenizerYY"
21 %option reentrant
22 %option extra-type="struct doctokenizerYY_state *"
23 %top{
24 #include <stdint.h>
25 // forward declare yyscan_t to improve type safety
26 #define YY_TYPEDEF_YY_SCANNER_T
27 struct yyguts_t;
28 typedef yyguts_t *yyscan_t;
29 }
30 
31 %{
32 
33 #include <ctype.h>
34 #include <stack>
35 #include <string>
36 #include <cassert>
37 
38 #include "doctokenizer.h"
39 #include "cmdmapper.h"
40 #include "config.h"
41 #include "message.h"
42 #include "section.h"
43 #include "membergroup.h"
44 #include "definition.h"
45 #include "doxygen.h"
46 #include "portable.h"
47 #include "cite.h"
48 #include "regex.h"
49 
50 #define YY_NO_INPUT 1
51 #define YY_NO_UNISTD_H 1
52 
53 #define USE_STATE2STRING 0
54 
55 #define TK_COMMAND_SEL() (yytext[0] == '@' ? TK_COMMAND_AT : TK_COMMAND_BS)
56 
57 //--------------------------------------------------------------------------
58 
59 struct DocLexerContext
60 {
DocLexerContextDocLexerContext61   DocLexerContext(TokenInfo *tk,int r,int lvl,yy_size_t pos,const char *s,YY_BUFFER_STATE bs)
62     : token(tk), rule(r), autoListLevel(lvl), inputPos(pos), inputString(s), state(bs) {}
63   TokenInfo *token;
64   int rule;
65   int autoListLevel;
66   yy_size_t inputPos;
67   const char *inputString;
68   YY_BUFFER_STATE state;
69 };
70 
71 struct doctokenizerYY_state
72 {
73 
74   // context for tokenizer phase
75   int commentState;
76   TokenInfo *token = 0;
77   yy_size_t inputPos = 0;
78   const char *inputString = 0;
79   QCString fileName;
80   bool insidePre = false;
81   int sharpCount=0;
82   bool markdownSupport=TRUE;
83 
84   // context for section finding phase
85   const Definition  *definition = 0;
86   QCString     secLabel;
87   QCString     secTitle;
88   SectionType  secType;
89   QCString     endMarker;
90   int          autoListLevel;
91   std::stack< std::unique_ptr<DocLexerContext> > lexerStack;
92 
93   int yyLineNr = 0;
94 };
95 
96 #define lineCount(s,len) do { for(int i=0;i<(int)len;i++) if (s[i]=='\n') yyextra->yyLineNr++; } while(0)
97 
98 
99 #if USE_STATE2STRING
100 static const char *stateToString(int state);
101 #endif
102 
103 
104 static yy_size_t yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size);
105 static void handleHtmlTag(yyscan_t yyscanner,const char *text);
106 static void processSection(yyscan_t yyscanner);
107 
108 //--------------------------------------------------------------------------
109 
extractPartAfterNewLine(const QCString & text)110 QCString extractPartAfterNewLine(const QCString &text)
111 {
112   int nl1 = text.find('\n');
113   int nl2 = text.find("\\ilinebr");
114   if (nl1!=-1 && nl1<nl2)
115   {
116     return text.mid(nl1+1);
117   }
118   if (nl2!=-1)
119   {
120     if (text.at(nl2+8)==' ') nl2++; // skip space after \\ilinebr
121     return text.mid(nl2+8);
122   }
123   return text;
124 }
125 
126 //--------------------------------------------------------------------------
127 
tokToString(int token)128 const char *DocTokenizer::tokToString(int token)
129 {
130   switch (token)
131   {
132     case 0:              return "TK_EOF";
133     case TK_WORD:        return "TK_WORD";
134     case TK_LNKWORD:     return "TK_LNKWORD";
135     case TK_WHITESPACE:  return "TK_WHITESPACE";
136     case TK_LISTITEM:    return "TK_LISTITEM";
137     case TK_ENDLIST:     return "TK_ENDLIST";
138     case TK_COMMAND_AT:  return "TK_COMMAND_AT";
139     case TK_HTMLTAG:     return "TK_HTMLTAG";
140     case TK_SYMBOL:      return "TK_SYMBOL";
141     case TK_NEWPARA:     return "TK_NEWPARA";
142     case TK_RCSTAG:      return "TK_RCSTAG";
143     case TK_URL:         return "TK_URL";
144     case TK_COMMAND_BS:  return "TK_COMMAND_BS";
145   }
146   return "ERROR";
147 }
148 
retvalToString(int retval)149 const char *DocTokenizer::retvalToString(int retval)
150 {
151   switch (retval)
152   {
153     case RetVal_OK:            return "RetVal_OK";
154     case RetVal_SimpleSec:     return "RetVal_SimpleSec";
155     case RetVal_ListItem:      return "RetVal_ListItem";
156     case RetVal_Section:       return "RetVal_Section";
157     case RetVal_Subsection:    return "RetVal_Subsection";
158     case RetVal_Subsubsection: return "RetVal_Subsubsection";
159     case RetVal_Paragraph:     return "RetVal_Paragraph";
160     case RetVal_SubParagraph:  return "RetVal_SubParagraph";
161     case RetVal_EndList:       return "RetVal_EndList";
162     case RetVal_EndPre:        return "RetVal_EndPre";
163     case RetVal_DescData:      return "RetVal_DescData";
164     case RetVal_DescTitle :    return "RetVal_DescTitle";
165     case RetVal_EndDesc:       return "RetVal_EndDesc";
166     case RetVal_TableRow:      return "RetVal_TableRow";
167     case RetVal_TableCell:     return "RetVal_TableCell";
168     case RetVal_TableHCell:    return "RetVal_TableHCell";
169     case RetVal_EndTable:      return "RetVal_EndTable";
170     case RetVal_Internal:      return "RetVal_Internal";
171   }
172   return "ERROR";
173 }
174 
computeIndent(const char * str,size_t length)175 static int computeIndent(const char *str,size_t length)
176 {
177   if (str==0 || length==std::string::npos) return 0;
178   size_t i;
179   int indent=0;
180   static int tabSize=Config_getInt(TAB_SIZE);
181   for (i=0;i<length;i++)
182   {
183     if (str[i]=='\t')
184     {
185       indent+=tabSize - (indent%tabSize);
186     }
187     else if (str[i]=='\n')
188     {
189       indent=0;
190     }
191     else
192     {
193       indent++;
194     }
195   }
196   return indent;
197 }
198 
199 //--------------------------------------------------------------------------
200 
stripEmptyLines(const QCString & s)201 static QCString stripEmptyLines(const QCString &s)
202 {
203   if (s.isEmpty()) return QCString();
204   int end=s.length();
205   int start=0,p=0;
206   // skip leading empty lines
207   for (;;)
208   {
209     int c;
210     while ((c=s[p]) && (c==' ' || c=='\t')) p++;
211     if (s[p]=='\n')
212     {
213       start=++p;
214     }
215     else
216     {
217       break;
218     }
219   }
220   // skip trailing empty lines
221   p=end-1;
222   if (p>=start && s.at(p)=='\n') p--;
223   while (p>=start)
224   {
225     int c;
226     while ((c=s[p]) && (c==' ' || c=='\t')) p--;
227     if (s[p]=='\n')
228     {
229       end=p;
230     }
231     else
232     {
233       break;
234     }
235     p--;
236   }
237   //printf("stripEmptyLines(%d-%d)\n",start,end);
238   return s.mid(start,end-start);
239 }
240 
241 #define unput_string(yytext,yyleng) do { for (int i=(int)yyleng-1;i>=0;i--) unput(yytext[i]); } while(0)
242 //--------------------------------------------------------------------------
243 
244 #undef  YY_INPUT
245 #define YY_INPUT(buf,result,max_size) result=yyread(yyscanner,buf,max_size);
246 
247 // otherwise the filename would be the name of the converted file (*.cpp instead of *.l)
getLexerFILE()248 static inline const char *getLexerFILE() {return __FILE__;}
249 #include "doxygen_lex.h"
250 
251 //--------------------------------------------------------------------------
252 //#define REAL_YY_DECL int doctokenizerYYlex (void)
253 //#define YY_DECL static int local_doctokenizer(void)
254 //#define LOCAL_YY_DECL local_doctokenizer()
255 
256 %}
257 
258 CMD       ("\\"|"@")
259 WS        [ \t\r\n]
260 NONWS     [^ \t\r\n]
261 BLANK     [ \t\r]
262 BLANKopt  {BLANK}*
263 ID        [$a-z_A-Z\x80-\xFF][$a-z_A-Z0-9\x80-\xFF]*
264 LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
265 PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
266 CITESCHAR [a-z_A-Z0-9\x80-\xFF\-\?]
267 CITEECHAR [a-z_A-Z0-9\x80-\xFF\-\+:\/\?]
268 CITEID    {CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*|"\""{CITESCHAR}{CITEECHAR}*("."{CITESCHAR}{CITEECHAR}*)*"\""
269 MAILADDR  ("mailto:")?[a-z_A-Z0-9\x80-\xFF.+-]+"@"[a-z_A-Z0-9\x80-\xFf-]+("."[a-z_A-Z0-9\x80-\xFf\-]+)+[a-z_A-Z0-9\x80-\xFf\-]+
270 MAILWS    [\t a-z_A-Z0-9\x80-\xFF+-]
271 MAILADDR2 {MAILWS}+{BLANK}+("at"|"AT"|"_at_"|"_AT_"){BLANK}+{MAILWS}+("dot"|"DOT"|"_dot_"|"_DOT_"){BLANK}+{MAILWS}+
272 OPTSTARS  ("/""/"{BLANK}*)?"*"*{BLANK}*
273 LISTITEM  {BLANK}*[-]("#")?{WS}
274 MLISTITEM {BLANK}*[+*]{WS}
275 OLISTITEM {BLANK}*[1-9][0-9]*"."{BLANK}
276 ENDLIST   {BLANK}*"."{BLANK}*\n
277 ATTRNAME  [a-z_A-Z\x80-\xFF][:a-z_A-Z0-9\x80-\xFF\-]*
278 ATTRIB    {ATTRNAME}{WS}*("="{WS}*(("\""[^\"]*"\"")|("'"[^\']*"'")|[^ \t\r\n'"><]+))?
279 URLCHAR   [a-z_A-Z0-9\!\~\,\:\;\'\$\?\@\&\%\#\.\-\+\/\=\x80-\xFF]
280 URLMASK   ({URLCHAR}+([({]{URLCHAR}*[)}])?)+
281 URLPROTOCOL ("http:"|"https:"|"ftp:"|"ftps:"|"sftp:"|"file:"|"news:"|"irc:"|"ircs:")
282 FILEICHAR [a-z_A-Z0-9\\:\\\/\-\+&#@]
283 FILEECHAR [a-z_A-Z0-9\-\+&#@]
284 FILECHARS {FILEICHAR}*{FILEECHAR}+
285 HFILEMASK {FILEICHAR}*("."{FILEICHAR}+)+{FILECHARS}*
286 VFILEMASK {FILECHARS}("."{FILECHARS})*
287 FILEMASK  {VFILEMASK}|{HFILEMASK}
288 LINKMASK  [^ \t\n\r\\@<&${}]+("("[^\n)]*")")?({BLANK}*("const"|"volatile"){BLANK}+)?
289 VERBATIM  "verbatim"{BLANK}*
290 SPCMD1    {CMD}([a-z_A-Z][a-z_A-Z0-9]*|{VERBATIM}|"--"|"---")
291 SPCMD2    {CMD}[\\@<>&$#%~".+=|-]
292 SPCMD3    {CMD}_form#[0-9]+
293 SPCMD4    {CMD}"::"
294 SPCMD5    {CMD}":"
295 INOUT     "in"|"out"|("in"{BLANK}*","?{BLANK}*"out")|("out"{BLANK}*","?{BLANK}*"in")
296 PARAMIO   {CMD}param{BLANK}*"["{BLANK}*{INOUT}{BLANK}*"]"
297 VARARGS   "..."
298 TEMPCHAR  [a-z_A-Z0-9.,: \t\*\&\(\)\[\]]
299 FUNCCHAR  [a-z_A-Z0-9,:\<\> \t\n\^\*\&\[\]]|{VARARGS}|"\\ilinebr"
300 FUNCPART  {FUNCCHAR}*("("{FUNCCHAR}*")"{FUNCCHAR}*)?
301 SCOPESEP  "::"|"#"|"."
302 TEMPLPART "<"{TEMPCHAR}*("<"{TEMPCHAR}*("<"{TEMPCHAR}*">")?">")?">"
303 ANONNS    "anonymous_namespace{"[^}]*"}"
304 SCOPEPRE  (({ID}{TEMPLPART}?)|{ANONNS}){SCOPESEP}
305 SCOPEKEYS ":"({ID}":")*
306 SCOPECPP  {SCOPEPRE}*(~)?{ID}{TEMPLPART}?
307 SCOPEOBJC {SCOPEPRE}?{ID}{SCOPEKEYS}?
308 SCOPEMASK {SCOPECPP}|{SCOPEOBJC}
309 FUNCARG   "("{FUNCPART}")"({BLANK}*("volatile"|"const"){BLANK})?
310 FUNCARG2  "("{FUNCPART}")"({BLANK}*("volatile"|"const"))?
311 OPNEW     {BLANK}+"new"({BLANK}*"[]")?
312 OPDEL     {BLANK}+"delete"({BLANK}*"[]")?
313 OPNORM    {OPNEW}|{OPDEL}|"+"|"-"|"*"|"/"|"%"|"^"|"&"|"|"|"~"|"!"|"="|"<"|">"|"+="|"-="|"*="|"/="|"%="|"^="|"&="|"|="|"<<"|">>"|"<<="|">>="|"=="|"!="|"<="|">="|"&&"|"||"|"++"|"--"|","|"->*"|"->"|"[]"|"()"|"<=>"
314 OPCAST    {BLANK}+[^<(\r\n.,][^(\r\n.,]*
315 OPMASK    ({BLANK}*{OPNORM}{FUNCARG})
316 OPMASKOPT ({BLANK}*{OPNORM}{FUNCARG}?)|({OPCAST}{FUNCARG})
317 OPMASKOP2 ({BLANK}*{OPNORM}{FUNCARG2}?)|({OPCAST}{FUNCARG2})
318 LNKWORD1  ("::"|"#")?{SCOPEMASK}
319 CVSPEC    {BLANK}*("const"|"volatile")
320 LNKWORD2  (({SCOPEPRE}*"operator"{OPMASK})|({SCOPEPRE}"operator"{OPMASKOPT})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOPT})){CVSPEC}?
321 LNKWORD3  ([0-9a-z_A-Z\-]+("/"|"\\"))*[0-9a-z_A-Z\-]+("."[0-9a-z_A-Z]+)+
322 CHARWORDQ [^ \t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
323 ESCWORD   ("%"{ID}(("::"|"."){ID})*)|("%'")
324 CHARWORDQ1 [^ \-+0-9\t\n\r\\@<>()\[\]:;\?{}&%$#,."=']
325 WORD1     {ESCWORD}|{CHARWORDQ1}{CHARWORDQ}*|"{"|"}"|"'\"'"|("\""([^"\n]*(\\\"|\n)?)*[^"\n]*"\"")
326 WORD2     "."|","|"("|")"|"["|"]"|"::"|":"|";"|"\?"|"="|"'"
327 WORD1NQ   {ESCWORD}|{CHARWORDQ}+|"{"|"}"
328 WORD2NQ   "."|","|"("|")"|"["|"]"|"::"|":"|";"|"\?"|"="|"'"
329 CAPTION   [cC][aA][pP][tT][iI][oO][nN]
330 HTMLTAG   "<"(("/")?){ID}({WS}+{ATTRIB})*{WS}*(("/")?)">"
331 HTMLKEYL  "strong"|"center"|"table"|"caption"|"small"|"code"|"dfn"|"var"|"img"|"pre"|"sub"|"sup"|"tr"|"td"|"th"|"ol"|"ul"|"li"|"tt"|"kbd"|"em"|"hr"|"dl"|"dt"|"dd"|"br"|"i"|"a"|"b"|"p"|"strike"|"u"|"del"|"ins"|"s"
332 HTMLKEYU  "STRONG"|"CENTER"|"TABLE"|"CAPTION"|"SMALL"|"CODE"|"DFN"|"VAR"|"IMG"|"PRE"|"SUB"|"SUP"|"TR"|"TD"|"TH"|"OL"|"UL"|"LI"|"TT"|"KBD"|"EM"|"HR"|"DL"|"DT"|"DD"|"BR"|"I"|"A"|"B"|"P"|"STRIKE"|"U"|"DEL"|"INS"|"S"
333 HTMLKEYW  {HTMLKEYL}|{HTMLKEYU}
334 REFWORD2_PRE   ("#"|"::")?((({ID}{TEMPLPART}?)|{ANONNS})("."|"#"|"::"|"-"|"/"))*({ID}{TEMPLPART}?(":")?)
335 REFWORD2       {REFWORD2_PRE}{FUNCARG2}?
336 REFWORD2_NOCV  {REFWORD2_PRE}("("{FUNCPART}")")?
337 REFWORD3       ({ID}":")*{ID}":"?
338 REFWORD4_NOCV  (({SCOPEPRE}*"operator"{OPMASKOP2})|(("::"|"#"){SCOPEPRE}*"operator"{OPMASKOP2}))
339 REFWORD4       {REFWORD4_NOCV}{CVSPEC}?
340 REFWORD        {FILEMASK}|{LABELID}|{REFWORD2}|{REFWORD3}|{REFWORD4}
341 REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
342 RCSID "$"("Author"|"Date"|"Header"|"Id"|"Locker"|"Log"|"Name"|"RCSfile"|"Revision"|"Source"|"State")":"[^:\n$][^\n$]*"$"
343 LINENR {BLANK}*[1-9][0-9]*
344 
345 %option noyywrap
346 
347 %x St_Para
348 %x St_Comment
349 %x St_Title
350 %x St_TitleN
351 %x St_TitleQ
352 %x St_TitleA
353 %x St_TitleV
354 %x St_Code
355 %x St_CodeOpt
356 %x St_XmlCode
357 %x St_HtmlOnly
358 %x St_HtmlOnlyOption
359 %x St_ManOnly
360 %x St_LatexOnly
361 %x St_RtfOnly
362 %x St_XmlOnly
363 %x St_DbOnly
364 %x St_Verbatim
365 %x St_ILiteral
366 %x St_ILiteralOpt
367 %x St_Dot
368 %x St_Msc
369 %x St_PlantUMLOpt
370 %x St_PlantUML
371 %x St_Param
372 %x St_XRefItem
373 %x St_XRefItem2
374 %x St_File
375 %x St_Pattern
376 %x St_Link
377 %x St_Cite
378 %x St_Ref
379 %x St_Ref2
380 %x St_IntRef
381 %x St_Text
382 %x St_SkipTitle
383 %x St_Anchor
384 %x St_Snippet
385 %x St_SetScope
386 %x St_SetScopeEnd
387 %x St_Options
388 %x St_Block
389 %x St_Emoji
390 %x St_Iline
391 
392 %x St_Sections
393 %s St_SecLabel1
394 %s St_SecLabel2
395 %s St_SecTitle
396 %x St_SecSkip
397 
398 %%
399 <St_Para>\r            /* skip carriage return */
400 <St_Para>^{LISTITEM}   { /* list item */
401                          lineCount(yytext,yyleng);
402                          QCString text(yytext);
403                          uint dashPos = static_cast<uint>(text.findRev('-'));
404                          assert(dashPos!=static_cast<uint>(-1));
405                          yyextra->token->isEnumList = text.at(dashPos+1)=='#';
406                          yyextra->token->id         = -1;
407                          yyextra->token->indent     = computeIndent(yytext,dashPos);
408                          return TK_LISTITEM;
409                        }
410 <St_Para>^{MLISTITEM}  { /* list item */
411                          if (!yyextra->markdownSupport || yyextra->insidePre)
412                          {
413                            REJECT;
414                          }
415                          else
416                          {
417                            lineCount(yytext,yyleng);
418                            std::string text(yytext);
419                            static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
420                            reg::Match match;
421                            reg::search(text,match,re);
422                            size_t listPos = match.position();
423                            assert(listPos!=std::string::npos);
424                            yyextra->token->isEnumList = FALSE;
425                            yyextra->token->id         = -1;
426                            yyextra->token->indent     = computeIndent(yytext,listPos);
427                            return TK_LISTITEM;
428                          }
429                        }
430 <St_Para>^{OLISTITEM}  { /* numbered list item */
431                          if (!yyextra->markdownSupport || yyextra->insidePre)
432                          {
433                            REJECT;
434                          }
435                          else
436                          {
437                            std::string text(yytext);
438                            static const reg::Ex re(R"(\d+)");
439                            reg::Match match;
440                            reg::search(text,match,re);
441                            size_t markPos = match.position();
442                            assert(markPos!=std::string::npos);
443                            yyextra->token->isEnumList = true;
444                            bool ok = false;
445                            int id = QCString(match.str()).toInt(&ok);
446                            yyextra->token->id = ok ? id : -1;
447                            if (!ok)
448                            {
449                              warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '%s' ",match.str().c_str());
450                            }
451                            yyextra->token->indent     = computeIndent(yytext,markPos);
452                            return TK_LISTITEM;
453                          }
454                        }
455 <St_Para>{BLANK}*(\n|"\\ilinebr"){LISTITEM}     { /* list item on next line */
456                          lineCount(yytext,yyleng);
457                          QCString text=extractPartAfterNewLine(QCString(yytext));
458                          uint dashPos = static_cast<uint>(text.findRev('-'));
459                          assert(dashPos!=static_cast<uint>(-1));
460                          yyextra->token->isEnumList = text.at(dashPos+1)=='#';
461                          yyextra->token->id         = -1;
462                          yyextra->token->indent     = computeIndent(text.data(),dashPos);
463                          return TK_LISTITEM;
464                        }
465 <St_Para>{BLANK}*(\n|"\\ilinebr"){MLISTITEM}     { /* list item on next line */
466                          if (!yyextra->markdownSupport || yyextra->insidePre)
467                          {
468                            REJECT;
469                          }
470                          else
471                          {
472                            lineCount(yytext,yyleng);
473                            std::string text=extractPartAfterNewLine(QCString(yytext)).str();
474                            static const reg::Ex re(R"([*+][^*+]*$)"); // find last + or *
475                            reg::Match match;
476                            reg::search(text,match,re);
477                            size_t markPos = match.position();
478                            assert(markPos!=std::string::npos);
479                            yyextra->token->isEnumList = FALSE;
480                            yyextra->token->id         = -1;
481                            yyextra->token->indent     = computeIndent(text.c_str(),markPos);
482                            return TK_LISTITEM;
483                          }
484                        }
485 <St_Para>{BLANK}*(\n|"\\ilinebr"){OLISTITEM}     { /* list item on next line */
486                          if (!yyextra->markdownSupport || yyextra->insidePre)
487                          {
488                            REJECT;
489                          }
490                          else
491                          {
492                            lineCount(yytext,yyleng);
493                            std::string text=extractPartAfterNewLine(QCString(yytext)).str();
494                            static const reg::Ex re(R"(\d+)");
495                            reg::Match match;
496                            reg::search(text,match,re);
497                            size_t markPos = match.position();
498                            assert(markPos!=std::string::npos);
499                            yyextra->token->isEnumList = true;
500                            bool ok = false;
501                            int id = QCString(match.str()).toInt(&ok);
502                            yyextra->token->id = ok ? id : -1;
503                            if (!ok)
504                            {
505                              warn(yyextra->fileName,yyextra->yyLineNr,"Invalid number for list item '%s' ",match.str().c_str());
506                            }
507                            yyextra->token->indent     = computeIndent(text.c_str(),markPos);
508                            return TK_LISTITEM;
509                          }
510                        }
511 <St_Para>^{ENDLIST}       { /* end list */
512                          lineCount(yytext,yyleng);
513                          size_t dotPos = static_cast<size_t>(QCString(yytext).findRev('.'));
514                          yyextra->token->indent     = computeIndent(yytext,dotPos);
515                          return TK_ENDLIST;
516                        }
517 <St_Para>{BLANK}*(\n|"\\ilinebr"){ENDLIST}      { /* end list on next line */
518                          lineCount(yytext,yyleng);
519                          QCString text=extractPartAfterNewLine(QCString(yytext));
520                          size_t dotPos = static_cast<size_t>(text.findRev('.'));
521                          yyextra->token->indent = computeIndent(text.data(),dotPos);
522                          return TK_ENDLIST;
523                        }
524 <St_Para>"{"{BLANK}*"@linkplain"/{WS}+ {
525                          yyextra->token->name = "javalinkplain";
526                          return TK_COMMAND_AT;
527                        }
528 <St_Para>"{"{BLANK}*"@link"/{WS}+ {
529                          yyextra->token->name = "javalink";
530                          return TK_COMMAND_AT;
531                        }
532 <St_Para>"{"{BLANK}*"@inheritDoc"{BLANK}*"}" {
533                          yyextra->token->name = "inheritdoc";
534                          return TK_COMMAND_AT;
535                        }
536 <St_Para>"@_fakenl"    { // artificial new line
537                          //yyextra->yyLineNr++;
538                        }
539 <St_Para>{SPCMD3}      {
540                          yyextra->token->name = "_form";
541                          bool ok;
542                          yyextra->token->id = QCString(yytext).right((int)yyleng-7).toInt(&ok);
543                          ASSERT(ok);
544                          return TK_COMMAND_SEL();
545                        }
546 <St_Para>{CMD}"n"\n    { /* \n followed by real newline */
547                          lineCount(yytext,yyleng);
548                          //yyextra->yyLineNr++;
549                          yyextra->token->name = yytext+1;
550                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
551                          yyextra->token->paramDir=TokenInfo::Unspecified;
552                          return TK_COMMAND_SEL();
553                        }
554 <St_Para>"\\ilinebr"   {
555                        }
556 <St_Para>{SPCMD1}      |
557 <St_Para>{SPCMD2}      |
558 <St_Para>{SPCMD5}      |
559 <St_Para>{SPCMD4}      { /* special command */
560                          yyextra->token->name = yytext+1;
561                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
562                          yyextra->token->paramDir=TokenInfo::Unspecified;
563                          return TK_COMMAND_SEL();
564                        }
565 <St_Para>{PARAMIO}     { /* param [in,out] command */
566                          yyextra->token->name = "param";
567                          QCString s(yytext);
568                          bool isIn  = s.find("in")!=-1;
569                          bool isOut = s.find("out")!=-1;
570                          if (isIn)
571                          {
572                            if (isOut)
573                            {
574                              yyextra->token->paramDir=TokenInfo::InOut;
575                            }
576                            else
577                            {
578                              yyextra->token->paramDir=TokenInfo::In;
579                            }
580                          }
581                          else if (isOut)
582                          {
583                            yyextra->token->paramDir=TokenInfo::Out;
584                          }
585                          else
586                          {
587                            yyextra->token->paramDir=TokenInfo::Unspecified;
588                          }
589                          return TK_COMMAND_SEL();
590                        }
591 <St_Para>{URLPROTOCOL}{URLMASK}/[,\.] { // URL, or URL.
592                          yyextra->token->name=yytext;
593                          yyextra->token->isEMailAddr=FALSE;
594                          return TK_URL;
595                        }
596 <St_Para>{URLPROTOCOL}{URLMASK} { // URL
597                          yyextra->token->name=yytext;
598                          yyextra->token->isEMailAddr=FALSE;
599                          return TK_URL;
600                        }
601 <St_Para>"<"{URLPROTOCOL}{URLMASK}">" { // URL
602                          yyextra->token->name=yytext;
603                          yyextra->token->name = yyextra->token->name.mid(1,yyextra->token->name.length()-2);
604                          yyextra->token->isEMailAddr=FALSE;
605                          return TK_URL;
606                        }
607 <St_Para>{MAILADDR}    { // Mail address
608                          yyextra->token->name=yytext;
609                          yyextra->token->name.stripPrefix("mailto:");
610                          yyextra->token->isEMailAddr=TRUE;
611                          return TK_URL;
612                        }
613 <St_Para>"<"{MAILADDR}">" { // Mail address
614                          yyextra->token->name=yytext;
615                          yyextra->token->name = yyextra->token->name.mid(1,yyextra->token->name.length()-2);
616                          yyextra->token->name.stripPrefix("mailto:");
617                          yyextra->token->isEMailAddr=TRUE;
618                          return TK_URL;
619                        }
620 <St_Para>"<"{MAILADDR2}">" { // anti spam mail address
621                          yyextra->token->name=yytext;
622                          return TK_WORD;
623                        }
624 <St_Para>{RCSID} { /* RCS tag */
625                          QCString tagName(yytext+1);
626                          int index=tagName.find(':');
627                          if (index<0) index=0; // should never happen
628                          yyextra->token->name = tagName.left(index);
629                          int text_begin = index+2;
630                          int text_end = tagName.length()-1;
631                          if (tagName[text_begin-1]==':') /* check for Subversion fixed-length keyword */
632                          {
633                                  ++text_begin;
634                                  if (tagName[text_end-1]=='#')
635                                          --text_end;
636                          }
637                          yyextra->token->text = tagName.mid(text_begin,text_end-text_begin);
638                          return TK_RCSTAG;
639                        }
640 <St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}")"   | /* environment variable */
641 <St_Para,St_HtmlOnly,St_ManOnly,St_LatexOnly,St_RtfOnly,St_XmlOnly,St_DbOnly>"$("{ID}"("{ID}"))"   { /* environment variable */
642                          QCString name(&yytext[2]);
643                          name = name.left(name.length()-1);
644                          QCString value = Portable::getenv(name);
645                          for (int i=value.length()-1;i>=0;i--) unput(value.at(i));
646                        }
647 <St_Para>{HTMLTAG}     { /* html tag */
648                          lineCount(yytext,yyleng);
649                          handleHtmlTag(yyscanner,yytext);
650                          return TK_HTMLTAG;
651                        }
652 <St_Para,St_Text>"&"{ID}";" { /* special symbol */
653                          yyextra->token->name = yytext;
654                          return TK_SYMBOL;
655                        }
656 
657   /********* patterns for linkable words ******************/
658 
659 <St_Para>{ID}/"<"{HTMLKEYW}">" { /* this rule is to prevent opening html
660                                   * tag to be recognized as a templated classes
661                                   */
662                          yyextra->token->name = yytext;
663                          return TK_LNKWORD;
664                         }
665 <St_Para>{LNKWORD1}/"<tt>" { // prevent <tt> html tag to be parsed as template arguments
666                          yyextra->token->name = yytext;
667                          return TK_LNKWORD;
668                        }
669 <St_Para>{LNKWORD1}/"<br>"           | // prevent <br> html tag to be parsed as template arguments
670 <St_Para>{LNKWORD1}                  |
671 <St_Para>{LNKWORD1}{FUNCARG}         |
672 <St_Para>{LNKWORD2}                  |
673 <St_Para>{LNKWORD3}    {
674                          yyextra->token->name = yytext;
675                          return TK_LNKWORD;
676                        }
677 <St_Para>{LNKWORD1}{FUNCARG}{CVSPEC}[^a-z_A-Z0-9] {
678                          yyextra->token->name = yytext;
679                          yyextra->token->name = yyextra->token->name.left(yyextra->token->name.length()-1);
680                          unput(yytext[(int)yyleng-1]);
681                          return TK_LNKWORD;
682                        }
683   /********* patterns for normal words ******************/
684 
685 <St_Para,St_Text>[\-+0-9] |
686 <St_Para,St_Text>{WORD1} |
687 <St_Para,St_Text>{WORD2} { /* function call */
688                          if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
689                          lineCount(yytext,yyleng);
690                          if (yytext[0]=='%') // strip % if present
691                            yyextra->token->name = &yytext[1];
692                          else
693                            yyextra->token->name = yytext;
694                          return TK_WORD;
695                        }
696 <St_Text>({ID}".")+{ID} {
697                           yyextra->token->name = yytext;
698                           return TK_WORD;
699                         }
700 <St_Para,St_Text>"operator"/{BLANK}*"<"[a-zA-Z_0-9]+">" { // Special case: word "operator" followed by a HTML command
701                                                           // avoid interpretation as "operator <"
702                            yyextra->token->name = yytext;
703                            return TK_WORD;
704                          }
705 
706   /*******************************************************/
707 
708 <St_Para,St_Text>{BLANK}+      |
709 <St_Para,St_Text>{BLANK}*\n{BLANK}* { /* white space */
710                          lineCount(yytext,yyleng);
711                          yyextra->token->chars=yytext;
712                          return TK_WHITESPACE;
713                        }
714 <St_Text>[\\@<>&$#%~]  {
715                          yyextra->token->name = yytext;
716                          return TK_COMMAND_SEL();
717                        }
718 <St_Para>({BLANK}*\n)+{BLANK}*\n/{LISTITEM} { /* skip trailing paragraph followed by new list item */
719                          if (yyextra->insidePre || yyextra->autoListLevel==0)
720                          {
721                            REJECT;
722                          }
723                          lineCount(yytext,yyleng);
724                        }
725 <St_Para>({BLANK}*\n)+{BLANK}*\n/{MLISTITEM} { /* skip trailing paragraph followed by new list item */
726                          if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
727                          {
728                            REJECT;
729                          }
730                          lineCount(yytext,yyleng);
731                        }
732 <St_Para>({BLANK}*\n)+{BLANK}*\n/{OLISTITEM} { /* skip trailing paragraph followed by new list item */
733                          if (!yyextra->markdownSupport || yyextra->insidePre || yyextra->autoListLevel==0)
734                          {
735                            REJECT;
736                          }
737                          lineCount(yytext,yyleng);
738                        }
739 <St_Para,St_Param>({BLANK}*(\n|"\\ilinebr"))+{BLANK}*(\n|"\\ilinebr"){BLANK}* {
740                          lineCount(yytext,yyleng);
741                          if (yyextra->insidePre)
742                          {
743                            yyextra->token->chars=yytext;
744                            return TK_WHITESPACE;
745                          }
746                          else
747                          {
748                            yyextra->token->indent=computeIndent(yytext,yyleng);
749                            int i;
750                            // put back the indentation (needed for list items)
751                            for (i=0;i<yyextra->token->indent;i++)
752                            {
753                              unput(' ');
754                            }
755                            // tell flex that after putting the last indent
756                            // back we are at the beginning of the line
757                            YY_CURRENT_BUFFER->yy_at_bol=1;
758                            // start of a new paragraph
759                            return TK_NEWPARA;
760                          }
761                        }
762 <St_CodeOpt>{BLANK}*"{"(".")?{LABELID}"}" {
763                          yyextra->token->name = yytext;
764                          int i=yyextra->token->name.find('{'); /* } to keep vi happy */
765                          yyextra->token->name = yyextra->token->name.mid(i+1,yyextra->token->name.length()-i-2);
766                          BEGIN(St_Code);
767                        }
768 <St_CodeOpt>"\\ilinebr" |
769 <St_CodeOpt>\n         |
770 <St_CodeOpt>.          {
771                          unput_string(yytext,yyleng);
772                          BEGIN(St_Code);
773                        }
774 <St_Code>{WS}*{CMD}"endcode" {
775                          lineCount(yytext,yyleng);
776                          return RetVal_OK;
777                        }
778 <St_XmlCode>{WS}*"</code>" {
779                          lineCount(yytext,yyleng);
780                          return RetVal_OK;
781                        }
782 <St_Code,St_XmlCode>[^\\@\n<]+  |
783 <St_Code,St_XmlCode>\n          |
784 <St_Code,St_XmlCode>.           {
785                          lineCount(yytext,yyleng);
786                          yyextra->token->verb+=yytext;
787                        }
788 <St_HtmlOnlyOption>" [block]" { // the space is added in commentscan.l
789                          yyextra->token->name="block";
790                          BEGIN(St_HtmlOnly);
791                         }
792 <St_HtmlOnlyOption>.|\n {
793                          unput(*yytext);
794                          BEGIN(St_HtmlOnly);
795                         }
796 <St_HtmlOnlyOption>"\\ilinebr" {
797                          unput_string(yytext,yyleng);
798                          BEGIN(St_HtmlOnly);
799                         }
800 <St_HtmlOnly>{CMD}"endhtmlonly" {
801                          return RetVal_OK;
802                        }
803 <St_HtmlOnly>[^\\@\n$]+    |
804 <St_HtmlOnly>\n            |
805 <St_HtmlOnly>.             {
806                          lineCount(yytext,yyleng);
807                          yyextra->token->verb+=yytext;
808                        }
809 <St_ManOnly>{CMD}"endmanonly" {
810                          return RetVal_OK;
811                        }
812 <St_ManOnly>[^\\@\n$]+    |
813 <St_ManOnly>\n            |
814 <St_ManOnly>.             {
815                          lineCount(yytext,yyleng);
816                          yyextra->token->verb+=yytext;
817                        }
818 <St_RtfOnly>{CMD}"endrtfonly" {
819                          return RetVal_OK;
820                        }
821 <St_RtfOnly>[^\\@\n$]+    |
822 <St_RtfOnly>\n            |
823 <St_RtfOnly>.             {
824                          lineCount(yytext,yyleng);
825                          yyextra->token->verb+=yytext;
826                        }
827 <St_LatexOnly>{CMD}"endlatexonly" {
828                          return RetVal_OK;
829                        }
830 <St_LatexOnly>[^\\@\n]+     |
831 <St_LatexOnly>\n            |
832 <St_LatexOnly>.             {
833                          lineCount(yytext,yyleng);
834                          yyextra->token->verb+=yytext;
835                        }
836 <St_XmlOnly>{CMD}"endxmlonly" {
837                          return RetVal_OK;
838                        }
839 <St_XmlOnly>[^\\@\n]+  |
840 <St_XmlOnly>\n         |
841 <St_XmlOnly>.          {
842                          lineCount(yytext,yyleng);
843                          yyextra->token->verb+=yytext;
844                        }
845 <St_DbOnly>{CMD}"enddocbookonly" {
846                          return RetVal_OK;
847                        }
848 <St_DbOnly>[^\\@\n]+  |
849 <St_DbOnly>\n         |
850 <St_DbOnly>.          {
851                          lineCount(yytext,yyleng);
852                          yyextra->token->verb+=yytext;
853                        }
854 <St_Verbatim>{CMD}"endverbatim" {
855                          yyextra->token->verb=stripEmptyLines(yyextra->token->verb);
856                          return RetVal_OK;
857                        }
858 <St_ILiteral>{CMD}"endiliteral " { // note extra space as this is artificially added
859                          // remove spaces that have been added
860                          yyextra->token->verb=yyextra->token->verb.mid(1,yyextra->token->verb.length()-2);
861                          return RetVal_OK;
862                        }
863 <St_Verbatim,St_ILiteral>[^\\@\n]+ |
864 <St_Verbatim,St_ILiteral>\n        |
865 <St_Verbatim,St_ILiteral>.         { /* Verbatim / javadac literal/code text */
866                          lineCount(yytext,yyleng);
867                          yyextra->token->verb+=yytext;
868                        }
869 <St_ILiteralOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // option(s) present
870                          yyextra->token->verb = QCString(yytext).stripWhiteSpace();
871                          return RetVal_OK;
872                        }
873 <St_ILiteralOpt>"\\ilinebr" |
874 <St_ILiteralOpt>"\n"   |
875 <St_ILiteralOpt>.      {
876                          yyextra->token->sectionId = "";
877                          unput_string(yytext,yyleng);
878                          return RetVal_OK;
879                        }
880 <St_Dot>{CMD}"enddot"  {
881                          return RetVal_OK;
882                        }
883 <St_Dot>[^\\@\n]+      |
884 <St_Dot>\n             |
885 <St_Dot>.              { /* dot text */
886                          lineCount(yytext,yyleng);
887                          yyextra->token->verb+=yytext;
888                        }
889 <St_Msc>{CMD}("endmsc")  {
890                          return RetVal_OK;
891                        }
892 <St_Msc>[^\\@\n]+      |
893 <St_Msc>\n             |
894 <St_Msc>.              { /* msc text */
895                          lineCount(yytext,yyleng);
896                          yyextra->token->verb+=yytext;
897                        }
898 <St_PlantUMLOpt>{BLANK}*"{"[a-zA-Z_,:0-9\. ]*"}" { // case 1: options present
899                          yyextra->token->sectionId = QCString(yytext).stripWhiteSpace();
900                          return RetVal_OK;
901                        }
902 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/{ID}"=" { // case 2: plain file name specified followed by an attribute
903                          yyextra->token->sectionId = QCString(yytext).stripWhiteSpace();
904                          return RetVal_OK;
905                        }
906 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANK}+/"\"" { // case 3: plain file name specified followed by a quoted title
907                          yyextra->token->sectionId = QCString(yytext).stripWhiteSpace();
908                          return RetVal_OK;
909                        }
910 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/\n { // case 4: plain file name specified without title or attributes
911                          yyextra->token->sectionId = QCString(yytext).stripWhiteSpace();
912                          return RetVal_OK;
913                        }
914 <St_PlantUMLOpt>{BLANK}*{FILEMASK}{BLANKopt}/"\\ilinebr" { // case 5: plain file name specified without title or attributes
915                          yyextra->token->sectionId = QCString(yytext).stripWhiteSpace();
916                          return RetVal_OK;
917                        }
918 <St_PlantUMLOpt>"\\ilinebr" |
919 <St_PlantUMLOpt>"\n"   |
920 <St_PlantUMLOpt>.      {
921                          yyextra->token->sectionId = "";
922                          unput_string(yytext,yyleng);
923                          return RetVal_OK;
924                        }
925 <St_PlantUML>{CMD}"enduml"  {
926                          return RetVal_OK;
927                        }
928 <St_PlantUML>[^\\@\n]+ |
929 <St_PlantUML>\n        |
930 <St_PlantUML>.         { /* plantuml text */
931                          lineCount(yytext,yyleng);
932                          yyextra->token->verb+=yytext;
933                        }
934 <St_Title>"\""         { // quoted title
935                          BEGIN(St_TitleQ);
936                        }
937 <St_Title>[ \t]+       {
938                          yyextra->token->chars=yytext;
939                          return TK_WHITESPACE;
940                        }
941 <St_Title>.            { // non-quoted title
942                          unput(*yytext);
943                          BEGIN(St_TitleN);
944                        }
945 <St_Title>\n           {
946                          unput(*yytext);
947                          return 0;
948                        }
949 <St_Title>"\\ilinebr"  {
950                          unput_string(yytext,yyleng);
951                          return 0;
952                        }
953 <St_TitleN>"&"{ID}";"  { /* symbol */
954                          yyextra->token->name = yytext;
955                          return TK_SYMBOL;
956                        }
957 <St_TitleN>{HTMLTAG}   {
958                          lineCount(yytext,yyleng);
959                        }
960 <St_TitleN>\n          { /* new line => end of title */
961                          unput(*yytext);
962                          return 0;
963                        }
964 <St_TitleN>"\\ilinebr" { /* new line => end of title */
965                          unput_string(yytext,yyleng);
966                          return 0;
967                        }
968 <St_TitleN>{SPCMD1}    |
969 <St_TitleN>{SPCMD2}    { /* special command */
970                          yyextra->token->name = yytext+1;
971                          yyextra->token->paramDir=TokenInfo::Unspecified;
972                          return TK_COMMAND_SEL();
973                        }
974 <St_TitleN>{ID}"="     { /* attribute */
975                          if (yytext[0]=='%') // strip % if present
976                            yyextra->token->name = &yytext[1];
977                          else
978                            yyextra->token->name = yytext;
979                          return TK_WORD;
980                        }
981 <St_TitleN>[\-+0-9]    |
982 <St_TitleN>{WORD1}     |
983 <St_TitleN>{WORD2}     { /* word */
984                          if (QCString(yytext).find("\\ilinebr")!=-1) REJECT; // see issue #8311
985                          lineCount(yytext,yyleng);
986                          if (yytext[0]=='%') // strip % if present
987                            yyextra->token->name = &yytext[1];
988                          else
989                            yyextra->token->name = yytext;
990                          return TK_WORD;
991                        }
992 <St_TitleN>[ \t]+      {
993                          yyextra->token->chars=yytext;
994                          return TK_WHITESPACE;
995                        }
996 <St_TitleQ>"&"{ID}";"  { /* symbol */
997                          yyextra->token->name = yytext;
998                          return TK_SYMBOL;
999                        }
1000 <St_TitleQ>(\n|"\\ilinebr") { /* new line => end of title */
1001                          unput_string(yytext,yyleng);
1002                          return 0;
1003                        }
1004 <St_TitleQ>{SPCMD1}    |
1005 <St_TitleQ>{SPCMD2}    { /* special command */
1006                          yyextra->token->name = yytext+1;
1007                          yyextra->token->paramDir=TokenInfo::Unspecified;
1008                          return TK_COMMAND_SEL();
1009                        }
1010 <St_TitleQ>{WORD1NQ}   |
1011 <St_TitleQ>{WORD2NQ}   { /* word */
1012                          yyextra->token->name = yytext;
1013                          return TK_WORD;
1014                        }
1015 <St_TitleQ>[ \t]+      {
1016                          yyextra->token->chars=yytext;
1017                          return TK_WHITESPACE;
1018                        }
1019 <St_TitleQ>"\""        { /* closing quote => end of title */
1020                          BEGIN(St_TitleA);
1021                          return 0;
1022                        }
1023 <St_TitleA>{BLANK}*{ID}{BLANK}*"="{BLANK}* { // title attribute
1024                          yyextra->token->name = yytext;
1025                          int pos = yyextra->token->name.find('=');
1026                          if (pos<0) pos=0; // should never happen
1027                          yyextra->token->name = yyextra->token->name.left(pos).stripWhiteSpace();
1028                          BEGIN(St_TitleV);
1029                        }
1030 <St_TitleV>[^ \t\r\n]+ { // attribute value
1031                          lineCount(yytext,yyleng);
1032                          yyextra->token->chars = yytext;
1033                          BEGIN(St_TitleN);
1034                          return TK_WORD;
1035                        }
1036 <St_TitleV,St_TitleA>. {
1037                          unput(*yytext);
1038                          return 0;
1039                        }
1040 <St_TitleV,St_TitleA>(\n|"\\ilinebr") {
1041                          unput_string(yytext,yyleng);
1042                          return 0;
1043                        }
1044 
1045 <St_Anchor>{LABELID}{WS}? { // anchor
1046                          lineCount(yytext,yyleng);
1047                          yyextra->token->name = QCString(yytext).stripWhiteSpace();
1048                          return TK_WORD;
1049                        }
1050 <St_Anchor>.           {
1051                          unput(*yytext);
1052                          return 0;
1053                        }
1054 <St_Cite>{CITEID}      { // label to cite
1055                          if (yytext[0] =='"')
1056                          {
1057                            yyextra->token->name=yytext+1;
1058                            yyextra->token->name=yyextra->token->name.left(static_cast<uint>(yyleng)-2);
1059                          }
1060                          else
1061                          {
1062                            yyextra->token->name=yytext;
1063                          }
1064                          return TK_WORD;
1065                        }
1066 <St_Cite>{BLANK}       { // white space
1067                          unput(' ');
1068                          return 0;
1069                        }
1070 <St_Cite>(\n|"\\ilinebr")  { // new line
1071                          unput_string(yytext,yyleng);
1072                          return 0;
1073                        }
1074 <St_Cite>.             { // any other character
1075                          unput(*yytext);
1076                          return 0;
1077                        }
1078 <St_Ref>{REFWORD_NOCV}/{BLANK}("const")[a-z_A-Z0-9] { // see bug776988
1079                          yyextra->token->name=yytext;
1080                          return TK_WORD;
1081                        }
1082 <St_Ref>{REFWORD_NOCV}/{BLANK}("volatile")[a-z_A-Z0-9] { // see bug776988
1083                          yyextra->token->name=yytext;
1084                          return TK_WORD;
1085                        }
1086 <St_Ref>{REFWORD}      { // label to refer to
1087                          yyextra->token->name=yytext;
1088                          return TK_WORD;
1089                        }
1090 <St_Ref>{BLANK}        { // white space
1091                          unput(' ');
1092                          return 0;
1093                        }
1094 <St_Ref>{WS}+"\""{WS}* { // white space following by quoted string
1095                          lineCount(yytext,yyleng);
1096                          BEGIN(St_Ref2);
1097                        }
1098 <St_Ref>(\n|"\\ilinebr") { // new line
1099                          unput_string(yytext,yyleng);
1100                          return 0;
1101                        }
1102 <St_Ref>.              { // any other character
1103                          unput(*yytext);
1104                          return 0;
1105                        }
1106 <St_IntRef>[A-Z_a-z0-9.:/#\-\+\(\)]+ {
1107                          yyextra->token->name = yytext;
1108                          return TK_WORD;
1109                        }
1110 <St_IntRef>{BLANK}+"\"" {
1111                          BEGIN(St_Ref2);
1112                        }
1113 <St_SetScope>({SCOPEMASK}|{ANONNS}){BLANK}|{FILEMASK} {
1114                          yyextra->token->name = yytext;
1115                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
1116                          return TK_WORD;
1117                        }
1118 <St_SetScope>{SCOPEMASK}"<" {
1119                          yyextra->token->name = yytext;
1120                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
1121                          yyextra->sharpCount=1;
1122                          BEGIN(St_SetScopeEnd);
1123                        }
1124 <St_SetScope>{BLANK}   {
1125                        }
1126 <St_SetScopeEnd>"<"    {
1127                          yyextra->token->name += yytext;
1128                          yyextra->sharpCount++;
1129                        }
1130 <St_SetScopeEnd>">"    {
1131                          yyextra->token->name += yytext;
1132                          yyextra->sharpCount--;
1133                          if (yyextra->sharpCount<=0)
1134                          {
1135                            return TK_WORD;
1136                          }
1137                        }
1138 <St_SetScopeEnd>.      {
1139                          yyextra->token->name += yytext;
1140                        }
1141 <St_Ref2>"&"{ID}";"    { /* symbol */
1142                          yyextra->token->name = yytext;
1143                          return TK_SYMBOL;
1144                        }
1145 <St_Ref2>"\""|\n|"\\ilinebr" { /* " or \n => end of title */
1146                          lineCount(yytext,yyleng);
1147                          return 0;
1148                        }
1149 <St_Ref2>{SPCMD1}      |
1150 <St_Ref2>{SPCMD2}      { /* special command */
1151                          yyextra->token->name = yytext+1;
1152                          yyextra->token->paramDir=TokenInfo::Unspecified;
1153                          return TK_COMMAND_SEL();
1154                        }
1155 <St_Ref2>{WORD1NQ}     |
1156 <St_Ref2>{WORD2NQ}     {
1157                          /* word */
1158                          yyextra->token->name = yytext;
1159                          return TK_WORD;
1160                        }
1161 <St_Ref2>[ \t]+        {
1162                          yyextra->token->chars=yytext;
1163                          return TK_WHITESPACE;
1164                        }
1165 <St_XRefItem>{LABELID} {
1166                          yyextra->token->name=yytext;
1167                        }
1168 <St_XRefItem>" "       {
1169                          BEGIN(St_XRefItem2);
1170                        }
1171 <St_XRefItem2>[0-9]+"." {
1172                          QCString numStr(yytext);
1173                          numStr=numStr.left((int)yyleng-1);
1174                          yyextra->token->id=numStr.toInt();
1175                          return RetVal_OK;
1176                        }
1177 <St_Para,St_Title,St_Ref2>"<!--"     { /* html style comment block */
1178                          yyextra->commentState = YY_START;
1179                          BEGIN(St_Comment);
1180                        }
1181 <St_Param>"\""[^\n\"]+"\"" {
1182                          yyextra->token->name = yytext+1;
1183                          yyextra->token->name = yyextra->token->name.left((int)yyleng-2);
1184                          return TK_WORD;
1185                        }
1186 <St_Param>({PHPTYPE}{BLANK}*("["{BLANK}*"]")*{BLANK}*"|"{BLANK}*)*{PHPTYPE}{BLANK}*("["{BLANK}*"]")*{WS}+("&")?"$"{LABELID} {
1187                          lineCount(yytext,yyleng);
1188                          QCString params(yytext);
1189                          int j = params.find('&');
1190                          int i = params.find('$');
1191                          if (i<0) i=0; // should never happen
1192                          if (j<i && j>=0) i=j;
1193                          QCString types = params.left(i).stripWhiteSpace();
1194                          yyextra->token->name = types+"#"+params.mid(i);
1195                          return TK_WORD;
1196                        }
1197 <St_Param>[^ \t\n,@\\]+  {
1198                          yyextra->token->name = yytext;
1199                          if (yyextra->token->name.at(static_cast<uint>(yyleng)-1)==':')
1200                          {
1201                            yyextra->token->name=yyextra->token->name.left(static_cast<uint>(yyleng)-1);
1202                          }
1203                          return TK_WORD;
1204                        }
1205 <St_Param>{WS}*","{WS}*  /* param separator */
1206 <St_Param>{WS}         {
1207                          lineCount(yytext,yyleng);
1208                          yyextra->token->chars=yytext;
1209                          return TK_WHITESPACE;
1210                        }
1211 <St_Options>{ID}       {
1212                          yyextra->token->name+=yytext;
1213                        }
1214 <St_Options>{WS}*":"{WS}* {
1215                          lineCount(yytext,yyleng);
1216                          yyextra->token->name+=":";
1217                        }
1218 <St_Options>{WS}*","{WS}* |
1219 <St_Options>{WS}       { /* option separator */
1220                          lineCount(yytext,yyleng);
1221                          yyextra->token->name+=",";
1222                        }
1223 <St_Options>"}"        {
1224                          return TK_WORD;
1225                        }
1226 <St_Block>{ID}         {
1227                          yyextra->token->name+=yytext;
1228                        }
1229 <St_Block>"]"          {
1230                          return TK_WORD;
1231                        }
1232 <St_Emoji>[:0-9_a-z+-]+  {
1233                          yyextra->token->name=yytext;
1234                          return TK_WORD;
1235                        }
1236 <St_Emoji>.            {
1237                          return 0;
1238                        }
1239 <St_Iline>{LINENR}/[\n\.]     |
1240 <St_Iline>{LINENR}{BLANK}     {
1241                          bool ok = false;
1242                          int nr = QCString(yytext).toInt(&ok);
1243                          if (!ok)
1244                          {
1245                            warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext);
1246                          }
1247                          else
1248                          {
1249                            yyextra->yyLineNr = nr;
1250                          }
1251                          return TK_WORD;
1252                        }
1253 <St_Iline>.            {
1254                          return 0;
1255                        }
1256 <St_File>{FILEMASK}    {
1257                          yyextra->token->name = yytext;
1258                          return TK_WORD;
1259                        }
1260 <St_File>"\""[^\n\"]+"\"" {
1261                          QCString text(yytext);
1262                          yyextra->token->name = text.mid(1,text.length()-2);
1263                          return TK_WORD;
1264                        }
1265 <St_Pattern>[^\\\r\n]+   {
1266                          yyextra->token->name += yytext;
1267                        }
1268 <St_Pattern>"\\ilinebr" {
1269                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
1270                          return TK_WORD;
1271                        }
1272 <St_Pattern>\n         {
1273                          lineCount(yytext,yyleng);
1274                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
1275                          return TK_WORD;
1276                        }
1277 <St_Pattern>.          {
1278                          yyextra->token->name += yytext;
1279                        }
1280 <St_Link>{LINKMASK}|{REFWORD}    {
1281                          yyextra->token->name = yytext;
1282                          return TK_WORD;
1283                        }
1284 <St_Comment>"-->"      { /* end of html comment */
1285                          BEGIN(yyextra->commentState);
1286                        }
1287 <St_Comment>[^-]+      /* inside html comment */
1288 <St_Comment>.          /* inside html comment */
1289 
1290      /* State for skipping title (all chars until the end of the line) */
1291 
1292 <St_SkipTitle>.
1293 <St_SkipTitle>(\n|"\\ilinebr") {
1294                          lineCount(yytext,yyleng);
1295                          return 0;
1296                          }
1297 
1298      /* State for the pass used to find the anchors and sections */
1299 
1300 <St_Sections>[^\n@\\<]+
1301 <St_Sections>{CMD}("<"|{CMD})
1302 <St_Sections>"<"{CAPTION}({WS}+{ATTRIB})*">" {
1303                                       lineCount(yytext,yyleng);
1304                                       QCString tag(yytext);
1305                                       int s=tag.find("id=");
1306                                       if (s!=-1) // command has id attribute
1307                                       {
1308                                         char c=tag[s+3];
1309                                         if (c=='\'' || c=='"') // valid start
1310                                         {
1311                                           int e=tag.find(c,s+4);
1312                                           if (e!=-1) // found matching end
1313                                           {
1314                                             yyextra->secType = SectionType::Table;
1315                                             yyextra->secLabel=tag.mid(s+4,e-s-4); // extract id
1316                                             processSection(yyscanner);
1317                                           }
1318                                         }
1319                                       }
1320                                     }
1321 <St_Sections>{CMD}"anchor"{BLANK}+  {
1322                                       yyextra->secType = SectionType::Anchor;
1323                                       BEGIN(St_SecLabel1);
1324                                     }
1325 <St_Sections>{CMD}"section"{BLANK}+ {
1326                                       yyextra->secType = SectionType::Section;
1327                                       BEGIN(St_SecLabel2);
1328                                     }
1329 <St_Sections>{CMD}"subsection"{BLANK}+ {
1330                                       yyextra->secType = SectionType::Subsection;
1331                                       BEGIN(St_SecLabel2);
1332                                     }
1333 <St_Sections>{CMD}"subsubsection"{BLANK}+ {
1334                                       yyextra->secType = SectionType::Subsubsection;
1335                                       BEGIN(St_SecLabel2);
1336                                     }
1337 <St_Sections>{CMD}"paragraph"{BLANK}+ {
1338                                       yyextra->secType = SectionType::Paragraph;
1339                                       BEGIN(St_SecLabel2);
1340                                     }
1341 <St_Sections>{CMD}"verbatim"/[^a-z_A-Z0-9]  {
1342                                       yyextra->endMarker="endverbatim";
1343                                       BEGIN(St_SecSkip);
1344                                     }
1345 <St_Sections>{CMD}"iliteral"/[^a-z_A-Z0-9]  {
1346                                       yyextra->endMarker="endiliteral";
1347                                       BEGIN(St_SecSkip);
1348                                     }
1349 <St_Sections>{CMD}"dot"/[^a-z_A-Z0-9] {
1350                                       yyextra->endMarker="enddot";
1351                                       BEGIN(St_SecSkip);
1352                                     }
1353 <St_Sections>{CMD}"msc"/[^a-z_A-Z0-9] {
1354                                       yyextra->endMarker="endmsc";
1355                                       BEGIN(St_SecSkip);
1356                                     }
1357 <St_Sections>{CMD}"startuml"/[^a-z_A-Z0-9] {
1358                                       yyextra->endMarker="enduml";
1359                                       BEGIN(St_SecSkip);
1360                                     }
1361 <St_Sections>{CMD}"htmlonly"/[^a-z_A-Z0-9] {
1362                                       yyextra->endMarker="endhtmlonly";
1363                                       BEGIN(St_SecSkip);
1364                                     }
1365 <St_Sections>{CMD}"latexonly"/[^a-z_A-Z0-9] {
1366                                       yyextra->endMarker="endlatexonly";
1367                                       BEGIN(St_SecSkip);
1368                                     }
1369 <St_Sections>{CMD}"manonly"/[^a-z_A-Z0-9] {
1370                                       yyextra->endMarker="endmanonly";
1371                                       BEGIN(St_SecSkip);
1372                                     }
1373 <St_Sections>{CMD}"rtfonly"/[^a-z_A-Z0-9] {
1374                                       yyextra->endMarker="endrtfonly";
1375                                       BEGIN(St_SecSkip);
1376                                     }
1377 <St_Sections>{CMD}"xmlonly"/[^a-z_A-Z0-9] {
1378                                       yyextra->endMarker="endxmlonly";
1379                                       BEGIN(St_SecSkip);
1380                                     }
1381 <St_Sections>{CMD}"docbookonly"/[^a-z_A-Z0-9] {
1382                                       yyextra->endMarker="enddocbookonly";
1383                                       BEGIN(St_SecSkip);
1384                                     }
1385 <St_Sections>{CMD}"code"/[^a-z_A-Z0-9] {
1386                                       yyextra->endMarker="endcode";
1387                                       BEGIN(St_SecSkip);
1388                                     }
1389 <St_Sections>"<!--"                 {
1390                                       yyextra->endMarker="-->";
1391                                       BEGIN(St_SecSkip);
1392                                     }
1393 <St_SecSkip>{CMD}{ID}               {
1394                                       if (yyextra->endMarker==yytext+1)
1395                                       {
1396                                         BEGIN(St_Sections);
1397                                       }
1398                                     }
1399 <St_SecSkip>"-->"                   {
1400                                       if (yyextra->endMarker==yytext)
1401                                       {
1402                                         BEGIN(St_Sections);
1403                                       }
1404                                     }
1405 <St_SecSkip>[^a-z_A-Z0-9\-\\\@]+
1406 <St_SecSkip>.
1407 <St_SecSkip>(\n|"\\ilinebr")
1408 <St_Sections>.
1409 <St_Sections>(\n|"\\ilinebr")
1410 <St_SecLabel1>{LABELID} {
1411                          lineCount(yytext,yyleng);
1412                          yyextra->secLabel = yytext;
1413                          processSection(yyscanner);
1414                          BEGIN(St_Sections);
1415                        }
1416 <St_SecLabel2>{LABELID}{BLANK}+ |
1417 <St_SecLabel2>{LABELID}         {
1418                          yyextra->secLabel = yytext;
1419                          yyextra->secLabel = yyextra->secLabel.stripWhiteSpace();
1420                          BEGIN(St_SecTitle);
1421                        }
1422 <St_SecTitle>[^\n]+    |
1423 <St_SecTitle>[^\n]*\n  {
1424                          lineCount(yytext,yyleng);
1425                          yyextra->secTitle = yytext;
1426                          yyextra->secTitle = yyextra->secTitle.stripWhiteSpace();
1427                          if (yyextra->secTitle.right(8)=="\\ilinebr")
1428                          {
1429                            yyextra->secTitle.left(yyextra->secTitle.length()-8);
1430                          }
1431                          processSection(yyscanner);
1432                          BEGIN(St_Sections);
1433                        }
1434 <St_SecTitle,St_SecLabel1,St_SecLabel2>. {
1435                          warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s' while looking for section label or title",yytext);
1436                        }
1437 
1438 <St_Snippet>[^\\\n]+   {
1439                          yyextra->token->name += yytext;
1440                        }
1441 <St_Snippet>"\\"       {
1442                          yyextra->token->name += yytext;
1443                        }
1444 <St_Snippet>(\n|"\\ilinebr")  {
1445                          unput_string(yytext,yyleng);
1446                          yyextra->token->name = yyextra->token->name.stripWhiteSpace();
1447                          return TK_WORD;
1448                        }
1449 
1450      /* Generic rules that work for all states */
1451 <*>\n                  {
1452                          lineCount(yytext,yyleng);
1453                          warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected new line character");
1454                        }
1455 <*>"\\ilinebr"         {
1456                        }
1457 <*>[\\@<>&$#%~"=]      { /* unescaped special character */
1458                          //warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext);
1459                          yyextra->token->name = yytext;
1460                          return TK_COMMAND_SEL();
1461                        }
1462 <*>.                   {
1463                          warn(yyextra->fileName,yyextra->yyLineNr,"Unexpected character '%s'",yytext);
1464                        }
1465 %%
1466 
1467 //--------------------------------------------------------------------------
1468 
1469 static yy_size_t yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size)
1470 {
1471   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1472   yy_size_t c=0;
1473   const char *p = yyextra->inputString + yyextra->inputPos;
1474   while ( c < max_size && *p ) { *buf++ = *p++; c++; }
1475   yyextra->inputPos+=c;
1476   return c;
1477 }
1478 
1479 static void processSection(yyscan_t yyscanner)
1480 {
1481   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1482   //printf("%s: found section/anchor with name '%s'\n",qPrint(g_fileName),qPrint(g_secLabel));
1483   QCString file;
1484   if (yyextra->definition)
1485   {
1486     file = yyextra->definition->getOutputFileBase();
1487   }
1488   else
1489   {
1490     warn(yyextra->fileName,yyextra->yyLineNr,"Found section/anchor %s without context\n",qPrint(yyextra->secLabel));
1491   }
1492   SectionInfo *si = SectionManager::instance().find(yyextra->secLabel);
1493   if (si)
1494   {
1495     si->setFileName(file);
1496     si->setType(yyextra->secType);
1497   }
1498 }
1499 
1500 static void handleHtmlTag(yyscan_t yyscanner,const char *text)
1501 {
1502   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1503 
1504   QCString tagText(text);
1505   yyextra->token->attribs.clear();
1506   yyextra->token->endTag = FALSE;
1507   yyextra->token->emptyTag = FALSE;
1508 
1509   // Check for end tag
1510   int startNamePos=1;
1511   if (tagText.at(1)=='/')
1512   {
1513     yyextra->token->endTag = TRUE;
1514     startNamePos++;
1515   }
1516 
1517   // Parse the name portion
1518   int i = startNamePos;
1519   for (i=startNamePos; i < (int)yyleng; i++)
1520   {
1521     // Check for valid HTML/XML name chars (including namespaces)
1522     char c = tagText.at(i);
1523     if (!(isalnum(c) || c=='-' || c=='_' || c==':')) break;
1524   }
1525   yyextra->token->name = tagText.mid(startNamePos,i-startNamePos);
1526 
1527   // Parse the attributes. Each attribute is a name, value pair
1528   // The result is stored in yyextra->token->attribs.
1529   int startName,endName,startAttrib,endAttrib;
1530   int startAttribList = i;
1531   while (i<(int)yyleng)
1532   {
1533     char c=tagText.at(i);
1534     // skip spaces
1535     while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
1536     // check for end of the tag
1537     if (c == '>') break;
1538     // Check for XML style "empty" tag.
1539     if (c == '/')
1540     {
1541       yyextra->token->emptyTag = TRUE;
1542       break;
1543     }
1544     startName=i;
1545     // search for end of name
1546     while (i<(int)yyleng && !isspace((uchar)c) && c!='=' && c!= '>') { c=tagText.at(++i); }
1547     endName=i;
1548     HtmlAttrib opt;
1549     opt.name  = tagText.mid(startName,endName-startName).lower();
1550     // skip spaces
1551     while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
1552     if (tagText.at(i)=='=') // option has value
1553     {
1554       c=tagText.at(++i);
1555       // skip spaces
1556       while (i<(int)yyleng && isspace((uchar)c)) { c=tagText.at(++i); }
1557       if (tagText.at(i)=='\'') // option '...'
1558       {
1559         c=tagText.at(++i);
1560         startAttrib=i;
1561 
1562         // search for matching quote
1563         while (i<(int)yyleng && c!='\'') { c=tagText.at(++i); }
1564         endAttrib=i;
1565         if (i<(int)yyleng) { c=tagText.at(++i);}
1566       }
1567       else if (tagText.at(i)=='"') // option "..."
1568       {
1569         c=tagText.at(++i);
1570         startAttrib=i;
1571         // search for matching quote
1572         while (i<(int)yyleng && c!='"') { c=tagText.at(++i); }
1573         endAttrib=i;
1574         if (i<(int)yyleng) { c=tagText.at(++i);}
1575       }
1576       else // value without any quotes
1577       {
1578         startAttrib=i;
1579         // search for separator or end symbol
1580         while (i<(int)yyleng && !isspace((uchar)c) && c!='>') { c=tagText.at(++i); }
1581         endAttrib=i;
1582         if (i<(int)yyleng) { c=tagText.at(++i);}
1583       }
1584       opt.value  = tagText.mid(startAttrib,endAttrib-startAttrib);
1585       if (opt.name == "align") opt.value = opt.value.lower();
1586       else if (opt.name == "valign")
1587       {
1588         opt.value = opt.value.lower();
1589         if (opt.value == "center") opt.value="middle";
1590       }
1591     }
1592     else // start next option
1593     {
1594     }
1595     //printf("=====> Adding option name=<%s> value=<%s>\n",
1596     //    qPrint(opt.name),qPrint(opt.value));
1597     yyextra->token->attribs.push_back(opt);
1598   }
1599   yyextra->token->attribsStr = tagText.mid(startAttribList,i-startAttribList);
1600 }
1601 
1602 struct DocTokenizer::Private
1603 {
1604   yyscan_t yyscanner;
1605   doctokenizerYY_state extra;
1606 };
1607 
1608 
1609 void DocTokenizer::pushContext()
1610 {
1611   yyscan_t yyscanner = p->yyscanner;
1612   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1613   //printf("DocTokenizer::pushContext() stack=%zu\n",yyextra->lexerStack.size());
1614   yyextra->lexerStack.push(
1615       std::make_unique<DocLexerContext>(
1616         yyextra->token,YY_START,
1617         yyextra->autoListLevel,
1618         yyextra->inputPos,
1619         yyextra->inputString,
1620         YY_CURRENT_BUFFER));
1621   yy_switch_to_buffer(yy_create_buffer(0, YY_BUF_SIZE, yyscanner), yyscanner);
1622 }
1623 
1624 bool DocTokenizer::popContext()
1625 {
1626   yyscan_t yyscanner = p->yyscanner;
1627   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1628   //printf("DocTokenizer::popContext() stack=%zu\n",yyextra->lexerStack.size());
1629   if (yyextra->lexerStack.empty()) return FALSE;
1630   const auto &ctx = yyextra->lexerStack.top();
1631   yyextra->autoListLevel = ctx->autoListLevel;
1632   yyextra->inputPos      = ctx->inputPos;
1633   yyextra->inputString   = ctx->inputString;
1634 
1635   yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
1636   yy_switch_to_buffer(ctx->state, yyscanner);
1637 
1638   BEGIN(ctx->rule);
1639   yyextra->lexerStack.pop();
1640   return TRUE;
1641 }
1642 
1643 
1644 DocTokenizer::DocTokenizer() : p(std::make_unique<Private>())
1645 {
1646   //printf("%p:DocTokenizer::DocTokenizer()\n",(void*)this);
1647   doctokenizerYYlex_init_extra(&p->extra,&p->yyscanner);
1648 #ifdef FLEX_DEBUG
1649   doctokenizerYYset_debug(1,p->yyscanner);
1650 #endif
1651 }
1652 
1653 DocTokenizer::~DocTokenizer()
1654 {
1655   //printf("%p:DocTokenizer::~DocTokenizer()\n",(void*)this);
1656   doctokenizerYYlex_destroy(p->yyscanner);
1657 }
1658 
1659 int DocTokenizer::lex()
1660 {
1661   return doctokenizerYYlex(p->yyscanner);
1662 }
1663 
1664 void DocTokenizer::findSections(const QCString &input,const Definition *d,
1665                                 const QCString &fileName)
1666 {
1667   yyscan_t yyscanner = p->yyscanner;
1668   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1669 
1670   if (input.isEmpty()) return;
1671   printlex(yy_flex_debug, TRUE, __FILE__, qPrint(fileName));
1672   yyextra->inputString = input.data();
1673   //printf("parsing --->'%s'<---\n",input);
1674   yyextra->inputPos    = 0;
1675   yyextra->definition  = d;
1676   yyextra->fileName    = fileName;
1677   BEGIN(St_Sections);
1678   yyextra->yyLineNr = 1;
1679   doctokenizerYYlex(yyscanner);
1680   printlex(yy_flex_debug, FALSE, __FILE__, qPrint(fileName));
1681 }
1682 
1683 void DocTokenizer::init(const char *input,const QCString &fileName,bool markdownSupport)
1684 {
1685   yyscan_t yyscanner = p->yyscanner;
1686   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1687   yyextra->autoListLevel = 0;
1688   yyextra->inputString = input;
1689   yyextra->inputPos    = 0;
1690   yyextra->fileName    = fileName;
1691   yyextra->insidePre   = FALSE;
1692   yyextra->markdownSupport = markdownSupport;
1693   BEGIN(St_Para);
1694 }
1695 
1696 TokenInfo *DocTokenizer::newToken()
1697 {
1698   yyscan_t yyscanner = p->yyscanner;
1699   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1700   return yyextra->token = new TokenInfo;
1701 }
1702 
1703 void DocTokenizer::replaceToken(TokenInfo *newToken)
1704 {
1705   yyscan_t yyscanner = p->yyscanner;
1706   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1707   delete yyextra->token;
1708   yyextra->token = newToken;
1709 }
1710 
1711 void DocTokenizer::setStatePara()
1712 {
1713   yyscan_t yyscanner = p->yyscanner;
1714   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1715   BEGIN(St_Para);
1716 }
1717 
1718 void DocTokenizer::setStateTitle()
1719 {
1720   yyscan_t yyscanner = p->yyscanner;
1721   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1722   BEGIN(St_Title);
1723 }
1724 
1725 void DocTokenizer::setStateTitleAttrValue()
1726 {
1727   yyscan_t yyscanner = p->yyscanner;
1728   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1729   BEGIN(St_TitleV);
1730 }
1731 
1732 void DocTokenizer::setStateCode()
1733 {
1734   yyscan_t yyscanner = p->yyscanner;
1735   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1736   yyextra->token->verb="";
1737   yyextra->token->name="";
1738   BEGIN(St_CodeOpt);
1739 }
1740 
1741 void DocTokenizer::setStateXmlCode()
1742 {
1743   yyscan_t yyscanner = p->yyscanner;
1744   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1745   yyextra->token->verb="";
1746   yyextra->token->name="";
1747   BEGIN(St_XmlCode);
1748 }
1749 
1750 void DocTokenizer::setStateHtmlOnly()
1751 {
1752   yyscan_t yyscanner = p->yyscanner;
1753   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1754   yyextra->token->verb="";
1755   yyextra->token->name="";
1756   BEGIN(St_HtmlOnlyOption);
1757 }
1758 
1759 void DocTokenizer::setStateManOnly()
1760 {
1761   yyscan_t yyscanner = p->yyscanner;
1762   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1763   yyextra->token->verb="";
1764   BEGIN(St_ManOnly);
1765 }
1766 
1767 void DocTokenizer::setStateRtfOnly()
1768 {
1769   yyscan_t yyscanner = p->yyscanner;
1770   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1771   yyextra->token->verb="";
1772   BEGIN(St_RtfOnly);
1773 }
1774 
1775 void DocTokenizer::setStateXmlOnly()
1776 {
1777   yyscan_t yyscanner = p->yyscanner;
1778   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1779   yyextra->token->verb="";
1780   BEGIN(St_XmlOnly);
1781 }
1782 
1783 void DocTokenizer::setStateDbOnly()
1784 {
1785   yyscan_t yyscanner = p->yyscanner;
1786   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1787   yyextra->token->verb="";
1788   BEGIN(St_DbOnly);
1789 }
1790 
1791 void DocTokenizer::setStateLatexOnly()
1792 {
1793   yyscan_t yyscanner = p->yyscanner;
1794   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1795   yyextra->token->verb="";
1796   BEGIN(St_LatexOnly);
1797 }
1798 
1799 void DocTokenizer::setStateILiteral()
1800 {
1801   yyscan_t yyscanner = p->yyscanner;
1802   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1803   yyextra->token->verb="";
1804   BEGIN(St_ILiteral);
1805 }
1806 
1807 void DocTokenizer::setStateILiteralOpt()
1808 {
1809   yyscan_t yyscanner = p->yyscanner;
1810   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1811   yyextra->token->verb="";
1812   BEGIN(St_ILiteralOpt);
1813 }
1814 
1815 void DocTokenizer::setStateVerbatim()
1816 {
1817   yyscan_t yyscanner = p->yyscanner;
1818   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1819   yyextra->token->verb="";
1820   BEGIN(St_Verbatim);
1821 }
1822 
1823 void DocTokenizer::setStateDot()
1824 {
1825   yyscan_t yyscanner = p->yyscanner;
1826   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1827   yyextra->token->verb="";
1828   BEGIN(St_Dot);
1829 }
1830 
1831 void DocTokenizer::setStateMsc()
1832 {
1833   yyscan_t yyscanner = p->yyscanner;
1834   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1835   yyextra->token->verb="";
1836   BEGIN(St_Msc);
1837 }
1838 
1839 void DocTokenizer::setStatePlantUMLOpt()
1840 {
1841   yyscan_t yyscanner = p->yyscanner;
1842   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1843   yyextra->token->verb="";
1844   yyextra->token->sectionId="";
1845   BEGIN(St_PlantUMLOpt);
1846 }
1847 
1848 void DocTokenizer::setStatePlantUML()
1849 {
1850   yyscan_t yyscanner = p->yyscanner;
1851   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1852   yyextra->token->verb="";
1853   BEGIN(St_PlantUML);
1854 }
1855 
1856 void DocTokenizer::setStateParam()
1857 {
1858   yyscan_t yyscanner = p->yyscanner;
1859   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1860   BEGIN(St_Param);
1861 }
1862 
1863 void DocTokenizer::setStateXRefItem()
1864 {
1865   yyscan_t yyscanner = p->yyscanner;
1866   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1867   BEGIN(St_XRefItem);
1868 }
1869 
1870 void DocTokenizer::setStateFile()
1871 {
1872   yyscan_t yyscanner = p->yyscanner;
1873   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1874   BEGIN(St_File);
1875 }
1876 
1877 void DocTokenizer::setStatePattern()
1878 {
1879   yyscan_t yyscanner = p->yyscanner;
1880   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1881   yyextra->token->name = "";
1882   BEGIN(St_Pattern);
1883 }
1884 
1885 void DocTokenizer::setStateLink()
1886 {
1887   yyscan_t yyscanner = p->yyscanner;
1888   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1889   BEGIN(St_Link);
1890 }
1891 
1892 void DocTokenizer::setStateCite()
1893 {
1894   yyscan_t yyscanner = p->yyscanner;
1895   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1896   BEGIN(St_Cite);
1897 }
1898 
1899 void DocTokenizer::setStateRef()
1900 {
1901   yyscan_t yyscanner = p->yyscanner;
1902   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1903   BEGIN(St_Ref);
1904 }
1905 
1906 void DocTokenizer::setStateInternalRef()
1907 {
1908   yyscan_t yyscanner = p->yyscanner;
1909   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1910   BEGIN(St_IntRef);
1911 }
1912 
1913 void DocTokenizer::setStateText()
1914 {
1915   yyscan_t yyscanner = p->yyscanner;
1916   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1917   BEGIN(St_Text);
1918 }
1919 
1920 void DocTokenizer::setStateSkipTitle()
1921 {
1922   yyscan_t yyscanner = p->yyscanner;
1923   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1924   BEGIN(St_SkipTitle);
1925 }
1926 
1927 void DocTokenizer::setStateAnchor()
1928 {
1929   yyscan_t yyscanner = p->yyscanner;
1930   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1931   BEGIN(St_Anchor);
1932 }
1933 
1934 void DocTokenizer::setStateSnippet()
1935 {
1936   yyscan_t yyscanner = p->yyscanner;
1937   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1938   yyextra->token->name="";
1939   BEGIN(St_Snippet);
1940 }
1941 
1942 void DocTokenizer::setStateSetScope()
1943 {
1944   yyscan_t yyscanner = p->yyscanner;
1945   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1946   BEGIN(St_SetScope);
1947 }
1948 
1949 void DocTokenizer::setStateOptions()
1950 {
1951   yyscan_t yyscanner = p->yyscanner;
1952   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1953   yyextra->token->name="";
1954   BEGIN(St_Options);
1955 }
1956 
1957 void DocTokenizer::setStateBlock()
1958 {
1959   yyscan_t yyscanner = p->yyscanner;
1960   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1961   yyextra->token->name="";
1962   BEGIN(St_Block);
1963 }
1964 
1965 void DocTokenizer::setStateEmoji()
1966 {
1967   yyscan_t yyscanner = p->yyscanner;
1968   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1969   yyextra->token->name="";
1970   BEGIN(St_Emoji);
1971 }
1972 
1973 void DocTokenizer::setStateIline()
1974 {
1975   yyscan_t yyscanner = p->yyscanner;
1976   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1977   BEGIN(St_Iline);
1978 }
1979 
1980 void DocTokenizer::cleanup()
1981 {
1982   yyscan_t yyscanner = p->yyscanner;
1983   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1984   yy_delete_buffer( YY_CURRENT_BUFFER, yyscanner );
1985 }
1986 
1987 void DocTokenizer::setInsidePre(bool b)
1988 {
1989   yyscan_t yyscanner = p->yyscanner;
1990   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1991   yyextra->insidePre = b;
1992 }
1993 
1994 void DocTokenizer::pushBackHtmlTag(const QCString &tag)
1995 {
1996   yyscan_t yyscanner = p->yyscanner;
1997   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
1998   QCString tagName = tag;
1999   int i,l = tagName.length();
2000   unput('>');
2001   for (i=l-1;i>=0;i--)
2002   {
2003     unput(tag[i]);
2004   }
2005   unput('<');
2006 }
2007 
2008 void DocTokenizer::startAutoList()
2009 {
2010   yyscan_t yyscanner = p->yyscanner;
2011   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2012   yyextra->autoListLevel++;
2013 }
2014 
2015 void DocTokenizer::endAutoList()
2016 {
2017   yyscan_t yyscanner = p->yyscanner;
2018   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2019   yyextra->autoListLevel--;
2020 }
2021 
2022 //REAL_YY_DECL
2023 //{
2024 //  printlex(yy_flex_debug, TRUE, __FILE__, g_fileName);
2025 //  int retval = LOCAL_YY_DECL;
2026 //  printlex(yy_flex_debug, FALSE, __FILE__, g_fileName);
2027 //  return retval;
2028 //}
2029 
2030 void DocTokenizer::setLineNr(int lineno)
2031 {
2032   yyscan_t yyscanner = p->yyscanner;
2033   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2034   yyextra->yyLineNr = lineno;
2035 }
2036 
2037 int DocTokenizer::getLineNr(void)
2038 {
2039   yyscan_t yyscanner = p->yyscanner;
2040   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;
2041   return yyextra->yyLineNr;
2042 }
2043 
2044 #if USE_STATE2STRING
2045 #include "doctokenizer.l.h"
2046 #endif
2047