1 /***************************************************************************
2                           codegenerator.h  -  description
3                              -------------------
4     begin                : Die Jul 9 2002
5     copyright            : (C) 2002-2021 by Andre Simon
6     email                : a.simon@mailbox.org
7  ***************************************************************************/
8 
9 
10 /*
11 This file is part of Highlight.
12 
13 Highlight is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
17 
18 Highlight is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
22 
23 You should have received a copy of the GNU General Public License
24 along with Highlight.  If not, see <http://www.gnu.org/licenses/>.
25 */
26 
27 #ifndef CODEPARSER_H
28 #define CODEPARSER_H
29 
30 #include <iostream>
31 #include <sstream>
32 #include <string>
33 #include <iomanip>
34 #include <stack>
35 #include <vector>
36 
37 #include "regextoken.h"
38 
39 #include "syntaxreader.h"
40 #include "themereader.h"
41 
42 #include "astyle/astyle.h"
43 #include "astyle/ASStreamIterator.h"
44 
45 #include "preformatter.h"
46 
47 #include "stringtools.h"
48 
49 #include "lspclient.h"
50 
51 /// The highlight namespace contains all classes and data structures needed for parsing input data.
52 
53 namespace highlight
54 {
55 
56 
57 /** \brief Base class for parsing. Works like a finite state machine.
58 
59     The virtual class provides source code parsing functionality, based on
60     information stored in language definitions.<br>
61     The derived classes have to define the output format.<br>
62     The colour information is stored in a ThemeReader instance.<br>
63     Codegenerator is a singleton class.<br>
64     Use getInstance for a singleton class instance. Then call the init* methods
65     and loadLanguage to initialize the parser. Init methods have to be called first.
66     Call generate* methods to get results.<br><br>
67 
68     Parser workflow (neglecting embedded syntax and other details):
69 
70     This flow chart shows the method calls, starting in processRootState:
71 
72     -> lineIndex=0
73     -> processRootState()
74        -> state = getCurrentState()
75           -> c = getInputChar()
76              -> increase lineIndex
77              -> if new line: matchRegex()
78                 -> cycle through all regexes of Syntaxreader:
79                    KEYWORD, COMMENT, IDENTIFIER, NUMBER, STRING, INTERPOLATION,
80                    PREPROCESSOR, OPERATOR, NESTEDSECTIONS
81                 -> save position and length of a match for the current line
82           -> is there a match at line[lineIndex]?
83              -> YES: - token = match substring of line
84                      - lineIndex = lineIndex + length of token
85                      - call OnStateChange chunk if available
86                      - state = state of the associated regex or OnStateChange return value
87              -> NO:  - token=c
88                      - state = STANDARD
89         -> state is used to determine the next process* method to call. There
90            are process methods for each state (ie processKeywordState, processNumberState).
91            These methods open and close the delimiter tags in the output format, and most
92            importantly decide when the state ends. They also limit the allowed state changes,
93            like INTERPOLATION in STRING.
94 
95 * @author Andre Simon
96 */
97 
98 class CodeGenerator
99 {
100 
101 public:
102 
103     virtual ~CodeGenerator();
104 
105     /**
106       Get appropriate Codegenerator instance (should be used with auto_ptr)
107       \param type Output file type (HTML, XHTML, RTF, LATEX, TEX, ANSI, XTERM256)
108       \return CodeGenerator
109     */
110     static CodeGenerator* getInstance ( OutputType type );
111 
112     /**
113       Delete CodeGenerator instance (this is intended for SWIG integration only,
114       in normal C++ code the result of getInstance() should be saved in an auto_ptr)
115       \param inst CodeGenerator instance
116     */
deleteInstance(CodeGenerator * inst)117     static void deleteInstance ( CodeGenerator* inst )
118     {
119         if ( inst ) delete inst;
120     }
121 
122     /**
123      Define colour theme information; needs to be called before using a generate* method.
124      Call this method before loadLanguage().
125      \param themePath Path of style description file
126      \param loadSemanticStyles set true if semantic styles should be read
127      \return true if successful
128     */
129     bool initTheme ( const string& themePath, bool loadSemanticStyles=false );
130 
131     LSResult initLanguageServer ( const string& executable, const vector<string> &options, const string& workspace,
132                                   const string& syntax, int delay, int logLevel );
133 
134     void exitLanguageServer ();
135 
136 
137     /**
138      \return description of the theme init error
139     */
140     string getThemeInitError();
141 
142     /**
143      \return description of the plug-in script error
144     */
145     string getPluginScriptError();
146 
147     /** initialize source code indentation and reformatting scheme;
148         needs to be called before using a generate* method
149         \param indentScheme Name of indentation scheme
150         \return true if successful
151      */
152     bool initIndentationScheme ( const string& indentScheme );
153 
154     /** \param langDefPath Absolute path to language definition, may be used multiple times for a generator instance
155      *  \param embedded set True if method is called to load an embedded language
156         \return   LOAD_OK, LOAD_FAILED, LOAD_FAILED_REGEX, LOAD_FAILED_LUA
157     */
158     LoadResult loadLanguage ( const string& langDefPath, bool embedded=false );
159 
160     /**
161      Generate output file from input file
162      \param inFileName Path of input file (if empty use stdin)
163      \param outFileName Path of output file (if empty use stdout)
164      \return ParseError
165     */
166     ParseError generateFile ( const string &inFileName, const string &outFileName );
167 
168     /**
169      Generate output string from input string
170      \param input input code string
171      \return formatted output code
172     */
173     string generateString ( const string &input );
174 
175     /**
176      Generate output string from input file
177      \param inFileName file path
178      \return formatted output code
179     */
180     string generateStringFromFile ( const string &inFileName );
181 
182     /** Print style definitions to external file or stdout
183       \param outFile Path of external style definition; print to stdout if empty
184        \return true if successful
185       */
186     bool printExternalStyle ( const string &outFile );
187 
188     /** Print persistent state snippets to external file
189       \param outFile Path of plugin file to generate
190        \return true if successful
191       */
192     bool printPersistentState ( const string &outFile );
193 
194     /** Print index file with all input file names
195        \param fileList List of output file names
196        \param outPath Output path
197         \return true if successful
198      */
199     virtual bool printIndexFile ( const vector<string> & fileList,
200                                   const string &outPath );
201 
202     /** define the preformatting parameters. Preformatting takes place before
203         the optional astyle reformatting and indenting is performed (defined by initIndentationScheme)
204        \param lineWrappingStyle wrapping style (WRAP_DISABLED, WRAP_SIMPLE, WRAP_DEFAULT)
205        \param lineLength max line length
206        \param numberSpaces number of spaces which replace a tab
207     */
208     void setPreformatting ( WrapMode lineWrappingStyle, unsigned int lineLength,int numberSpaces );
209 
210     /** \return True if document style was found */
211     bool styleFound();
212 
213     /** \return True if reformatting of current input is disabled */
214     bool formattingDisabled();
215 
216     /** \return True if reformatting of current input is possible */
217     bool formattingIsPossible();
218 
219     /** output line numbers
220        \param flag true if line numbers should be printed
221        \param startCnt line number starting count
222     */
223     void setPrintLineNumbers ( bool flag, unsigned int startCnt=1 );
224 
225     /** \return line number flag */
226     bool getPrintLineNumbers();
227 
228     /** output line numbers filled with zeroes
229         \param  flag true if zeroes should be printed
230     */
231     void setPrintZeroes ( bool flag );
232 
233     /** \return print zeroes flag */
234     bool getPrintZeroes();
235 
236     /** omit document header and footer
237        \param  flag true if output should be fragmented
238     */
239     void setFragmentCode ( bool flag );
240 
241     /** \return fragment flag */
242     bool getFragmentCode();
243 
244     /** define line number width
245        \param  w width
246     */
247     void setLineNumberWidth ( int w );
248 
249     /** \return line number width */
250     int getLineNumberWidth();
251 
252     /** check if input is binary or text
253        \param  flag true if input should be checked
254     */
255     void setValidateInput ( bool flag );
256 
257     /** \return input validation flag */
258     bool getValidateInput();
259 
260     /** set keep injections flag
261      * \param  flag true if plug-in injections should be outputted if header
262      * and footer are omitted (fragmentCode is true)
263      */
264     void setKeepInjections( bool flag );;
265 
266     /** \return keep injection flag */
267     bool getKeepInjections();
268 
269     bool requiresTwoPassParsing() const;
270 
271     /** \param  flag true if wrapped lines receive unique line numbers.
272      *  otherwise wrapped lines don't have line numbers at all. */
273     void setNumberWrappedLines ( bool flag );
274 
275     /** return number wrapped lines flag */
276     bool getNumberWrappedLines ();
277 
278     /** \param flag true if version info comment should be omitted */
279     void setOmitVersionComment ( bool flag );
280 
281     /** return version info comment flag */
282     bool getOmitVersionComment ();
283 
284     /** \param flag true if token of the same syntax category should be outputted in separate tags */
285     void setIsolateTags ( bool flag );
286 
287     /** return version info comment flag */
288     bool getIsolateTags ();
289 
290     /** \return style path */
291     const string& getStyleName();
292 
293     /** use this font as base font
294       * \param fontName the font name, e.g. "Courier New"
295      */
296     void setBaseFont ( const string& fontName);
297 
298     /** \return base font */
299     const string getBaseFont() const ;
300 
301     /** use this size as base font size
302       * \param fontSize the font size, e.g. "12"
303      */
304     void setBaseFontSize ( const string& fontSize );
305 
306     /** \return base font size*/
307     const string getBaseFontSize();
308 
309     /** tell parser the include style definition in output
310         \param flag true if style should be included
311      */
312     void setIncludeStyle ( bool flag );
313 
314     /** tell parser to omit trailing newline character
315         \param flag 1 if no trailing newline should be printed,
316                     2 omit newline only for empty input
317      */
318     void disableTrailingNL ( int flag );
319 
320     /** Set style input path
321         \param path path to style input file
322       */
323     void setStyleInputPath ( const string& path );
324 
325     /** Set style output path
326       \param path path to style output file
327     */
328     void setStyleOutputPath ( const string& path );
329 
330     /** Set encoding (output encoding must match input file)
331       \param encodingName encoding name
332     */
333     void setEncoding ( const string& encodingName );
334 
335     /** \return style input file path */
336     const string&  getStyleInputPath();
337 
338     /** \return style output file path */
339     const string&  getStyleOutputPath();
340 
341     /** \param title Document title */
342     void setTitle ( const string & title );
343 
344     /** \return Document title */
345     string getTitle();
346 
347     /** \param begin number of the first input line to be processed */
348     void setStartingInputLine ( unsigned int begin );
349 
350     /** \param cnt upper limit of input lines to be processed */
351     void setMaxInputLineCnt ( unsigned int cnt );
352 
353     /** \param cnt number of input files */
354     void setFilesCnt ( unsigned int cnt );
355 
356     /** \param keyCase Keyword case */
357     void setKeyWordCase ( StringTools::KeywordCase keyCase );
358 
359     /** \param delim End of line delimiter (default: NL)
360     */
361     void setEOLDelimiter(char delim);
362 
363     /** \param param path of plugin input file
364     */
365     void setPluginParameter ( const string& param );
366 
367     void resetSyntaxReaders();
368 
369     /** Load the Lua functions og the plug-in script
370     	\param script path of the plug-in script
371     */
372     bool initPluginScript(const string& script);
373 
374     bool syntaxRequiresTwoPassRun();
375 
376     void clearPersistentSnippets();
377 
378     /** \param flag true if styles should be cached for repeated file output calls */
379     void setStyleCaching ( bool flag );
380 
381     /** \return Syntaxreader Regex error message */
382     string getSyntaxRegexError();
383 
384     /** \return Syntaxreader Lua error message */
385     string getSyntaxLuaError();
386 
387     /** \return Syntaxreader description */
388     string getSyntaxDescription();
389 
390     /** \return Encoding hint */
391     string getSyntaxEncodingHint();
392 
393     /** \return Theme description */
394     string getThemeDescription();
395 
396     /** \return Syntaxreader category description */
397     string getSyntaxCatDescription();
398 
399     /** \return Encoding contrast of Canvas and Default colours */
400     float getThemeContrast();
401 
402     /** \return Theme category description */
403     string getThemeCatDescription();
404 
getPosTestErrors()405     vector<string> getPosTestErrors() { return failedPosTests; }
406 
407     /** \return Pointer Syntaxreader, intended for debug output  */
getSyntaxReader()408     SyntaxReader* getSyntaxReader()
409     {
410         return currentSyntax;
411     }
412 
413     /** \return Content of user defined input style */
414     string readUserStyleDef();
415 
416     /** \return Style definition of the chosen output format */
getStyleDefinition()417     virtual string getStyleDefinition()
418     {
419         return "";
420     }
421 
getHoverTagOpen(const string & hoverText)422     virtual string getHoverTagOpen(const string & hoverText)
423     {
424         return "";
425     }
426 
getHoverTagClose()427     virtual string getHoverTagClose()
428     {
429         return "";
430     }
431 
432     bool lsOpenDocument(const string& fileName, const string & suffix);
433 
434     bool lsCloseDocument(const string& fileName, const string & suffix);
435 
436     bool lsAddSemanticInfo(const string& fileName, const string & suffix);
437 
438     void lsAddHoverInfo(bool hover);
439 
440     void lsAddSyntaxErrorInfo(bool error);
441 
442     bool isHoverProvider();
443 
444     bool isSemanticTokensProvider();
445 
446     /** set HTML output anchor flag
447      */
setHTMLAttachAnchors(bool)448     virtual void setHTMLAttachAnchors ( bool )  {};
449 
450     /** set HTML output ordered list flag
451      */
setHTMLOrderedList(bool)452     virtual void setHTMLOrderedList ( bool )  {};
453 
454     /** set HTML output inline CSS flag
455      */
setHTMLInlineCSS(bool)456     virtual void setHTMLInlineCSS ( bool )  {};
457 
458     /** set HTML output enclose pre tag flag
459      */
setHTMLEnclosePreTag(bool)460     virtual void setHTMLEnclosePreTag ( bool )  {};
461 
462     /** set HTML output replace space by &nbsp; flag
463      */
setHTMLUseNonBreakingSpace(bool)464     virtual void setHTMLUseNonBreakingSpace ( bool )  {};
465 
466     /** set HTML output anchor prefix
467      */
setHTMLAnchorPrefix(const string &)468     virtual void setHTMLAnchorPrefix ( const string& )  {};
469 
470     /** set HTML output class name
471      */
setHTMLClassName(const string &)472     virtual void setHTMLClassName ( const string& )  {};
473 
474     /** set LaTeX replace quotes flag
475      */
setLATEXReplaceQuotes(bool)476     virtual void setLATEXReplaceQuotes ( bool )  {};
477 
478     /** set LaTeX no Babel shorthands flag
479      */
setLATEXNoShorthands(bool)480     virtual void setLATEXNoShorthands ( bool )  {};
481 
482     /** set LaTeX pretty Symbols flag
483      */
setLATEXPrettySymbols(bool)484     virtual void setLATEXPrettySymbols ( bool )  {};
485 
setLATEXBeamerMode(bool)486     virtual void setLATEXBeamerMode ( bool )  {};
487 
488     /** set RTF page size
489      */
setRTFPageSize(const string &)490     virtual void setRTFPageSize ( const string& )  {};
491 
492     /** set RTF output character styles flag
493      */
setRTFCharStyles(bool)494     virtual void setRTFCharStyles ( bool )  {};
495 
496     /** set RTF page color flag
497      */
setRTFPageColor(bool)498     virtual void setRTFPageColor ( bool )  {};
499 
500     /** set SVG page size
501      */
setSVGSize(const string &,const string &)502     virtual void setSVGSize ( const string&, const string& )  {};
503 
504     /** set True Color flag
505      */
setESCTrueColor(bool)506     virtual void setESCTrueColor ( bool )  {};
507 
508      /** set background padding width (<=0 to disable)
509      */
setESCCanvasPadding(unsigned int)510     virtual void setESCCanvasPadding ( unsigned int )  {};
511 
512 protected:
513 
514     static const unsigned int NUMBER_BUILTIN_STATES;  ///< number of token states (without keyword group IDs)
515 
516     static const string STY_NAME_STD;
517     static const string STY_NAME_STR;
518     static const string STY_NAME_NUM;
519     static const string STY_NAME_SLC;
520     static const string STY_NAME_COM;
521     static const string STY_NAME_ESC;
522     static const string STY_NAME_DIR;
523     static const string STY_NAME_DST;
524     static const string STY_NAME_LIN;
525     static const string STY_NAME_SYM;
526     static const string STY_NAME_IPL;
527 
528     static const string STY_NAME_HVR;
529     static const string STY_NAME_ERR;
530     static const string STY_NAME_ERM;
531 
532     /** \param type Output type */
533     CodeGenerator ( highlight::OutputType type );
CodeGenerator()534     CodeGenerator() {};
535 
536     /** \param c Character to be masked
537         \return Escape sequence of output format */
538     virtual string maskCharacter ( unsigned char c ) = 0;
539 
540     /** \param ss destination stream
541         \param s string */
542     void maskString ( ostream& ss, const string &s ) ;
543 
544     void printSyntaxError ( ostream& ss ) ;
545 
546     /** Get current line number
547       \return line number  */
548     unsigned int getLineNumber();
549 
550     vector <string> openTags,   ///< list of format delimiters (open new format descriptions)
551            closeTags;   ///< list of format delimiters (close format descriptions)
552 
553     /** Description of document colour style*/
554     ThemeReader docStyle;
555 
556     /** Language definition*/
557     SyntaxReader* currentSyntax;
558 
559     /** Tag for inserting line feeds*/
560     string newLineTag;
561 
562     /** String that represents a white space in output */
563     string spacer, initialSpacer;
564 
565     /** file input*/
566     istream *in;
567 
568     /** file output*/
569     ostream *out;
570 
571     string maskWsBegin,  ///< open whitespace mask
572            maskWsEnd;    ///< close whitespace mask
573 
574     string styleCommentOpen,  ///< open comment delimiter
575            styleCommentClose; ///< close comment delimiter
576 
577     string embedBlockOpen,  ///< open block delimiter to highlight embedded code
578            embedBlockClose; ///< close block delimiter
579 
580     /** Encoding name */
581     string encoding;
582 
583     /** document title */
584     string docTitle;
585 
586     string inFile,   ///< input file name
587            outFile;  ///< output file name
588 
589     /// LSP syntax error description
590     string lsSyntaxErrorDesc;
591 
592     /** Test if maskWsBegin and maskWsEnd should be applied */
593     bool maskWs;
594 
595     /** Test if whitespace should always be separated from enclosing tokens */
596     bool excludeWs;
597 
598     /** Test if header and footer should be omitted */
599     bool fragmentOutput;
600 
601     /** Test if plugin injections should be printed if fragmentOutput is true */
602     bool keepInjections;
603 
604     /** Test if line numbers should be printed */
605     bool showLineNumbers;
606 
607     /** Test if leading spyce of line number should be filled with zeroes*/
608     bool lineNumberFillZeroes;
609 
610     /** Flag to test if newlines should be printed */
611     bool printNewLines;
612 
613     /** Test if version info comment printed */
614     bool omitVersionComment;
615 
616     /** Flag to output token of the same syntax category in separate tags */
617     bool isolateTags;
618 
619     bool disableStyleCache;
620 
621     /** The base font to use */
622     string baseFont ;
623 
624     /** The base font size to use */
625     string baseFontSize ;
626 
627     /** Current line of input file*/
628     string line;
629 
630     /** Current line number */
631     unsigned int lineNumber;
632 
633     /**output line number count start */
634     int lineNumberOffset;
635 
636     /** Current state*/
637     State currentState;
638 
639     /** keyword class id, used to apply the corresponding keyword style*/
640     unsigned int currentKeywordClass;
641 
642     /** Processes origin state */
643     void processRootState();
644 
645     /** \return line break sequence */
646     virtual string getNewLine();
647 
648     /**
649        \param s current state
650        \param kwClassID keyword class (has to be set when s=KEYWORD)
651        \return Index of style tag corresponding to the states
652     */
653     unsigned int getStyleID ( State s, unsigned int kwClassID = 0 );
654 
655     /** \return current line index */
656     unsigned int getLineIndex();
657 
658     /** \return last line index */
659     unsigned int getLastLineLength();
660 
661     /** print all remaining white space*/
662     void flushWs(int );
663 
664     /** \return true id encoding is defined */
encodingDefined()665     bool encodingDefined()
666     {
667         return StringTools::change_case ( encoding ) !="none";
668     }
669 
670     /** Invoke plugin decorate user function */
671     Diluculum::LuaValueList callDecorateFct(const string& token);
672 
673     /** Invoke plugin decorate line user function
674      @param isLineStart set true if line start function should be called,
675             otherwise line end function is invoked */
676     Diluculum::LuaValueList callDecorateLineFct(bool isLineStart);
677 
678     /** contains white space, which will be printed after a closing tag */
679     string wsBuffer;
680 
681     /** Flag to test if style definition should be included in output document */
682     bool includeStyleDef;
683 
684     /** Class for line wrapping and tab replacement*/
685     PreFormatter preFormatter;
686 
687     /** Stores if the current line should receive a line number.
688      *  If the line is just the continuation of a wrapped line,
689      *  and numberWrappedLines is false, this is set true. */
690     bool numberCurrentLine;
691 
692     /** method to fix output type in Lua state for XHTML and Truecolor
693         @param output type */
setOutputType(OutputType t)694     void setOutputType(OutputType t) { outputType = t; }
695 
696 private:
697 
698     void printTrace(const string &s);
699 
CodeGenerator(const CodeGenerator &)700     CodeGenerator ( const CodeGenerator& ) {}
701 
702     CodeGenerator& operator= ( CodeGenerator& )
703     {
704         return *this;
705     }
706 
707     /** Insert line number at the beginning of current output line
708         @param insertNewLine set true if newline should be outputted
709      */
710     virtual void insertLineNumber ( bool insertNewLine=true);
711 
712     /** returns output specific document footer
713         @return footer */
714     virtual string getFooter() = 0;
715 
716     /** returns output specific document header
717       * @return header */
718     virtual string getHeader() = 0;
719 
720     /** Prints document header*/
721     void printHeader();
722 
723     /** Prints document body*/
724     virtual void printBody() = 0;
725 
726     /** Prints document footer*/
727     void printFooter();
728 
729     /** initialize tags in specific format according to colouring information
730      *  provided in DucumentStyle */
731     virtual void initOutputTags() = 0;
732 
733     /** \param styleID keyword style id
734     	\return  open tag  */
735     virtual string getKeywordOpenTag ( unsigned int styleID) = 0;
736 
737     /** \param styleID keyword style id
738     	\return  close tag  */
739     virtual string getKeywordCloseTag ( unsigned int styleID) = 0;
740 
741     /** open a new tag, set current state to s*/
742     void openTag ( State s );
743 
744     /** close opened tag, clear current state */
745     void closeTag ( State s );
746 
747     /** close Keyword tag of corresponding style ID */
748     void closeKWTag ( unsigned int styleID );
749 
750     /** open Keyword tag of corresponding style ID */
751     void openKWTag ( unsigned int styleID );
752 
753     /// path to style definition file
754     string themePath;
755 
756     /// path to host language definition
757     string hostLangDefPath;
758 
759     /// path to embedded language definition
760     string embedLangDefPath;
761 
762     /// path to plugin input file
763     string pluginParameter;
764 
765     /// contains current position in line
766     unsigned int lineIndex;
767 
768     /// contains length of last line
769     unsigned int lastLineLength;
770 
771     /// line index where syntax change takes place
772     unsigned int syntaxChangeIndex;
773 
774     /// line number where syntax change takes place
775     unsigned int syntaxChangeLineNo;
776 
777     /// width of line numbering column
778     unsigned int lineNumberWidth;
779 
780     /**first input line to be processed*/
781     unsigned int startLineCnt;
782     unsigned int startLineCntCurFile;
783 
784     /**maximum count of input lines to be processed*/
785     unsigned int maxLineCnt;
786 
787 
788     /**number of files to be processed and counter*/
789     unsigned int inputFilesCnt;
790     unsigned int processedFilesCnt;
791 
792     int kwOffset;
793 
794     /** Flag to test if trailing newline should be printed */
795     int noTrailingNewLine;
796 
797     /**last character of the last line*/
798     unsigned char terminatingChar;
799 
800     /** Class for reformatting */
801     astyle::ASFormatter *formatter;
802     astyle::ASStreamIterator *streamIterator;
803 
804     /** Flag to test if formatting is enabled with current input document*/
805     bool formattingEnabled;
806 
807     /** Flag to test if formatting is possible with current input document*/
808     bool formattingPossible;
809 
810     /** Flag to test if input should be validated (binary or text) */
811     bool validateInput;
812 
813     /** Flag if wrapped lines should receive unique line numbers as well */
814     bool numberWrappedLines;
815 
816     /** indicator if current state was set by Lua hook function */
817     bool resultOfHook;
818 
819     bool lineContainedTestCase;
820 
821     bool lineContainedStmt;
822 
823     bool applySyntaxTestCase;
824 
825     bool toggleDynRawString;
826 
827     bool lsEnableHoverRequests;
828 
829     bool lsCheckSemanticTokens;
830 
831     bool lsCheckSyntaxErrors;
832 
833     /** flag which determines keyword output (unchangeed, uppercase, lowercase)*/
834     StringTools::KeywordCase keywordCase;
835 
836     /** contains the current token*/
837     string token;
838 
839     string styleInputPath,   ///< style input file path
840            styleOutputPath;  ///< style output file path
841 
842     string userScriptError;  ///< Plug-In script error message
843 
844     string lsDocumentPath;   ///< Language Server input file name
845 
846     /** end-of-line delimiter*/
847     char eolDelimiter;
848 
849     /** Resets parser to origin state, call this after every file conversion */
850     void reset();
851 
852     /** read new line from in stream */
853     bool readNewLine ( string &newLine );
854 
855     /** return next character from in stream */
856     unsigned char getInputChar();
857 
858     /** output file type */
859     OutputType outputType;
860 
861     /** return new state */
862     State getCurrentState (State oldState);
863 
864     std::stack<std::string> nestedLangs;
865 
866     /* Methods that represent a parsing state */
867     bool processKeywordState ( State myState );  ///< process keywords
868     bool processNumberState() ;               ///< process numbers
869     bool processMultiLineCommentState();      ///< process multi line comments
870     bool processSingleLineCommentState();     ///< process single line comments
871     bool processStringState ( State oldState ); ///< process strings
872     bool processEscapeCharState();            ///< process escape characters
873     bool processInterpolationState();         ///< process string interpolation sequences
874     bool processDirectiveState();             ///< process directives
875     bool processTagState();                   ///< process tags
876     bool processSymbolState();                ///< process symbols
877     void processWsState();                    ///< process whitespace
878     bool processSyntaxChangeState(State myState ); ///< process syntax change of embedded languages
879     bool processSyntaxErrorState();           ///< process syntax errors
880 
881     /* checks whether the given state was defined in the same column of the last parsed input line */
882     void runSyntaxTestcases(unsigned int column);
883 
884     /* returns name of testcase state */
885     string getTestcaseName(State s, unsigned int kwClass);
886 
887     /** print escaped token and clears it
888        \param flushWhiteSpace set true if white space should be flushed
889        \param tcase keyword case
890     */
891     void printMaskedToken ( bool flushWhiteSpace = true,
892             StringTools::KeywordCase tcase = StringTools::CASE_UNCHANGED );
893 
894     void updateKeywordClasses();
895 
896     /** association of matched regexes and the corresponding keyword class ids*/
897     map <int, RegexToken> regexGroups;
898 
899     /** history of states per line position in the current line of input code (max 200 entries) */
900 
901     struct PositionState {
902         State state;
903         unsigned int kwClass;
904         bool isWhiteSpace;
PositionStatePositionState905         PositionState(const State s, const unsigned int kwc, bool ws)
906         : state(s), kwClass(kwc), isWhiteSpace(ws) {
907             if (s!=KEYWORD) kwClass=0;
908         }
909     };
910     vector<PositionState> stateTraceCurrent, stateTraceTest;
911 
912     vector<string> failedPosTests;
913 
914     /** association of syntax definitions and their paths*/
915     map <string, SyntaxReader*> syntaxReaders;
916 
917     /** test for regular expressions
918         \param line current input line
919         \param skipState state which should be ignored*/
920     void matchRegex ( const string &line, State skipState=_UNKNOWN );
921 
922     /** \return true if input is no binary stream */
923     bool validateInputStream();
924 
925     /** load syntax description of embedded snippet's language
926      \param embedLangDefPath path to language definition
927       \return true if successful */
928     bool loadEmbeddedLang(const string&embedLangDefPath);
929 
930     /** call user script OnStateChange function if defined to confirm state change
931      \param newState new state
932      \param oldState old state
933      */
934     State validateState(State newState, State oldState);
935 
936     unsigned int getCurrentKeywordClassId();
937 
938     /**
939     	\param chunk Lua function to be added to the function list
940     */
addUserChunk(const Diluculum::LuaFunction & chunk)941     void addUserChunk(const Diluculum::LuaFunction& chunk)
942     {
943         pluginChunks.push_back(new Diluculum::LuaFunction(chunk));
944     }
945 
946     void applyPluginChunk(const string& fctName, string *result, bool *keepDefault);
947 
948     void initASStream();
949 
950     void setOverrideParams();
951 
952     static vector<Diluculum::LuaFunction*> pluginChunks;
953 
954     highlight::LSPClient LSPClient;
955 };
956 
957 }
958 
959 #endif
960