1 /*
2 * This file is part of the FortranProject plugin for Code::Blocks IDE
3 * and licensed under the GNU General Public License, version 3
4 * http://www.gnu.org/licenses/gpl-3.0.html
5 */
6
7 #include "tokenizerf.h"
8
9 #include <sdk.h>
10 #ifndef CB_PRECOMP
11 #include <wx/file.h>
12 #endif
13
ReadFileToString(wxFile & file,wxString & st)14 bool ReadFileToString(wxFile& file,wxString& st)
15 {
16 st.Empty();
17 if (!file.IsOpened())
18 return false;
19 int len = file.Length();
20 if(!len)
21 {
22 file.Close();
23 return true;
24 }
25 #if wxUSE_UNICODE
26 char* buff = new char[len+1];
27 if (!buff)
28 {
29 file.Close();
30 return false;
31 }
32 file.Read((void*)buff, len);
33 file.Close();
34 buff[len]='\0';
35
36 st = wxString((const char *)buff, wxConvUTF8);
37 if (st.Length() == 0)
38 {
39 // could not read as utf-8 encoding, try iso8859-1
40 st = wxString((const char *)buff, wxConvISO8859_1);
41 }
42 delete[] buff;
43
44 #else
45 char* buff = st.GetWriteBuf(len); // GetWriteBuf already handles the extra '\0'.
46 file.Read((void*)buff, len);
47 file.Close();
48 st.UngetWriteBuf();
49 #endif
50 return true;
51 }
52
53 ///---------------------------------------
54
Tokenizerf(const wxString & filename,FortranSourceForm sourceForm)55 Tokenizerf::Tokenizerf(const wxString& filename, FortranSourceForm sourceForm)
56 : m_Filename(filename),
57 m_BufferLen(0),
58 m_TokenIndex(0),
59 m_UndoTokenIndex(0),
60 m_PeekedTokenIndex(0),
61 m_LineNumber(1),
62 m_LineNumberStart(1),
63 m_UndoLineNumber(1),
64 m_UndoLineNumberStart(1),
65 m_PeekedLineNumber(1),
66 m_PeekedLineNumberStart(1),
67 m_Column(1),
68 m_UndoColumn(1),
69 m_PeekedColumn(1),
70 m_WasNextLine(false),
71 m_UndoWasNextLine(false),
72 m_PeekedWasNextLine(false),
73 m_WasPeeked(false),
74 m_IsOK(false),
75 m_SourceForm(sourceForm),
76 m_PeekedToken(),
77 m_DetailedParsing(false)
78 {
79 if (!m_Filename.IsEmpty())
80 Init(m_Filename, m_SourceForm);
81 m_LineStartIdx.push_back(0);
82 }
83
~Tokenizerf()84 Tokenizerf::~Tokenizerf()
85 {
86 }
87
Init(const wxString & filename,FortranSourceForm sourceForm)88 bool Tokenizerf::Init(const wxString& filename, FortranSourceForm sourceForm)
89 {
90 BaseInit();
91 if (filename.IsEmpty())
92 {
93 if (m_Filename.IsEmpty())
94 return false;
95 }
96 else
97 m_Filename = filename;
98
99 if (!wxFileExists(m_Filename))
100 return false;
101
102 if (!ReadFile())
103 return false;
104
105 if (!m_BufferLen)
106 return false;
107
108 m_SourceForm = sourceForm;
109 AdjustLineNumber();
110
111 m_IsOK = true;
112 return true;
113 }
114
InitFromBuffer(const wxString & buffer,FortranSourceForm sourceForm)115 bool Tokenizerf::InitFromBuffer(const wxString& buffer, FortranSourceForm sourceForm)
116 {
117 BaseInit();
118 m_Buffer = buffer;
119 m_BufferLen = buffer.Length();
120 m_IsOK = true;
121 m_Filename.Clear();
122 m_SourceForm = sourceForm;
123 AdjustLineNumber();
124 return true;
125 }
126
BaseInit()127 void Tokenizerf::BaseInit()
128 {
129 m_TokenIndex = 0;
130 m_UndoTokenIndex = 0;
131 m_PeekedTokenIndex = 0;
132 m_LineNumber = 1;
133 m_UndoLineNumber = 1;
134 m_PeekedLineNumber = 1;
135 m_LineNumberStart = 1;
136 m_UndoLineNumberStart = 1;
137 m_PeekedLineNumberStart = 1;
138 m_Column = 1;
139 m_UndoColumn = 1;
140 m_PeekedColumn = 1;
141 m_BufferLen = 0;
142 m_Buffer.Clear();
143 m_IsOK = false;
144 m_WasNextLine = false;
145 m_UndoWasNextLine = false;
146 m_PeekedWasNextLine = false;
147 m_WasPeeked = false;
148 m_LineStartIdx.clear();
149 m_LineStartIdx.push_back(0);
150 }
151
ReadFile()152 bool Tokenizerf::ReadFile()
153 {
154 if (!wxFileExists(m_Filename))
155 return false;
156
157 // open file
158 wxFile file(m_Filename);
159
160 //if (!cbRead(file,m_Buffer))
161 if (!ReadFileToString(file,m_Buffer))
162 {
163 return false;
164 }
165 m_BufferLen = m_Buffer.Length();
166
167 return true;
168 }
169
CurrentChar()170 wxChar Tokenizerf::CurrentChar()
171 {
172 return m_Buffer.GetChar(m_TokenIndex);
173 }
174
NextChar()175 wxChar Tokenizerf::NextChar()
176 {
177 if ((m_TokenIndex + 1) < 0 || (m_TokenIndex + 1) >= m_BufferLen)
178 return 0;
179 return m_Buffer.GetChar(m_TokenIndex + 1);
180 }
181
PreviousChar()182 wxChar Tokenizerf::PreviousChar()
183 {
184 if ((m_TokenIndex - 1) < 0 || (m_TokenIndex - 1) >= m_BufferLen)
185 return 0;
186 return m_Buffer.GetChar(m_TokenIndex - 1);
187 }
188
AdjustLineNumber()189 void Tokenizerf::AdjustLineNumber()
190 {
191 if (m_WasNextLine)
192 {
193 ++m_LineNumber;
194 m_WasNextLine = false;
195 if (m_LineStartIdx.size() < m_LineNumber)
196 {
197 m_LineStartIdx.push_back(m_TokenIndex);
198 }
199 }
200 if (CurrentChar() == '\n')
201 {
202 m_WasNextLine = true;
203 m_Column = 0;
204 }
205 }
206
MoveToNextChar()207 bool Tokenizerf::MoveToNextChar()
208 {
209 ++m_TokenIndex;
210 ++m_Column;
211 if (!IsEOF())
212 {
213 AdjustLineNumber();
214 return true;
215 }
216 return false;
217 }
218
SkipWhiteSpace()219 bool Tokenizerf::SkipWhiteSpace()
220 {
221 // skip spaces, tabs, etc.
222 while (!IsEOF() && isspace(CurrentChar()))
223 MoveToNextChar();
224 if (IsEOF())
225 return false;
226 return true;
227 }
228
SkipToChar(const wxChar & ch,bool toLineEnd)229 bool Tokenizerf::SkipToChar(const wxChar& ch, bool toLineEnd)
230 {
231 // skip everything until we find ch
232 while (1)
233 {
234 while (!IsEOF() && CurrentChar() != ch && (!toLineEnd || (toLineEnd && CurrentChar() != '\n')))
235 MoveToNextChar();
236 break;
237 }
238 if (IsEOF())
239 return false;
240 return true;
241 }
242
CharInString(const char ch,const char * chars)243 bool Tokenizerf::CharInString(const char ch, const char* chars)
244 {
245 int len = strlen(chars);
246 for (int i = 0; i < len; ++i)
247 {
248 if (ch == chars[i])
249 return true;
250 }
251 return false;
252 }
253
254
SkipToOneOfChars(const char * chars,bool toLineEnd)255 bool Tokenizerf::SkipToOneOfChars(const char* chars, bool toLineEnd)
256 {
257 m_WasPeeked = false;
258 // skip everything until we find any one of chars or end of line if toLineEnd
259 while (1)
260 {
261 wxChar cch = CurrentChar();
262 if(toLineEnd && cch == '\n')
263 {
264 break;
265 }
266 if(cch == '&')
267 {
268 MoveToNextChar();
269 SkipWhiteSpace();
270 cch = CurrentChar();
271 if (cch == '\n' || cch == '\r' || cch == '!')
272 {
273 SkipToEOL();
274 cch = CurrentChar();
275 }
276 }
277 if(toLineEnd && cch == '!')
278 {
279 SkipToEOL();
280 break;
281 }
282 else if(cch == '!')
283 {
284 SkipToEOL();
285 cch = CurrentChar();
286 }
287 if(IsEOF() || CharInString(cch, chars))
288 {
289 break;
290 }
291 else
292 {
293 if (cch == '"' || cch == '\'')
294 {
295 // this is the case that match is inside a string!
296 char ch = CurrentChar();
297 MoveToNextChar();
298 SkipToChar(ch, true);
299 }
300 MoveToNextChar();
301 }
302 }
303 if (IsEOF())
304 return false;
305 return true;
306 }
307
SkipToEOL()308 bool Tokenizerf::SkipToEOL()
309 {
310 // skip everything until we find EOL
311 while (1)
312 {
313 while (!IsEOF() && CurrentChar() != '\n')
314 MoveToNextChar();
315 break;
316 }
317 if (IsEOF())
318 return false;
319 return true;
320 }
321
SkipBlock(const wxChar & ch,int maxLines)322 bool Tokenizerf::SkipBlock(const wxChar& ch, int maxLines)
323 {
324 // skip blocks () [] {} <>
325 wxChar match;
326 switch (ch)
327 {
328 case '(':
329 match = ')';
330 break;
331 case '[':
332 match = ']';
333 break;
334 case '{':
335 match = '}';
336 break;
337 case '<':
338 match = '>';
339 break;
340 default :
341 return false;
342 }
343
344 MoveToNextChar();
345 int n_lines = 1;
346 int count = 1; // counter for nested blocks (xxx())
347 bool wasCont= false;
348 while (!IsEOF())
349 {
350 while (!IsEOF())
351 {
352 if (CurrentChar() == '"' || CurrentChar() == '\'')
353 {
354 // this is the case that match is inside a string!
355 char chOne = CurrentChar();
356 MoveToNextChar();
357 SkipToChar(chOne, true);
358 MoveToNextChar();
359 }
360 else
361 break;
362 }
363 if (CurrentChar() == ch)
364 ++count;
365 else if (CurrentChar() == match)
366 --count;
367 else if (CurrentChar() == '&' && m_SourceForm == fsfFree && !wasCont)
368 {
369 SkipToEOL();
370 wasCont = true;
371 }
372 else if (maxLines > 0 && CurrentChar() == '\n')
373 {
374 n_lines++;
375 if (n_lines > maxLines)
376 count = 0;
377 }
378 else if (wasCont && m_SourceForm == fsfFree && !isspace(CurrentChar()))
379 wasCont = false;
380
381 MoveToNextChar();
382 if (count == 0)
383 break;
384 }
385 if (IsEOF())
386 return false;
387 return true;
388 }
389
SkipUnwanted()390 bool Tokenizerf::SkipUnwanted()
391 {
392 while ( //(CurrentChar() == '=' && !m_DetailedParsing) ||
393 CurrentChar() == '!' ||
394 ((CurrentChar() == 'c' || CurrentChar() == 'C' || CurrentChar() == '*') && m_Column == 1 && m_SourceForm == fsfFixed))
395 {
396 while ((CurrentChar() == 'c' || CurrentChar() == 'C' ||CurrentChar() == '*') && m_Column == 1 && m_SourceForm == fsfFixed)
397 {
398 if (IsBindTo())
399 return true;
400 SkipToEOL();
401 if (!SkipWhiteSpace())
402 return false;
403 }
404 while (CurrentChar() == '!')
405 {
406 if (IsBindTo())
407 return true;
408 SkipToEOL();
409 if (!SkipWhiteSpace())
410 return false;
411 }
412
413 // while (CurrentChar() == '=')
414 // {
415 // if (NextChar() != '>')
416 // {
417 // MoveToNextChar();
418 // if (!SkipWhiteSpace())
419 // return false;
420 // // skip assignments
421 // if (CurrentChar() == '[' || CurrentChar() == '(')
422 // break;
423 // if (!SkipToOneOfChars(";", true))
424 // return false;
425 // if (!SkipWhiteSpace())
426 // return false;
427 // }
428 // else
429 // {
430 // return true;
431 // }
432 // }
433 }
434 return true;
435 }
436
GetToken()437 wxString Tokenizerf::GetToken()
438 {
439 m_UndoTokenIndex = m_TokenIndex;
440 m_UndoLineNumber = m_LineNumber;
441 m_UndoLineNumberStart = m_LineNumberStart;
442 m_UndoColumn = m_Column;
443 m_UndoWasNextLine = m_WasNextLine;
444 if (m_WasPeeked)
445 {
446 m_TokenIndex = m_PeekedTokenIndex;
447 m_LineNumber = m_PeekedLineNumber;
448 m_LineNumberStart = m_PeekedLineNumberStart;
449 m_Column = m_PeekedColumn;
450 m_WasNextLine = m_PeekedWasNextLine;
451 m_WasPeeked = false;
452 return m_PeekedToken;
453 }
454 else
455 return DoGetToken();
456 }
457
GetTokenSameLine()458 wxString Tokenizerf::GetTokenSameLine()
459 {
460 unsigned int oldTokenIndex = m_TokenIndex;
461 unsigned int oldLineNumber = m_LineNumber;
462 unsigned int oldLineNumberStart = m_LineNumberStart;
463 unsigned int oldColumn = m_Column;
464 bool oldWasNextLine = m_WasNextLine;
465
466 wxString token;
467
468 if (m_WasPeeked)
469 {
470 m_TokenIndex = m_PeekedTokenIndex;
471 m_LineNumber = m_PeekedLineNumber;
472 m_LineNumberStart = m_PeekedLineNumberStart;
473 m_Column = m_PeekedColumn;
474 m_WasNextLine = m_PeekedWasNextLine;
475 m_WasPeeked = false;
476 token = m_PeekedToken;
477 }
478 else
479 token = DoGetToken();
480
481
482 if (oldLineNumber != m_LineNumberStart)
483 {
484 m_TokenIndex = oldTokenIndex;
485 m_LineNumber = oldLineNumber;
486 m_LineNumberStart = oldLineNumberStart;
487 m_Column = oldColumn;
488 m_WasNextLine = oldWasNextLine;
489 token = wxEmptyString;
490 }
491 else
492 {
493 m_UndoTokenIndex = oldTokenIndex;
494 m_UndoLineNumber = oldLineNumber;
495 m_UndoLineNumberStart = oldLineNumberStart;
496 m_UndoColumn = oldColumn;
497 m_UndoWasNextLine = oldWasNextLine;
498 }
499
500 return token;
501 }
502
GetTokenSameFortranLine()503 wxString Tokenizerf::GetTokenSameFortranLine()
504 {
505 wxString token;
506
507 if (m_SourceForm == fsfFree)
508 {
509 token = GetTokenSameLine();
510 while (token.IsSameAs(_T("&")))
511 {
512 token = GetToken();
513 }
514 }
515 else
516 {
517 token = PeekToken();
518 if (m_LineNumberStart == m_PeekedLineNumberStart)
519 token = GetToken();
520 else
521 {
522 if ( (m_PeekedColumn >= 7 && (m_PeekedColumn - token.Length()) >= 7) ||
523 ((m_PeekedColumn - token.Length()) < 6) ||
524 (token.Mid((token.Length() - (m_PeekedColumn - 6)),1).IsSameAs(_("0"))) )
525 token = wxEmptyString;
526 else
527 {
528 token = GetToken();
529 if (m_Column > 7)
530 token = token.Mid(token.Length() - (m_Column - 7));
531 else
532 {
533 token = PeekToken();
534 if (m_LineNumberStart == m_PeekedLineNumberStart)
535 token = GetToken();
536 else
537 token = wxEmptyString;
538 }
539 }
540 }
541 }
542 return token;
543 }
544
545
PeekToken()546 wxString Tokenizerf::PeekToken()
547 {
548 unsigned int undoTokenIndex = m_TokenIndex;
549 unsigned int undoLineNumber = m_LineNumber;
550 unsigned int undoLineNumberStart = m_LineNumberStart;
551 unsigned int undoColumn = m_Column;
552 bool undoWasNextLine = m_WasNextLine;
553
554 m_PeekedToken = DoGetToken();
555
556 m_WasPeeked = true;
557 m_PeekedTokenIndex = m_TokenIndex;
558 m_PeekedLineNumber = m_LineNumber;
559 m_PeekedLineNumberStart = m_LineNumberStart;
560 m_PeekedColumn = m_Column;
561 m_PeekedWasNextLine = m_WasNextLine;
562
563 m_TokenIndex = undoTokenIndex;
564 m_LineNumber = undoLineNumber;
565 m_LineNumberStart = undoLineNumberStart;
566 m_Column = undoColumn;
567 m_WasNextLine = undoWasNextLine;
568
569 return m_PeekedToken;
570 }
571
PeekTokenSameFortranLine()572 wxString Tokenizerf::PeekTokenSameFortranLine()
573 {
574 unsigned int undoTokenIndex = m_TokenIndex;
575 unsigned int undoLineNumber = m_LineNumber;
576 unsigned int undoLineNumberStart = m_LineNumberStart;
577 unsigned int undoColumn = m_Column;
578 bool undoWasNextLine = m_WasNextLine;
579
580 wxString token = GetTokenSameFortranLine();
581
582 m_WasPeeked = false;
583
584 m_TokenIndex = undoTokenIndex;
585 m_LineNumber = undoLineNumber;
586 m_LineNumberStart = undoLineNumberStart;
587 m_Column = undoColumn;
588 m_WasNextLine = undoWasNextLine;
589
590 return token;
591 }
592
UngetToken()593 void Tokenizerf::UngetToken()
594 {
595 // m_WasPeeked = true;
596 // m_PeekedTokenIndex = m_TokenIndex;
597 // m_PeekedLineNumber = m_LineNumber;
598 // m_PeekedLineNumberStart = m_LineNumberStart;
599 // m_PeekedColumn = m_Column;
600 // m_PeekedWasNextLine = m_WasNextLine;
601
602 m_WasPeeked = false;
603
604 m_TokenIndex = m_UndoTokenIndex;
605 m_LineNumber = m_UndoLineNumber;
606 m_LineNumberStart = m_UndoLineNumberStart;
607 m_Column = m_UndoColumn;
608 m_WasNextLine = m_UndoWasNextLine;
609 }
610
DoGetToken()611 wxString Tokenizerf::DoGetToken()
612 {
613 if (IsEOF())
614 return wxEmptyString;
615
616 if (!SkipWhiteSpace())
617 return wxEmptyString;
618
619 if (!SkipUnwanted())
620 return wxEmptyString;
621
622 m_LineNumberStart = m_LineNumber;
623
624 int start = m_TokenIndex;
625 wxString ret_Str;
626
627 if (IsBindTo())
628 {
629 m_TokenIndex += 8;
630 ret_Str = _T("!bindto");
631 }
632 else if (isalpha(CurrentChar()) || CurrentChar() == '_' || CurrentChar() == '$' || CurrentChar() == '#')
633 {
634 // keywords, identifiers, etc.
635 while (!IsEOF() &&
636 (isalnum(CurrentChar()) ||
637 CurrentChar() == '_' ||
638 CurrentChar() == '$' ||
639 CurrentChar() == '#'))
640 MoveToNextChar();
641 if (IsEOF())
642 return wxEmptyString;
643 ret_Str = m_Buffer.Mid(start, m_TokenIndex - start);
644 }
645 else if (isdigit(CurrentChar()))
646 {
647 // numbers
648 while (!IsEOF() && CharInString(CurrentChar(), "0123456789.abcdefABCDEFfXxLl"))
649 MoveToNextChar();
650 if (IsEOF())
651 return wxEmptyString;
652 ret_Str = m_Buffer.Mid(start, m_TokenIndex - start);
653 }
654 else if (CurrentChar() == '"' ||
655 CurrentChar() == '\'')
656 {
657 // string, char, etc.
658 wxChar match = CurrentChar();
659 MoveToNextChar(); // skip starting ' or "
660 if (!SkipToChar(match, true))
661 return wxEmptyString;
662 MoveToNextChar(); // skip ending ' or "
663 ret_Str = m_Buffer.Mid(start, m_TokenIndex - start);
664 }
665 else if (CurrentChar() == ':')
666 {
667 if (NextChar() == ':')
668 {
669 MoveToNextChar();
670 MoveToNextChar();
671 ret_Str = _T("::");
672 }
673 else
674 {
675 MoveToNextChar();
676 ret_Str = _T(":");
677 }
678 }
679 else if (CurrentChar() == '=' && NextChar() == '>')
680 {
681 MoveToNextChar();
682 MoveToNextChar();
683 ret_Str = _T("=>");
684 }
685 else if (CurrentChar() == '(' || CurrentChar() == '[')
686 {
687 // skip block ()
688 wxChar chBlock = CurrentChar();
689 wxString tmp;
690 if (m_SourceForm == fsfFree)
691 {
692 if (!SkipBlock(chBlock,1))
693 return wxEmptyString;
694 tmp = m_Buffer.Mid(start, m_TokenIndex - start);
695
696 // skip fortran comments
697 for (unsigned int i = 0; i < tmp.Length() - 1; ++i)
698 {
699 if (tmp.GetChar(i) == '!')
700 {
701 // replace comment line with spaces
702 tmp.SetChar(i,' ');
703 for(++i; i < tmp.Length() - 1; ++i)
704 {
705 if (tmp.GetChar(i) == '\n')
706 {
707 tmp.SetChar(i,' ');
708 break;
709 }
710 else
711 {
712 tmp.SetChar(i,' ');
713 }
714 }
715 }
716 }
717 }
718 else
719 {
720 // fsfFixed
721 if (!SkipBlock(chBlock, 20))
722 return wxEmptyString;
723 tmp = m_Buffer.Mid(start, m_TokenIndex - start);
724
725 // skip fixed-form fortran comments
726 int col = -1;
727 for (unsigned int i = 0; i < tmp.Length() - 1; ++i)
728 {
729 if (col != -1)
730 {
731 col++;
732 if (col == 6 && tmp.GetChar(i) != ' ' && tmp.GetChar(i) != '0')
733 {
734 //this line is continuation line
735 tmp.SetChar(i,' ');
736 }
737 else if (col == 6)
738 {
739 // something is wrong
740 return wxEmptyString;
741 }
742 }
743 if ( (tmp.GetChar(i) == '!') ||
744 (col == 1 && (tmp.GetChar(i) == 'c' || tmp.GetChar(i) == 'C' || tmp.GetChar(i) == '*')) )
745 {
746 // replace comment line with spaces
747 tmp.SetChar(i,' ');
748 for(++i; i < tmp.Length() - 1; ++i)
749 {
750 if (tmp.GetChar(i) == '\n')
751 {
752 col = 0;
753 tmp.SetChar(i,' ');
754 break;
755 }
756 else
757 {
758 tmp.SetChar(i,' ');
759 }
760 }
761 }
762 else if (tmp.GetChar(i) == '\n')
763 {
764 col = 0;
765 }
766 }
767 }
768 tmp.Replace(_T("\t"), _T(" ")); // replace tabs with spaces
769 tmp.Replace(_T("\n"), _T(" ")); // replace LF with spaces
770 tmp.Replace(_T("\r"), _T(" ")); // replace CR with spaces
771 tmp.Replace(_T("&"), _T(" ")); // replace fortran line continuation with spaces
772 // fix-up arguments (remove excessive spaces/tabs/newlines)
773 for (unsigned int i = 0; i < tmp.Length()-1; ++i)
774 {
775 if (i < tmp.Length()-1 && tmp.GetChar(i) == ' ' && tmp.GetChar(i + 1) == ' ')
776 continue; // skip excessive spaces
777 ret_Str << tmp.GetChar(i);
778 }
779 if (chBlock == '(')
780 {
781 ret_Str << _T(')'); // add closing parenthesis (see "i < tmp.Length() - 1" in previous "for")
782 ret_Str.Replace(_T(" "), _T(" ")); // replace two-spaces with single-space (introduced if it skipped comments or assignments)
783 ret_Str.Replace(_T("( "), _T("("));
784 ret_Str.Replace(_T(" )"), _T(")"));
785 }
786 else
787 {
788 ret_Str << _T(']'); // add closing parenthesis (see "i < tmp.Length() - 1" in previous "for")
789 ret_Str.Replace(_T(" "), _T(" ")); // replace two-spaces with single-space (introduced if it skipped comments or assignments)
790 ret_Str.Replace(_T("[ "), _T("["));
791 ret_Str.Replace(_T(" ]"), _T("]"));
792 }
793 }
794 else
795 {
796 ret_Str = CurrentChar();
797 MoveToNextChar();
798 }
799
800 return ret_Str;
801 }
802
803
GetTokensToEOL(wxArrayString * arrStrLines)804 wxArrayString Tokenizerf::GetTokensToEOL(wxArrayString* arrStrLines)
805 {
806 // get all tokens on line until EOL
807 wxArrayString arrStr;
808 wxString o_tok;
809 wxString tok;
810 bool newLineNext = false;
811 while (1)
812 {
813 unsigned int line = m_LineNumber;
814 o_tok = tok;
815 tok = GetToken();
816
817 if (tok.IsEmpty())
818 break;
819 unsigned int n_line = m_LineNumber;
820 if (m_SourceForm == fsfFree)
821 {
822 if ( (n_line > line) && !o_tok.IsSameAs(_T("&")) )
823 {
824 UngetToken();
825 break;
826 }
827 else if (tok.IsSameAs(_T(";")))
828 {
829 break;
830 }
831 else if (!tok.IsSameAs(_T("&")) && ((!m_DetailedParsing && !tok.IsSameAs(_T(","))) || m_DetailedParsing) )
832 {
833 arrStr.Add(tok);
834 if (arrStrLines)
835 arrStrLines->Add(GetCurrentLine());
836 }
837 }
838 else
839 {
840 if ( (((n_line > line) && (m_Column != 0)) || newLineNext) && (m_Column != 7 || tok.Length() > 1) )
841 {
842 UngetToken();
843 break;
844 }
845 else if (tok.IsSameAs(_T(";")) || (m_Column < 7 && m_Column != 0))
846 {
847 break;
848 }
849 else if ((m_Column > 7 || m_Column == 0) && ((!m_DetailedParsing && !tok.IsSameAs(_T(","))) || m_DetailedParsing) )
850 {
851 arrStr.Add(tok);
852 if (arrStrLines)
853 arrStrLines->Add(GetCurrentLine());
854 }
855
856 if (m_Column == 0)
857 newLineNext = true;
858 else
859 newLineNext = false;
860 }
861 }
862 return arrStr;
863 }
864
865
PeekTokensToEOL()866 wxArrayString Tokenizerf::PeekTokensToEOL()
867 {
868 // peek all tokens on line until EOL
869 unsigned int undoTokenIndex = m_TokenIndex;
870 unsigned int undoLineNumber = m_LineNumber;
871 unsigned int undoLineNumberStart = m_LineNumberStart;
872 unsigned int undoColumn = m_Column;
873 bool undoWasNextLine = m_WasNextLine;
874
875 wxArrayString arrStr = GetTokensToEOL();
876
877 m_WasPeeked = false;
878 m_TokenIndex = undoTokenIndex;
879 m_LineNumber = undoLineNumber;
880 m_LineNumberStart = undoLineNumberStart;
881 m_Column = undoColumn;
882 m_WasNextLine = undoWasNextLine;
883
884 return arrStr;
885 }
886
GetCurrentLine()887 wxString Tokenizerf::GetCurrentLine()
888 {
889 int curLineStart = GetLineStartIndex(m_TokenIndex);
890 int curLineEnd = GetLineEndIndex(m_TokenIndex);
891 wxString curLine = m_Buffer.Mid(curLineStart, curLineEnd - curLineStart);
892 return curLine;
893 }
894
GetLineFortran()895 wxString Tokenizerf::GetLineFortran()
896 {
897 // get current statements' line including continuation lines
898 int curLineStart = GetLineStartIndex(m_TokenIndex);
899 int curInd = m_TokenIndex - curLineStart;
900 int curLineEnd = GetLineEndIndex(m_TokenIndex);
901 wxString curLine = m_Buffer.Mid(curLineStart, curLineEnd - curLineStart);
902
903 int comInd = curLine.Find('!');
904 if (comInd != wxNOT_FOUND)
905 curLine = curLine.Mid(0,comInd);
906
907 bool startFound = false;
908 bool endFound = false;
909 int sc_ind = curLine.Find(';');
910 if (sc_ind != wxNOT_FOUND)
911 {
912 if (sc_ind >= curInd)
913 {
914 curLine = curLine.Mid(0,sc_ind);
915 endFound = true;
916 }
917 else
918 {
919 curLine = curLine.Mid(sc_ind+1);
920 startFound = true;
921 }
922
923 }
924 curLine = curLine.Trim().Trim(false);
925
926 if (curLineStart != 0 && !startFound)
927 {
928 int beforeLineStart;
929 int beforeLineEnd = curLineStart - 1;
930 beforeLineStart = GetLineStartIndex(beforeLineEnd);
931
932 while (beforeLineStart != 0)
933 {
934 wxString beforeLine = m_Buffer.Mid(beforeLineStart, beforeLineEnd - beforeLineStart);
935 comInd = beforeLine.Find('!');
936 if (comInd != wxNOT_FOUND)
937 beforeLine = beforeLine.Mid(0,comInd);
938 beforeLine = beforeLine.Trim().Trim(false);
939 if (beforeLine.EndsWith(_T("&")))
940 {
941 curLine = beforeLine.BeforeLast('&').Trim() + _T(" ") + curLine;
942 sc_ind = curLine.Find(';');
943 if (sc_ind != wxNOT_FOUND)
944 {
945 curLine = curLine.Mid(sc_ind+1).Trim(false);
946 break;
947 }
948 beforeLineEnd = beforeLineStart - 1;
949 beforeLineStart = GetLineStartIndex(beforeLineEnd);
950 }
951 else
952 {
953 break;
954 }
955 }
956 }
957
958 if (!endFound && curLine.EndsWith(_T("&")))
959 {
960 curLine = curLine.BeforeLast('&').Trim();
961 unsigned int afterLineStart = curLineEnd + 1;
962 unsigned int afterLineEnd = GetLineEndIndex(afterLineStart);
963 while (afterLineStart < afterLineEnd)
964 {
965 wxString afterLine = m_Buffer.Mid(afterLineStart, afterLineEnd - afterLineStart);
966 comInd = afterLine.Find('!');
967 if (comInd != wxNOT_FOUND)
968 afterLine = afterLine.Mid(0,comInd);
969 curLine = curLine + _T(" ") + afterLine.Trim().Trim(false);
970 sc_ind = curLine.Find(';');
971 if (sc_ind != wxNOT_FOUND)
972 {
973 curLine = curLine.Mid(0,sc_ind).Trim();
974 break;
975 }
976 if (curLine.EndsWith(_T("&")))
977 {
978 curLine = curLine.BeforeLast('&');
979 afterLineStart = afterLineEnd + 1;
980 afterLineEnd = GetLineEndIndex(afterLineStart);
981 }
982 else
983 {
984 break;
985 }
986 }
987 }
988 return curLine;
989 }
990
GetLineStartIndex(unsigned int indexInLine)991 unsigned int Tokenizerf::GetLineStartIndex(unsigned int indexInLine)
992 {
993 unsigned int startIndex;
994 bool foundEnd = false;
995 for (int i=indexInLine-1; i>=0; i--)
996 {
997 if (m_Buffer.GetChar(i) == '\n')
998 {
999 startIndex = i+1;
1000 foundEnd = true;
1001 break;
1002 }
1003 }
1004 if (!foundEnd)
1005 startIndex = 0;
1006
1007 return startIndex;
1008 }
1009
GetLineEndIndex(unsigned int indexInLine)1010 unsigned int Tokenizerf::GetLineEndIndex(unsigned int indexInLine)
1011 {
1012 unsigned int endIndex;
1013 bool foundEnd = false;
1014 for (unsigned int i=indexInLine; i<m_Buffer.Len(); i++)
1015 {
1016 if (m_Buffer.GetChar(i) == '\n')
1017 {
1018 endIndex = i;
1019 foundEnd = true;
1020 break;
1021 }
1022 }
1023 if (!foundEnd)
1024 endIndex = m_Buffer.Len() - 1;
1025
1026 return endIndex;
1027 }
1028
SetDetailedParsing(bool detPars)1029 void Tokenizerf::SetDetailedParsing(bool detPars)
1030 {
1031 m_DetailedParsing = detPars;
1032 }
1033
SetFilename(const wxString & filename)1034 void Tokenizerf::SetFilename(const wxString& filename)
1035 {
1036 m_Filename = filename;
1037 }
1038
GetLine(unsigned int nl)1039 wxString Tokenizerf::GetLine(unsigned int nl)
1040 {
1041 // get line nl. Note: line numbers starts from 1.
1042 if (nl == 0 || nl > m_LineStartIdx.size())
1043 return wxEmptyString;
1044
1045 unsigned int endIndex;
1046 if (nl >= m_LineStartIdx.size())
1047 endIndex = m_Buffer.Len() - 1;
1048 else
1049 endIndex = m_LineStartIdx[nl];
1050
1051 wxString linenl = m_Buffer.Mid(m_LineStartIdx[nl-1], endIndex - m_LineStartIdx[nl-1]);
1052 return linenl;
1053 }
1054
IsBindTo()1055 bool Tokenizerf::IsBindTo()
1056 {
1057 if (CurrentChar() == '!' ||
1058 ((CurrentChar() == 'c' || CurrentChar() == 'C' || CurrentChar() == '*') && m_Column == 1 && m_SourceForm == fsfFixed))
1059 {
1060 if (m_TokenIndex + 7 >= m_BufferLen)
1061 return false;
1062
1063 wxString str = m_Buffer.Mid(m_TokenIndex+1,6);
1064 if (str.IsSameAs(_T("bindto"),false))
1065 {
1066 unsigned int idx = m_TokenIndex + 7;
1067 if (m_Buffer.GetChar(idx) == ' ' || m_Buffer.GetChar(idx) == '\t')
1068 return true;
1069 }
1070 }
1071 return false;
1072 }
1073