1 // Tokenizer.cpp
2 #ifndef NOT_USING_MODULES
3 // !!! if you are not using modules, read BELOW !!!
4 #include "module.h" // if you are not using modules,
5 // create an empty Module.h in your
6 // project -- use of modules allows
7 // the error handler to be overridden
8 // with a custom CErrHandler
9 #endif
10 #include "tokenizer.h"
11
12 #ifdef WIN32_FILE_IO
13 #include <stdio.h>
14 #include <stdlib.h>
15 #endif
16
17 enum
18 {
19 DIR_INCLUDE = TK_USERDEF,
20 DIR_IFDEF,
21 DIR_IFNDEF,
22 DIR_ENDIF,
23 DIR_ELSE,
24 DIR_DEFINE,
25 DIR_UNDEFINE,
26 };
27
28 keywordArray_t CTokenizer::directiveKeywords[] =
29 {
30 "include", DIR_INCLUDE,
31 "ifdef", DIR_IFDEF,
32 "ifndef", DIR_IFNDEF,
33 "endif", DIR_ENDIF,
34 "else", DIR_ELSE,
35 "define", DIR_DEFINE,
36 "undefine", DIR_UNDEFINE,
37 "", TK_EOF,
38 };
39
40 keywordArray_t CTokenizer::errorMessages[] =
41 {
42 "No Error", TKERR_NONE,
43 "Unknown Error", TKERR_UNKNOWN,
44 "Buffer creation failed", TKERR_BUFFERCREATE,
45 "Unrecognized symbol", TKERR_UNRECOGNIZEDSYMBOL,
46 "Duplicate symbol", TKERR_DUPLICATESYMBOL,
47 "String length exceeded", TKERR_STRINGLENGTHEXCEEDED,
48 "Identifier length exceeded", TKERR_IDENTIFIERLENGTHEXCEEDED,
49 "Expected integer", TKERR_EXPECTED_INTEGER,
50 "Expected identifier", TKERR_EXPECTED_IDENTIFIER,
51 "Expected string", TKERR_EXPECTED_STRING,
52 "Expected char", TKERR_EXPECTED_CHAR,
53 "Expected float", TKERR_EXPECTED_FLOAT,
54 "Unexpected token", TKERR_UNEXPECTED_TOKEN,
55 "Invalid directive", TKERR_INVALID_DIRECTIVE,
56 "Include file not found", TKERR_INCLUDE_FILE_NOTFOUND,
57 "Unmatched directive", TKERR_UNMATCHED_DIRECTIVE,
58 "", TKERR_USERERROR,
59 };
60
61 //
62 // CSymbol
63 //
64
CSymbol()65 CSymbol::CSymbol()
66 {
67 }
68
~CSymbol()69 CSymbol::~CSymbol()
70 {
71 }
72
Create(LPCTSTR symbolName)73 CSymbol* CSymbol::Create(LPCTSTR symbolName)
74 {
75 CSymbol* retval = new CSymbol();
76 retval->Init(symbolName);
77 return retval;
78 }
79
GetName()80 LPCTSTR CSymbol::GetName()
81 {
82 if (m_symbolName == NULL)
83 {
84 return "";
85 }
86 return m_symbolName;
87 }
88
InitBaseSymbol(LPCTSTR symbolName)89 void CSymbol::InitBaseSymbol(LPCTSTR symbolName)
90 {
91 m_symbolName = (char*)malloc(strlen(symbolName) + 1);
92 // ASSERT(m_symbolName);
93 strcpy(m_symbolName, symbolName);
94 }
95
Delete()96 void CSymbol::Delete()
97 {
98 if (m_symbolName != NULL)
99 {
100 free(m_symbolName);
101 m_symbolName = NULL;
102 }
103 delete this;
104 }
105
106 //
107 // CDirectiveSymbol
108 //
109
CDirectiveSymbol()110 CDirectiveSymbol::CDirectiveSymbol()
111 {
112 }
113
~CDirectiveSymbol()114 CDirectiveSymbol::~CDirectiveSymbol()
115 {
116 }
117
Create(LPCTSTR symbolName)118 CDirectiveSymbol* CDirectiveSymbol::Create(LPCTSTR symbolName)
119 {
120 CDirectiveSymbol* retval = new CDirectiveSymbol();
121 retval->Init(symbolName);
122 return retval;
123 }
124
Init(LPCTSTR symbolName)125 void CDirectiveSymbol::Init(LPCTSTR symbolName)
126 {
127 CSymbol::InitBaseSymbol(symbolName);
128 m_value = NULL;
129 }
130
Delete()131 void CDirectiveSymbol::Delete()
132 {
133 if (m_value != NULL)
134 {
135 free(m_value);
136 m_value = NULL;
137 }
138 CSymbol::Delete();
139 }
140
SetValue(LPCTSTR value)141 void CDirectiveSymbol::SetValue(LPCTSTR value)
142 {
143 if (m_value != NULL)
144 {
145 free(m_value);
146 }
147 m_value = (char*)malloc(strlen(value) + 1);
148 strcpy(m_value, value);
149 }
150
GetValue()151 LPCTSTR CDirectiveSymbol::GetValue()
152 {
153 return m_value;
154 }
155
156 //
157 // CIntSymbol
158 //
159
CIntSymbol()160 CIntSymbol::CIntSymbol()
161 {
162 }
163
Create(LPCTSTR symbolName,int value)164 CIntSymbol* CIntSymbol::Create(LPCTSTR symbolName, int value)
165 {
166 CIntSymbol* retval = new CIntSymbol();
167 retval->Init(symbolName, value);
168 return retval;
169 }
170
Delete()171 void CIntSymbol::Delete()
172 {
173 CSymbol::Delete();
174 }
175
Init(LPCTSTR symbolName,int value)176 void CIntSymbol::Init(LPCTSTR symbolName, int value)
177 {
178 CSymbol::InitBaseSymbol(symbolName);
179 m_value = value;
180 }
181
GetValue()182 int CIntSymbol::GetValue()
183 {
184 return m_value;
185 }
186
187 //
188 // CSymbolTable
189 //
190
CSymbolTable()191 CSymbolTable::CSymbolTable()
192 {
193 Init();
194 }
195
~CSymbolTable()196 CSymbolTable::~CSymbolTable()
197 {
198 }
199
Create()200 CSymbolTable* CSymbolTable::Create()
201 {
202 CSymbolTable* retval = new CSymbolTable();
203 retval->Init();
204 return retval;
205 }
206
Init()207 void CSymbolTable::Init()
208 {
209 }
210
DiscardSymbols()211 void CSymbolTable::DiscardSymbols()
212 {
213 for (symbolmap_t::iterator isymbol = m_symbols.begin(); isymbol != m_symbols.end(); ++isymbol)
214 {
215 (*isymbol).second->Delete();
216 }
217 m_symbols.erase(m_symbols.begin(), m_symbols.end());
218 }
219
Delete()220 void CSymbolTable::Delete()
221 {
222 DiscardSymbols();
223 delete this;
224 }
225
AddSymbol(CSymbol * theSymbol)226 bool CSymbolTable::AddSymbol(CSymbol* theSymbol)
227 {
228 LPCTSTR name = theSymbol->GetName();
229
230 symbolmap_t::iterator iter = m_symbols.find(name);
231 if (iter != m_symbols.end())
232 {
233 return false;
234 }
235 m_symbols.insert(symbolmap_t::value_type(name, theSymbol));
236 return true;
237 }
238
FindSymbol(LPCTSTR symbolName)239 CSymbol* CSymbolTable::FindSymbol(LPCTSTR symbolName)
240 {
241 symbolmap_t::iterator iter = m_symbols.find(symbolName);
242 if (iter != m_symbols.end())
243 {
244 return (*iter).second;
245 }
246 return NULL;
247 }
248
ExtractSymbol(LPCTSTR symbolName)249 CSymbol* CSymbolTable::ExtractSymbol(LPCTSTR symbolName)
250 {
251 symbolmap_t::iterator iter = m_symbols.find(symbolName);
252 if (iter != m_symbols.end())
253 {
254 CSymbol* retval = (*iter).second;
255 m_symbols.erase(iter);
256 }
257 return NULL;
258 }
259
RemoveSymbol(LPCTSTR symbolName)260 void CSymbolTable::RemoveSymbol(LPCTSTR symbolName)
261 {
262 m_symbols.erase(symbolName);
263 }
264
265 //
266 // CParseStream
267 //
268
CParseStream()269 CParseStream::CParseStream()
270 {
271 }
272
~CParseStream()273 CParseStream::~CParseStream()
274 {
275 }
276
Create()277 CParseStream* CParseStream::Create()
278 {
279 return NULL;
280 }
281
Delete()282 void CParseStream::Delete()
283 {
284 delete this;
285 }
286
InitBaseStream()287 bool CParseStream::InitBaseStream()
288 {
289 m_next = NULL;
290
291 return true;
292 }
293
NextChar(byte & theByte)294 bool CParseStream::NextChar(byte& theByte)
295 {
296 return false;
297 }
298
GetRemainingSize()299 long CParseStream::GetRemainingSize()
300 {
301 return 0;
302 }
303
GetNext()304 CParseStream* CParseStream::GetNext()
305 {
306 return m_next;
307 }
308
SetNext(CParseStream * next)309 void CParseStream::SetNext(CParseStream* next)
310 {
311 m_next = next;
312 }
313
GetCurLine()314 int CParseStream::GetCurLine()
315 {
316 return 0;
317 }
318
GetCurFilename(char ** theBuff)319 void CParseStream::GetCurFilename(char** theBuff)
320 {
321 *theBuff = NULL;
322 }
323
IsThisDefinition(void * theDefinition)324 bool CParseStream::IsThisDefinition(void* theDefinition)
325 {
326 return false;
327 }
328
329 //
330 // CParsePutBack
331 //
332
CParsePutBack()333 CParsePutBack::CParsePutBack()
334 {
335 }
336
~CParsePutBack()337 CParsePutBack::~CParsePutBack()
338 {
339 }
340
Create(byte theByte,int curLine,LPCTSTR filename)341 CParsePutBack* CParsePutBack::Create(byte theByte, int curLine, LPCTSTR filename)
342 {
343 CParsePutBack* curParsePutBack = new CParsePutBack();
344 curParsePutBack->Init(theByte, curLine, filename);
345 return curParsePutBack;
346 }
347
Delete()348 void CParsePutBack::Delete()
349 {
350 if (m_curFile != NULL)
351 {
352 free(m_curFile);
353 m_curFile = NULL;
354 }
355 delete this;
356 }
357
NextChar(byte & theByte)358 bool CParsePutBack::NextChar(byte& theByte)
359 {
360 if (m_consumed)
361 {
362 return false;
363 }
364 theByte = m_byte;
365 m_consumed = true;
366 return true;
367 }
368
Init(byte theByte,int curLine,LPCTSTR filename)369 void CParsePutBack::Init(byte theByte, int curLine, LPCTSTR filename)
370 {
371 CParseStream::InitBaseStream();
372 m_consumed = false;
373 m_byte = theByte;
374 m_curLine = curLine;
375 if (filename != NULL)
376 {
377 m_curFile = (char*)malloc(strlen(filename) + 1);
378 strcpy(m_curFile, filename);
379 }
380 else
381 {
382 m_curFile = NULL;
383 }
384 }
385
GetRemainingSize()386 long CParsePutBack::GetRemainingSize()
387 {
388 if (m_consumed)
389 {
390 return 0;
391 }
392 else
393 {
394 return 1;
395 }
396 }
397
GetCurLine()398 int CParsePutBack::GetCurLine()
399 {
400 return m_curLine;
401 }
402
GetCurFilename(char ** theBuff)403 void CParsePutBack::GetCurFilename(char** theBuff)
404 {
405 if (m_curFile == NULL)
406 {
407 *theBuff = NULL;
408 return;
409 }
410 *theBuff = (char*)malloc(strlen(m_curFile) + 1);
411 strcpy(*theBuff, m_curFile);
412 }
413
414 //
415 // CParseFile
416 //
417
CParseFile()418 CParseFile::CParseFile()
419 {
420 }
421
~CParseFile()422 CParseFile::~CParseFile()
423 {
424 }
425
Create()426 CParseFile* CParseFile::Create()
427 {
428 CParseFile* theParseFile = new CParseFile();
429
430 if ( !theParseFile->Init() )
431 {
432 delete theParseFile;
433 return NULL;
434 }
435
436 return theParseFile;
437 }
438
Create(LPCTSTR filename,CTokenizer * tokenizer)439 CParseFile* CParseFile::Create(LPCTSTR filename, CTokenizer* tokenizer)
440 {
441 CParseFile* theParseFile = new CParseFile();
442
443 if ( theParseFile->Init(filename, tokenizer) )
444 return theParseFile;
445
446 return NULL;
447 }
448
Delete()449 void CParseFile::Delete()
450 {
451 if (m_buff != NULL)
452 {
453 free(m_buff);
454 m_buff = NULL;
455 }
456 if (m_ownsFile && (m_fileHandle != NULL))
457 {
458 #ifdef WIN32_FILE_IO
459 CloseHandle(m_fileHandle);
460 #else
461 fclose( m_fileHandle );
462 #endif
463 m_fileHandle = NULL;
464 }
465 if (m_fileName != NULL)
466 {
467 free(m_fileName);
468 m_fileName = NULL;
469 }
470 delete this;
471 }
472
Init()473 bool CParseFile::Init()
474 {
475 m_fileHandle = NULL;
476 m_buff = NULL;
477 m_ownsFile = false;
478 m_curByte = NULL;
479 m_curLine = 1;
480 m_fileName = NULL;
481 return CParseStream::InitBaseStream();
482 }
483
GetFileSize()484 unsigned int CParseFile::GetFileSize()
485 {
486 #ifdef WIN32_FILE_IO
487 unsigned int dwCur = SetFilePointer(m_fileHandle, 0L, NULL, FILE_CURRENT);
488 unsigned int dwLen = SetFilePointer(m_fileHandle, 0, NULL, FILE_END);
489 SetFilePointer(m_fileHandle, dwCur, NULL, FILE_BEGIN);
490 #else
491 fseek( m_fileHandle, 0L, SEEK_END );
492 unsigned int dwLen = ftell( m_fileHandle );
493 fseek( m_fileHandle, 0L, SEEK_SET );
494 #endif
495 return dwLen;
496 }
497
Read(void * buff,UINT buffsize)498 void CParseFile::Read(void* buff, UINT buffsize)
499 {
500 unsigned int bytesRead;
501 #ifdef WIN32_FILE_IO
502 ReadFile(m_fileHandle, buff, buffsize, &bytesRead, NULL);
503 #else
504 fread( buff, buffsize, 1, m_fileHandle );
505 #endif
506 }
507
Init(LPCTSTR filename,CTokenizer * tokenizer)508 bool CParseFile::Init(LPCTSTR filename, CTokenizer* tokenizer)
509 {
510 CParseStream::InitBaseStream();
511 m_fileName = (char*)malloc(strlen(filename) + 1);
512 strcpy(m_fileName, filename);
513 #ifdef WIN32_FILE_IO
514 DWORD dwAccess = GENERIC_READ;
515 DWORD dwShareMode = FILE_SHARE_WRITE | FILE_SHARE_READ;
516 SECURITY_ATTRIBUTES sa;
517 sa.nLength = sizeof(sa);
518 sa.lpSecurityDescriptor = NULL;
519 sa.bInheritHandle = 0;
520 DWORD dwCreateFlag = OPEN_EXISTING;
521
522 m_fileHandle = CreateFile(filename, dwAccess, dwShareMode, &sa, dwCreateFlag, FILE_ATTRIBUTE_NORMAL, NULL);
523
524 if (m_fileHandle == (HANDLE)-1)
525 {
526 tokenizer->Error(TKERR_INCLUDE_FILE_NOTFOUND);
527 Init();
528 return false;
529 }
530 #else
531 m_fileHandle = fopen( filename, "r+" );
532
533 if ( !m_fileHandle ) {
534 tokenizer->Error( TKERR_INCLUDE_FILE_NOTFOUND );
535 Init();
536 return false;
537 }
538 #endif
539
540 m_filesize = GetFileSize();
541 m_buff = (byte*)malloc(m_filesize);
542 if (m_buff == NULL)
543 {
544 tokenizer->Error(TKERR_BUFFERCREATE);
545 Init();
546 return false;
547 }
548 Read(m_buff, m_filesize);
549 m_curByte = 0;
550 m_curPos = 1;
551 m_ownsFile = true;
552 m_curLine = 1;
553
554 return true;
555 }
556
GetRemainingSize()557 long CParseFile::GetRemainingSize()
558 {
559 return m_filesize - m_curByte;
560 }
561
NextChar(byte & theByte)562 bool CParseFile::NextChar(byte& theByte)
563 {
564 if (m_curByte < m_filesize)
565 {
566 if (m_buff[m_curByte] == '\n')
567 {
568 m_curLine += 1;
569 m_curPos = 1;
570 }
571 else
572 {
573 m_curPos++;
574 }
575 theByte = m_buff[m_curByte++];
576 return true;
577 }
578 else
579 {
580 return false;
581 }
582 }
583
GetCurLine()584 int CParseFile::GetCurLine()
585 {
586 return m_curLine;
587 }
588
GetCurFilename(char ** theBuff)589 void CParseFile::GetCurFilename(char** theBuff)
590 {
591 *theBuff = NULL;
592 if (m_fileName != NULL)
593 {
594 *theBuff = (char*)malloc(strlen(m_fileName) + 1);
595 strcpy(*theBuff, m_fileName);
596 }
597 }
598
599 //
600 // CParseMemory
601 //
602
CParseMemory()603 CParseMemory::CParseMemory()
604 {
605 }
606
~CParseMemory()607 CParseMemory::~CParseMemory()
608 {
609 }
610
Create(byte * data,long datasize)611 CParseMemory* CParseMemory::Create(byte* data, long datasize)
612 {
613 CParseMemory* curParse = new CParseMemory();
614 curParse->Init(data, datasize);
615 return curParse;
616 }
617
Delete()618 void CParseMemory::Delete()
619 {
620 delete this;
621 }
622
NextChar(byte & theByte)623 bool CParseMemory::NextChar(byte& theByte)
624 {
625 if (m_offset < m_datasize)
626 {
627 if (m_data[m_offset] == '\n')
628 {
629 m_curLine += 1;
630 m_curPos = 1;
631 }
632 else
633 {
634 m_curPos++;
635 }
636 theByte = m_data[m_offset++];
637 return true;
638 }
639 else
640 {
641 return false;
642 }
643 }
644
Init(byte * data,long datasize)645 void CParseMemory::Init(byte* data, long datasize)
646 {
647 m_data = data;
648 m_curLine = 1;
649 m_curPos = 1;
650 m_offset = 0;
651 m_datasize = datasize;
652 }
653
GetRemainingSize()654 long CParseMemory::GetRemainingSize()
655 {
656 return m_datasize - m_offset;
657 }
658
GetCurLine()659 int CParseMemory::GetCurLine()
660 {
661 return m_curLine;
662 }
663
GetCurFilename(char ** theBuff)664 void CParseMemory::GetCurFilename(char** theBuff)
665 {
666 *theBuff = NULL;
667 }
668
669 //
670 // CParseBlock
671 //
672
CParseBlock()673 CParseBlock::CParseBlock()
674 {
675 }
676
~CParseBlock()677 CParseBlock::~CParseBlock()
678 {
679 }
680
Create(byte * data,long datasize)681 CParseBlock* CParseBlock::Create(byte* data, long datasize)
682 {
683 CParseBlock* curParse = new CParseBlock();
684 curParse->Init(data, datasize);
685 return curParse;
686 }
687
Delete()688 void CParseBlock::Delete()
689 {
690 if (m_data != NULL)
691 {
692 free(m_data);
693 m_data = NULL;
694 }
695 delete this;
696 }
697
Init(byte * data,long datasize)698 void CParseBlock::Init(byte* data, long datasize)
699 {
700 m_data = (byte*)malloc(datasize);
701 memcpy(m_data, data, datasize);
702 m_curLine = 1;
703 m_curPos = 1;
704 m_offset = 0;
705 m_datasize = datasize;
706 }
707
708 //
709 // CParseToken
710 //
711
CParseToken()712 CParseToken::CParseToken()
713 {
714 }
715
~CParseToken()716 CParseToken::~CParseToken()
717 {
718 }
719
Create(CToken * token)720 CParseToken* CParseToken::Create(CToken* token)
721 {
722 CParseToken* curParse = new CParseToken();
723 curParse->Init(token);
724 return curParse;
725 }
726
Delete()727 void CParseToken::Delete()
728 {
729 if (m_data != NULL)
730 {
731 free(m_data);
732 m_data = NULL;
733 }
734 delete this;
735 }
736
NextChar(byte & theByte)737 bool CParseToken::NextChar(byte& theByte)
738 {
739 if (m_offset < m_datasize)
740 {
741 if (m_data[m_offset] == '\n')
742 {
743 m_curLine += 1;
744 m_curPos = 1;
745 }
746 else
747 {
748 m_curPos++;
749 }
750 theByte = m_data[m_offset++];
751 return true;
752 }
753 else
754 {
755 return false;
756 }
757 }
758
Init(CToken * token)759 void CParseToken::Init(CToken* token)
760 {
761 LPCTSTR tokenString = token->GetStringValue();
762 m_datasize = strlen(tokenString);
763 if (m_datasize > 0)
764 {
765 m_data = (byte*) malloc(m_datasize);
766 memcpy(m_data, tokenString, m_datasize);
767 }
768 else
769 {
770 m_data = NULL;
771 }
772 m_curLine = 1;
773 m_curPos = 1;
774 m_offset = 0;
775 token->Delete();
776 }
777
GetRemainingSize()778 long CParseToken::GetRemainingSize()
779 {
780 return m_datasize - m_offset;
781 }
782
GetCurLine()783 int CParseToken::GetCurLine()
784 {
785 return m_curLine;
786 }
787
GetCurFilename(char ** theBuff)788 void CParseToken::GetCurFilename(char** theBuff)
789 {
790 *theBuff = NULL;
791 }
792
793 //
794 // CParseDefine
795 //
796
CParseDefine()797 CParseDefine::CParseDefine()
798 {
799 }
800
~CParseDefine()801 CParseDefine::~CParseDefine()
802 {
803 }
804
Create(CDirectiveSymbol * definesymbol)805 CParseDefine* CParseDefine::Create(CDirectiveSymbol* definesymbol)
806 {
807 CParseDefine* retval = new CParseDefine();
808 retval->Init(definesymbol);
809 return retval;
810 }
811
Delete()812 void CParseDefine::Delete()
813 {
814 CParseMemory::Delete();
815 }
816
Init(CDirectiveSymbol * definesymbol)817 void CParseDefine::Init(CDirectiveSymbol* definesymbol)
818 {
819 CParseMemory::Init((byte*)definesymbol->GetValue(), strlen(definesymbol->GetValue()));
820 m_defineSymbol = definesymbol;
821 }
822
IsThisDefinition(void * theDefinition)823 bool CParseDefine::IsThisDefinition(void* theDefinition)
824 {
825 return (CDirectiveSymbol*)theDefinition == m_defineSymbol;
826 }
827
828 //
829 // CToken
830 //
831
CToken()832 CToken::CToken()
833 {
834 }
835
~CToken()836 CToken::~CToken()
837 {
838 }
839
Create()840 CToken* CToken::Create()
841 {
842 CToken* theToken = new CToken();
843 theToken->InitBaseToken();
844 return theToken;
845 }
846
Delete()847 void CToken::Delete()
848 {
849 if (m_string != NULL)
850 {
851 free(m_string);
852 m_string = NULL;
853 }
854 delete this;
855 }
856
Init()857 void CToken::Init()
858 {
859 m_next = NULL;
860 m_string = NULL;
861 }
862
SetNext(CToken * theToken)863 void CToken::SetNext(CToken* theToken)
864 {
865 m_next = theToken;
866 }
867
GetNext()868 CToken* CToken::GetNext()
869 {
870 return m_next;
871 }
872
GetType()873 int CToken::GetType()
874 {
875 return TK_EOF;
876 }
877
GetIntValue()878 int CToken::GetIntValue()
879 {
880 return 0;
881 }
882
GetStringValue()883 LPCTSTR CToken::GetStringValue()
884 {
885 if (m_string == NULL)
886 {
887 return "";
888 }
889 return m_string;
890 }
891
GetFloatValue()892 float CToken::GetFloatValue()
893 {
894 return 0.0;
895 }
896
897 //
898 // CCharToken
899 //
900
CCharToken()901 CCharToken::CCharToken()
902 {
903 }
904
~CCharToken()905 CCharToken::~CCharToken()
906 {
907 }
908
Create(byte theByte)909 CCharToken* CCharToken::Create(byte theByte)
910 {
911 CCharToken* theToken = new CCharToken();
912 theToken->Init(theByte);
913 return theToken;
914 }
915
Delete()916 void CCharToken::Delete()
917 {
918 CToken::Delete();
919 }
920
Init(byte theByte)921 void CCharToken::Init(byte theByte)
922 {
923 CToken::InitBaseToken();
924 char charString[10];
925 switch(theByte)
926 {
927 case '\0':
928 strcpy(charString, "\\0");
929 break;
930 case '\n':
931 strcpy(charString, "\\n");
932 break;
933 case '\\':
934 strcpy(charString, "\\\\");
935 break;
936 case '\'':
937 strcpy(charString, "\\'");
938 break;
939 case '\?':
940 strcpy(charString, "\\?");
941 break;
942 case '\a':
943 strcpy(charString, "\\a");
944 break;
945 case '\b':
946 strcpy(charString, "\\b");
947 break;
948 case '\f':
949 strcpy(charString, "\\f");
950 break;
951 case '\r':
952 strcpy(charString, "\\r");
953 break;
954 case '\t':
955 strcpy(charString, "\\t");
956 break;
957 case '\v':
958 strcpy(charString, "\\v");
959 break;
960 default:
961 charString[0] = (char)theByte;
962 charString[1] = '\0';
963 break;
964 }
965 m_string = (char*)malloc(strlen(charString) + 1);
966 strcpy(m_string, charString);
967 }
968
GetType()969 int CCharToken::GetType()
970 {
971 return TK_CHAR;
972 }
973
974 //
975 // CStringToken
976 //
977
CStringToken()978 CStringToken::CStringToken()
979 {
980 }
981
~CStringToken()982 CStringToken::~CStringToken()
983 {
984 }
985
Create(LPCTSTR theString)986 CStringToken* CStringToken::Create(LPCTSTR theString)
987 {
988 CStringToken* theToken = new CStringToken();
989 theToken->Init(theString);
990 return theToken;
991 }
992
Delete()993 void CStringToken::Delete()
994 {
995 CToken::Delete();
996 }
997
Init(LPCTSTR theString)998 void CStringToken::Init(LPCTSTR theString)
999 {
1000 CToken::InitBaseToken();
1001 m_string = (char*)malloc(strlen(theString) + 1);
1002 // ASSERT(m_string);
1003 strcpy(m_string, theString);
1004 }
1005
GetType()1006 int CStringToken::GetType()
1007 {
1008 return TK_STRING;
1009 }
1010
1011 //
1012 // CIntToken
1013 //
1014
CIntToken()1015 CIntToken::CIntToken()
1016 {
1017 }
1018
~CIntToken()1019 CIntToken::~CIntToken()
1020 {
1021 }
1022
Create(long value)1023 CIntToken* CIntToken::Create(long value)
1024 {
1025 CIntToken* theToken = new CIntToken();
1026 theToken->Init(value);
1027 return theToken;
1028 }
1029
Delete()1030 void CIntToken::Delete()
1031 {
1032 CToken::Delete();
1033 }
1034
Init(long value)1035 void CIntToken::Init(long value)
1036 {
1037 CToken::InitBaseToken();
1038 m_value = value;
1039 }
1040
GetType()1041 int CIntToken::GetType()
1042 {
1043 return TK_INT;
1044 }
1045
GetIntValue()1046 int CIntToken::GetIntValue()
1047 {
1048 return m_value;
1049 }
1050
GetFloatValue()1051 float CIntToken::GetFloatValue()
1052 {
1053 return (float)m_value;
1054 }
1055
GetStringValue()1056 LPCTSTR CIntToken::GetStringValue()
1057 {
1058 if (m_string != NULL)
1059 {
1060 free(m_string);
1061 m_string = NULL;
1062 }
1063 char temp[128];
1064 sprintf(temp, "%d", m_value);
1065 m_string = (char*)malloc(strlen(temp) + 1);
1066 strcpy(m_string, temp);
1067 return m_string;
1068 }
1069
1070 //
1071 // CFloatToken
1072 //
1073
CFloatToken()1074 CFloatToken::CFloatToken()
1075 {
1076 }
1077
~CFloatToken()1078 CFloatToken::~CFloatToken()
1079 {
1080 }
1081
Create(float value)1082 CFloatToken* CFloatToken::Create(float value)
1083 {
1084 CFloatToken* theToken = new CFloatToken();
1085 theToken->Init(value);
1086 return theToken;
1087 }
1088
Delete()1089 void CFloatToken::Delete()
1090 {
1091 CToken::Delete();
1092 }
1093
Init(float value)1094 void CFloatToken::Init(float value)
1095 {
1096 CToken::InitBaseToken();
1097 m_value = value;
1098 }
1099
GetType()1100 int CFloatToken::GetType()
1101 {
1102 return TK_FLOAT;
1103 }
1104
GetFloatValue()1105 float CFloatToken::GetFloatValue()
1106 {
1107 return m_value;
1108 }
1109
GetStringValue()1110 LPCTSTR CFloatToken::GetStringValue()
1111 {
1112 if (m_string != NULL)
1113 {
1114 free(m_string);
1115 m_string = NULL;
1116 }
1117 char temp[128];
1118 sprintf(temp, "%g", m_value);
1119 m_string = (char*)malloc(strlen(temp) + 1);
1120 strcpy(m_string, temp);
1121 return m_string;
1122 }
1123
1124 //
1125 // CIdentifierToken
1126 //
1127
CIdentifierToken()1128 CIdentifierToken::CIdentifierToken()
1129 {
1130 }
1131
~CIdentifierToken()1132 CIdentifierToken::~CIdentifierToken()
1133 {
1134 }
1135
Create(LPCTSTR name)1136 CIdentifierToken* CIdentifierToken::Create(LPCTSTR name)
1137 {
1138 CIdentifierToken* theToken = new CIdentifierToken();
1139 theToken->Init(name);
1140 return theToken;
1141 }
1142
Delete()1143 void CIdentifierToken::Delete()
1144 {
1145 CToken::Delete();
1146 }
1147
Init(LPCTSTR name)1148 void CIdentifierToken::Init(LPCTSTR name)
1149 {
1150 CToken::InitBaseToken();
1151 m_string = (char*)malloc(strlen(name) + 1);
1152 // ASSERT(m_string);
1153 strcpy(m_string, name);
1154 }
1155
GetType()1156 int CIdentifierToken::GetType()
1157 {
1158 return TK_IDENTIFIER;
1159 }
1160
1161 //
1162 // CCommentToken
1163 //
1164
CCommentToken()1165 CCommentToken::CCommentToken()
1166 {
1167 }
1168
~CCommentToken()1169 CCommentToken::~CCommentToken()
1170 {
1171 }
1172
Create(LPCTSTR name)1173 CCommentToken* CCommentToken::Create(LPCTSTR name)
1174 {
1175 CCommentToken* theToken = new CCommentToken();
1176 theToken->Init(name);
1177 return theToken;
1178 }
1179
Delete()1180 void CCommentToken::Delete()
1181 {
1182 CToken::Delete();
1183 }
1184
Init(LPCTSTR name)1185 void CCommentToken::Init(LPCTSTR name)
1186 {
1187 CToken::InitBaseToken();
1188 m_string = (char*)malloc(strlen(name) + 1);
1189 // ASSERT(m_string);
1190 strcpy(m_string, name);
1191 }
1192
GetType()1193 int CCommentToken::GetType()
1194 {
1195 return TK_COMMENT;
1196 }
1197
1198 //
1199 // CUserToken
1200 //
1201
CUserToken()1202 CUserToken::CUserToken()
1203 {
1204 }
1205
~CUserToken()1206 CUserToken::~CUserToken()
1207 {
1208 }
1209
Create(int value,LPCTSTR string)1210 CUserToken* CUserToken::Create(int value, LPCTSTR string)
1211 {
1212 CUserToken* theToken = new CUserToken();
1213 theToken->Init(value, string);
1214 return theToken;
1215 }
1216
Delete()1217 void CUserToken::Delete()
1218 {
1219 CToken::Delete();
1220 }
1221
Init(int value,LPCTSTR string)1222 void CUserToken::Init(int value, LPCTSTR string)
1223 {
1224 CToken::InitBaseToken();
1225 m_value = value;
1226 m_string = (char*)malloc(strlen(string) + 1);
1227 strcpy(m_string, string);
1228 }
1229
GetType()1230 int CUserToken::GetType()
1231 {
1232 return m_value;
1233 }
1234
1235 //
1236 // CUndefinedToken
1237 //
1238
CUndefinedToken()1239 CUndefinedToken::CUndefinedToken()
1240 {
1241 }
1242
~CUndefinedToken()1243 CUndefinedToken::~CUndefinedToken()
1244 {
1245 }
1246
Create(LPCTSTR string)1247 CUndefinedToken* CUndefinedToken::Create(LPCTSTR string)
1248 {
1249 CUndefinedToken* theToken = new CUndefinedToken();
1250 theToken->Init(string);
1251 return theToken;
1252 }
1253
Delete()1254 void CUndefinedToken::Delete()
1255 {
1256 CToken::Delete();
1257 }
1258
Init(LPCTSTR string)1259 void CUndefinedToken::Init(LPCTSTR string)
1260 {
1261 CToken::InitBaseToken();
1262 m_string = (char*)malloc(strlen(string) + 1);
1263 strcpy(m_string, string);
1264 }
1265
GetType()1266 int CUndefinedToken::GetType()
1267 {
1268 return TK_UNDEFINED;
1269 }
1270
1271 //
1272 // CTokenizerState
1273 //
1274
CTokenizerState()1275 CTokenizerState::CTokenizerState()
1276 {
1277 }
1278
~CTokenizerState()1279 CTokenizerState::~CTokenizerState()
1280 {
1281 }
1282
Create(bool skip)1283 CTokenizerState* CTokenizerState::Create(bool skip)
1284 {
1285 CTokenizerState* retval = new CTokenizerState();
1286 retval->Init(skip);
1287 return retval;
1288 }
1289
Init(bool skip)1290 void CTokenizerState::Init(bool skip)
1291 {
1292 m_next = NULL;
1293 m_skip = skip;
1294 m_elseHit = false;
1295 }
1296
Delete()1297 void CTokenizerState::Delete()
1298 {
1299 delete this;
1300 }
1301
GetNext()1302 CTokenizerState* CTokenizerState::GetNext()
1303 {
1304 return m_next;
1305 }
1306
ProcessElse()1307 bool CTokenizerState::ProcessElse()
1308 {
1309 if (!m_elseHit)
1310 {
1311 m_elseHit = true;
1312 m_skip = !m_skip;
1313 }
1314 return m_elseHit;
1315 }
1316
SetNext(CTokenizerState * next)1317 void CTokenizerState::SetNext(CTokenizerState* next)
1318 {
1319 m_next = next;
1320 }
1321
Skipping()1322 bool CTokenizerState::Skipping()
1323 {
1324 return m_skip;
1325 }
1326
1327 //
1328 // CTokenizerHolderState
1329 //
1330
CTokenizerHolderState()1331 CTokenizerHolderState::CTokenizerHolderState()
1332 {
1333 }
1334
~CTokenizerHolderState()1335 CTokenizerHolderState::~CTokenizerHolderState()
1336 {
1337 }
1338
Create()1339 CTokenizerHolderState* CTokenizerHolderState::Create()
1340 {
1341 CTokenizerHolderState* retval = new CTokenizerHolderState();
1342 retval->Init();
1343 return retval;
1344 }
1345
Init()1346 void CTokenizerHolderState::Init()
1347 {
1348 CTokenizerState::Init(true);
1349 }
1350
Delete()1351 void CTokenizerHolderState::Delete()
1352 {
1353 delete this;
1354 }
1355
ProcessElse()1356 bool CTokenizerHolderState::ProcessElse()
1357 {
1358 if (!m_elseHit)
1359 {
1360 m_elseHit = true;
1361 }
1362 return m_elseHit;
1363 }
1364
1365 //
1366 // CKeywordTable
1367 //
1368
CKeywordTable(CTokenizer * tokenizer,keywordArray_t * keywords)1369 CKeywordTable::CKeywordTable(CTokenizer* tokenizer, keywordArray_t* keywords)
1370 {
1371 m_tokenizer = tokenizer;
1372 m_holdKeywords = tokenizer->SetKeywords(keywords);
1373 }
1374
~CKeywordTable()1375 CKeywordTable::~CKeywordTable()
1376 {
1377 m_tokenizer->SetKeywords(m_holdKeywords);
1378 }
1379
1380 //
1381 // CTokenizer
1382 //
1383
CTokenizer()1384 CTokenizer::CTokenizer()
1385 {
1386 }
1387
~CTokenizer()1388 CTokenizer::~CTokenizer()
1389 {
1390 }
1391
Create(UINT dwFlags)1392 CTokenizer* CTokenizer::Create(UINT dwFlags)
1393 {
1394 CTokenizer* theTokenizer = new CTokenizer();
1395 theTokenizer->Init(dwFlags);
1396 return theTokenizer;
1397 }
1398
Delete()1399 void CTokenizer::Delete()
1400 {
1401 while (m_curParseStream != NULL)
1402 {
1403 CParseStream* curStream = m_curParseStream;
1404 m_curParseStream = curStream->GetNext();
1405 curStream->Delete();
1406 }
1407 if (m_symbolLookup != NULL)
1408 {
1409 m_symbolLookup->Delete();
1410 m_symbolLookup = NULL;
1411 }
1412 while (m_nextToken != NULL)
1413 {
1414 CToken* curToken = m_nextToken;
1415 m_nextToken = curToken->GetNext();
1416 curToken->Delete();
1417 }
1418 while (m_state != NULL)
1419 {
1420 Error(TKERR_UNMATCHED_DIRECTIVE);
1421 CTokenizerState* curState = m_state;
1422 m_state = curState->GetNext();
1423 curState->Delete();
1424 }
1425
1426 /* if (m_lastErrMsg != NULL)
1427 {
1428 free(m_lastErrMsg);
1429 m_lastErrMsg = NULL;
1430 }*/
1431 delete this;
1432 }
1433
Error(int theError)1434 void CTokenizer::Error(int theError)
1435 {
1436 char errString[128];
1437 char lookupstring[128];
1438 int i = 0;
1439 while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1440 {
1441 i++;
1442 }
1443 if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1444 {
1445 i = 0;
1446 while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1447 {
1448 i++;
1449 }
1450 strcpy(lookupstring, m_errors[i].m_keyword);
1451 }
1452 else
1453 {
1454 strcpy(lookupstring, errorMessages[i].m_keyword);
1455 }
1456 sprintf(errString, "Error -- %d, %s", theError, lookupstring);
1457 Error(errString, theError);
1458 }
1459
Error(int theError,LPCTSTR errString)1460 void CTokenizer::Error(int theError, LPCTSTR errString)
1461 {
1462 char errstring[128];
1463 char lookupstring[128];
1464 int i = 0;
1465 while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1466 {
1467 i++;
1468 }
1469 if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1470 {
1471 i = 0;
1472 while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1473 {
1474 i++;
1475 }
1476 strcpy(lookupstring, m_errors[i].m_keyword);
1477 }
1478 else
1479 {
1480 strcpy(lookupstring, errorMessages[i].m_keyword);
1481 }
1482 sprintf(errstring, "Error -- %d, %s - %s", theError, lookupstring, errString);
1483 Error(errstring, theError);
1484 }
1485
Error(LPCTSTR errString,int theError)1486 void CTokenizer::Error(LPCTSTR errString, int theError)
1487 {
1488 if (m_errorProc != NULL)
1489 {
1490 m_errorProc(errString);
1491 }
1492 #ifdef USES_MODULES
1493 else
1494 {
1495 ReportError(theError, errString);
1496 }
1497 #endif
1498 }
1499
AddParseFile(LPCTSTR filename)1500 bool CTokenizer::AddParseFile(LPCTSTR filename)
1501 {
1502 CParseStream* newStream = CParseFile::Create(filename, this);
1503
1504 if ( newStream != NULL )
1505 {
1506 newStream->SetNext(m_curParseStream);
1507 m_curParseStream = newStream;
1508 return true;
1509 }
1510
1511 return false;
1512 }
1513
AddParseStream(byte * data,long datasize)1514 void CTokenizer::AddParseStream(byte* data, long datasize)
1515 {
1516 CParseStream* newStream = CParseMemory::Create(data, datasize);
1517 newStream->SetNext(m_curParseStream);
1518 m_curParseStream = newStream;
1519 }
1520
GetRemainingSize()1521 long CTokenizer::GetRemainingSize()
1522 {
1523 long retval = 0;
1524 CParseStream* curStream = m_curParseStream;
1525 while (curStream != NULL)
1526 {
1527 retval += curStream->GetRemainingSize();
1528 curStream = curStream->GetNext();
1529 }
1530 return retval;
1531 }
1532
LookupToken(int tokenID,keywordArray_t * theTable)1533 LPCTSTR CTokenizer::LookupToken(int tokenID, keywordArray_t* theTable)
1534 {
1535 if (theTable == NULL)
1536 {
1537 theTable = m_keywords;
1538 }
1539 if (theTable == NULL)
1540 {
1541 return NULL;
1542 }
1543
1544 int i = 0;
1545 while (theTable[i].m_tokenvalue != TK_EOF)
1546 {
1547 if (theTable[i].m_tokenvalue == tokenID)
1548 {
1549 return theTable[i].m_keyword;
1550 }
1551 i++;
1552 }
1553 return NULL;
1554 }
1555
PutBackToken(CToken * theToken,bool commented,LPCTSTR addedChars,bool bIgnoreThisTokenType)1556 void CTokenizer::PutBackToken(CToken* theToken, bool commented, LPCTSTR addedChars, bool bIgnoreThisTokenType)
1557 {
1558 if (commented)
1559 {
1560 CParseToken* newStream = CParseToken::Create(theToken);
1561 newStream->SetNext(m_curParseStream);
1562 m_curParseStream = newStream;
1563
1564 if (addedChars != NULL)
1565 {
1566 CParsePutBack* spacer = CParsePutBack::Create(' ', 0, NULL);
1567 spacer->SetNext(m_curParseStream);
1568 m_curParseStream = spacer;
1569
1570 CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1571 newBlock->SetNext(m_curParseStream);
1572 m_curParseStream = newBlock;
1573 }
1574
1575 char temp[] = "// * ";
1576 CParseBlock* newBlock = CParseBlock::Create((byte*)temp, strlen(temp));
1577 newBlock->SetNext(m_curParseStream);
1578 m_curParseStream = newBlock;
1579 return;
1580 }
1581
1582 switch(theToken->GetType())
1583 {
1584 case TK_INT:
1585 case TK_EOF:
1586 case TK_UNDEFINED:
1587 case TK_FLOAT:
1588 case TK_CHAR:
1589 case TK_STRING:
1590 case TK_EOL:
1591 case TK_COMMENT:
1592 if (!bIgnoreThisTokenType)
1593 {
1594 theToken->SetNext(m_nextToken);
1595 m_nextToken = theToken;
1596 break;
1597 }
1598 default:
1599 CParseToken* newStream = CParseToken::Create(theToken);
1600 newStream->SetNext(m_curParseStream);
1601 m_curParseStream = newStream;
1602 break;
1603 }
1604
1605 if (addedChars != NULL)
1606 {
1607 CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1608 newBlock->SetNext(m_curParseStream);
1609 m_curParseStream = newBlock;
1610 }
1611 }
1612
GetToken(keywordArray_t * keywords,UINT onFlags,UINT offFlags)1613 CToken* CTokenizer::GetToken(keywordArray_t* keywords, UINT onFlags, UINT offFlags)
1614 {
1615 keywordArray_t* holdKeywords = SetKeywords(keywords);
1616 CToken* retval = GetToken(onFlags, offFlags);
1617 SetKeywords(holdKeywords);
1618 return retval;
1619 }
1620
GetToken(UINT onFlags,UINT offFlags)1621 CToken* CTokenizer::GetToken(UINT onFlags, UINT offFlags)
1622 {
1623 UINT holdFlags = m_flags;
1624
1625 m_flags |= onFlags;
1626 m_flags &= (~offFlags);
1627 CToken* theToken = NULL;
1628 while (theToken == NULL)
1629 {
1630 theToken = FetchToken();
1631 if (theToken == NULL)
1632 {
1633 continue;
1634 }
1635 if (theToken->GetType() == TK_EOF)
1636 {
1637 break;
1638 }
1639 if (m_state != NULL)
1640 {
1641 if (m_state->Skipping())
1642 {
1643 theToken->Delete();
1644 theToken = NULL;
1645 }
1646 }
1647 }
1648
1649 m_flags = holdFlags;
1650 return theToken;
1651 }
1652
GetToEndOfLine(int tokenType)1653 CToken* CTokenizer::GetToEndOfLine(int tokenType)
1654 {
1655 // update, if you just want the whole line returned as a string, then allow a much bigger size than
1656 // the default string size of only 128 chars...
1657 //
1658 if (tokenType == TK_STRING)
1659 {
1660 #define iRETURN_STRING_SIZE 2048
1661 char theString[iRETURN_STRING_SIZE];
1662 theString[0] = ' ';
1663
1664 for (int i = 1; i < iRETURN_STRING_SIZE; i++)
1665 {
1666 if (NextChar((byte&)theString[i]))
1667 {
1668 if (theString[i] != '\n')
1669 {
1670 continue;
1671 }
1672 PutBackChar(theString[i]);
1673 }
1674 theString[i] = '\0';
1675
1676 return CStringToken::Create(theString);
1677 }
1678
1679 // line would maks a string too big to fit in buffer...
1680 //
1681 Error(TKERR_STRINGLENGTHEXCEEDED);
1682 }
1683 else
1684 {
1685 char theString[MAX_IDENTIFIER_LENGTH];
1686 theString[0] = ' ';
1687 while (theString[0] == ' ')
1688 {
1689 if (!NextChar((byte&)theString[0]))
1690 {
1691 return NULL;
1692 }
1693 }
1694 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
1695 {
1696 if (NextChar((byte&)theString[i]))
1697 {
1698 if (theString[i] != '\n')
1699 {
1700 continue;
1701 }
1702 PutBackChar(theString[i]);
1703 }
1704 theString[i] = '\0';
1705 switch(tokenType)
1706 {
1707 case TK_COMMENT:
1708 return CCommentToken::Create(theString);
1709 case TK_IDENTIFIER:
1710 default:
1711 return CIdentifierToken::Create(theString);
1712 }
1713 }
1714 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
1715 }
1716 return NULL;
1717 }
1718
SkipToLineEnd()1719 void CTokenizer::SkipToLineEnd()
1720 {
1721 byte theByte;
1722 while(NextChar(theByte))
1723 {
1724 if (theByte == '\n')
1725 {
1726 break;
1727 }
1728 }
1729 }
1730
FetchToken()1731 CToken* CTokenizer::FetchToken()
1732 {
1733 if (m_nextToken != NULL)
1734 {
1735 CToken* curToken = m_nextToken;
1736 m_nextToken = curToken->GetNext();
1737 curToken->SetNext(NULL);
1738 return curToken;
1739 }
1740 byte theByte;
1741 CToken* theToken = NULL;
1742
1743 while (true)
1744 {
1745 if (!NextChar(theByte))
1746 {
1747 return CToken::Create();
1748 }
1749 if (theByte <= ' ')
1750 {
1751 if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
1752 {
1753 return CUserToken::Create(TK_EOL, "-EOLN-");
1754 }
1755 continue;
1756 }
1757 switch(theByte)
1758 {
1759 case '#':
1760 if ((m_flags & TKF_IGNOREDIRECTIVES) == 0)
1761 {
1762 theToken = HandleDirective();
1763 }
1764 else
1765 {
1766 if ((m_flags & TKF_NODIRECTIVES) != 0)
1767 {
1768 return HandleSymbol('#');
1769 }
1770 SkipToLineEnd();
1771 }
1772 break;
1773 case '/':
1774 theToken = HandleSlash();
1775 break;
1776 case '"':
1777 theToken = HandleString();
1778 break;
1779 case '\'':
1780 theToken = HandleQuote();
1781 break;
1782 default:
1783 if (((theByte >= 'a') && (theByte <= 'z'))
1784 || ((theByte >= 'A') && (theByte <= 'Z'))
1785 || ((theByte == '_') && ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)))
1786 {
1787 theToken = HandleIdentifier(theByte);
1788 }
1789 else if (((m_flags & TKF_NUMERICIDENTIFIERSTART) != 0) && (theByte >= '0') && (theByte <= '9'))
1790 {
1791 theToken = HandleIdentifier(theByte);
1792 }
1793 else if (((theByte >= '0') && (theByte <= '9')) || (theByte == '-'))
1794 {
1795 theToken = HandleNumeric(theByte);
1796 }
1797 else if (theByte == '.')
1798 {
1799 theToken = HandleDecimal();
1800 }
1801 else if (theByte <= ' ')
1802 {
1803 break;
1804 }
1805 else
1806 {
1807 theToken = HandleSymbol(theByte);
1808 }
1809 }
1810 if (theToken != NULL)
1811 {
1812 return theToken;
1813 }
1814 }
1815 }
1816
NextChar(byte & theByte)1817 bool CTokenizer::NextChar(byte& theByte)
1818 {
1819 while (m_curParseStream != NULL)
1820 {
1821 if (m_curParseStream->NextChar(theByte))
1822 {
1823 return true;
1824 }
1825 CParseStream* curParseStream = m_curParseStream;
1826 m_curParseStream = curParseStream->GetNext();
1827 curParseStream->Delete();
1828 }
1829 return false;
1830 }
1831
RequireToken(int tokenType)1832 bool CTokenizer::RequireToken(int tokenType)
1833 {
1834 CToken* theToken = GetToken();
1835 bool retValue = theToken->GetType() == tokenType;
1836 theToken->Delete();
1837 return retValue;
1838 }
1839
ScanUntilToken(int tokenType)1840 void CTokenizer::ScanUntilToken(int tokenType)
1841 {
1842 CToken* curToken;
1843 int tokenValue = TK_UNDEFINED;
1844 while (tokenValue != tokenType)
1845 {
1846 curToken = GetToken();
1847 tokenValue = curToken->GetType();
1848 if (tokenValue == TK_EOF)
1849 {
1850 PutBackToken(curToken);
1851 break;
1852 }
1853 if (tokenValue == tokenType)
1854 {
1855 PutBackToken(curToken);
1856 }
1857 else
1858 {
1859 curToken->Delete();
1860 }
1861 }
1862 }
1863
Init(UINT dwFlags)1864 void CTokenizer::Init(UINT dwFlags)
1865 {
1866 m_symbolLookup = NULL;
1867 m_nextToken = NULL;
1868 m_curParseStream = NULL;
1869 m_keywords = NULL;
1870 m_symbols = NULL;
1871 m_errors = NULL;
1872 m_state = NULL;
1873 m_flags = dwFlags;
1874 m_errorProc = NULL;
1875 }
1876
SetErrorProc(LPTokenizerErrorProc errorProc)1877 void CTokenizer::SetErrorProc(LPTokenizerErrorProc errorProc)
1878 {
1879 m_errorProc = errorProc;
1880 }
1881
SetAdditionalErrors(keywordArray_t * theErrors)1882 void CTokenizer::SetAdditionalErrors(keywordArray_t* theErrors)
1883 {
1884 m_errors = theErrors;
1885 }
1886
SetKeywords(keywordArray_t * theKeywords)1887 keywordArray_t* CTokenizer::SetKeywords(keywordArray_t* theKeywords)
1888 {
1889 keywordArray_t* retval = m_keywords;
1890 m_keywords = theKeywords;
1891 return retval;
1892 }
1893
SetSymbols(keywordArray_t * theSymbols)1894 void CTokenizer::SetSymbols(keywordArray_t* theSymbols)
1895 {
1896 m_symbols = theSymbols;
1897 if (m_symbolLookup != NULL)
1898 {
1899 m_symbolLookup->Delete();
1900 m_symbolLookup = NULL;
1901 }
1902 int i = 0;
1903 if (theSymbols == NULL)
1904 {
1905 return;
1906 }
1907 while(theSymbols[i].m_tokenvalue != TK_EOF)
1908 {
1909 InsertSymbol(theSymbols[i].m_keyword, theSymbols[i].m_tokenvalue);
1910 i++;
1911 }
1912 }
1913
InsertSymbol(LPCTSTR theSymbol,int theValue)1914 void CTokenizer::InsertSymbol(LPCTSTR theSymbol, int theValue)
1915 {
1916 CSymbolLookup** curHead = &m_symbolLookup;
1917 CSymbolLookup* curParent = NULL;
1918 CSymbolLookup* curLookup = NULL;
1919
1920 for (UINT i = 0; i < strlen(theSymbol); i++)
1921 {
1922 bool found = false;
1923 curLookup = *curHead;
1924 while (curLookup != NULL)
1925 {
1926 if (curLookup->GetByte() == theSymbol[i])
1927 {
1928 found = true;
1929 break;
1930 }
1931 curLookup = curLookup->GetNext();
1932 }
1933 if (!found)
1934 {
1935 curLookup = CSymbolLookup::Create(theSymbol[i]);
1936 curLookup->SetParent(curParent);
1937 curLookup->SetNext(*curHead);
1938 *curHead = curLookup;
1939 }
1940 curHead = curLookup->GetChildAddress();
1941 curParent = curLookup;
1942 }
1943 if (curLookup->GetValue() != -1)
1944 {
1945 Error(TKERR_DUPLICATESYMBOL);
1946 }
1947 curLookup->SetValue(theValue);
1948 }
1949
HandleString()1950 CToken* CTokenizer::HandleString()
1951 {
1952 char theString[MAX_STRING_LENGTH];
1953 for (int i = 0; i < MAX_STRING_LENGTH; i++)
1954 {
1955 if (!NextChar((byte&)theString[i]))
1956 {
1957 return NULL;
1958 }
1959 if (theString[i] == '"')
1960 {
1961 theString[i] = '\0';
1962 return CStringToken::Create(theString);
1963 }
1964 if (theString[i] == '\\')
1965 {
1966 theString[i] = Escapement();
1967 }
1968 }
1969 Error(TKERR_STRINGLENGTHEXCEEDED);
1970 return NULL;
1971 }
1972
GetCurFilename(char ** filename)1973 void CTokenizer::GetCurFilename(char** filename)
1974 {
1975 if (m_curParseStream == NULL)
1976 {
1977 *filename = (char*)malloc(1);
1978 *filename[0] = '\0';
1979 return;
1980 }
1981 m_curParseStream->GetCurFilename(filename);
1982 }
1983
GetCurLine()1984 int CTokenizer::GetCurLine()
1985 {
1986 if (m_curParseStream == NULL)
1987 {
1988 return 0;
1989 }
1990 return m_curParseStream->GetCurLine();
1991 }
1992
PutBackChar(byte theByte,int curLine,LPCTSTR filename)1993 void CTokenizer::PutBackChar(byte theByte, int curLine, LPCTSTR filename)
1994 {
1995 CParseStream* newStream;
1996 if (filename == NULL)
1997 {
1998 curLine = m_curParseStream->GetCurLine();
1999 char* theFile = NULL;
2000 m_curParseStream->GetCurFilename(&theFile);
2001 newStream = CParsePutBack::Create(theByte, curLine, theFile);
2002 if (theFile != NULL)
2003 {
2004 free(theFile);
2005 }
2006 }
2007 else
2008 {
2009 newStream = CParsePutBack::Create(theByte, curLine, filename);
2010 }
2011 newStream->SetNext(m_curParseStream);
2012 m_curParseStream = newStream;
2013 }
2014
Escapement()2015 byte CTokenizer::Escapement()
2016 {
2017 byte theByte;
2018 if (NextChar(theByte))
2019 {
2020 switch(theByte)
2021 {
2022 case 'n':
2023 return '\n';
2024 case '\\':
2025 return '\\';
2026 case '\'':
2027 return '\'';
2028 case '?':
2029 return '\?';
2030 case 'a':
2031 return '\a';
2032 case 'b':
2033 return '\b';
2034 case 'f':
2035 return '\f';
2036 case 'r':
2037 return '\r';
2038 case 't':
2039 return '\t';
2040 case 'v':
2041 return '\v';
2042 case '0':
2043 return '\0';
2044 // support for octal or hex sequences?? \000 or \xhhh
2045 default:
2046 PutBackChar(theByte);
2047 }
2048 }
2049 return '\\';
2050 }
2051
AddDefineSymbol(CDirectiveSymbol * definesymbol)2052 bool CTokenizer::AddDefineSymbol(CDirectiveSymbol* definesymbol)
2053 {
2054 CParseStream* curStream = m_curParseStream;
2055 while(curStream != NULL)
2056 {
2057 if (curStream->IsThisDefinition(definesymbol))
2058 {
2059 return false;
2060 }
2061 curStream = curStream->GetNext();
2062 }
2063 CParseStream* newStream = CParseDefine::Create(definesymbol);
2064 newStream->SetNext(m_curParseStream);
2065 m_curParseStream = newStream;
2066 return true;
2067 }
2068
TokenFromName(LPCTSTR name)2069 CToken* CTokenizer::TokenFromName(LPCTSTR name)
2070 {
2071 CDirectiveSymbol* defineSymbol = (CDirectiveSymbol*)m_defines.FindSymbol(name);
2072 if (defineSymbol != NULL)
2073 {
2074 if (AddDefineSymbol(defineSymbol))
2075 {
2076 return FetchToken();
2077 }
2078 }
2079 if ((m_keywords != NULL) && ((m_flags & TKF_IGNOREKEYWORDS) == 0))
2080 {
2081 int i = 0;
2082 if ((m_flags & TKF_NOCASEKEYWORDS) == 0)
2083 {
2084 while (m_keywords[i].m_tokenvalue != TK_EOF)
2085 {
2086 if (strcmp(m_keywords[i].m_keyword, name) == 0)
2087 {
2088 return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2089 }
2090 i++;
2091 }
2092 }
2093 else
2094 {
2095 while (m_keywords[i].m_tokenvalue != TK_EOF)
2096 {
2097 if (stricmp(m_keywords[i].m_keyword, name) == 0)
2098 {
2099 return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2100 }
2101 i++;
2102 }
2103 }
2104 }
2105 return CIdentifierToken::Create(name);
2106 }
2107
DirectiveFromName(LPCTSTR name)2108 int CTokenizer::DirectiveFromName(LPCTSTR name)
2109 {
2110 if (directiveKeywords != NULL)
2111 {
2112 int i = 0;
2113 while (directiveKeywords[i].m_tokenvalue != TK_EOF)
2114 {
2115 if (strcmp(directiveKeywords[i].m_keyword, name) == 0)
2116 {
2117 return directiveKeywords[i].m_tokenvalue;
2118 }
2119 i++;
2120 }
2121 }
2122 return -1;
2123 }
2124
HandleIdentifier(byte theByte)2125 CToken* CTokenizer::HandleIdentifier(byte theByte)
2126 {
2127 char theString[MAX_IDENTIFIER_LENGTH];
2128 theString[0] = theByte;
2129 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2130 {
2131 if (NextChar((byte&)theString[i]))
2132 {
2133 if (((theString[i] != '_') || ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)) &&
2134 ((theString[i] != '-') || ((m_flags & TKF_NODASHINIDENTIFIER) == 0)))
2135 {
2136 if (((theString[i] >= 'A') && (theString[i] <= 'Z'))
2137 || ((theString[i] >= 'a') && (theString[i] <= 'z'))
2138 || ((theString[i] >= '0') && (theString[i] <= '9'))
2139 || (theString[i] == '_') || (theString[i] == '-'))
2140 {
2141 continue;
2142 }
2143 }
2144 PutBackChar(theString[i]);
2145 }
2146 theString[i] = '\0';
2147 return TokenFromName(theString);
2148 }
2149 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2150 return NULL;
2151 }
2152
HandleSlash()2153 CToken* CTokenizer::HandleSlash()
2154 {
2155 byte theByte;
2156 if (!NextChar(theByte))
2157 {
2158 return NULL;
2159 }
2160 if (theByte == '/')
2161 {
2162 if (m_flags & TKF_COMMENTTOKENS)
2163 {
2164 return GetToEndOfLine(TK_COMMENT);
2165 }
2166 SkipToLineEnd();
2167 return NULL;
2168 }
2169 if (theByte == '*')
2170 {
2171 if (m_flags & TKF_COMMENTTOKENS)
2172 {
2173 char theString[MAX_IDENTIFIER_LENGTH + 1];
2174 theString[0] = ' ';
2175 while (theString[0] == ' ')
2176 {
2177 if (!NextChar((byte&)theString[0]))
2178 {
2179 return NULL;
2180 }
2181 }
2182 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2183 {
2184 if (NextChar((byte&)theString[i]))
2185 {
2186 if (theString[i] != '*')
2187 {
2188 continue;
2189 }
2190 i++;
2191 if (NextChar((byte&)theString[i]))
2192 {
2193 if (theString[i] == '/')
2194 {
2195 i--;
2196 theString[i] = '\0';
2197 return CCommentToken::Create(theString);
2198 }
2199 }
2200 }
2201 }
2202 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2203 return NULL;
2204 }
2205 while(NextChar(theByte))
2206 {
2207 while(theByte == '*')
2208 {
2209 if (!NextChar(theByte))
2210 {
2211 break;
2212 }
2213 if (theByte == '/')
2214 {
2215 return NULL;
2216 }
2217 }
2218 }
2219 return NULL;
2220 }
2221 PutBackChar(theByte);
2222 return HandleSymbol('/');
2223 }
2224
HandleNumeric(byte theByte)2225 CToken* CTokenizer::HandleNumeric(byte theByte)
2226 {
2227 bool thesign = theByte == '-';
2228 if (thesign)
2229 {
2230 if (!NextChar(theByte))
2231 {
2232 return HandleSymbol('-');
2233 }
2234 if (theByte == '.')
2235 {
2236 return HandleDecimal(thesign);
2237 }
2238 if ((theByte < '0') || (theByte > '9'))
2239 {
2240 PutBackChar(theByte);
2241 return HandleSymbol('-');
2242 }
2243 }
2244 if (theByte == '0')
2245 {
2246 return HandleOctal(thesign);
2247 }
2248 long value = 0;
2249 bool digithit = false;
2250 while((theByte >= '0') && (theByte <= '9'))
2251 {
2252 value = (value * 10) + (theByte - '0');
2253 if (!NextChar(theByte))
2254 {
2255 if (thesign)
2256 {
2257 value = -value;
2258 }
2259 return CIntToken::Create(value);
2260 }
2261 digithit = true;
2262 }
2263 if (theByte == '.')
2264 {
2265 if (digithit)
2266 {
2267 return HandleFloat(thesign, value);
2268 }
2269 if (thesign)
2270 {
2271 PutBackChar(theByte);
2272 theByte = '-';
2273 }
2274 return HandleSymbol(theByte);
2275 }
2276 PutBackChar(theByte);
2277 if (thesign)
2278 {
2279 value = -value;
2280 }
2281 return CIntToken::Create(value);
2282 }
2283
HandleOctal(bool thesign)2284 CToken* CTokenizer::HandleOctal(bool thesign)
2285 {
2286 byte theByte;
2287 int value = 0;
2288
2289 if (!NextChar(theByte))
2290 {
2291 return CIntToken::Create(value);
2292 }
2293 if (theByte == '.')
2294 {
2295 return HandleDecimal(thesign);
2296 }
2297 if ((theByte == 'x') || (theByte == 'X'))
2298 {
2299 return HandleHex(thesign);
2300 }
2301 while(true)
2302 {
2303 if((theByte >= '0') && (theByte <='7'))
2304 {
2305 value = (value * 8) + (theByte - '0');
2306 }
2307 else
2308 {
2309 PutBackChar(theByte);
2310 if (thesign)
2311 {
2312 value = -value;
2313 }
2314 return CIntToken::Create(value);
2315 }
2316 if (!NextChar(theByte))
2317 {
2318 if (thesign)
2319 {
2320 value = -value;
2321 }
2322 return CIntToken::Create(value);
2323 }
2324 }
2325 }
2326
HandleHex(bool thesign)2327 CToken* CTokenizer::HandleHex(bool thesign)
2328 {
2329 int value = 0;
2330
2331 while (true)
2332 {
2333 byte theByte;
2334 if (!NextChar(theByte))
2335 {
2336 if (thesign)
2337 {
2338 value = -value;
2339 }
2340 return CIntToken::Create(value);
2341 }
2342 switch (theByte)
2343 {
2344 case '0':
2345 case '1':
2346 case '2':
2347 case '3':
2348 case '4':
2349 case '5':
2350 case '6':
2351 case '7':
2352 case '8':
2353 case '9':
2354 theByte = theByte - '0';
2355 break;
2356 case 'A':
2357 case 'B':
2358 case 'C':
2359 case 'D':
2360 case 'E':
2361 case 'F':
2362 theByte = theByte - 'A' + 10;
2363 break;
2364 case 'a':
2365 case 'b':
2366 case 'c':
2367 case 'd':
2368 case 'e':
2369 case 'f':
2370 theByte = theByte - 'a' + 10;
2371 break;
2372 default:
2373 PutBackChar(theByte);
2374 if (thesign)
2375 {
2376 value = -value;
2377 }
2378 return CIntToken::Create(value);
2379 }
2380 value = (value * 16) + theByte;
2381 }
2382 }
2383
HandleDecimal(bool thesign)2384 CToken* CTokenizer::HandleDecimal(bool thesign)
2385 {
2386 byte theByte;
2387 if (!NextChar(theByte))
2388 {
2389 if (thesign)
2390 {
2391 PutBackChar('.');
2392 return HandleSymbol('-');
2393 }
2394 HandleSymbol('.');
2395 }
2396 PutBackChar(theByte);
2397 if ((theByte <= '9') && (theByte >= '0'))
2398 {
2399 return HandleFloat(thesign);
2400 }
2401 if (thesign)
2402 {
2403 PutBackChar('.');
2404 theByte = '-';
2405 }
2406 else
2407 {
2408 theByte = '.';
2409 }
2410 return HandleSymbol(theByte);
2411 }
2412
HandleFloat(bool thesign,long value)2413 CToken* CTokenizer::HandleFloat(bool thesign, long value)
2414 {
2415 float lower = 1.0;
2416 float newValue = (float)value;
2417 byte theByte;
2418 while(NextChar(theByte))
2419 {
2420 if ((theByte >= '0') && (theByte <= '9'))
2421 {
2422 lower = lower / 10;
2423 newValue = newValue + ((theByte - '0') * lower);
2424 continue;
2425 }
2426 PutBackChar(theByte);
2427 break;
2428 }
2429 if (thesign)
2430 {
2431 newValue = -newValue;
2432 }
2433 return CFloatToken::Create(newValue);
2434 }
2435
HandleQuote()2436 CToken* CTokenizer::HandleQuote()
2437 {
2438 byte theByte;
2439 if (!NextChar(theByte))
2440 {
2441 Error(TKERR_EXPECTED_CHAR);
2442 return NULL;
2443 }
2444 if (theByte == '\\')
2445 {
2446 theByte = Escapement();
2447 }
2448 byte dummy;
2449 if (!NextChar(dummy))
2450 {
2451 Error(TKERR_EXPECTED_CHAR);
2452 return NULL;
2453 }
2454 if (dummy != '\'')
2455 {
2456 PutBackChar(dummy);
2457 PutBackChar(theByte);
2458 Error(TKERR_EXPECTED_CHAR);
2459 return NULL;
2460 }
2461 return CCharToken::Create(theByte);
2462 }
2463
SetFlags(UINT flags)2464 void CTokenizer::SetFlags(UINT flags)
2465 {
2466 m_flags = flags;
2467 }
2468
GetFlags()2469 UINT CTokenizer::GetFlags()
2470 {
2471 return m_flags;
2472 }
2473
HandleSymbol(byte theByte)2474 CToken* CTokenizer::HandleSymbol(byte theByte)
2475 {
2476 char symbolString[128];
2477 int curStrLen = 0;
2478 symbolString[0] = '\0';
2479 bool consumed = false;
2480
2481 CSymbolLookup* curLookup;
2482 if ((m_flags & TKF_RAWSYMBOLSONLY) == 0)
2483 {
2484 curLookup = m_symbolLookup;
2485 }
2486 else
2487 {
2488 curLookup = NULL;
2489 }
2490 CSymbolLookup* lastLookup = NULL;
2491 while(curLookup != NULL)
2492 {
2493 if (curLookup->GetByte() == theByte)
2494 {
2495 symbolString[curStrLen++] = theByte;
2496 symbolString[curStrLen] = '\0';
2497 lastLookup = curLookup;
2498 consumed = true;
2499 if (curLookup->GetChild() == NULL)
2500 {
2501 break;
2502 }
2503 if (!NextChar(theByte))
2504 {
2505 PutBackToken(CToken::Create());
2506 break;
2507 }
2508 consumed = false;
2509 curLookup = curLookup->GetChild();
2510 continue;
2511 }
2512 curLookup = curLookup->GetNext();
2513 }
2514 if ((!consumed) && (lastLookup != NULL))
2515 {
2516 PutBackChar(theByte);
2517 }
2518 while ((lastLookup != NULL) && (lastLookup->GetValue() == -1))
2519 {
2520 curStrLen--;
2521 symbolString[curStrLen] = '\0';
2522 // symbolString = symbolString.Left(symbolString.GetLength() - 1);
2523 if (lastLookup->GetParent() == NULL)
2524 {
2525 if ((m_flags & TKF_WANTUNDEFINED) == 0)
2526 {
2527 Error(TKERR_UNRECOGNIZEDSYMBOL);
2528 }
2529 }
2530 else
2531 {
2532 PutBackChar(lastLookup->GetByte());
2533 }
2534 lastLookup = lastLookup->GetParent();
2535 }
2536 if (lastLookup == NULL)
2537 {
2538 if ((m_flags & TKF_WANTUNDEFINED) == 0)
2539 {
2540 return NULL;
2541 }
2542 curStrLen = 0;
2543 symbolString[curStrLen++] = char(theByte);
2544 symbolString[curStrLen] = '\0';
2545 if ((m_flags & TKF_WIDEUNDEFINEDSYMBOLS) != 0)
2546 {
2547 while (true)
2548 {
2549 if (!NextChar(theByte))
2550 {
2551 PutBackToken(CToken::Create());
2552 break;
2553 }
2554 if (theByte == ' ')
2555 {
2556 break;
2557 }
2558 if (((theByte >= 'a') && (theByte <= 'z')) || ((theByte >= 'A') && (theByte <= 'Z')) ||
2559 ((theByte >= '0') && (theByte <= '9')))
2560 {
2561 PutBackChar(theByte);
2562 break;
2563 }
2564 if (theByte < ' ')
2565 {
2566 if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
2567 {
2568 PutBackToken(CUserToken::Create(TK_EOL, "-EOLN-"));
2569 break;
2570 }
2571 continue;
2572 }
2573 symbolString[curStrLen++] = theByte;
2574 symbolString[curStrLen] = '\0';
2575 }
2576 }
2577 return CUndefinedToken::Create(symbolString);
2578 }
2579 return CUserToken::Create(lastLookup->GetValue(), symbolString);
2580 }
2581
HandleDirective()2582 CToken* CTokenizer::HandleDirective()
2583 {
2584 int tokenValue = 0;
2585 CToken* theToken = FetchToken();
2586 if (theToken->GetType() == TK_EOF)
2587 {
2588 return theToken;
2589 }
2590 if (theToken->GetType() != TK_IDENTIFIER)
2591 {
2592 Error(TKERR_INVALID_DIRECTIVE);
2593 theToken->Delete();
2594 SkipToLineEnd();
2595 return NULL;
2596 }
2597
2598 CDirectiveSymbol* curSymbol;
2599 CTokenizerState* state;
2600 int theDirective = DirectiveFromName(theToken->GetStringValue());
2601 theToken->Delete();
2602 byte theByte;
2603 switch(theDirective)
2604 {
2605 case DIR_INCLUDE:
2606 if ((m_state != NULL) && (m_state->Skipping()))
2607 {
2608 break;
2609 }
2610 theToken = GetToken();
2611 if (theToken->GetType() != TK_STRING)
2612 {
2613 Error(TKERR_INCLUDE_FILE_NOTFOUND);
2614 theToken->Delete();
2615 SkipToLineEnd();
2616 break;
2617 }
2618 AddParseFile(theToken->GetStringValue());
2619 theToken->Delete();
2620 break;
2621 case DIR_IFDEF:
2622 if ((m_state != NULL) && (m_state->Skipping()))
2623 {
2624 state = CTokenizerHolderState::Create();
2625 state->SetNext(m_state);
2626 m_state = state;
2627 break;
2628 }
2629 theToken = GetToken();
2630 if (theToken->GetType() != TK_IDENTIFIER)
2631 {
2632 Error(TKERR_EXPECTED_IDENTIFIER);
2633 theToken->Delete();
2634 SkipToLineEnd();
2635 break;
2636 }
2637 state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) == NULL);
2638 theToken->Delete();
2639 state->SetNext(m_state);
2640 m_state = state;
2641 break;
2642 case DIR_IFNDEF:
2643 if ((m_state != NULL) && (m_state->Skipping()))
2644 {
2645 state = CTokenizerHolderState::Create();
2646 state->SetNext(m_state);
2647 m_state = state;
2648 break;
2649 }
2650 theToken = GetToken();
2651 if (theToken->GetType() != TK_IDENTIFIER)
2652 {
2653 Error(TKERR_EXPECTED_IDENTIFIER);
2654 theToken->Delete();
2655 SkipToLineEnd();
2656 break;
2657 }
2658 state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) != NULL);
2659 theToken->Delete();
2660 state->SetNext(m_state);
2661 m_state = state;
2662 break;
2663 case DIR_ENDIF:
2664 if (m_state == NULL)
2665 {
2666 Error(TKERR_UNMATCHED_DIRECTIVE);
2667 break;
2668 }
2669 state = m_state;
2670 m_state = state->GetNext();
2671 state->Delete();
2672 break;
2673 case DIR_ELSE:
2674 if (m_state == NULL)
2675 {
2676 Error(TKERR_UNMATCHED_DIRECTIVE);
2677 break;
2678 }
2679 if (!m_state->ProcessElse())
2680 {
2681 Error(TKERR_UNMATCHED_DIRECTIVE);
2682 break;
2683 }
2684 break;
2685 case DIR_DEFINE:
2686 if ((m_state != NULL) && (m_state->Skipping()))
2687 {
2688 break;
2689 }
2690 theToken = GetToken();
2691 if (theToken->GetType() != TK_IDENTIFIER)
2692 {
2693 Error(TKERR_EXPECTED_IDENTIFIER);
2694 theToken->Delete();
2695 SkipToLineEnd();
2696 break;
2697 }
2698 // blind add - should check first for value changes
2699 {
2700 curSymbol = CDirectiveSymbol::Create(theToken->GetStringValue());
2701 char temp[128];
2702 int tempsize = 0;
2703 while(NextChar(theByte))
2704 {
2705 if (theByte == '\n')
2706 {
2707 break;
2708 }
2709 temp[tempsize++] = char(theByte);
2710 }
2711 temp[tempsize] = '\0';
2712 curSymbol->SetValue(temp);
2713 if (!m_defines.AddSymbol(curSymbol))
2714 {
2715 curSymbol->Delete();
2716 }
2717 }
2718 break;
2719 case DIR_UNDEFINE:
2720 if ((m_state != NULL) && (m_state->Skipping()))
2721 {
2722 break;
2723 }
2724 theToken = GetToken();
2725 if (theToken->GetType() != TK_IDENTIFIER)
2726 {
2727 Error(TKERR_EXPECTED_IDENTIFIER);
2728 theToken->Delete();
2729 SkipToLineEnd();
2730 break;
2731 }
2732 m_defines.RemoveSymbol(theToken->GetStringValue());
2733 break;
2734 default:
2735 Error(TKERR_INVALID_DIRECTIVE);
2736 SkipToLineEnd();
2737 break;
2738 }
2739 return NULL;
2740 }
2741
ParseRGB()2742 COLORREF CTokenizer::ParseRGB()
2743 {
2744 CToken* theToken = GetToken();
2745 if (theToken->GetType() != TK_INT)
2746 {
2747 Error(TKERR_EXPECTED_INTEGER);
2748 theToken->Delete();
2749 return RGB(0, 0, 0);
2750 }
2751 int red = theToken->GetIntValue();
2752 theToken->Delete();
2753 theToken = GetToken();
2754 if (theToken->GetType() != TK_INT)
2755 {
2756 Error(TKERR_EXPECTED_INTEGER);
2757 theToken->Delete();
2758 return RGB(0, 0, 0);
2759 }
2760 int green = theToken->GetIntValue();
2761 theToken->Delete();
2762 theToken = GetToken();
2763 if (theToken->GetType() != TK_INT)
2764 {
2765 Error(TKERR_EXPECTED_INTEGER);
2766 theToken->Delete();
2767 return RGB(0, 0, 0);
2768 }
2769 int blue = theToken->GetIntValue();
2770 theToken->Delete();
2771 return RGB(red, green, blue);
2772 }
2773
2774 //
2775 // CSymbolLookup
2776 //
2777
CSymbolLookup()2778 CSymbolLookup::CSymbolLookup()
2779 {
2780 }
2781
~CSymbolLookup()2782 CSymbolLookup::~CSymbolLookup()
2783 {
2784 }
2785
Create(byte theByte)2786 CSymbolLookup* CSymbolLookup::Create(byte theByte)
2787 {
2788 CSymbolLookup* curLookup = new CSymbolLookup();
2789 curLookup->Init(theByte);
2790 return curLookup;
2791 }
2792
Delete()2793 void CSymbolLookup::Delete()
2794 {
2795 if (m_sibling != NULL)
2796 {
2797 m_sibling->Delete();
2798 m_sibling = NULL;
2799 }
2800 if (m_child != NULL)
2801 {
2802 m_child->Delete();
2803 m_child = NULL;
2804 }
2805 delete this;
2806 }
2807
Init(byte theByte)2808 void CSymbolLookup::Init(byte theByte)
2809 {
2810 m_parent = NULL;
2811 m_child = NULL;
2812 m_sibling = NULL;
2813 m_value = -1;
2814 m_byte = theByte;
2815 }
2816
GetNext()2817 CSymbolLookup* CSymbolLookup::GetNext()
2818 {
2819 return m_sibling;
2820 }
2821
SetNext(CSymbolLookup * next)2822 void CSymbolLookup::SetNext(CSymbolLookup* next)
2823 {
2824 m_sibling = next;
2825 }
2826
SetParent(CSymbolLookup * parent)2827 void CSymbolLookup::SetParent(CSymbolLookup* parent)
2828 {
2829 m_parent = parent;
2830 }
2831
SetValue(int value)2832 void CSymbolLookup::SetValue(int value)
2833 {
2834 m_value = value;
2835 }
2836
GetValue()2837 int CSymbolLookup::GetValue()
2838 {
2839 return m_value;
2840 }
2841
GetByte()2842 byte CSymbolLookup::GetByte()
2843 {
2844 return m_byte;
2845 }
2846
GetChildAddress()2847 CSymbolLookup** CSymbolLookup::GetChildAddress()
2848 {
2849 return &m_child;
2850 }
2851
GetChild()2852 CSymbolLookup* CSymbolLookup::GetChild()
2853 {
2854 return m_child;
2855 }
2856
GetParent()2857 CSymbolLookup* CSymbolLookup::GetParent()
2858 {
2859 return m_parent;
2860 }
2861