1 // Tokenizer.cpp
2 #ifndef NOT_USING_MODULES
3 // !!! if you are not using modules, read BELOW !!!
4 #include "module.h" // if you are not using modules,
5 // create an empty Module.h in your
6 // project -- use of modules allows
7 // the error handler to be overridden
8 // with a custom CErrHandler
9 #endif
10 #include "tokenizer.h"
11
12 #pragma warning(disable : 4100) //unreferenced formal parameter
13 #pragma warning(disable : 4127) //conditional expression is constant
14 #pragma warning(disable : 4189) //local variable is initialized but not referenced
15 #pragma warning(disable : 4244) //conversion from x to x, possible loss of data
16
17 #ifndef _WIN32
18 #include <stdio.h>
19 #include <stdlib.h>
20 #endif
21
22 enum
23 {
24 DIR_INCLUDE = TK_USERDEF,
25 DIR_IFDEF,
26 DIR_IFNDEF,
27 DIR_ENDIF,
28 DIR_ELSE,
29 DIR_DEFINE,
30 DIR_UNDEFINE,
31 };
32
33 keywordArray_t CTokenizer::directiveKeywords[] =
34 {
35 "include", DIR_INCLUDE,
36 "ifdef", DIR_IFDEF,
37 "ifndef", DIR_IFNDEF,
38 "endif", DIR_ENDIF,
39 "else", DIR_ELSE,
40 "define", DIR_DEFINE,
41 "undefine", DIR_UNDEFINE,
42 "", TK_EOF,
43 };
44
45 keywordArray_t CTokenizer::errorMessages[] =
46 {
47 "No Error", TKERR_NONE,
48 "Unknown Error", TKERR_UNKNOWN,
49 "Buffer creation failed", TKERR_BUFFERCREATE,
50 "Unrecognized symbol", TKERR_UNRECOGNIZEDSYMBOL,
51 "Duplicate symbol", TKERR_DUPLICATESYMBOL,
52 "String length exceeded", TKERR_STRINGLENGTHEXCEEDED,
53 "Identifier length exceeded", TKERR_IDENTIFIERLENGTHEXCEEDED,
54 "Expected integer", TKERR_EXPECTED_INTEGER,
55 "Expected identifier", TKERR_EXPECTED_IDENTIFIER,
56 "Expected string", TKERR_EXPECTED_STRING,
57 "Expected char", TKERR_EXPECTED_CHAR,
58 "Expected float", TKERR_EXPECTED_FLOAT,
59 "Unexpected token", TKERR_UNEXPECTED_TOKEN,
60 "Invalid directive", TKERR_INVALID_DIRECTIVE,
61 "Include file not found", TKERR_INCLUDE_FILE_NOTFOUND,
62 "Unmatched directive", TKERR_UNMATCHED_DIRECTIVE,
63 "", TKERR_USERERROR,
64 };
65
66 //
67 // CSymbol
68 //
69
CSymbol()70 CSymbol::CSymbol()
71 {
72 }
73
~CSymbol()74 CSymbol::~CSymbol()
75 {
76 }
77
Create(LPCTSTR symbolName)78 CSymbol* CSymbol::Create(LPCTSTR symbolName)
79 {
80 CSymbol* retval = new CSymbol();
81 retval->Init(symbolName);
82 return retval;
83 }
84
GetName()85 LPCTSTR CSymbol::GetName()
86 {
87 if (m_symbolName == NULL)
88 {
89 return "";
90 }
91 return m_symbolName;
92 }
93
InitBaseSymbol(LPCTSTR symbolName)94 void CSymbol::InitBaseSymbol(LPCTSTR symbolName)
95 {
96 m_symbolName = (char*)malloc(strlen(symbolName) + 1);
97 // ASSERT(m_symbolName);
98 strcpy(m_symbolName, symbolName);
99 }
100
Delete()101 void CSymbol::Delete()
102 {
103 if (m_symbolName != NULL)
104 {
105 free(m_symbolName);
106 m_symbolName = NULL;
107 }
108 delete this;
109 }
110
111 //
112 // CDirectiveSymbol
113 //
114
CDirectiveSymbol()115 CDirectiveSymbol::CDirectiveSymbol()
116 {
117 }
118
~CDirectiveSymbol()119 CDirectiveSymbol::~CDirectiveSymbol()
120 {
121 }
122
Create(LPCTSTR symbolName)123 CDirectiveSymbol* CDirectiveSymbol::Create(LPCTSTR symbolName)
124 {
125 CDirectiveSymbol* retval = new CDirectiveSymbol();
126 retval->Init(symbolName);
127 return retval;
128 }
129
Init(LPCTSTR symbolName)130 void CDirectiveSymbol::Init(LPCTSTR symbolName)
131 {
132 CSymbol::InitBaseSymbol(symbolName);
133 m_value = NULL;
134 }
135
Delete()136 void CDirectiveSymbol::Delete()
137 {
138 if (m_value != NULL)
139 {
140 free(m_value);
141 m_value = NULL;
142 }
143 CSymbol::Delete();
144 }
145
SetValue(LPCTSTR value)146 void CDirectiveSymbol::SetValue(LPCTSTR value)
147 {
148 if (m_value != NULL)
149 {
150 free(m_value);
151 }
152 m_value = (char*)malloc(strlen(value) + 1);
153 strcpy(m_value, value);
154 }
155
GetValue()156 LPCTSTR CDirectiveSymbol::GetValue()
157 {
158 return m_value;
159 }
160
161 //
162 // CIntSymbol
163 //
164
CIntSymbol()165 CIntSymbol::CIntSymbol()
166 {
167 }
168
Create(LPCTSTR symbolName,int value)169 CIntSymbol* CIntSymbol::Create(LPCTSTR symbolName, int value)
170 {
171 CIntSymbol* retval = new CIntSymbol();
172 retval->Init(symbolName, value);
173 return retval;
174 }
175
Delete()176 void CIntSymbol::Delete()
177 {
178 CSymbol::Delete();
179 }
180
Init(LPCTSTR symbolName,int value)181 void CIntSymbol::Init(LPCTSTR symbolName, int value)
182 {
183 CSymbol::InitBaseSymbol(symbolName);
184 m_value = value;
185 }
186
GetValue()187 int CIntSymbol::GetValue()
188 {
189 return m_value;
190 }
191
192 //
193 // CSymbolTable
194 //
195
CSymbolTable()196 CSymbolTable::CSymbolTable()
197 {
198 Init();
199 }
200
~CSymbolTable()201 CSymbolTable::~CSymbolTable()
202 {
203 }
204
Create()205 CSymbolTable* CSymbolTable::Create()
206 {
207 CSymbolTable* retval = new CSymbolTable();
208 retval->Init();
209 return retval;
210 }
211
Init()212 void CSymbolTable::Init()
213 {
214 }
215
DiscardSymbols()216 void CSymbolTable::DiscardSymbols()
217 {
218 for (symbolmap_t::iterator isymbol = m_symbols.begin(); isymbol != m_symbols.end(); ++isymbol)
219 {
220 (*isymbol).second->Delete();
221 }
222 m_symbols.erase(m_symbols.begin(), m_symbols.end());
223 }
224
Delete()225 void CSymbolTable::Delete()
226 {
227 DiscardSymbols();
228 delete this;
229 }
230
AddSymbol(CSymbol * theSymbol)231 bool CSymbolTable::AddSymbol(CSymbol* theSymbol)
232 {
233 LPCTSTR name = theSymbol->GetName();
234
235 symbolmap_t::iterator iter = m_symbols.find(name);
236 if (iter != m_symbols.end())
237 {
238 return false;
239 }
240 m_symbols.insert(symbolmap_t::value_type(name, theSymbol));
241 return true;
242 }
243
FindSymbol(LPCTSTR symbolName)244 CSymbol* CSymbolTable::FindSymbol(LPCTSTR symbolName)
245 {
246 symbolmap_t::iterator iter = m_symbols.find(symbolName);
247 if (iter != m_symbols.end())
248 {
249 return (*iter).second;
250 }
251 return NULL;
252 }
253
ExtractSymbol(LPCTSTR symbolName)254 CSymbol* CSymbolTable::ExtractSymbol(LPCTSTR symbolName)
255 {
256 symbolmap_t::iterator iter = m_symbols.find(symbolName);
257 if (iter != m_symbols.end())
258 {
259 CSymbol* retval = (*iter).second;
260 m_symbols.erase(iter);
261 }
262 return NULL;
263 }
264
RemoveSymbol(LPCTSTR symbolName)265 void CSymbolTable::RemoveSymbol(LPCTSTR symbolName)
266 {
267 m_symbols.erase(symbolName);
268 }
269
270 //
271 // CParseStream
272 //
273
CParseStream()274 CParseStream::CParseStream()
275 {
276 }
277
~CParseStream()278 CParseStream::~CParseStream()
279 {
280 }
281
Create()282 CParseStream* CParseStream::Create()
283 {
284 return NULL;
285 }
286
Delete()287 void CParseStream::Delete()
288 {
289 delete this;
290 }
291
InitBaseStream()292 bool CParseStream::InitBaseStream()
293 {
294 m_next = NULL;
295
296 return true;
297 }
298
NextChar(byte & theByte)299 bool CParseStream::NextChar(byte& theByte)
300 {
301 return false;
302 }
303
GetRemainingSize()304 long CParseStream::GetRemainingSize()
305 {
306 return 0;
307 }
308
GetNext()309 CParseStream* CParseStream::GetNext()
310 {
311 return m_next;
312 }
313
SetNext(CParseStream * next)314 void CParseStream::SetNext(CParseStream* next)
315 {
316 m_next = next;
317 }
318
GetCurLine()319 int CParseStream::GetCurLine()
320 {
321 return 0;
322 }
323
GetCurFilename(char ** theBuff)324 void CParseStream::GetCurFilename(char** theBuff)
325 {
326 *theBuff = NULL;
327 }
328
IsThisDefinition(void * theDefinition)329 bool CParseStream::IsThisDefinition(void* theDefinition)
330 {
331 return false;
332 }
333
334 //
335 // CParsePutBack
336 //
337
CParsePutBack()338 CParsePutBack::CParsePutBack()
339 {
340 }
341
~CParsePutBack()342 CParsePutBack::~CParsePutBack()
343 {
344 }
345
Create(byte theByte,int curLine,LPCTSTR filename)346 CParsePutBack* CParsePutBack::Create(byte theByte, int curLine, LPCTSTR filename)
347 {
348 CParsePutBack* curParsePutBack = new CParsePutBack();
349 curParsePutBack->Init(theByte, curLine, filename);
350 return curParsePutBack;
351 }
352
Delete()353 void CParsePutBack::Delete()
354 {
355 if (m_curFile != NULL)
356 {
357 free(m_curFile);
358 m_curFile = NULL;
359 }
360 delete this;
361 }
362
NextChar(byte & theByte)363 bool CParsePutBack::NextChar(byte& theByte)
364 {
365 if (m_consumed)
366 {
367 return false;
368 }
369 theByte = m_byte;
370 m_consumed = true;
371 return true;
372 }
373
Init(byte theByte,int curLine,LPCTSTR filename)374 void CParsePutBack::Init(byte theByte, int curLine, LPCTSTR filename)
375 {
376 CParseStream::InitBaseStream();
377 m_consumed = false;
378 m_byte = theByte;
379 m_curLine = curLine;
380 if (filename != NULL)
381 {
382 m_curFile = (char*)malloc(strlen(filename) + 1);
383 strcpy(m_curFile, filename);
384 }
385 else
386 {
387 m_curFile = NULL;
388 }
389 }
390
GetRemainingSize()391 long CParsePutBack::GetRemainingSize()
392 {
393 if (m_consumed)
394 {
395 return 0;
396 }
397 else
398 {
399 return 1;
400 }
401 }
402
GetCurLine()403 int CParsePutBack::GetCurLine()
404 {
405 return m_curLine;
406 }
407
GetCurFilename(char ** theBuff)408 void CParsePutBack::GetCurFilename(char** theBuff)
409 {
410 if (m_curFile == NULL)
411 {
412 *theBuff = NULL;
413 return;
414 }
415 *theBuff = (char*)malloc(strlen(m_curFile) + 1);
416 strcpy(*theBuff, m_curFile);
417 }
418
419 //
420 // CParseFile
421 //
422
CParseFile()423 CParseFile::CParseFile()
424 {
425 }
426
~CParseFile()427 CParseFile::~CParseFile()
428 {
429 }
430
Create()431 CParseFile* CParseFile::Create()
432 {
433 CParseFile* theParseFile = new CParseFile();
434
435 if ( !theParseFile->Init() )
436 {
437 delete theParseFile;
438 return NULL;
439 }
440
441 return theParseFile;
442 }
443
Create(LPCTSTR filename,CTokenizer * tokenizer)444 CParseFile* CParseFile::Create(LPCTSTR filename, CTokenizer* tokenizer)
445 {
446 CParseFile* theParseFile = new CParseFile();
447
448 if ( theParseFile->Init(filename, tokenizer) )
449 return theParseFile;
450
451 return NULL;
452 }
453
Delete()454 void CParseFile::Delete()
455 {
456 if (m_buff != NULL)
457 {
458 free(m_buff);
459 m_buff = NULL;
460 }
461 if (m_ownsFile && (m_fileHandle != NULL))
462 {
463 #ifdef _WIN32
464 CloseHandle(m_fileHandle);
465 #else
466 fclose(m_fileHandle);
467 #endif
468 m_fileHandle = NULL;
469 }
470 if (m_fileName != NULL)
471 {
472 free(m_fileName);
473 m_fileName = NULL;
474 }
475 delete this;
476 }
477
Init()478 bool CParseFile::Init()
479 {
480 m_fileHandle = NULL;
481 m_buff = NULL;
482 m_ownsFile = false;
483 m_curByte = NULL;
484 m_curLine = 1;
485 m_fileName = NULL;
486 return CParseStream::InitBaseStream();
487 }
488
GetFileSize()489 unsigned int CParseFile::GetFileSize()
490 {
491 #ifdef _WIN32
492 unsigned int dwCur = SetFilePointer(m_fileHandle, 0L, NULL, FILE_CURRENT);
493 unsigned int dwLen = SetFilePointer(m_fileHandle, 0, NULL, FILE_END);
494 SetFilePointer(m_fileHandle, dwCur, NULL, FILE_BEGIN);
495 #else
496 fseek(m_fileHandle, 0L, SEEK_END);
497 unsigned int dwLen = ftell(m_fileHandle);
498 fseek(m_fileHandle, 0L, SEEK_SET);
499 #endif
500 return dwLen;
501 }
502
Read(void * buff,UINT buffsize)503 void CParseFile::Read(void* buff, UINT buffsize)
504 {
505 unsigned int bytesRead;
506 #ifdef _WIN32
507 ReadFile(m_fileHandle, buff, buffsize, &bytesRead, NULL);
508 #else
509 fread(buff, buffsize, 1, m_fileHandle);
510 #endif
511 }
512
Init(LPCTSTR filename,CTokenizer * tokenizer)513 bool CParseFile::Init(LPCTSTR filename, CTokenizer* tokenizer)
514 {
515 CParseStream::InitBaseStream();
516 m_fileName = (char*)malloc(strlen(filename) + 1);
517 strcpy(m_fileName, filename);
518
519 #ifdef _WIN32
520 unsigned int dwAccess = GENERIC_READ;
521 unsigned int dwShareMode = FILE_SHARE_WRITE | FILE_SHARE_READ;
522 SECURITY_ATTRIBUTES sa;
523 sa.nLength = sizeof(sa);
524 sa.lpSecurityDescriptor = NULL;
525 sa.bInheritHandle = 0;
526 unsigned int dwCreateFlag = OPEN_EXISTING;
527
528 m_fileHandle = CreateFile(filename, dwAccess, dwShareMode, &sa, dwCreateFlag, FILE_ATTRIBUTE_NORMAL, NULL);
529
530 if (m_fileHandle == (HANDLE)-1)
531 {
532 tokenizer->Error(TKERR_INCLUDE_FILE_NOTFOUND);
533 Init();
534
535 return false;
536 }
537 #else
538 m_fileHandle = fopen(filename, "r+");
539
540 if (m_fileHandle == NULL)
541 {
542 tokenizer->Error(TKERR_INCLUDE_FILE_NOTFOUND);
543 Init();
544
545 return false;
546 }
547 #endif
548 m_filesize = GetFileSize();
549 m_buff = (byte*)malloc(m_filesize);
550 if (m_buff == NULL)
551 {
552 tokenizer->Error(TKERR_BUFFERCREATE);
553 Init();
554 return false;
555 }
556 Read(m_buff, m_filesize);
557 m_curByte = 0;
558 m_curPos = 1;
559 m_ownsFile = true;
560 m_curLine = 1;
561
562 return true;
563 }
564
GetRemainingSize()565 long CParseFile::GetRemainingSize()
566 {
567 return m_filesize - m_curByte;
568 }
569
NextChar(byte & theByte)570 bool CParseFile::NextChar(byte& theByte)
571 {
572 if (m_curByte < m_filesize)
573 {
574 if (m_buff[m_curByte] == '\n')
575 {
576 m_curLine += 1;
577 m_curPos = 1;
578 }
579 else
580 {
581 m_curPos++;
582 }
583 theByte = m_buff[m_curByte++];
584 return true;
585 }
586 else
587 {
588 return false;
589 }
590 }
591
GetCurLine()592 int CParseFile::GetCurLine()
593 {
594 return m_curLine;
595 }
596
GetCurFilename(char ** theBuff)597 void CParseFile::GetCurFilename(char** theBuff)
598 {
599 *theBuff = NULL;
600 if (m_fileName != NULL)
601 {
602 *theBuff = (char*)malloc(strlen(m_fileName) + 1);
603 strcpy(*theBuff, m_fileName);
604 }
605 }
606
607 //
608 // CParseMemory
609 //
610
CParseMemory()611 CParseMemory::CParseMemory()
612 {
613 }
614
~CParseMemory()615 CParseMemory::~CParseMemory()
616 {
617 }
618
Create(byte * data,long datasize)619 CParseMemory* CParseMemory::Create(byte* data, long datasize)
620 {
621 CParseMemory* curParse = new CParseMemory();
622 curParse->Init(data, datasize);
623 return curParse;
624 }
625
Delete()626 void CParseMemory::Delete()
627 {
628 delete this;
629 }
630
NextChar(byte & theByte)631 bool CParseMemory::NextChar(byte& theByte)
632 {
633 if (m_offset < m_datasize)
634 {
635 if (m_data[m_offset] == '\n')
636 {
637 m_curLine += 1;
638 m_curPos = 1;
639 }
640 else
641 {
642 m_curPos++;
643 }
644 theByte = m_data[m_offset++];
645 return true;
646 }
647 else
648 {
649 return false;
650 }
651 }
652
Init(byte * data,long datasize)653 void CParseMemory::Init(byte* data, long datasize)
654 {
655 m_data = data;
656 m_curLine = 1;
657 m_curPos = 1;
658 m_offset = 0;
659 m_datasize = datasize;
660 }
661
GetRemainingSize()662 long CParseMemory::GetRemainingSize()
663 {
664 return m_datasize - m_offset;
665 }
666
GetCurLine()667 int CParseMemory::GetCurLine()
668 {
669 return m_curLine;
670 }
671
GetCurFilename(char ** theBuff)672 void CParseMemory::GetCurFilename(char** theBuff)
673 {
674 *theBuff = NULL;
675 }
676
677 //
678 // CParseBlock
679 //
680
CParseBlock()681 CParseBlock::CParseBlock()
682 {
683 }
684
~CParseBlock()685 CParseBlock::~CParseBlock()
686 {
687 }
688
Create(byte * data,long datasize)689 CParseBlock* CParseBlock::Create(byte* data, long datasize)
690 {
691 CParseBlock* curParse = new CParseBlock();
692 curParse->Init(data, datasize);
693 return curParse;
694 }
695
Delete()696 void CParseBlock::Delete()
697 {
698 if (m_data != NULL)
699 {
700 free(m_data);
701 m_data = NULL;
702 }
703 delete this;
704 }
705
Init(byte * data,long datasize)706 void CParseBlock::Init(byte* data, long datasize)
707 {
708 m_data = (byte*)malloc(datasize);
709 memcpy(m_data, data, datasize);
710 m_curLine = 1;
711 m_curPos = 1;
712 m_offset = 0;
713 m_datasize = datasize;
714 }
715
716 //
717 // CParseToken
718 //
719
CParseToken()720 CParseToken::CParseToken()
721 {
722 }
723
~CParseToken()724 CParseToken::~CParseToken()
725 {
726 }
727
Create(CToken * token)728 CParseToken* CParseToken::Create(CToken* token)
729 {
730 CParseToken* curParse = new CParseToken();
731 curParse->Init(token);
732 return curParse;
733 }
734
Delete()735 void CParseToken::Delete()
736 {
737 if (m_data != NULL)
738 {
739 free(m_data);
740 m_data = NULL;
741 }
742 delete this;
743 }
744
NextChar(byte & theByte)745 bool CParseToken::NextChar(byte& theByte)
746 {
747 if (m_offset < m_datasize)
748 {
749 if (m_data[m_offset] == '\n')
750 {
751 m_curLine += 1;
752 m_curPos = 1;
753 }
754 else
755 {
756 m_curPos++;
757 }
758 theByte = m_data[m_offset++];
759 return true;
760 }
761 else
762 {
763 return false;
764 }
765 }
766
Init(CToken * token)767 void CParseToken::Init(CToken* token)
768 {
769 LPCTSTR tokenString = token->GetStringValue();
770 m_datasize = strlen(tokenString);
771 if (m_datasize > 0)
772 {
773 m_data = (byte*) malloc(m_datasize);
774 memcpy(m_data, tokenString, m_datasize);
775 }
776 else
777 {
778 m_data = NULL;
779 }
780 m_curLine = 1;
781 m_curPos = 1;
782 m_offset = 0;
783 token->Delete();
784 }
785
GetRemainingSize()786 long CParseToken::GetRemainingSize()
787 {
788 return m_datasize - m_offset;
789 }
790
GetCurLine()791 int CParseToken::GetCurLine()
792 {
793 return m_curLine;
794 }
795
GetCurFilename(char ** theBuff)796 void CParseToken::GetCurFilename(char** theBuff)
797 {
798 *theBuff = NULL;
799 }
800
801 //
802 // CParseDefine
803 //
804
CParseDefine()805 CParseDefine::CParseDefine()
806 {
807 }
808
~CParseDefine()809 CParseDefine::~CParseDefine()
810 {
811 }
812
Create(CDirectiveSymbol * definesymbol)813 CParseDefine* CParseDefine::Create(CDirectiveSymbol* definesymbol)
814 {
815 CParseDefine* retval = new CParseDefine();
816 retval->Init(definesymbol);
817 return retval;
818 }
819
Delete()820 void CParseDefine::Delete()
821 {
822 CParseMemory::Delete();
823 }
824
Init(CDirectiveSymbol * definesymbol)825 void CParseDefine::Init(CDirectiveSymbol* definesymbol)
826 {
827 CParseMemory::Init((byte*)definesymbol->GetValue(), strlen(definesymbol->GetValue()));
828 m_defineSymbol = definesymbol;
829 }
830
IsThisDefinition(void * theDefinition)831 bool CParseDefine::IsThisDefinition(void* theDefinition)
832 {
833 return (CDirectiveSymbol*)theDefinition == m_defineSymbol;
834 }
835
836 //
837 // CToken
838 //
839
CToken()840 CToken::CToken()
841 {
842 }
843
~CToken()844 CToken::~CToken()
845 {
846 }
847
Create()848 CToken* CToken::Create()
849 {
850 CToken* theToken = new CToken();
851 theToken->InitBaseToken();
852 return theToken;
853 }
854
Delete()855 void CToken::Delete()
856 {
857 if (m_string != NULL)
858 {
859 free(m_string);
860 m_string = NULL;
861 }
862 delete this;
863 }
864
InitBaseToken()865 void CToken::InitBaseToken()
866 {
867 m_next = NULL;
868 m_string = NULL;
869 }
870
SetNext(CToken * theToken)871 void CToken::SetNext(CToken* theToken)
872 {
873 m_next = theToken;
874 }
875
GetNext()876 CToken* CToken::GetNext()
877 {
878 return m_next;
879 }
880
GetType()881 int CToken::GetType()
882 {
883 return TK_EOF;
884 }
885
GetIntValue()886 int CToken::GetIntValue()
887 {
888 return 0;
889 }
890
GetStringValue()891 LPCTSTR CToken::GetStringValue()
892 {
893 if (m_string == NULL)
894 {
895 return "";
896 }
897 return m_string;
898 }
899
GetFloatValue()900 float CToken::GetFloatValue()
901 {
902 return 0.0;
903 }
904
905 //
906 // CCharToken
907 //
908
CCharToken()909 CCharToken::CCharToken()
910 {
911 }
912
~CCharToken()913 CCharToken::~CCharToken()
914 {
915 }
916
Create(byte theByte)917 CCharToken* CCharToken::Create(byte theByte)
918 {
919 CCharToken* theToken = new CCharToken();
920 theToken->Init(theByte);
921 return theToken;
922 }
923
Delete()924 void CCharToken::Delete()
925 {
926 CToken::Delete();
927 }
928
Init(byte theByte)929 void CCharToken::Init(byte theByte)
930 {
931 CToken::InitBaseToken();
932 char charString[10];
933 switch(theByte)
934 {
935 case '\0':
936 strcpy(charString, "\\0");
937 break;
938 case '\n':
939 strcpy(charString, "\\n");
940 break;
941 case '\\':
942 strcpy(charString, "\\\\");
943 break;
944 case '\'':
945 strcpy(charString, "\\'");
946 break;
947 case '\?':
948 strcpy(charString, "\\?");
949 break;
950 case '\a':
951 strcpy(charString, "\\a");
952 break;
953 case '\b':
954 strcpy(charString, "\\b");
955 break;
956 case '\f':
957 strcpy(charString, "\\f");
958 break;
959 case '\r':
960 strcpy(charString, "\\r");
961 break;
962 case '\t':
963 strcpy(charString, "\\t");
964 break;
965 case '\v':
966 strcpy(charString, "\\v");
967 break;
968 default:
969 charString[0] = (char)theByte;
970 charString[1] = '\0';
971 break;
972 }
973 m_string = (char*)malloc(strlen(charString) + 1);
974 strcpy(m_string, charString);
975 }
976
GetType()977 int CCharToken::GetType()
978 {
979 return TK_CHAR;
980 }
981
982 //
983 // CStringToken
984 //
985
CStringToken()986 CStringToken::CStringToken()
987 {
988 }
989
~CStringToken()990 CStringToken::~CStringToken()
991 {
992 }
993
Create(LPCTSTR theString)994 CStringToken* CStringToken::Create(LPCTSTR theString)
995 {
996 CStringToken* theToken = new CStringToken();
997 theToken->Init(theString);
998 return theToken;
999 }
1000
Delete()1001 void CStringToken::Delete()
1002 {
1003 CToken::Delete();
1004 }
1005
Init(LPCTSTR theString)1006 void CStringToken::Init(LPCTSTR theString)
1007 {
1008 CToken::InitBaseToken();
1009 m_string = (char*)malloc(strlen(theString) + 1);
1010 // ASSERT(m_string);
1011 strcpy(m_string, theString);
1012 }
1013
GetType()1014 int CStringToken::GetType()
1015 {
1016 return TK_STRING;
1017 }
1018
1019 //
1020 // CIntToken
1021 //
1022
CIntToken()1023 CIntToken::CIntToken()
1024 {
1025 }
1026
~CIntToken()1027 CIntToken::~CIntToken()
1028 {
1029 }
1030
Create(long value)1031 CIntToken* CIntToken::Create(long value)
1032 {
1033 CIntToken* theToken = new CIntToken();
1034 theToken->Init(value);
1035 return theToken;
1036 }
1037
Delete()1038 void CIntToken::Delete()
1039 {
1040 CToken::Delete();
1041 }
1042
Init(long value)1043 void CIntToken::Init(long value)
1044 {
1045 CToken::InitBaseToken();
1046 m_value = value;
1047 }
1048
GetType()1049 int CIntToken::GetType()
1050 {
1051 return TK_INT;
1052 }
1053
GetIntValue()1054 int CIntToken::GetIntValue()
1055 {
1056 return m_value;
1057 }
1058
GetFloatValue()1059 float CIntToken::GetFloatValue()
1060 {
1061 return (float)m_value;
1062 }
1063
GetStringValue()1064 LPCTSTR CIntToken::GetStringValue()
1065 {
1066 if (m_string != NULL)
1067 {
1068 free(m_string);
1069 m_string = NULL;
1070 }
1071 char temp[128];
1072 sprintf(temp, "%d", m_value);
1073 m_string = (char*)malloc(strlen(temp) + 1);
1074 strcpy(m_string, temp);
1075 return m_string;
1076 }
1077
1078 //
1079 // CFloatToken
1080 //
1081
CFloatToken()1082 CFloatToken::CFloatToken()
1083 {
1084 }
1085
~CFloatToken()1086 CFloatToken::~CFloatToken()
1087 {
1088 }
1089
Create(float value)1090 CFloatToken* CFloatToken::Create(float value)
1091 {
1092 CFloatToken* theToken = new CFloatToken();
1093 theToken->Init(value);
1094 return theToken;
1095 }
1096
Delete()1097 void CFloatToken::Delete()
1098 {
1099 CToken::Delete();
1100 }
1101
Init(float value)1102 void CFloatToken::Init(float value)
1103 {
1104 CToken::InitBaseToken();
1105 m_value = value;
1106 }
1107
GetType()1108 int CFloatToken::GetType()
1109 {
1110 return TK_FLOAT;
1111 }
1112
GetFloatValue()1113 float CFloatToken::GetFloatValue()
1114 {
1115 return m_value;
1116 }
1117
GetStringValue()1118 LPCTSTR CFloatToken::GetStringValue()
1119 {
1120 if (m_string != NULL)
1121 {
1122 free(m_string);
1123 m_string = NULL;
1124 }
1125 char temp[128];
1126 sprintf(temp, "%g", m_value);
1127 m_string = (char*)malloc(strlen(temp) + 1);
1128 strcpy(m_string, temp);
1129 return m_string;
1130 }
1131
1132 //
1133 // CIdentifierToken
1134 //
1135
CIdentifierToken()1136 CIdentifierToken::CIdentifierToken()
1137 {
1138 }
1139
~CIdentifierToken()1140 CIdentifierToken::~CIdentifierToken()
1141 {
1142 }
1143
Create(LPCTSTR name)1144 CIdentifierToken* CIdentifierToken::Create(LPCTSTR name)
1145 {
1146 CIdentifierToken* theToken = new CIdentifierToken();
1147 theToken->Init(name);
1148 return theToken;
1149 }
1150
Delete()1151 void CIdentifierToken::Delete()
1152 {
1153 CToken::Delete();
1154 }
1155
Init(LPCTSTR name)1156 void CIdentifierToken::Init(LPCTSTR name)
1157 {
1158 CToken::InitBaseToken();
1159 m_string = (char*)malloc(strlen(name) + 1);
1160 // ASSERT(m_string);
1161 strcpy(m_string, name);
1162 }
1163
GetType()1164 int CIdentifierToken::GetType()
1165 {
1166 return TK_IDENTIFIER;
1167 }
1168
1169 //
1170 // CCommentToken
1171 //
1172
CCommentToken()1173 CCommentToken::CCommentToken()
1174 {
1175 }
1176
~CCommentToken()1177 CCommentToken::~CCommentToken()
1178 {
1179 }
1180
Create(LPCTSTR name)1181 CCommentToken* CCommentToken::Create(LPCTSTR name)
1182 {
1183 CCommentToken* theToken = new CCommentToken();
1184 theToken->Init(name);
1185 return theToken;
1186 }
1187
Delete()1188 void CCommentToken::Delete()
1189 {
1190 CToken::Delete();
1191 }
1192
Init(LPCTSTR name)1193 void CCommentToken::Init(LPCTSTR name)
1194 {
1195 CToken::InitBaseToken();
1196 m_string = (char*)malloc(strlen(name) + 1);
1197 // ASSERT(m_string);
1198 strcpy(m_string, name);
1199 }
1200
GetType()1201 int CCommentToken::GetType()
1202 {
1203 return TK_COMMENT;
1204 }
1205
1206 //
1207 // CUserToken
1208 //
1209
CUserToken()1210 CUserToken::CUserToken()
1211 {
1212 }
1213
~CUserToken()1214 CUserToken::~CUserToken()
1215 {
1216 }
1217
Create(int value,LPCTSTR string)1218 CUserToken* CUserToken::Create(int value, LPCTSTR string)
1219 {
1220 CUserToken* theToken = new CUserToken();
1221 theToken->Init(value, string);
1222 return theToken;
1223 }
1224
Delete()1225 void CUserToken::Delete()
1226 {
1227 CToken::Delete();
1228 }
1229
Init(int value,LPCTSTR string)1230 void CUserToken::Init(int value, LPCTSTR string)
1231 {
1232 CToken::InitBaseToken();
1233 m_value = value;
1234 m_string = (char*)malloc(strlen(string) + 1);
1235 strcpy(m_string, string);
1236 }
1237
GetType()1238 int CUserToken::GetType()
1239 {
1240 return m_value;
1241 }
1242
1243 //
1244 // CUndefinedToken
1245 //
1246
CUndefinedToken()1247 CUndefinedToken::CUndefinedToken()
1248 {
1249 }
1250
~CUndefinedToken()1251 CUndefinedToken::~CUndefinedToken()
1252 {
1253 }
1254
Create(LPCTSTR string)1255 CUndefinedToken* CUndefinedToken::Create(LPCTSTR string)
1256 {
1257 CUndefinedToken* theToken = new CUndefinedToken();
1258 theToken->Init(string);
1259 return theToken;
1260 }
1261
Delete()1262 void CUndefinedToken::Delete()
1263 {
1264 CToken::Delete();
1265 }
1266
Init(LPCTSTR string)1267 void CUndefinedToken::Init(LPCTSTR string)
1268 {
1269 CToken::InitBaseToken();
1270 m_string = (char*)malloc(strlen(string) + 1);
1271 strcpy(m_string, string);
1272 }
1273
GetType()1274 int CUndefinedToken::GetType()
1275 {
1276 return TK_UNDEFINED;
1277 }
1278
1279 //
1280 // CTokenizerState
1281 //
1282
CTokenizerState()1283 CTokenizerState::CTokenizerState()
1284 {
1285 }
1286
~CTokenizerState()1287 CTokenizerState::~CTokenizerState()
1288 {
1289 }
1290
Create(bool skip)1291 CTokenizerState* CTokenizerState::Create(bool skip)
1292 {
1293 CTokenizerState* retval = new CTokenizerState();
1294 retval->Init(skip);
1295 return retval;
1296 }
1297
Init(bool skip)1298 void CTokenizerState::Init(bool skip)
1299 {
1300 m_next = NULL;
1301 m_skip = skip;
1302 m_elseHit = false;
1303 }
1304
Delete()1305 void CTokenizerState::Delete()
1306 {
1307 delete this;
1308 }
1309
GetNext()1310 CTokenizerState* CTokenizerState::GetNext()
1311 {
1312 return m_next;
1313 }
1314
ProcessElse()1315 bool CTokenizerState::ProcessElse()
1316 {
1317 if (!m_elseHit)
1318 {
1319 m_elseHit = true;
1320 m_skip = !m_skip;
1321 }
1322 return m_elseHit;
1323 }
1324
SetNext(CTokenizerState * next)1325 void CTokenizerState::SetNext(CTokenizerState* next)
1326 {
1327 m_next = next;
1328 }
1329
Skipping()1330 bool CTokenizerState::Skipping()
1331 {
1332 return m_skip;
1333 }
1334
1335 //
1336 // CTokenizerHolderState
1337 //
1338
CTokenizerHolderState()1339 CTokenizerHolderState::CTokenizerHolderState()
1340 {
1341 }
1342
~CTokenizerHolderState()1343 CTokenizerHolderState::~CTokenizerHolderState()
1344 {
1345 }
1346
Create()1347 CTokenizerHolderState* CTokenizerHolderState::Create()
1348 {
1349 CTokenizerHolderState* retval = new CTokenizerHolderState();
1350 retval->Init();
1351 return retval;
1352 }
1353
Init()1354 void CTokenizerHolderState::Init()
1355 {
1356 CTokenizerState::Init(true);
1357 }
1358
Delete()1359 void CTokenizerHolderState::Delete()
1360 {
1361 delete this;
1362 }
1363
ProcessElse()1364 bool CTokenizerHolderState::ProcessElse()
1365 {
1366 if (!m_elseHit)
1367 {
1368 m_elseHit = true;
1369 }
1370 return m_elseHit;
1371 }
1372
1373 //
1374 // CKeywordTable
1375 //
1376
CKeywordTable(CTokenizer * tokenizer,keywordArray_t * keywords)1377 CKeywordTable::CKeywordTable(CTokenizer* tokenizer, keywordArray_t* keywords)
1378 {
1379 m_tokenizer = tokenizer;
1380 m_holdKeywords = tokenizer->SetKeywords(keywords);
1381 }
1382
~CKeywordTable()1383 CKeywordTable::~CKeywordTable()
1384 {
1385 m_tokenizer->SetKeywords(m_holdKeywords);
1386 }
1387
1388 //
1389 // CTokenizer
1390 //
1391
CTokenizer()1392 CTokenizer::CTokenizer()
1393 {
1394 }
1395
~CTokenizer()1396 CTokenizer::~CTokenizer()
1397 {
1398 }
1399
Create(UINT dwFlags)1400 CTokenizer* CTokenizer::Create(UINT dwFlags)
1401 {
1402 CTokenizer* theTokenizer = new CTokenizer();
1403 theTokenizer->Init(dwFlags);
1404 return theTokenizer;
1405 }
1406
Delete()1407 void CTokenizer::Delete()
1408 {
1409 while (m_curParseStream != NULL)
1410 {
1411 CParseStream* curStream = m_curParseStream;
1412 m_curParseStream = curStream->GetNext();
1413 curStream->Delete();
1414 }
1415 if (m_symbolLookup != NULL)
1416 {
1417 m_symbolLookup->Delete();
1418 m_symbolLookup = NULL;
1419 }
1420 while (m_nextToken != NULL)
1421 {
1422 CToken* curToken = m_nextToken;
1423 m_nextToken = curToken->GetNext();
1424 curToken->Delete();
1425 }
1426 while (m_state != NULL)
1427 {
1428 Error(TKERR_UNMATCHED_DIRECTIVE);
1429 CTokenizerState* curState = m_state;
1430 m_state = curState->GetNext();
1431 curState->Delete();
1432 }
1433
1434 /* if (m_lastErrMsg != NULL)
1435 {
1436 free(m_lastErrMsg);
1437 m_lastErrMsg = NULL;
1438 }*/
1439 delete this;
1440 }
1441
Error(int theError)1442 void CTokenizer::Error(int theError)
1443 {
1444 char errString[128];
1445 char lookupstring[128];
1446 int i = 0;
1447 while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1448 {
1449 i++;
1450 }
1451 if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1452 {
1453 i = 0;
1454 while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1455 {
1456 i++;
1457 }
1458 strcpy(lookupstring, m_errors[i].m_keyword);
1459 }
1460 else
1461 {
1462 strcpy(lookupstring, errorMessages[i].m_keyword);
1463 }
1464 sprintf(errString, "Error -- %d, %s", theError, lookupstring);
1465 Error(errString, theError);
1466 }
1467
Error(int theError,LPCTSTR errString)1468 void CTokenizer::Error(int theError, LPCTSTR errString)
1469 {
1470 char errstring[128];
1471 char lookupstring[128];
1472 int i = 0;
1473 while ((errorMessages[i].m_tokenvalue != TKERR_USERERROR) && (errorMessages[i].m_tokenvalue != theError))
1474 {
1475 i++;
1476 }
1477 if ((errorMessages[i].m_tokenvalue == TKERR_USERERROR) && (m_errors != NULL))
1478 {
1479 i = 0;
1480 while ((m_errors[i].m_tokenvalue != TK_EOF) && (m_errors[i].m_tokenvalue != theError))
1481 {
1482 i++;
1483 }
1484 strcpy(lookupstring, m_errors[i].m_keyword);
1485 }
1486 else
1487 {
1488 strcpy(lookupstring, errorMessages[i].m_keyword);
1489 }
1490 sprintf(errstring, "Error -- %d, %s - %s", theError, lookupstring, errString);
1491 Error(errstring, theError);
1492 }
1493
Error(LPCTSTR errString,int theError)1494 void CTokenizer::Error(LPCTSTR errString, int theError)
1495 {
1496 if (m_errorProc != NULL)
1497 {
1498 m_errorProc(errString);
1499 }
1500 #ifdef USES_MODULES
1501 else
1502 {
1503 ReportError(theError, errString);
1504 }
1505 #endif
1506 }
1507
AddParseFile(LPCTSTR filename)1508 bool CTokenizer::AddParseFile(LPCTSTR filename)
1509 {
1510 CParseStream* newStream = CParseFile::Create(filename, this);
1511
1512 if ( newStream != NULL )
1513 {
1514 newStream->SetNext(m_curParseStream);
1515 m_curParseStream = newStream;
1516 return true;
1517 }
1518
1519 return false;
1520 }
1521
AddParseStream(byte * data,long datasize)1522 void CTokenizer::AddParseStream(byte* data, long datasize)
1523 {
1524 CParseStream* newStream = CParseMemory::Create(data, datasize);
1525 newStream->SetNext(m_curParseStream);
1526 m_curParseStream = newStream;
1527 }
1528
GetRemainingSize()1529 long CTokenizer::GetRemainingSize()
1530 {
1531 long retval = 0;
1532 CParseStream* curStream = m_curParseStream;
1533 while (curStream != NULL)
1534 {
1535 retval += curStream->GetRemainingSize();
1536 curStream = curStream->GetNext();
1537 }
1538 return retval;
1539 }
1540
LookupToken(int tokenID,keywordArray_t * theTable)1541 LPCTSTR CTokenizer::LookupToken(int tokenID, keywordArray_t* theTable)
1542 {
1543 if (theTable == NULL)
1544 {
1545 theTable = m_keywords;
1546 }
1547 if (theTable == NULL)
1548 {
1549 return NULL;
1550 }
1551
1552 int i = 0;
1553 while (theTable[i].m_tokenvalue != TK_EOF)
1554 {
1555 if (theTable[i].m_tokenvalue == tokenID)
1556 {
1557 return theTable[i].m_keyword;
1558 }
1559 i++;
1560 }
1561 return NULL;
1562 }
1563
PutBackToken(CToken * theToken,bool commented,LPCTSTR addedChars,bool bIgnoreThisTokenType)1564 void CTokenizer::PutBackToken(CToken* theToken, bool commented, LPCTSTR addedChars, bool bIgnoreThisTokenType)
1565 {
1566 if (commented)
1567 {
1568 CParseToken* newStream = CParseToken::Create(theToken);
1569 newStream->SetNext(m_curParseStream);
1570 m_curParseStream = newStream;
1571
1572 if (addedChars != NULL)
1573 {
1574 CParsePutBack* spacer = CParsePutBack::Create(' ', 0, NULL);
1575 spacer->SetNext(m_curParseStream);
1576 m_curParseStream = spacer;
1577
1578 CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1579 newBlock->SetNext(m_curParseStream);
1580 m_curParseStream = newBlock;
1581 }
1582
1583 char temp[] = "// * ";
1584 CParseBlock* newBlock = CParseBlock::Create((byte*)temp, strlen(temp));
1585 newBlock->SetNext(m_curParseStream);
1586 m_curParseStream = newBlock;
1587 return;
1588 }
1589
1590 switch(theToken->GetType())
1591 {
1592 case TK_INT:
1593 case TK_EOF:
1594 case TK_UNDEFINED:
1595 case TK_FLOAT:
1596 case TK_CHAR:
1597 case TK_STRING:
1598 case TK_EOL:
1599 case TK_COMMENT:
1600 if (!bIgnoreThisTokenType)
1601 {
1602 theToken->SetNext(m_nextToken);
1603 m_nextToken = theToken;
1604 break;
1605 }
1606 default:
1607 CParseToken* newStream = CParseToken::Create(theToken);
1608 newStream->SetNext(m_curParseStream);
1609 m_curParseStream = newStream;
1610 break;
1611 }
1612
1613 if (addedChars != NULL)
1614 {
1615 CParseBlock* newBlock = CParseBlock::Create((byte*)addedChars, strlen(addedChars));
1616 newBlock->SetNext(m_curParseStream);
1617 m_curParseStream = newBlock;
1618 }
1619 }
1620
GetToken(keywordArray_t * keywords,UINT onFlags,UINT offFlags)1621 CToken* CTokenizer::GetToken(keywordArray_t* keywords, UINT onFlags, UINT offFlags)
1622 {
1623 keywordArray_t* holdKeywords = SetKeywords(keywords);
1624 CToken* retval = GetToken(onFlags, offFlags);
1625 SetKeywords(holdKeywords);
1626 return retval;
1627 }
1628
GetToken(UINT onFlags,UINT offFlags)1629 CToken* CTokenizer::GetToken(UINT onFlags, UINT offFlags)
1630 {
1631 UINT holdFlags = m_flags;
1632
1633 m_flags |= onFlags;
1634 m_flags &= (~offFlags);
1635 CToken* theToken = NULL;
1636 while (theToken == NULL)
1637 {
1638 theToken = FetchToken();
1639 if (theToken == NULL)
1640 {
1641 continue;
1642 }
1643 if (theToken->GetType() == TK_EOF)
1644 {
1645 break;
1646 }
1647 if (m_state != NULL)
1648 {
1649 if (m_state->Skipping())
1650 {
1651 theToken->Delete();
1652 theToken = NULL;
1653 }
1654 }
1655 }
1656
1657 m_flags = holdFlags;
1658 return theToken;
1659 }
1660
GetToEndOfLine(int tokenType)1661 CToken* CTokenizer::GetToEndOfLine(int tokenType)
1662 {
1663 // update, if you just want the whole line returned as a string, then allow a much bigger size than
1664 // the default string size of only 128 chars...
1665 //
1666 if (tokenType == TK_STRING)
1667 {
1668 #define iRETURN_STRING_SIZE 2048
1669 char theString[iRETURN_STRING_SIZE];
1670 theString[0] = ' ';
1671
1672 for (int i = 1; i < iRETURN_STRING_SIZE; i++)
1673 {
1674 if (NextChar((byte&)theString[i]))
1675 {
1676 if (theString[i] != '\n')
1677 {
1678 continue;
1679 }
1680 PutBackChar(theString[i]);
1681 }
1682 theString[i] = '\0';
1683
1684 return CStringToken::Create(theString);
1685 }
1686
1687 // line would maks a string too big to fit in buffer...
1688 //
1689 Error(TKERR_STRINGLENGTHEXCEEDED);
1690 }
1691 else
1692 {
1693 char theString[MAX_IDENTIFIER_LENGTH];
1694 theString[0] = ' ';
1695 while (theString[0] == ' ')
1696 {
1697 if (!NextChar((byte&)theString[0]))
1698 {
1699 return NULL;
1700 }
1701 }
1702 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
1703 {
1704 if (NextChar((byte&)theString[i]))
1705 {
1706 if (theString[i] != '\n')
1707 {
1708 continue;
1709 }
1710 PutBackChar(theString[i]);
1711 }
1712 theString[i] = '\0';
1713 switch(tokenType)
1714 {
1715 case TK_COMMENT:
1716 return CCommentToken::Create(theString);
1717 case TK_IDENTIFIER:
1718 default:
1719 return CIdentifierToken::Create(theString);
1720 }
1721 }
1722 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
1723 }
1724 return NULL;
1725 }
1726
SkipToLineEnd()1727 void CTokenizer::SkipToLineEnd()
1728 {
1729 byte theByte;
1730 while(NextChar(theByte))
1731 {
1732 if (theByte == '\n')
1733 {
1734 break;
1735 }
1736 }
1737 }
1738
FetchToken()1739 CToken* CTokenizer::FetchToken()
1740 {
1741 if (m_nextToken != NULL)
1742 {
1743 CToken* curToken = m_nextToken;
1744 m_nextToken = curToken->GetNext();
1745 curToken->SetNext(NULL);
1746 return curToken;
1747 }
1748 byte theByte;
1749 CToken* theToken = NULL;
1750
1751 while (true)
1752 {
1753 if (!NextChar(theByte))
1754 {
1755 return CToken::Create();
1756 }
1757 if (theByte <= ' ')
1758 {
1759 if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
1760 {
1761 return CUserToken::Create(TK_EOL, "-EOLN-");
1762 }
1763 continue;
1764 }
1765 switch(theByte)
1766 {
1767 case '#':
1768 if ((m_flags & TKF_IGNOREDIRECTIVES) == 0)
1769 {
1770 theToken = HandleDirective();
1771 }
1772 else
1773 {
1774 if ((m_flags & TKF_NODIRECTIVES) != 0)
1775 {
1776 return HandleSymbol('#');
1777 }
1778 SkipToLineEnd();
1779 }
1780 break;
1781 case '/':
1782 theToken = HandleSlash();
1783 break;
1784 case '"':
1785 theToken = HandleString();
1786 break;
1787 case '\'':
1788 theToken = HandleQuote();
1789 break;
1790 default:
1791 if (((theByte >= 'a') && (theByte <= 'z'))
1792 || ((theByte >= 'A') && (theByte <= 'Z'))
1793 || ((theByte == '_') && ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)))
1794 {
1795 theToken = HandleIdentifier(theByte);
1796 }
1797 else if (((m_flags & TKF_NUMERICIDENTIFIERSTART) != 0) && (theByte >= '0') && (theByte <= '9'))
1798 {
1799 theToken = HandleIdentifier(theByte);
1800 }
1801 else if (((theByte >= '0') && (theByte <= '9')) || (theByte == '-'))
1802 {
1803 theToken = HandleNumeric(theByte);
1804 }
1805 else if (theByte == '.')
1806 {
1807 theToken = HandleDecimal();
1808 }
1809 else if (theByte <= ' ')
1810 {
1811 break;
1812 }
1813 else
1814 {
1815 theToken = HandleSymbol(theByte);
1816 }
1817 }
1818 if (theToken != NULL)
1819 {
1820 return theToken;
1821 }
1822 }
1823 }
1824
NextChar(byte & theByte)1825 bool CTokenizer::NextChar(byte& theByte)
1826 {
1827 while (m_curParseStream != NULL)
1828 {
1829 if (m_curParseStream->NextChar(theByte))
1830 {
1831 return true;
1832 }
1833 CParseStream* curParseStream = m_curParseStream;
1834 m_curParseStream = curParseStream->GetNext();
1835 curParseStream->Delete();
1836 }
1837 return false;
1838 }
1839
RequireToken(int tokenType)1840 bool CTokenizer::RequireToken(int tokenType)
1841 {
1842 CToken* theToken = GetToken();
1843 bool retValue = theToken->GetType() == tokenType;
1844 theToken->Delete();
1845 return retValue;
1846 }
1847
ScanUntilToken(int tokenType)1848 void CTokenizer::ScanUntilToken(int tokenType)
1849 {
1850 CToken* curToken;
1851 int tokenValue = TK_UNDEFINED;
1852 while (tokenValue != tokenType)
1853 {
1854 curToken = GetToken();
1855 tokenValue = curToken->GetType();
1856 if (tokenValue == TK_EOF)
1857 {
1858 PutBackToken(curToken);
1859 break;
1860 }
1861 if (tokenValue == tokenType)
1862 {
1863 PutBackToken(curToken);
1864 }
1865 else
1866 {
1867 curToken->Delete();
1868 }
1869 }
1870 }
1871
Init(UINT dwFlags)1872 void CTokenizer::Init(UINT dwFlags)
1873 {
1874 m_symbolLookup = NULL;
1875 m_nextToken = NULL;
1876 m_curParseStream = NULL;
1877 m_keywords = NULL;
1878 m_symbols = NULL;
1879 m_errors = NULL;
1880 m_state = NULL;
1881 m_flags = dwFlags;
1882 m_errorProc = NULL;
1883 }
1884
SetErrorProc(LPTokenizerErrorProc errorProc)1885 void CTokenizer::SetErrorProc(LPTokenizerErrorProc errorProc)
1886 {
1887 m_errorProc = errorProc;
1888 }
1889
SetAdditionalErrors(keywordArray_t * theErrors)1890 void CTokenizer::SetAdditionalErrors(keywordArray_t* theErrors)
1891 {
1892 m_errors = theErrors;
1893 }
1894
SetKeywords(keywordArray_t * theKeywords)1895 keywordArray_t* CTokenizer::SetKeywords(keywordArray_t* theKeywords)
1896 {
1897 keywordArray_t* retval = m_keywords;
1898 m_keywords = theKeywords;
1899 return retval;
1900 }
1901
SetSymbols(keywordArray_t * theSymbols)1902 void CTokenizer::SetSymbols(keywordArray_t* theSymbols)
1903 {
1904 m_symbols = theSymbols;
1905 if (m_symbolLookup != NULL)
1906 {
1907 m_symbolLookup->Delete();
1908 m_symbolLookup = NULL;
1909 }
1910 int i = 0;
1911 if (theSymbols == NULL)
1912 {
1913 return;
1914 }
1915 while(theSymbols[i].m_tokenvalue != TK_EOF)
1916 {
1917 InsertSymbol(theSymbols[i].m_keyword, theSymbols[i].m_tokenvalue);
1918 i++;
1919 }
1920 }
1921
InsertSymbol(LPCTSTR theSymbol,int theValue)1922 void CTokenizer::InsertSymbol(LPCTSTR theSymbol, int theValue)
1923 {
1924 CSymbolLookup** curHead = &m_symbolLookup;
1925 CSymbolLookup* curParent = NULL;
1926 CSymbolLookup* curLookup = NULL;
1927
1928 for (UINT i = 0; i < strlen(theSymbol); i++)
1929 {
1930 bool found = false;
1931 curLookup = *curHead;
1932 while (curLookup != NULL)
1933 {
1934 if (curLookup->GetByte() == theSymbol[i])
1935 {
1936 found = true;
1937 break;
1938 }
1939 curLookup = curLookup->GetNext();
1940 }
1941 if (!found)
1942 {
1943 curLookup = CSymbolLookup::Create(theSymbol[i]);
1944 curLookup->SetParent(curParent);
1945 curLookup->SetNext(*curHead);
1946 *curHead = curLookup;
1947 }
1948 curHead = curLookup->GetChildAddress();
1949 curParent = curLookup;
1950 }
1951 if (curLookup->GetValue() != -1)
1952 {
1953 Error(TKERR_DUPLICATESYMBOL);
1954 }
1955 curLookup->SetValue(theValue);
1956 }
1957
HandleString()1958 CToken* CTokenizer::HandleString()
1959 {
1960 char theString[MAX_STRING_LENGTH];
1961 for (int i = 0; i < MAX_STRING_LENGTH; i++)
1962 {
1963 if (!NextChar((byte&)theString[i]))
1964 {
1965 return NULL;
1966 }
1967 if (theString[i] == '"')
1968 {
1969 theString[i] = '\0';
1970 return CStringToken::Create(theString);
1971 }
1972 if (theString[i] == '\\')
1973 {
1974 theString[i] = Escapement();
1975 }
1976 }
1977 Error(TKERR_STRINGLENGTHEXCEEDED);
1978 return NULL;
1979 }
1980
GetCurFilename(char ** filename)1981 void CTokenizer::GetCurFilename(char** filename)
1982 {
1983 if (m_curParseStream == NULL)
1984 {
1985 *filename = (char*)malloc(1);
1986 *filename[0] = '\0';
1987 return;
1988 }
1989 m_curParseStream->GetCurFilename(filename);
1990 }
1991
GetCurLine()1992 int CTokenizer::GetCurLine()
1993 {
1994 if (m_curParseStream == NULL)
1995 {
1996 return 0;
1997 }
1998 return m_curParseStream->GetCurLine();
1999 }
2000
PutBackChar(byte theByte,int curLine,LPCTSTR filename)2001 void CTokenizer::PutBackChar(byte theByte, int curLine, LPCTSTR filename)
2002 {
2003 CParseStream* newStream;
2004 if (filename == NULL)
2005 {
2006 curLine = m_curParseStream->GetCurLine();
2007 char* theFile = NULL;
2008 m_curParseStream->GetCurFilename(&theFile);
2009 newStream = CParsePutBack::Create(theByte, curLine, theFile);
2010 if (theFile != NULL)
2011 {
2012 free(theFile);
2013 }
2014 }
2015 else
2016 {
2017 newStream = CParsePutBack::Create(theByte, curLine, filename);
2018 }
2019 newStream->SetNext(m_curParseStream);
2020 m_curParseStream = newStream;
2021 }
2022
Escapement()2023 byte CTokenizer::Escapement()
2024 {
2025 byte theByte;
2026 if (NextChar(theByte))
2027 {
2028 switch(theByte)
2029 {
2030 case 'n':
2031 return '\n';
2032 case '\\':
2033 return '\\';
2034 case '\'':
2035 return '\'';
2036 case '?':
2037 return '\?';
2038 case 'a':
2039 return '\a';
2040 case 'b':
2041 return '\b';
2042 case 'f':
2043 return '\f';
2044 case 'r':
2045 return '\r';
2046 case 't':
2047 return '\t';
2048 case 'v':
2049 return '\v';
2050 case '0':
2051 return '\0';
2052 // support for octal or hex sequences?? \000 or \xhhh
2053 default:
2054 PutBackChar(theByte);
2055 }
2056 }
2057 return '\\';
2058 }
2059
AddDefineSymbol(CDirectiveSymbol * definesymbol)2060 bool CTokenizer::AddDefineSymbol(CDirectiveSymbol* definesymbol)
2061 {
2062 CParseStream* curStream = m_curParseStream;
2063 while(curStream != NULL)
2064 {
2065 if (curStream->IsThisDefinition(definesymbol))
2066 {
2067 return false;
2068 }
2069 curStream = curStream->GetNext();
2070 }
2071 CParseStream* newStream = CParseDefine::Create(definesymbol);
2072 newStream->SetNext(m_curParseStream);
2073 m_curParseStream = newStream;
2074 return true;
2075 }
2076
TokenFromName(LPCTSTR name)2077 CToken* CTokenizer::TokenFromName(LPCTSTR name)
2078 {
2079 CDirectiveSymbol* defineSymbol = (CDirectiveSymbol*)m_defines.FindSymbol(name);
2080 if (defineSymbol != NULL)
2081 {
2082 if (AddDefineSymbol(defineSymbol))
2083 {
2084 return FetchToken();
2085 }
2086 }
2087 if ((m_keywords != NULL) && ((m_flags & TKF_IGNOREKEYWORDS) == 0))
2088 {
2089 int i = 0;
2090 if ((m_flags & TKF_NOCASEKEYWORDS) == 0)
2091 {
2092 while (m_keywords[i].m_tokenvalue != TK_EOF)
2093 {
2094 if (strcmp(m_keywords[i].m_keyword, name) == 0)
2095 {
2096 return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2097 }
2098 i++;
2099 }
2100 }
2101 else
2102 {
2103 while (m_keywords[i].m_tokenvalue != TK_EOF)
2104 {
2105 if (stricmp(m_keywords[i].m_keyword, name) == 0)
2106 {
2107 return CUserToken::Create(m_keywords[i].m_tokenvalue, name);
2108 }
2109 i++;
2110 }
2111 }
2112 }
2113 return CIdentifierToken::Create(name);
2114 }
2115
DirectiveFromName(LPCTSTR name)2116 int CTokenizer::DirectiveFromName(LPCTSTR name)
2117 {
2118 if (directiveKeywords != NULL)
2119 {
2120 int i = 0;
2121 while (directiveKeywords[i].m_tokenvalue != TK_EOF)
2122 {
2123 if (strcmp(directiveKeywords[i].m_keyword, name) == 0)
2124 {
2125 return directiveKeywords[i].m_tokenvalue;
2126 }
2127 i++;
2128 }
2129 }
2130 return -1;
2131 }
2132
HandleIdentifier(byte theByte)2133 CToken* CTokenizer::HandleIdentifier(byte theByte)
2134 {
2135 char theString[MAX_IDENTIFIER_LENGTH];
2136 theString[0] = theByte;
2137 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2138 {
2139 if (NextChar((byte&)theString[i]))
2140 {
2141 if (((theString[i] != '_') || ((m_flags & TKF_NOUNDERSCOREINIDENTIFIER) == 0)) &&
2142 ((theString[i] != '-') || ((m_flags & TKF_NODASHINIDENTIFIER) == 0)))
2143 {
2144 if (((theString[i] >= 'A') && (theString[i] <= 'Z'))
2145 || ((theString[i] >= 'a') && (theString[i] <= 'z'))
2146 || ((theString[i] >= '0') && (theString[i] <= '9'))
2147 || (theString[i] == '_') || (theString[i] == '-'))
2148 {
2149 continue;
2150 }
2151 }
2152 PutBackChar(theString[i]);
2153 }
2154 theString[i] = '\0';
2155 return TokenFromName(theString);
2156 }
2157 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2158 return NULL;
2159 }
2160
HandleSlash()2161 CToken* CTokenizer::HandleSlash()
2162 {
2163 byte theByte;
2164 if (!NextChar(theByte))
2165 {
2166 return NULL;
2167 }
2168 if (theByte == '/')
2169 {
2170 if (m_flags & TKF_COMMENTTOKENS)
2171 {
2172 return GetToEndOfLine(TK_COMMENT);
2173 }
2174 SkipToLineEnd();
2175 return NULL;
2176 }
2177 if (theByte == '*')
2178 {
2179 if (m_flags & TKF_COMMENTTOKENS)
2180 {
2181 char theString[MAX_IDENTIFIER_LENGTH + 1];
2182 theString[0] = ' ';
2183 while (theString[0] == ' ')
2184 {
2185 if (!NextChar((byte&)theString[0]))
2186 {
2187 return NULL;
2188 }
2189 }
2190 for (int i = 1; i < MAX_IDENTIFIER_LENGTH; i++)
2191 {
2192 if (NextChar((byte&)theString[i]))
2193 {
2194 if (theString[i] != '*')
2195 {
2196 continue;
2197 }
2198 i++;
2199 if (NextChar((byte&)theString[i]))
2200 {
2201 if (theString[i] == '/')
2202 {
2203 i--;
2204 theString[i] = '\0';
2205 return CCommentToken::Create(theString);
2206 }
2207 }
2208 }
2209 }
2210 Error(TKERR_IDENTIFIERLENGTHEXCEEDED);
2211 return NULL;
2212 }
2213 while(NextChar(theByte))
2214 {
2215 while(theByte == '*')
2216 {
2217 if (!NextChar(theByte))
2218 {
2219 break;
2220 }
2221 if (theByte == '/')
2222 {
2223 return NULL;
2224 }
2225 }
2226 }
2227 return NULL;
2228 }
2229 PutBackChar(theByte);
2230 return HandleSymbol('/');
2231 }
2232
HandleNumeric(byte theByte)2233 CToken* CTokenizer::HandleNumeric(byte theByte)
2234 {
2235 bool thesign = theByte == '-';
2236 if (thesign)
2237 {
2238 if (!NextChar(theByte))
2239 {
2240 return HandleSymbol('-');
2241 }
2242 if (theByte == '.')
2243 {
2244 return HandleDecimal(thesign);
2245 }
2246 if ((theByte < '0') || (theByte > '9'))
2247 {
2248 PutBackChar(theByte);
2249 return HandleSymbol('-');
2250 }
2251 }
2252 if (theByte == '0')
2253 {
2254 return HandleOctal(thesign);
2255 }
2256 long value = 0;
2257 bool digithit = false;
2258 while((theByte >= '0') && (theByte <= '9'))
2259 {
2260 value = (value * 10) + (theByte - '0');
2261 if (!NextChar(theByte))
2262 {
2263 if (thesign)
2264 {
2265 value = -value;
2266 }
2267 return CIntToken::Create(value);
2268 }
2269 digithit = true;
2270 }
2271 if (theByte == '.')
2272 {
2273 if (digithit)
2274 {
2275 return HandleFloat(thesign, value);
2276 }
2277 if (thesign)
2278 {
2279 PutBackChar(theByte);
2280 theByte = '-';
2281 }
2282 return HandleSymbol(theByte);
2283 }
2284 PutBackChar(theByte);
2285 if (thesign)
2286 {
2287 value = -value;
2288 }
2289 return CIntToken::Create(value);
2290 }
2291
HandleOctal(bool thesign)2292 CToken* CTokenizer::HandleOctal(bool thesign)
2293 {
2294 byte theByte;
2295 int value = 0;
2296
2297 if (!NextChar(theByte))
2298 {
2299 return CIntToken::Create(value);
2300 }
2301 if (theByte == '.')
2302 {
2303 return HandleDecimal(thesign);
2304 }
2305 if ((theByte == 'x') || (theByte == 'X'))
2306 {
2307 return HandleHex(thesign);
2308 }
2309 while(true)
2310 {
2311 if((theByte >= '0') && (theByte <='7'))
2312 {
2313 value = (value * 8) + (theByte - '0');
2314 }
2315 else
2316 {
2317 PutBackChar(theByte);
2318 if (thesign)
2319 {
2320 value = -value;
2321 }
2322 return CIntToken::Create(value);
2323 }
2324 if (!NextChar(theByte))
2325 {
2326 if (thesign)
2327 {
2328 value = -value;
2329 }
2330 return CIntToken::Create(value);
2331 }
2332 }
2333 }
2334
HandleHex(bool thesign)2335 CToken* CTokenizer::HandleHex(bool thesign)
2336 {
2337 int value = 0;
2338
2339 while (true)
2340 {
2341 byte theByte;
2342 if (!NextChar(theByte))
2343 {
2344 if (thesign)
2345 {
2346 value = -value;
2347 }
2348 return CIntToken::Create(value);
2349 }
2350 switch (theByte)
2351 {
2352 case '0':
2353 case '1':
2354 case '2':
2355 case '3':
2356 case '4':
2357 case '5':
2358 case '6':
2359 case '7':
2360 case '8':
2361 case '9':
2362 theByte = theByte - '0';
2363 break;
2364 case 'A':
2365 case 'B':
2366 case 'C':
2367 case 'D':
2368 case 'E':
2369 case 'F':
2370 theByte = theByte - 'A' + 10;
2371 break;
2372 case 'a':
2373 case 'b':
2374 case 'c':
2375 case 'd':
2376 case 'e':
2377 case 'f':
2378 theByte = theByte - 'a' + 10;
2379 break;
2380 default:
2381 PutBackChar(theByte);
2382 if (thesign)
2383 {
2384 value = -value;
2385 }
2386 return CIntToken::Create(value);
2387 }
2388 value = (value * 16) + theByte;
2389 }
2390 }
2391
HandleDecimal(bool thesign)2392 CToken* CTokenizer::HandleDecimal(bool thesign)
2393 {
2394 byte theByte;
2395 if (!NextChar(theByte))
2396 {
2397 if (thesign)
2398 {
2399 PutBackChar('.');
2400 return HandleSymbol('-');
2401 }
2402 HandleSymbol('.');
2403 }
2404 PutBackChar(theByte);
2405 if ((theByte <= '9') && (theByte >= '0'))
2406 {
2407 return HandleFloat(thesign);
2408 }
2409 if (thesign)
2410 {
2411 PutBackChar('.');
2412 theByte = '-';
2413 }
2414 else
2415 {
2416 theByte = '.';
2417 }
2418 return HandleSymbol(theByte);
2419 }
2420
HandleFloat(bool thesign,long value)2421 CToken* CTokenizer::HandleFloat(bool thesign, long value)
2422 {
2423 float lower = 1.0;
2424 float newValue = (float)value;
2425 byte theByte;
2426 while(NextChar(theByte))
2427 {
2428 if ((theByte >= '0') && (theByte <= '9'))
2429 {
2430 lower = lower / 10;
2431 newValue = newValue + ((theByte - '0') * lower);
2432 continue;
2433 }
2434 PutBackChar(theByte);
2435 break;
2436 }
2437 if (thesign)
2438 {
2439 newValue = -newValue;
2440 }
2441 return CFloatToken::Create(newValue);
2442 }
2443
HandleQuote()2444 CToken* CTokenizer::HandleQuote()
2445 {
2446 byte theByte;
2447 if (!NextChar(theByte))
2448 {
2449 Error(TKERR_EXPECTED_CHAR);
2450 return NULL;
2451 }
2452 if (theByte == '\\')
2453 {
2454 theByte = Escapement();
2455 }
2456 byte dummy;
2457 if (!NextChar(dummy))
2458 {
2459 Error(TKERR_EXPECTED_CHAR);
2460 return NULL;
2461 }
2462 if (dummy != '\'')
2463 {
2464 PutBackChar(dummy);
2465 PutBackChar(theByte);
2466 Error(TKERR_EXPECTED_CHAR);
2467 return NULL;
2468 }
2469 return CCharToken::Create(theByte);
2470 }
2471
SetFlags(UINT flags)2472 void CTokenizer::SetFlags(UINT flags)
2473 {
2474 m_flags = flags;
2475 }
2476
GetFlags()2477 UINT CTokenizer::GetFlags()
2478 {
2479 return m_flags;
2480 }
2481
HandleSymbol(byte theByte)2482 CToken* CTokenizer::HandleSymbol(byte theByte)
2483 {
2484 char symbolString[128];
2485 int curStrLen = 0;
2486 symbolString[0] = '\0';
2487 bool consumed = false;
2488
2489 CSymbolLookup* curLookup;
2490 if ((m_flags & TKF_RAWSYMBOLSONLY) == 0)
2491 {
2492 curLookup = m_symbolLookup;
2493 }
2494 else
2495 {
2496 curLookup = NULL;
2497 }
2498 CSymbolLookup* lastLookup = NULL;
2499 while(curLookup != NULL)
2500 {
2501 if (curLookup->GetByte() == theByte)
2502 {
2503 symbolString[curStrLen++] = theByte;
2504 symbolString[curStrLen] = '\0';
2505 lastLookup = curLookup;
2506 consumed = true;
2507 if (curLookup->GetChild() == NULL)
2508 {
2509 break;
2510 }
2511 if (!NextChar(theByte))
2512 {
2513 PutBackToken(CToken::Create());
2514 break;
2515 }
2516 consumed = false;
2517 curLookup = curLookup->GetChild();
2518 continue;
2519 }
2520 curLookup = curLookup->GetNext();
2521 }
2522 if ((!consumed) && (lastLookup != NULL))
2523 {
2524 PutBackChar(theByte);
2525 }
2526 while ((lastLookup != NULL) && (lastLookup->GetValue() == -1))
2527 {
2528 curStrLen--;
2529 symbolString[curStrLen] = '\0';
2530 // symbolString = symbolString.Left(symbolString.GetLength() - 1);
2531 if (lastLookup->GetParent() == NULL)
2532 {
2533 if ((m_flags & TKF_WANTUNDEFINED) == 0)
2534 {
2535 Error(TKERR_UNRECOGNIZEDSYMBOL);
2536 }
2537 }
2538 else
2539 {
2540 PutBackChar(lastLookup->GetByte());
2541 }
2542 lastLookup = lastLookup->GetParent();
2543 }
2544 if (lastLookup == NULL)
2545 {
2546 if ((m_flags & TKF_WANTUNDEFINED) == 0)
2547 {
2548 return NULL;
2549 }
2550 curStrLen = 0;
2551 symbolString[curStrLen++] = char(theByte);
2552 symbolString[curStrLen] = '\0';
2553 if ((m_flags & TKF_WIDEUNDEFINEDSYMBOLS) != 0)
2554 {
2555 while (true)
2556 {
2557 if (!NextChar(theByte))
2558 {
2559 PutBackToken(CToken::Create());
2560 break;
2561 }
2562 if (theByte == ' ')
2563 {
2564 break;
2565 }
2566 if (((theByte >= 'a') && (theByte <= 'z')) || ((theByte >= 'A') && (theByte <= 'Z')) ||
2567 ((theByte >= '0') && (theByte <= '9')))
2568 {
2569 PutBackChar(theByte);
2570 break;
2571 }
2572 if (theByte < ' ')
2573 {
2574 if ((theByte == '\n') && ((TKF_USES_EOL & m_flags) != 0))
2575 {
2576 PutBackToken(CUserToken::Create(TK_EOL, "-EOLN-"));
2577 break;
2578 }
2579 continue;
2580 }
2581 symbolString[curStrLen++] = theByte;
2582 symbolString[curStrLen] = '\0';
2583 }
2584 }
2585 return CUndefinedToken::Create(symbolString);
2586 }
2587 return CUserToken::Create(lastLookup->GetValue(), symbolString);
2588 }
2589
HandleDirective()2590 CToken* CTokenizer::HandleDirective()
2591 {
2592 int tokenValue = 0;
2593 CToken* theToken = FetchToken();
2594 if (theToken->GetType() == TK_EOF)
2595 {
2596 return theToken;
2597 }
2598 if (theToken->GetType() != TK_IDENTIFIER)
2599 {
2600 Error(TKERR_INVALID_DIRECTIVE);
2601 theToken->Delete();
2602 SkipToLineEnd();
2603 return NULL;
2604 }
2605
2606 CDirectiveSymbol* curSymbol;
2607 CTokenizerState* state;
2608 int theDirective = DirectiveFromName(theToken->GetStringValue());
2609 theToken->Delete();
2610 byte theByte;
2611 switch(theDirective)
2612 {
2613 case DIR_INCLUDE:
2614 if ((m_state != NULL) && (m_state->Skipping()))
2615 {
2616 break;
2617 }
2618 theToken = GetToken();
2619 if (theToken->GetType() != TK_STRING)
2620 {
2621 Error(TKERR_INCLUDE_FILE_NOTFOUND);
2622 theToken->Delete();
2623 SkipToLineEnd();
2624 break;
2625 }
2626 AddParseFile(theToken->GetStringValue());
2627 theToken->Delete();
2628 break;
2629 case DIR_IFDEF:
2630 if ((m_state != NULL) && (m_state->Skipping()))
2631 {
2632 state = CTokenizerHolderState::Create();
2633 state->SetNext(m_state);
2634 m_state = state;
2635 break;
2636 }
2637 theToken = GetToken();
2638 if (theToken->GetType() != TK_IDENTIFIER)
2639 {
2640 Error(TKERR_EXPECTED_IDENTIFIER);
2641 theToken->Delete();
2642 SkipToLineEnd();
2643 break;
2644 }
2645 state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) == NULL);
2646 theToken->Delete();
2647 state->SetNext(m_state);
2648 m_state = state;
2649 break;
2650 case DIR_IFNDEF:
2651 if ((m_state != NULL) && (m_state->Skipping()))
2652 {
2653 state = CTokenizerHolderState::Create();
2654 state->SetNext(m_state);
2655 m_state = state;
2656 break;
2657 }
2658 theToken = GetToken();
2659 if (theToken->GetType() != TK_IDENTIFIER)
2660 {
2661 Error(TKERR_EXPECTED_IDENTIFIER);
2662 theToken->Delete();
2663 SkipToLineEnd();
2664 break;
2665 }
2666 state = CTokenizerState::Create(m_defines.FindSymbol(theToken->GetStringValue()) != NULL);
2667 theToken->Delete();
2668 state->SetNext(m_state);
2669 m_state = state;
2670 break;
2671 case DIR_ENDIF:
2672 if (m_state == NULL)
2673 {
2674 Error(TKERR_UNMATCHED_DIRECTIVE);
2675 break;
2676 }
2677 state = m_state;
2678 m_state = state->GetNext();
2679 state->Delete();
2680 break;
2681 case DIR_ELSE:
2682 if (m_state == NULL)
2683 {
2684 Error(TKERR_UNMATCHED_DIRECTIVE);
2685 break;
2686 }
2687 if (!m_state->ProcessElse())
2688 {
2689 Error(TKERR_UNMATCHED_DIRECTIVE);
2690 break;
2691 }
2692 break;
2693 case DIR_DEFINE:
2694 if ((m_state != NULL) && (m_state->Skipping()))
2695 {
2696 break;
2697 }
2698 theToken = GetToken();
2699 if (theToken->GetType() != TK_IDENTIFIER)
2700 {
2701 Error(TKERR_EXPECTED_IDENTIFIER);
2702 theToken->Delete();
2703 SkipToLineEnd();
2704 break;
2705 }
2706 // blind add - should check first for value changes
2707 {
2708 curSymbol = CDirectiveSymbol::Create(theToken->GetStringValue());
2709 char temp[128];
2710 int tempsize = 0;
2711 while(NextChar(theByte))
2712 {
2713 if (theByte == '\n')
2714 {
2715 break;
2716 }
2717 temp[tempsize++] = char(theByte);
2718 }
2719 temp[tempsize] = '\0';
2720 curSymbol->SetValue(temp);
2721 if (!m_defines.AddSymbol(curSymbol))
2722 {
2723 curSymbol->Delete();
2724 }
2725 }
2726 break;
2727 case DIR_UNDEFINE:
2728 if ((m_state != NULL) && (m_state->Skipping()))
2729 {
2730 break;
2731 }
2732 theToken = GetToken();
2733 if (theToken->GetType() != TK_IDENTIFIER)
2734 {
2735 Error(TKERR_EXPECTED_IDENTIFIER);
2736 theToken->Delete();
2737 SkipToLineEnd();
2738 break;
2739 }
2740 m_defines.RemoveSymbol(theToken->GetStringValue());
2741 break;
2742 default:
2743 Error(TKERR_INVALID_DIRECTIVE);
2744 SkipToLineEnd();
2745 break;
2746 }
2747 return NULL;
2748 }
2749
ParseRGB()2750 COLORREF CTokenizer::ParseRGB()
2751 {
2752 CToken* theToken = GetToken();
2753 if (theToken->GetType() != TK_INT)
2754 {
2755 Error(TKERR_EXPECTED_INTEGER);
2756 theToken->Delete();
2757 return RGB(0, 0, 0);
2758 }
2759 int red = theToken->GetIntValue();
2760 theToken->Delete();
2761 theToken = GetToken();
2762 if (theToken->GetType() != TK_INT)
2763 {
2764 Error(TKERR_EXPECTED_INTEGER);
2765 theToken->Delete();
2766 return RGB(0, 0, 0);
2767 }
2768 int green = theToken->GetIntValue();
2769 theToken->Delete();
2770 theToken = GetToken();
2771 if (theToken->GetType() != TK_INT)
2772 {
2773 Error(TKERR_EXPECTED_INTEGER);
2774 theToken->Delete();
2775 return RGB(0, 0, 0);
2776 }
2777 int blue = theToken->GetIntValue();
2778 theToken->Delete();
2779 return RGB(red, green, blue);
2780 }
2781
2782 //
2783 // CSymbolLookup
2784 //
2785
CSymbolLookup()2786 CSymbolLookup::CSymbolLookup()
2787 {
2788 }
2789
~CSymbolLookup()2790 CSymbolLookup::~CSymbolLookup()
2791 {
2792 }
2793
Create(byte theByte)2794 CSymbolLookup* CSymbolLookup::Create(byte theByte)
2795 {
2796 CSymbolLookup* curLookup = new CSymbolLookup();
2797 curLookup->Init(theByte);
2798 return curLookup;
2799 }
2800
Delete()2801 void CSymbolLookup::Delete()
2802 {
2803 if (m_sibling != NULL)
2804 {
2805 m_sibling->Delete();
2806 m_sibling = NULL;
2807 }
2808 if (m_child != NULL)
2809 {
2810 m_child->Delete();
2811 m_child = NULL;
2812 }
2813 delete this;
2814 }
2815
Init(byte theByte)2816 void CSymbolLookup::Init(byte theByte)
2817 {
2818 m_parent = NULL;
2819 m_child = NULL;
2820 m_sibling = NULL;
2821 m_value = -1;
2822 m_byte = theByte;
2823 }
2824
GetNext()2825 CSymbolLookup* CSymbolLookup::GetNext()
2826 {
2827 return m_sibling;
2828 }
2829
SetNext(CSymbolLookup * next)2830 void CSymbolLookup::SetNext(CSymbolLookup* next)
2831 {
2832 m_sibling = next;
2833 }
2834
SetParent(CSymbolLookup * parent)2835 void CSymbolLookup::SetParent(CSymbolLookup* parent)
2836 {
2837 m_parent = parent;
2838 }
2839
SetValue(int value)2840 void CSymbolLookup::SetValue(int value)
2841 {
2842 m_value = value;
2843 }
2844
GetValue()2845 int CSymbolLookup::GetValue()
2846 {
2847 return m_value;
2848 }
2849
GetByte()2850 byte CSymbolLookup::GetByte()
2851 {
2852 return m_byte;
2853 }
2854
GetChildAddress()2855 CSymbolLookup** CSymbolLookup::GetChildAddress()
2856 {
2857 return &m_child;
2858 }
2859
GetChild()2860 CSymbolLookup* CSymbolLookup::GetChild()
2861 {
2862 return m_child;
2863 }
2864
GetParent()2865 CSymbolLookup* CSymbolLookup::GetParent()
2866 {
2867 return m_parent;
2868 }
2869