1## @file
2# This file is used to parse DEC file. It will consumed by DecParser
3#
4# Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
5#
6# SPDX-License-Identifier: BSD-2-Clause-Patent
7'''
8DecParser
9'''
10## Import modules
11#
12import Logger.Log as Logger
13from Logger.ToolError import FILE_PARSE_FAILURE
14from Logger.ToolError import FILE_OPEN_FAILURE
15from Logger import StringTable as ST
16from Logger.ToolError import FORMAT_INVALID
17
18import Library.DataType as DT
19from Library.ParserValidate import IsValidToken
20from Library.ParserValidate import IsValidPath
21from Library.ParserValidate import IsValidCFormatGuid
22from Library.ParserValidate import IsValidIdString
23from Library.ParserValidate import IsValidUserId
24from Library.ParserValidate import IsValidArch
25from Library.ParserValidate import IsValidWord
26from Library.ParserValidate import IsValidDecVersionVal
27from Parser.DecParserMisc import TOOL_NAME
28from Parser.DecParserMisc import CleanString
29from Parser.DecParserMisc import IsValidPcdDatum
30from Parser.DecParserMisc import ParserHelper
31from Parser.DecParserMisc import StripRoot
32from Parser.DecParserMisc import VERSION_PATTERN
33from Parser.DecParserMisc import CVAR_PATTERN
34from Parser.DecParserMisc import PCD_TOKEN_PATTERN
35from Parser.DecParserMisc import MACRO_PATTERN
36from Parser.DecParserMisc import FileContent
37from Object.Parser.DecObject import _DecComments
38from Object.Parser.DecObject import DecDefineObject
39from Object.Parser.DecObject import DecDefineItemObject
40from Object.Parser.DecObject import DecIncludeObject
41from Object.Parser.DecObject import DecIncludeItemObject
42from Object.Parser.DecObject import DecLibraryclassObject
43from Object.Parser.DecObject import DecLibraryclassItemObject
44from Object.Parser.DecObject import DecGuidObject
45from Object.Parser.DecObject import DecPpiObject
46from Object.Parser.DecObject import DecProtocolObject
47from Object.Parser.DecObject import DecGuidItemObject
48from Object.Parser.DecObject import DecUserExtensionObject
49from Object.Parser.DecObject import DecUserExtensionItemObject
50from Object.Parser.DecObject import DecPcdObject
51from Object.Parser.DecObject import DecPcdItemObject
52from Library.Misc import GuidStructureStringToGuidString
53from Library.Misc import CheckGuidRegFormat
54from Library.StringUtils import ReplaceMacro
55from Library.StringUtils import GetSplitValueList
56from Library.StringUtils import gMACRO_PATTERN
57from Library.StringUtils import ConvertSpecialChar
58from Library.CommentParsing import ParsePcdErrorCode
59
60##
61# _DecBase class for parsing
62#
63class _DecBase:
64    def __init__(self, RawData):
65        self._RawData = RawData
66        self._ItemDict = {}
67        self._LocalMacro = {}
68        #
69        # Data parsed by 'self' are saved to this object
70        #
71        self.ItemObject = None
72
73    def GetDataObject(self):
74        return self.ItemObject
75
76    def GetLocalMacro(self):
77        return self._LocalMacro
78
79    ## BlockStart
80    #
81    # Called if a new section starts
82    #
83    def BlockStart(self):
84        self._LocalMacro = {}
85
86    ## _CheckReDefine
87    #
88    # @param Key: to be checked if multi-defined
89    # @param Scope: Format: [[SectionName, Arch], ...].
90    #               If scope is none, use global scope
91    #
92    def _CheckReDefine(self, Key, Scope = None):
93        if not Scope:
94            Scope = self._RawData.CurrentScope
95            return
96
97        SecArch = []
98        #
99        # Copy scope to SecArch, avoid Scope be changed outside
100        #
101        SecArch[0:1] = Scope[:]
102        if Key not in self._ItemDict:
103            self._ItemDict[Key] = [[SecArch, self._RawData.LineIndex]]
104            return
105
106        for Value in self._ItemDict[Key]:
107            for SubValue in Scope:
108                #
109                # If current is common section
110                #
111                if SubValue[-1] == 'COMMON':
112                    for Other in Value[0]:
113                        # Key in common cannot be redefined in other arches
114                        # [:-1] means stripping arch info
115                        if Other[:-1] == SubValue[:-1]:
116                            self._LoggerError(ST.ERR_DECPARSE_REDEFINE % (Key, Value[1]))
117                            return
118                    continue
119                CommonScope = []
120                CommonScope[0:1] = SubValue
121                CommonScope[-1] = 'COMMON'
122                #
123                # Cannot be redefined if this key already defined in COMMON Or defined in same arch
124                #
125                if SubValue in Value[0] or CommonScope in Value[0]:
126                    self._LoggerError(ST.ERR_DECPARSE_REDEFINE % (Key, Value[1]))
127                    return
128        self._ItemDict[Key].append([SecArch, self._RawData.LineIndex])
129
130    ## CheckRequiredFields
131    # Some sections need to check if some fields exist, define section for example
132    # Derived class can re-implement, top parser will call this function after all parsing done
133    #
134    def CheckRequiredFields(self):
135        if self._RawData:
136            pass
137        return True
138
139    ## IsItemRequired
140    # In DEC spec, sections must have at least one statement except user
141    # extension.
142    # For example: "[guids" [<attribs>] "]" <EOL> <statements>+
143    # sub class can override this method to indicate if statement is a must.
144    #
145    def _IsStatementRequired(self):
146        if self._RawData:
147            pass
148        return False
149
150    def _LoggerError(self, ErrorString):
151        Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE, File=self._RawData.Filename,
152                     Line = self._RawData.LineIndex,
153                     ExtraData=ErrorString + ST.ERR_DECPARSE_LINE % self._RawData.CurrentLine)
154
155    def _ReplaceMacro(self, String):
156        if gMACRO_PATTERN.findall(String):
157            String = ReplaceMacro(String, self._LocalMacro, False,
158                                  FileName = self._RawData.Filename,
159                                  Line = ['', self._RawData.LineIndex])
160            String = ReplaceMacro(String, self._RawData.Macros, False,
161                                  FileName = self._RawData.Filename,
162                                  Line = ['', self._RawData.LineIndex])
163            MacroUsed = gMACRO_PATTERN.findall(String)
164            if MacroUsed:
165                Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE,
166                             File=self._RawData.Filename,
167                             Line = self._RawData.LineIndex,
168                             ExtraData = ST.ERR_DECPARSE_MACRO_RESOLVE % (str(MacroUsed), String))
169        return String
170
171    def _MacroParser(self, String):
172        TokenList = GetSplitValueList(String, ' ', 1)
173        if len(TokenList) < 2 or TokenList[1] == '':
174            self._LoggerError(ST.ERR_DECPARSE_MACRO_PAIR)
175
176        TokenList = GetSplitValueList(TokenList[1], DT.TAB_EQUAL_SPLIT, 1)
177        if TokenList[0] == '':
178            self._LoggerError(ST.ERR_DECPARSE_MACRO_NAME)
179        elif not IsValidToken(MACRO_PATTERN, TokenList[0]):
180            self._LoggerError(ST.ERR_DECPARSE_MACRO_NAME_UPPER % TokenList[0])
181
182        if len(TokenList) == 1:
183            self._LocalMacro[TokenList[0]] = ''
184        else:
185            self._LocalMacro[TokenList[0]] = self._ReplaceMacro(TokenList[1])
186
187    ## _ParseItem
188    #
189    # Parse specified item, this function must be derived by subclass
190    #
191    def _ParseItem(self):
192        if self._RawData:
193            pass
194        #
195        # Should never be called
196        #
197        return None
198
199
200    ## _TailCommentStrategy
201    #
202    # This function can be derived to parse tail comment
203    # default is it will not consume any lines
204    #
205    # @param Comment: Comment of current line
206    #
207    def _TailCommentStrategy(self, Comment):
208        if Comment:
209            pass
210        if self._RawData:
211            pass
212        return False
213
214    ## _StopCurrentParsing
215    #
216    # Called in Parse if current parsing should be stopped when encounter some
217    # keyword
218    # Default is section start and end
219    #
220    # @param Line: Current line
221    #
222    def _StopCurrentParsing(self, Line):
223        if self._RawData:
224            pass
225        return Line[0] == DT.TAB_SECTION_START and Line[-1] == DT.TAB_SECTION_END
226
227    ## _TryBackSlash
228    #
229    # Split comment and DEC content, concatenate lines if end of char is '\'
230    #
231    # @param ProcessedLine: ProcessedLine line
232    # @param ProcessedComments: ProcessedComments line
233    #
234    def _TryBackSlash(self, ProcessedLine, ProcessedComments):
235        CatLine = ''
236        Comment = ''
237        Line = ProcessedLine
238        CommentList = ProcessedComments
239        while not self._RawData.IsEndOfFile():
240            if Line == '':
241                self._LoggerError(ST.ERR_DECPARSE_BACKSLASH_EMPTY)
242                break
243
244            if Comment:
245                CommentList.append((Comment, self._RawData.LineIndex))
246            if Line[-1] != DT.TAB_SLASH:
247                CatLine += Line
248                break
249            elif len(Line) < 2 or Line[-2] != ' ':
250                self._LoggerError(ST.ERR_DECPARSE_BACKSLASH)
251            else:
252                CatLine += Line[:-1]
253                Line, Comment = CleanString(self._RawData.GetNextLine())
254        #
255        # Reach end of content
256        #
257        if self._RawData.IsEndOfFile():
258            if not CatLine:
259                if ProcessedLine[-1] == DT.TAB_SLASH:
260                    self._LoggerError(ST.ERR_DECPARSE_BACKSLASH_EMPTY)
261                CatLine = ProcessedLine
262            else:
263                if not Line or Line[-1] == DT.TAB_SLASH:
264                    self._LoggerError(ST.ERR_DECPARSE_BACKSLASH_EMPTY)
265                CatLine += Line
266
267        #
268        # All MACRO values defined by the DEFINE statements in any section
269        # (except [Userextensions] sections for Intel) of the INF or DEC file
270        # must be expanded before processing of the file.
271        #
272        __IsReplaceMacro = True
273        Header = self._RawData.CurrentScope[0] if self._RawData.CurrentScope else None
274        if Header and len(Header) > 2:
275            if Header[0].upper() == 'USEREXTENSIONS' and not (Header[1] == 'TianoCore' and Header[2] == '"ExtraFiles"'):
276                __IsReplaceMacro = False
277        if __IsReplaceMacro:
278            self._RawData.CurrentLine = self._ReplaceMacro(CatLine)
279        else:
280            self._RawData.CurrentLine = CatLine
281
282        return CatLine, CommentList
283
284    ## Parse
285    # This is a template method in which other member functions which might
286    # override by sub class are called. It is responsible for reading file
287    # line by line, and call other member functions to parse. This function
288    # should not be re-implement by sub class.
289    #
290    def Parse(self):
291        HeadComments = []
292        TailComments = []
293
294        #======================================================================
295        # CurComments may pointer to HeadComments or TailComments
296        #======================================================================
297        CurComments = HeadComments
298        CurObj = None
299        ItemNum = 0
300        FromBuf = False
301
302        #======================================================================
303        # Used to report error information if empty section found
304        #======================================================================
305        Index = self._RawData.LineIndex
306        LineStr = self._RawData.CurrentLine
307        while not self._RawData.IsEndOfFile() or self._RawData.NextLine:
308            if self._RawData.NextLine:
309                #==============================================================
310                # Have processed line in buffer
311                #==============================================================
312                Line = self._RawData.NextLine
313                HeadComments.extend(self._RawData.HeadComment)
314                TailComments.extend(self._RawData.TailComment)
315                self._RawData.ResetNext()
316                Comment = ''
317                FromBuf = True
318            else:
319                #==============================================================
320                # No line in buffer, read next line
321                #==============================================================
322                Line, Comment = CleanString(self._RawData.GetNextLine())
323                FromBuf = False
324            if Line:
325                if not FromBuf and CurObj and TailComments:
326                    #==========================================================
327                    # Set tail comments to previous statement if not empty.
328                    #==========================================================
329                    CurObj.SetTailComment(CurObj.GetTailComment()+TailComments)
330
331                if not FromBuf:
332                    del TailComments[:]
333                CurComments = TailComments
334                Comments = []
335                if Comment:
336                    Comments = [(Comment, self._RawData.LineIndex)]
337
338                #==============================================================
339                # Try if last char of line has backslash
340                #==============================================================
341                Line, Comments = self._TryBackSlash(Line, Comments)
342                CurComments.extend(Comments)
343
344                #==============================================================
345                # Macro found
346                #==============================================================
347                if Line.startswith('DEFINE '):
348                    self._MacroParser(Line)
349                    del HeadComments[:]
350                    del TailComments[:]
351                    CurComments = HeadComments
352                    continue
353
354                if self._StopCurrentParsing(Line):
355                    #==========================================================
356                    # This line does not belong to this parse,
357                    # Save it, can be used by next parse
358                    #==========================================================
359                    self._RawData.SetNext(Line, HeadComments, TailComments)
360                    break
361
362                Obj = self._ParseItem()
363                ItemNum += 1
364                if Obj:
365                    Obj.SetHeadComment(Obj.GetHeadComment()+HeadComments)
366                    Obj.SetTailComment(Obj.GetTailComment()+TailComments)
367                    del HeadComments[:]
368                    del TailComments[:]
369                    CurObj = Obj
370                else:
371                    CurObj = None
372            else:
373                if id(CurComments) == id(TailComments):
374                    #==========================================================
375                    # Check if this comment belongs to tail comment
376                    #==========================================================
377                    if not self._TailCommentStrategy(Comment):
378                        CurComments = HeadComments
379
380                if Comment:
381                    CurComments.append(((Comment, self._RawData.LineIndex)))
382                else:
383                    del CurComments[:]
384
385        if self._IsStatementRequired() and ItemNum == 0:
386            Logger.Error(
387                    TOOL_NAME, FILE_PARSE_FAILURE,
388                    File=self._RawData.Filename,
389                    Line=Index,
390                    ExtraData=ST.ERR_DECPARSE_STATEMENT_EMPTY % LineStr
391            )
392
393## _DecDefine
394# Parse define section
395#
396class _DecDefine(_DecBase):
397    def __init__(self, RawData):
398        _DecBase.__init__(self, RawData)
399        self.ItemObject = DecDefineObject(RawData.Filename)
400        self._LocalMacro = self._RawData.Macros
401        self._DefSecNum = 0
402
403        #
404        # Each field has a function to validate
405        #
406        self.DefineValidation = {
407            DT.TAB_DEC_DEFINES_DEC_SPECIFICATION   :   self._SetDecSpecification,
408            DT.TAB_DEC_DEFINES_PACKAGE_NAME        :   self._SetPackageName,
409            DT.TAB_DEC_DEFINES_PACKAGE_GUID        :   self._SetPackageGuid,
410            DT.TAB_DEC_DEFINES_PACKAGE_VERSION     :   self._SetPackageVersion,
411            DT.TAB_DEC_DEFINES_PKG_UNI_FILE        :   self._SetPackageUni,
412        }
413
414    def BlockStart(self):
415        self._DefSecNum += 1
416        if self._DefSecNum > 1:
417            self._LoggerError(ST.ERR_DECPARSE_DEFINE_MULTISEC)
418
419    ## CheckRequiredFields
420    #
421    # Check required fields: DEC_SPECIFICATION, PACKAGE_NAME
422    #                        PACKAGE_GUID, PACKAGE_VERSION
423    #
424    def CheckRequiredFields(self):
425        Ret = False
426        if self.ItemObject.GetPackageSpecification() == '':
427            Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE, File=self._RawData.Filename,
428                         ExtraData=ST.ERR_DECPARSE_DEFINE_REQUIRED % DT.TAB_DEC_DEFINES_DEC_SPECIFICATION)
429        elif self.ItemObject.GetPackageName() == '':
430            Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE, File=self._RawData.Filename,
431                         ExtraData=ST.ERR_DECPARSE_DEFINE_REQUIRED % DT.TAB_DEC_DEFINES_PACKAGE_NAME)
432        elif self.ItemObject.GetPackageGuid() == '':
433            Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE, File=self._RawData.Filename,
434                         ExtraData=ST.ERR_DECPARSE_DEFINE_REQUIRED % DT.TAB_DEC_DEFINES_PACKAGE_GUID)
435        elif self.ItemObject.GetPackageVersion() == '':
436            Logger.Error(TOOL_NAME, FILE_PARSE_FAILURE, File=self._RawData.Filename,
437                         ExtraData=ST.ERR_DECPARSE_DEFINE_REQUIRED % DT.TAB_DEC_DEFINES_PACKAGE_VERSION)
438        else:
439            Ret = True
440        return Ret
441
442    def _ParseItem(self):
443        Line = self._RawData.CurrentLine
444        TokenList = GetSplitValueList(Line, DT.TAB_EQUAL_SPLIT, 1)
445        if TokenList[0] == DT.TAB_DEC_DEFINES_PKG_UNI_FILE:
446            self.DefineValidation[TokenList[0]](TokenList[1])
447        elif len(TokenList) < 2:
448            self._LoggerError(ST.ERR_DECPARSE_DEFINE_FORMAT)
449        elif TokenList[0] not in self.DefineValidation:
450            self._LoggerError(ST.ERR_DECPARSE_DEFINE_UNKNOWKEY % TokenList[0])
451        else:
452            self.DefineValidation[TokenList[0]](TokenList[1])
453
454        DefineItem = DecDefineItemObject()
455        DefineItem.Key   = TokenList[0]
456        DefineItem.Value = TokenList[1]
457        self.ItemObject.AddItem(DefineItem, self._RawData.CurrentScope)
458        return DefineItem
459
460    def _SetDecSpecification(self, Token):
461        if self.ItemObject.GetPackageSpecification():
462            self._LoggerError(ST.ERR_DECPARSE_DEFINE_DEFINED % DT.TAB_DEC_DEFINES_DEC_SPECIFICATION)
463        if not IsValidToken('0[xX][0-9a-fA-F]{8}', Token):
464            if not IsValidDecVersionVal(Token):
465                self._LoggerError(ST.ERR_DECPARSE_DEFINE_SPEC)
466        self.ItemObject.SetPackageSpecification(Token)
467
468    def _SetPackageName(self, Token):
469        if self.ItemObject.GetPackageName():
470            self._LoggerError(ST.ERR_DECPARSE_DEFINE_DEFINED % DT.TAB_DEC_DEFINES_PACKAGE_NAME)
471        if not IsValidWord(Token):
472            self._LoggerError(ST.ERR_DECPARSE_DEFINE_PKGNAME)
473        self.ItemObject.SetPackageName(Token)
474
475    def _SetPackageGuid(self, Token):
476        if self.ItemObject.GetPackageGuid():
477            self._LoggerError(ST.ERR_DECPARSE_DEFINE_DEFINED % DT.TAB_DEC_DEFINES_PACKAGE_GUID)
478        if not CheckGuidRegFormat(Token):
479            self._LoggerError(ST.ERR_DECPARSE_DEFINE_PKGGUID)
480        self.ItemObject.SetPackageGuid(Token)
481
482    def _SetPackageVersion(self, Token):
483        if self.ItemObject.GetPackageVersion():
484            self._LoggerError(ST.ERR_DECPARSE_DEFINE_DEFINED % DT.TAB_DEC_DEFINES_PACKAGE_VERSION)
485        if not IsValidToken(VERSION_PATTERN, Token):
486            self._LoggerError(ST.ERR_DECPARSE_DEFINE_PKGVERSION)
487        else:
488            if not DT.TAB_SPLIT in Token:
489                Token = Token + '.0'
490            self.ItemObject.SetPackageVersion(Token)
491
492    def _SetPackageUni(self, Token):
493        if self.ItemObject.GetPackageUniFile():
494            self._LoggerError(ST.ERR_DECPARSE_DEFINE_DEFINED % DT.TAB_DEC_DEFINES_PKG_UNI_FILE)
495        self.ItemObject.SetPackageUniFile(Token)
496
497## _DecInclude
498#
499# Parse include section
500#
501class _DecInclude(_DecBase):
502    def __init__(self, RawData):
503        _DecBase.__init__(self, RawData)
504        self.ItemObject = DecIncludeObject(RawData.Filename)
505
506    def _ParseItem(self):
507        Line = self._RawData.CurrentLine
508
509        if not IsValidPath(Line, self._RawData.PackagePath):
510            self._LoggerError(ST.ERR_DECPARSE_INCLUDE % Line)
511
512        Item = DecIncludeItemObject(StripRoot(self._RawData.PackagePath, Line), self._RawData.PackagePath)
513        self.ItemObject.AddItem(Item, self._RawData.CurrentScope)
514        return Item
515
516## _DecLibraryclass
517#
518# Parse library class section
519#
520class _DecLibraryclass(_DecBase):
521    def __init__(self, RawData):
522        _DecBase.__init__(self, RawData)
523        self.ItemObject = DecLibraryclassObject(RawData.Filename)
524
525    def _ParseItem(self):
526        Line = self._RawData.CurrentLine
527        TokenList = GetSplitValueList(Line, DT.TAB_VALUE_SPLIT)
528        if len(TokenList) != 2:
529            self._LoggerError(ST.ERR_DECPARSE_LIBCLASS_SPLIT)
530        if TokenList[0] == '' or TokenList[1] == '':
531            self._LoggerError(ST.ERR_DECPARSE_LIBCLASS_EMPTY)
532        if not IsValidToken('[A-Z][0-9A-Za-z]*', TokenList[0]):
533            self._LoggerError(ST.ERR_DECPARSE_LIBCLASS_LIB)
534
535        self._CheckReDefine(TokenList[0])
536
537        Value = TokenList[1]
538        #
539        # Must end with .h
540        #
541        if not Value.endswith('.h'):
542            self._LoggerError(ST.ERR_DECPARSE_LIBCLASS_PATH_EXT)
543
544        #
545        # Path must be existed
546        #
547        if not IsValidPath(Value, self._RawData.PackagePath):
548            self._LoggerError(ST.ERR_DECPARSE_INCLUDE % Value)
549
550        Item = DecLibraryclassItemObject(TokenList[0], StripRoot(self._RawData.PackagePath, Value),
551                                         self._RawData.PackagePath)
552        self.ItemObject.AddItem(Item, self._RawData.CurrentScope)
553        return Item
554
555## _DecPcd
556#
557# Parse PCD section
558#
559class _DecPcd(_DecBase):
560    def __init__(self, RawData):
561        _DecBase.__init__(self, RawData)
562        self.ItemObject = DecPcdObject(RawData.Filename)
563        #
564        # Used to check duplicate token
565        # Key is token space and token number (integer), value is C name
566        #
567        self.TokenMap = {}
568
569    def _ParseItem(self):
570        Line = self._RawData.CurrentLine
571        TokenList = Line.split(DT.TAB_VALUE_SPLIT)
572        if len(TokenList) < 4:
573            self._LoggerError(ST.ERR_DECPARSE_PCD_SPLIT)
574
575        #
576        # Token space guid C name
577        #
578        PcdName = GetSplitValueList(TokenList[0], DT.TAB_SPLIT)
579        if len(PcdName) != 2 or PcdName[0] == '' or PcdName[1] == '':
580            self._LoggerError(ST.ERR_DECPARSE_PCD_NAME)
581
582        Guid = PcdName[0]
583        if not IsValidToken(CVAR_PATTERN, Guid):
584            self._LoggerError(ST.ERR_DECPARSE_PCD_CVAR_GUID)
585
586        #
587        # PCD C name
588        #
589        CName = PcdName[1]
590        if not IsValidToken(CVAR_PATTERN, CName):
591            self._LoggerError(ST.ERR_DECPARSE_PCD_CVAR_PCDCNAME)
592
593        self._CheckReDefine(Guid + DT.TAB_SPLIT + CName)
594
595        #
596        # Default value, may be C array, string or number
597        #
598        Data = DT.TAB_VALUE_SPLIT.join(TokenList[1:-2]).strip()
599
600        #
601        # PCD data type
602        #
603        DataType = TokenList[-2].strip()
604        Valid, Cause = IsValidPcdDatum(DataType, Data)
605        if not Valid:
606            self._LoggerError(Cause)
607        PcdType = self._RawData.CurrentScope[0][0]
608        if PcdType == DT.TAB_PCDS_FEATURE_FLAG_NULL.upper() and DataType != 'BOOLEAN':
609            self._LoggerError(ST.ERR_DECPARSE_PCD_FEATUREFLAG)
610        #
611        # Token value is the last element in list.
612        #
613        Token = TokenList[-1].strip()
614        if not IsValidToken(PCD_TOKEN_PATTERN, Token):
615            self._LoggerError(ST.ERR_DECPARSE_PCD_TOKEN % Token)
616        elif not Token.startswith('0x') and not Token.startswith('0X'):
617            if int(Token) > 4294967295:
618                self._LoggerError(ST.ERR_DECPARSE_PCD_TOKEN_INT % Token)
619            Token = '0x%x' % int(Token)
620
621        IntToken = int(Token, 0)
622        if (Guid, IntToken) in self.TokenMap:
623            if self.TokenMap[Guid, IntToken] != CName:
624                self._LoggerError(ST.ERR_DECPARSE_PCD_TOKEN_UNIQUE%(Token))
625        else:
626            self.TokenMap[Guid, IntToken] = CName
627
628        Item = DecPcdItemObject(Guid, CName, Data, DataType, Token)
629        self.ItemObject.AddItem(Item, self._RawData.CurrentScope)
630        return Item
631
632## _DecGuid
633#
634# Parse GUID, PPI, Protocol section
635#
636class _DecGuid(_DecBase):
637    def __init__(self, RawData):
638        _DecBase.__init__(self, RawData)
639        self.GuidObj = DecGuidObject(RawData.Filename)
640        self.PpiObj = DecPpiObject(RawData.Filename)
641        self.ProtocolObj = DecProtocolObject(RawData.Filename)
642        self.ObjectDict = \
643        {
644            DT.TAB_GUIDS.upper()     :   self.GuidObj,
645            DT.TAB_PPIS.upper()      :   self.PpiObj,
646            DT.TAB_PROTOCOLS.upper() :   self.ProtocolObj
647        }
648
649    def GetDataObject(self):
650        if self._RawData.CurrentScope:
651            return self.ObjectDict[self._RawData.CurrentScope[0][0]]
652        return None
653
654    def GetGuidObject(self):
655        return self.GuidObj
656
657    def GetPpiObject(self):
658        return self.PpiObj
659
660    def GetProtocolObject(self):
661        return self.ProtocolObj
662
663    def _ParseItem(self):
664        Line = self._RawData.CurrentLine
665        TokenList = GetSplitValueList(Line, DT.TAB_EQUAL_SPLIT, 1)
666        if len(TokenList) < 2:
667            self._LoggerError(ST.ERR_DECPARSE_CGUID)
668        if TokenList[0] == '':
669            self._LoggerError(ST.ERR_DECPARSE_CGUID_NAME)
670        if TokenList[1] == '':
671            self._LoggerError(ST.ERR_DECPARSE_CGUID_GUID)
672        if not IsValidToken(CVAR_PATTERN, TokenList[0]):
673            self._LoggerError(ST.ERR_DECPARSE_PCD_CVAR_GUID)
674
675        self._CheckReDefine(TokenList[0])
676
677        if TokenList[1][0] != '{':
678            if not CheckGuidRegFormat(TokenList[1]):
679                self._LoggerError(ST.ERR_DECPARSE_DEFINE_PKGGUID)
680            GuidString = TokenList[1]
681        else:
682            #
683            # Convert C format GUID to GUID string and Simple error check
684            #
685            GuidString = GuidStructureStringToGuidString(TokenList[1])
686            if TokenList[1][0] != '{' or TokenList[1][-1] != '}' or GuidString == '':
687                self._LoggerError(ST.ERR_DECPARSE_CGUID_GUIDFORMAT)
688
689            #
690            # Check C format GUID
691            #
692            if not IsValidCFormatGuid(TokenList[1]):
693                self._LoggerError(ST.ERR_DECPARSE_CGUID_GUIDFORMAT)
694
695        Item = DecGuidItemObject(TokenList[0], TokenList[1], GuidString)
696        ItemObject = self.ObjectDict[self._RawData.CurrentScope[0][0]]
697        ItemObject.AddItem(Item, self._RawData.CurrentScope)
698        return Item
699
700## _DecUserExtension
701#
702# Parse user extension section
703#
704class _DecUserExtension(_DecBase):
705    def __init__(self, RawData):
706        _DecBase.__init__(self, RawData)
707        self.ItemObject = DecUserExtensionObject(RawData.Filename)
708        self._Headers = []
709        self._CurItems = []
710
711    def BlockStart(self):
712        self._CurItems = []
713        for Header in self._RawData.CurrentScope:
714            if Header in self._Headers:
715                self._LoggerError(ST.ERR_DECPARSE_UE_DUPLICATE)
716            else:
717                self._Headers.append(Header)
718
719            for Item in self._CurItems:
720                if Item.UserId == Header[1] and Item.IdString == Header[2]:
721                    Item.ArchAndModuleType.append(Header[3])
722                    break
723            else:
724                Item = DecUserExtensionItemObject()
725                Item.UserId = Header[1]
726                Item.IdString = Header[2]
727                Item.ArchAndModuleType.append(Header[3])
728                self._CurItems.append(Item)
729                self.ItemObject.AddItem(Item, None)
730        self._LocalMacro = {}
731
732    def _ParseItem(self):
733        Line = self._RawData.CurrentLine
734        Item = None
735        for Item in self._CurItems:
736            if Item.UserString:
737                Item.UserString = '\n'.join([Item.UserString, Line])
738            else:
739                Item.UserString = Line
740        return Item
741
742## Dec
743#
744# Top dec parser
745#
746class Dec(_DecBase, _DecComments):
747    def __init__(self, DecFile, Parse = True):
748        try:
749            Content = ConvertSpecialChar(open(DecFile, 'r').readlines())
750        except BaseException:
751            Logger.Error(TOOL_NAME, FILE_OPEN_FAILURE, File=DecFile,
752                         ExtraData=ST.ERR_DECPARSE_FILEOPEN % DecFile)
753
754        #
755        # Pre-parser for Private section
756        #
757        self._Private = ''
758        __IsFoundPrivate = False
759        NewContent = []
760        for Line in Content:
761            Line = Line.strip()
762            if Line.startswith(DT.TAB_SECTION_START) and Line.endswith(DT.TAB_PRIVATE + DT.TAB_SECTION_END):
763                __IsFoundPrivate = True
764            if Line.startswith(DT.TAB_SECTION_START) and Line.endswith(DT.TAB_SECTION_END)\
765               and not Line.endswith(DT.TAB_PRIVATE + DT.TAB_SECTION_END):
766                __IsFoundPrivate = False
767            if __IsFoundPrivate:
768                self._Private += Line + '\r'
769            if not __IsFoundPrivate:
770                NewContent.append(Line + '\r')
771
772        RawData = FileContent(DecFile, NewContent)
773
774        _DecComments.__init__(self)
775        _DecBase.__init__(self, RawData)
776
777        self.BinaryHeadComment = []
778        self.PcdErrorCommentDict = {}
779
780        self._Define    = _DecDefine(RawData)
781        self._Include   = _DecInclude(RawData)
782        self._Guid      = _DecGuid(RawData)
783        self._LibClass  = _DecLibraryclass(RawData)
784        self._Pcd       = _DecPcd(RawData)
785        self._UserEx    = _DecUserExtension(RawData)
786
787        #
788        # DEC file supported data types (one type per section)
789        #
790        self._SectionParser = {
791            DT.TAB_DEC_DEFINES.upper()                     :   self._Define,
792            DT.TAB_INCLUDES.upper()                        :   self._Include,
793            DT.TAB_LIBRARY_CLASSES.upper()                 :   self._LibClass,
794            DT.TAB_GUIDS.upper()                           :   self._Guid,
795            DT.TAB_PPIS.upper()                            :   self._Guid,
796            DT.TAB_PROTOCOLS.upper()                       :   self._Guid,
797            DT.TAB_PCDS_FIXED_AT_BUILD_NULL.upper()        :   self._Pcd,
798            DT.TAB_PCDS_PATCHABLE_IN_MODULE_NULL.upper()   :   self._Pcd,
799            DT.TAB_PCDS_FEATURE_FLAG_NULL.upper()          :   self._Pcd,
800            DT.TAB_PCDS_DYNAMIC_NULL.upper()               :   self._Pcd,
801            DT.TAB_PCDS_DYNAMIC_EX_NULL.upper()            :   self._Pcd,
802            DT.TAB_USER_EXTENSIONS.upper()                 :   self._UserEx
803        }
804
805        if Parse:
806            self.ParseDecComment()
807            self.Parse()
808            #
809            # Parsing done, check required fields
810            #
811            self.CheckRequiredFields()
812
813    def CheckRequiredFields(self):
814        for SectionParser in self._SectionParser.values():
815            if not SectionParser.CheckRequiredFields():
816                return False
817        return True
818
819    ##
820    # Parse DEC file
821    #
822    def ParseDecComment(self):
823        IsFileHeader = False
824        IsBinaryHeader = False
825        FileHeaderLineIndex = -1
826        BinaryHeaderLineIndex = -1
827        TokenSpaceGuidCName = ''
828
829        #
830        # Parse PCD error comment section
831        #
832        while not self._RawData.IsEndOfFile():
833            self._RawData.CurrentLine = self._RawData.GetNextLine()
834            if self._RawData.CurrentLine.startswith(DT.TAB_COMMENT_SPLIT) and \
835                DT.TAB_SECTION_START in self._RawData.CurrentLine and \
836                DT.TAB_SECTION_END in self._RawData.CurrentLine:
837                self._RawData.CurrentLine = self._RawData.CurrentLine.replace(DT.TAB_COMMENT_SPLIT, '').strip()
838
839                if self._RawData.CurrentLine[0] == DT.TAB_SECTION_START and \
840                    self._RawData.CurrentLine[-1] == DT.TAB_SECTION_END:
841                    RawSection = self._RawData.CurrentLine[1:-1].strip()
842                    if RawSection.upper().startswith(DT.TAB_PCD_ERROR.upper()+'.'):
843                        TokenSpaceGuidCName = RawSection.split(DT.TAB_PCD_ERROR+'.')[1].strip()
844                        continue
845
846            if TokenSpaceGuidCName and self._RawData.CurrentLine.startswith(DT.TAB_COMMENT_SPLIT):
847                self._RawData.CurrentLine = self._RawData.CurrentLine.replace(DT.TAB_COMMENT_SPLIT, '').strip()
848                if self._RawData.CurrentLine != '':
849                    if DT.TAB_VALUE_SPLIT not in self._RawData.CurrentLine:
850                        self._LoggerError(ST.ERR_DECPARSE_PCDERRORMSG_MISS_VALUE_SPLIT)
851
852                    PcdErrorNumber, PcdErrorMsg = GetSplitValueList(self._RawData.CurrentLine, DT.TAB_VALUE_SPLIT, 1)
853                    PcdErrorNumber = ParsePcdErrorCode(PcdErrorNumber, self._RawData.Filename, self._RawData.LineIndex)
854                    if not PcdErrorMsg.strip():
855                        self._LoggerError(ST.ERR_DECPARSE_PCD_MISS_ERRORMSG)
856
857                    self.PcdErrorCommentDict[(TokenSpaceGuidCName, PcdErrorNumber)] = PcdErrorMsg.strip()
858            else:
859                TokenSpaceGuidCName = ''
860
861        self._RawData.LineIndex = 0
862        self._RawData.CurrentLine = ''
863        self._RawData.NextLine = ''
864
865        while not self._RawData.IsEndOfFile():
866            Line, Comment = CleanString(self._RawData.GetNextLine())
867
868            #
869            # Header must be pure comment
870            #
871            if Line != '':
872                self._RawData.UndoNextLine()
873                break
874
875            if Comment and Comment.startswith(DT.TAB_SPECIAL_COMMENT) and Comment.find(DT.TAB_HEADER_COMMENT) > 0 \
876                and not Comment[2:Comment.find(DT.TAB_HEADER_COMMENT)].strip():
877                IsFileHeader = True
878                IsBinaryHeader = False
879                FileHeaderLineIndex = self._RawData.LineIndex
880
881            #
882            # Get license information before '@file'
883            #
884            if not IsFileHeader and not IsBinaryHeader and Comment and Comment.startswith(DT.TAB_COMMENT_SPLIT) and \
885            DT.TAB_BINARY_HEADER_COMMENT not in Comment:
886                self._HeadComment.append((Comment, self._RawData.LineIndex))
887
888            if Comment and IsFileHeader and \
889            not(Comment.startswith(DT.TAB_SPECIAL_COMMENT) \
890            and Comment.find(DT.TAB_BINARY_HEADER_COMMENT) > 0):
891                self._HeadComment.append((Comment, self._RawData.LineIndex))
892            #
893            # Double '#' indicates end of header comments
894            #
895            if (not Comment or Comment == DT.TAB_SPECIAL_COMMENT) and IsFileHeader:
896                IsFileHeader = False
897                continue
898
899            if Comment and Comment.startswith(DT.TAB_SPECIAL_COMMENT) \
900            and Comment.find(DT.TAB_BINARY_HEADER_COMMENT) > 0:
901                IsBinaryHeader = True
902                IsFileHeader = False
903                BinaryHeaderLineIndex = self._RawData.LineIndex
904
905            if Comment and IsBinaryHeader:
906                self.BinaryHeadComment.append((Comment, self._RawData.LineIndex))
907            #
908            # Double '#' indicates end of header comments
909            #
910            if (not Comment or Comment == DT.TAB_SPECIAL_COMMENT) and IsBinaryHeader:
911                IsBinaryHeader = False
912                break
913
914            if FileHeaderLineIndex > -1 and not IsFileHeader and not IsBinaryHeader:
915                break
916
917        if FileHeaderLineIndex > BinaryHeaderLineIndex and FileHeaderLineIndex > -1 and BinaryHeaderLineIndex > -1:
918            self._LoggerError(ST.ERR_BINARY_HEADER_ORDER)
919
920        if FileHeaderLineIndex == -1:
921#            self._LoggerError(ST.ERR_NO_SOURCE_HEADER)
922            Logger.Error(TOOL_NAME, FORMAT_INVALID,
923                         ST.ERR_NO_SOURCE_HEADER,
924                         File=self._RawData.Filename)
925        return
926
927    def _StopCurrentParsing(self, Line):
928        return False
929
930    def _ParseItem(self):
931        self._SectionHeaderParser()
932        if len(self._RawData.CurrentScope) == 0:
933            self._LoggerError(ST.ERR_DECPARSE_SECTION_EMPTY)
934        SectionObj = self._SectionParser[self._RawData.CurrentScope[0][0]]
935        SectionObj.BlockStart()
936        SectionObj.Parse()
937        return SectionObj.GetDataObject()
938
939    def _UserExtentionSectionParser(self):
940        self._RawData.CurrentScope = []
941        ArchList = set()
942        Section = self._RawData.CurrentLine[1:-1]
943        Par = ParserHelper(Section, self._RawData.Filename)
944        while not Par.End():
945            #
946            # User extention
947            #
948            Token = Par.GetToken()
949            if Token.upper() != DT.TAB_USER_EXTENSIONS.upper():
950                self._LoggerError(ST.ERR_DECPARSE_SECTION_UE)
951            UserExtension = Token.upper()
952            Par.AssertChar(DT.TAB_SPLIT, ST.ERR_DECPARSE_SECTION_UE, self._RawData.LineIndex)
953
954            #
955            # UserID
956            #
957            Token = Par.GetToken()
958            if not IsValidUserId(Token):
959                self._LoggerError(ST.ERR_DECPARSE_SECTION_UE_USERID)
960            UserId = Token
961            Par.AssertChar(DT.TAB_SPLIT, ST.ERR_DECPARSE_SECTION_UE, self._RawData.LineIndex)
962            #
963            # IdString
964            #
965            Token = Par.GetToken()
966            if not IsValidIdString(Token):
967                self._LoggerError(ST.ERR_DECPARSE_SECTION_UE_IDSTRING)
968            IdString = Token
969            Arch = 'COMMON'
970            if Par.Expect(DT.TAB_SPLIT):
971                Token = Par.GetToken()
972                Arch = Token.upper()
973                if not IsValidArch(Arch):
974                    self._LoggerError(ST.ERR_DECPARSE_ARCH)
975            ArchList.add(Arch)
976            if [UserExtension, UserId, IdString, Arch] not in \
977                self._RawData.CurrentScope:
978                self._RawData.CurrentScope.append(
979                    [UserExtension, UserId, IdString, Arch]
980                )
981            if not Par.Expect(DT.TAB_COMMA_SPLIT):
982                break
983            elif Par.End():
984                self._LoggerError(ST.ERR_DECPARSE_SECTION_COMMA)
985        Par.AssertEnd(ST.ERR_DECPARSE_SECTION_UE, self._RawData.LineIndex)
986        if 'COMMON' in ArchList and len(ArchList) > 1:
987            self._LoggerError(ST.ERR_DECPARSE_SECTION_COMMON)
988
989    ## Section header parser
990    #
991    # The section header is always in following format:
992    #
993    # [section_name.arch<.platform|module_type>]
994    #
995    def _SectionHeaderParser(self):
996        if self._RawData.CurrentLine[0] != DT.TAB_SECTION_START or self._RawData.CurrentLine[-1] != DT.TAB_SECTION_END:
997            self._LoggerError(ST.ERR_DECPARSE_SECTION_IDENTIFY)
998
999        RawSection = self._RawData.CurrentLine[1:-1].strip().upper()
1000        #
1001        # Check defines section which is only allowed to occur once and
1002        # no arch can be followed
1003        #
1004        if RawSection.startswith(DT.TAB_DEC_DEFINES.upper()):
1005            if RawSection != DT.TAB_DEC_DEFINES.upper():
1006                self._LoggerError(ST.ERR_DECPARSE_DEFINE_SECNAME)
1007        #
1008        # Check user extension section
1009        #
1010        if RawSection.startswith(DT.TAB_USER_EXTENSIONS.upper()):
1011            return self._UserExtentionSectionParser()
1012        self._RawData.CurrentScope = []
1013        SectionNames = []
1014        ArchList = set()
1015        for Item in GetSplitValueList(RawSection, DT.TAB_COMMA_SPLIT):
1016            if Item == '':
1017                self._LoggerError(ST.ERR_DECPARSE_SECTION_SUBEMPTY % self._RawData.CurrentLine)
1018
1019            ItemList = GetSplitValueList(Item, DT.TAB_SPLIT)
1020            #
1021            # different types of PCD are permissible in one section
1022            #
1023            SectionName = ItemList[0]
1024            if SectionName not in self._SectionParser:
1025                self._LoggerError(ST.ERR_DECPARSE_SECTION_UNKNOW % SectionName)
1026            if SectionName not in SectionNames:
1027                SectionNames.append(SectionName)
1028            #
1029            # In DEC specification, all section headers have at most two part:
1030            # SectionName.Arch except UserExtension
1031            #
1032            if len(ItemList) > 2:
1033                self._LoggerError(ST.ERR_DECPARSE_SECTION_SUBTOOMANY % Item)
1034
1035            if DT.TAB_PCDS_FEATURE_FLAG_NULL.upper() in SectionNames and len(SectionNames) > 1:
1036                self._LoggerError(ST.ERR_DECPARSE_SECTION_FEATUREFLAG % DT.TAB_PCDS_FEATURE_FLAG_NULL)
1037            #
1038            # S1 is always Arch
1039            #
1040            if len(ItemList) > 1:
1041                Str1 = ItemList[1]
1042                if not IsValidArch(Str1):
1043                    self._LoggerError(ST.ERR_DECPARSE_ARCH)
1044            else:
1045                Str1 = 'COMMON'
1046            ArchList.add(Str1)
1047
1048            if [SectionName, Str1] not in self._RawData.CurrentScope:
1049                self._RawData.CurrentScope.append([SectionName, Str1])
1050        #
1051        # 'COMMON' must not be used with specific ARCHs at the same section
1052        #
1053        if 'COMMON' in ArchList and len(ArchList) > 1:
1054            self._LoggerError(ST.ERR_DECPARSE_SECTION_COMMON)
1055        if len(SectionNames) == 0:
1056            self._LoggerError(ST.ERR_DECPARSE_SECTION_SUBEMPTY % self._RawData.CurrentLine)
1057        if len(SectionNames) != 1:
1058            for Sec in SectionNames:
1059                if not Sec.startswith(DT.TAB_PCDS.upper()):
1060                    self._LoggerError(ST.ERR_DECPARSE_SECTION_NAME % str(SectionNames))
1061
1062    def GetDefineSectionMacro(self):
1063        return self._Define.GetLocalMacro()
1064    def GetDefineSectionObject(self):
1065        return self._Define.GetDataObject()
1066    def GetIncludeSectionObject(self):
1067        return self._Include.GetDataObject()
1068    def GetGuidSectionObject(self):
1069        return self._Guid.GetGuidObject()
1070    def GetProtocolSectionObject(self):
1071        return self._Guid.GetProtocolObject()
1072    def GetPpiSectionObject(self):
1073        return self._Guid.GetPpiObject()
1074    def GetLibraryClassSectionObject(self):
1075        return self._LibClass.GetDataObject()
1076    def GetPcdSectionObject(self):
1077        return self._Pcd.GetDataObject()
1078    def GetUserExtensionSectionObject(self):
1079        return self._UserEx.GetDataObject()
1080    def GetPackageSpecification(self):
1081        return self._Define.GetDataObject().GetPackageSpecification()
1082    def GetPackageName(self):
1083        return self._Define.GetDataObject().GetPackageName()
1084    def GetPackageGuid(self):
1085        return self._Define.GetDataObject().GetPackageGuid()
1086    def GetPackageVersion(self):
1087        return self._Define.GetDataObject().GetPackageVersion()
1088    def GetPackageUniFile(self):
1089        return self._Define.GetDataObject().GetPackageUniFile()
1090    def GetPrivateSections(self):
1091        return self._Private
1092