1## @file
2# This file is used to define comment parsing interface
3#
4# Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
5#
6# SPDX-License-Identifier: BSD-2-Clause-Patent
7#
8
9'''
10CommentParsing
11'''
12
13##
14# Import Modules
15#
16import re
17
18from Library.StringUtils import GetSplitValueList
19from Library.StringUtils import CleanString2
20from Library.DataType import HEADER_COMMENT_NOT_STARTED
21from Library.DataType import TAB_COMMENT_SPLIT
22from Library.DataType import HEADER_COMMENT_LICENSE
23from Library.DataType import HEADER_COMMENT_ABSTRACT
24from Library.DataType import HEADER_COMMENT_COPYRIGHT
25from Library.DataType import HEADER_COMMENT_DESCRIPTION
26from Library.DataType import TAB_SPACE_SPLIT
27from Library.DataType import TAB_COMMA_SPLIT
28from Library.DataType import SUP_MODULE_LIST
29from Library.DataType import TAB_VALUE_SPLIT
30from Library.DataType import TAB_PCD_VALIDRANGE
31from Library.DataType import TAB_PCD_VALIDLIST
32from Library.DataType import TAB_PCD_EXPRESSION
33from Library.DataType import TAB_PCD_PROMPT
34from Library.DataType import TAB_CAPHEX_START
35from Library.DataType import TAB_HEX_START
36from Library.DataType import PCD_ERR_CODE_MAX_SIZE
37from Library.ExpressionValidate import IsValidRangeExpr
38from Library.ExpressionValidate import IsValidListExpr
39from Library.ExpressionValidate import IsValidLogicalExpr
40from Object.POM.CommonObject import TextObject
41from Object.POM.CommonObject import PcdErrorObject
42import Logger.Log as Logger
43from Logger.ToolError import FORMAT_INVALID
44from Logger.ToolError import FORMAT_NOT_SUPPORTED
45from Logger import StringTable as ST
46
47## ParseHeaderCommentSection
48#
49# Parse Header comment section lines, extract Abstract, Description, Copyright
50# , License lines
51#
52# @param CommentList:   List of (Comment, LineNumber)
53# @param FileName:      FileName of the comment
54#
55def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False):
56    Abstract = ''
57    Description = ''
58    Copyright = ''
59    License = ''
60    EndOfLine = "\n"
61    if IsBinaryHeader:
62        STR_HEADER_COMMENT_START = "@BinaryHeader"
63    else:
64        STR_HEADER_COMMENT_START = "@file"
65    HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
66
67    #
68    # first find the last copyright line
69    #
70    Last = 0
71    for Index in range(len(CommentList)-1, 0, -1):
72        Line = CommentList[Index][0]
73        if _IsCopyrightLine(Line):
74            Last = Index
75            break
76
77    for Item in CommentList:
78        Line = Item[0]
79        LineNo = Item[1]
80
81        if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
82            Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
83        Comment = CleanString2(Line)[1]
84        Comment = Comment.strip()
85        #
86        # if there are blank lines between License or Description, keep them as they would be
87        # indication of different block; or in the position that Abstract should be, also keep it
88        # as it indicates that no abstract
89        #
90        if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
91                                                      HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
92            continue
93
94        if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
95            if Comment.startswith(STR_HEADER_COMMENT_START):
96                HeaderCommentStage = HEADER_COMMENT_ABSTRACT
97            else:
98                License += Comment + EndOfLine
99        else:
100            if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
101                #
102                # in case there is no abstract and description
103                #
104                if not Comment:
105                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
106                elif _IsCopyrightLine(Comment):
107                    Result, ErrMsg = _ValidateCopyright(Comment)
108                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
109                    Copyright += Comment + EndOfLine
110                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
111                else:
112                    Abstract += Comment + EndOfLine
113                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
114            elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
115                #
116                # in case there is no description
117                #
118                if _IsCopyrightLine(Comment):
119                    Result, ErrMsg = _ValidateCopyright(Comment)
120                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
121                    Copyright += Comment + EndOfLine
122                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
123                else:
124                    Description += Comment + EndOfLine
125            elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
126                if _IsCopyrightLine(Comment):
127                    Result, ErrMsg = _ValidateCopyright(Comment)
128                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
129                    Copyright += Comment + EndOfLine
130                else:
131                    #
132                    # Contents after copyright line are license, those non-copyright lines in between
133                    # copyright line will be discarded
134                    #
135                    if LineNo > Last:
136                        if License:
137                            License += EndOfLine
138                        License += Comment + EndOfLine
139                        HeaderCommentStage = HEADER_COMMENT_LICENSE
140            else:
141                if not Comment and not License:
142                    continue
143                License += Comment + EndOfLine
144
145    return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
146
147## _IsCopyrightLine
148# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
149# followed by zero or more white space characters followed by a "(" character
150#
151# @param LineContent:  the line need to be checked
152# @return: True if current line is copyright line, False else
153#
154def _IsCopyrightLine (LineContent):
155    LineContent = LineContent.upper()
156    Result = False
157
158    ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
159    if ReIsCopyrightRe.search(LineContent):
160        Result = True
161
162    return Result
163
164## ParseGenericComment
165#
166# @param GenericComment: Generic comment list, element of
167#                        (CommentLine, LineNum)
168# @param ContainerFile:  Input value for filename of Dec file
169#
170def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
171    if ContainerFile:
172        pass
173    HelpTxt = None
174    HelpStr = ''
175
176    for Item in GenericComment:
177        CommentLine = Item[0]
178        Comment = CleanString2(CommentLine)[1]
179        if SkipTag is not None and Comment.startswith(SkipTag):
180            Comment = Comment.replace(SkipTag, '', 1)
181        HelpStr += Comment + '\n'
182
183    if HelpStr:
184        HelpTxt = TextObject()
185        if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
186            HelpStr = HelpStr[:-1]
187        HelpTxt.SetString(HelpStr)
188
189    return HelpTxt
190
191## ParsePcdErrorCode
192#
193# @param Value: original ErrorCode value
194# @param ContainerFile: Input value for filename of Dec file
195# @param LineNum: Line Num
196#
197def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None):
198    try:
199        if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)):
200            Base = 16
201        else:
202            Base = 10
203        ErrorCode = int(Value, Base)
204        if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0:
205            Logger.Error('Parser',
206                        FORMAT_NOT_SUPPORTED,
207                        "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
208                        File = ContainerFile,
209                        Line = LineNum)
210        ErrorCode = '0x%x' % ErrorCode
211        return ErrorCode
212    except ValueError as XStr:
213        if XStr:
214            pass
215        Logger.Error('Parser',
216                    FORMAT_NOT_SUPPORTED,
217                    "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
218                    File = ContainerFile,
219                    Line = LineNum)
220
221## ParseDecPcdGenericComment
222#
223# @param GenericComment: Generic comment list, element of (CommentLine,
224#                         LineNum)
225# @param ContainerFile:  Input value for filename of Dec file
226#
227def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict):
228    HelpStr = ''
229    PromptStr = ''
230    PcdErr = None
231    PcdErrList = []
232    ValidValueNum = 0
233    ValidRangeNum = 0
234    ExpressionNum = 0
235
236    for (CommentLine, LineNum) in GenericComment:
237        Comment = CleanString2(CommentLine)[1]
238        #
239        # To replace Macro
240        #
241        MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
242        MatchedStrs =  re.findall(MACRO_PATTERN, Comment)
243        for MatchedStr in MatchedStrs:
244            if MatchedStr:
245                Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip()
246                if Macro in MacroReplaceDict:
247                    Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro])
248        if Comment.startswith(TAB_PCD_VALIDRANGE):
249            if ValidValueNum > 0 or ExpressionNum > 0:
250                Logger.Error('Parser',
251                             FORMAT_NOT_SUPPORTED,
252                             ST.WRN_MULTI_PCD_RANGES,
253                             File = ContainerFile,
254                             Line = LineNum)
255            else:
256                PcdErr = PcdErrorObject()
257                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
258                PcdErr.SetCName(CName)
259                PcdErr.SetFileLine(Comment)
260                PcdErr.SetLineNum(LineNum)
261                ValidRangeNum += 1
262            ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip()
263            Valid, Cause = _CheckRangeExpression(ValidRange)
264            if Valid:
265                ValueList = ValidRange.split(TAB_VALUE_SPLIT)
266                if len(ValueList) > 1:
267                    PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
268                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
269                else:
270                    PcdErr.SetValidValueRange(ValidRange)
271                PcdErrList.append(PcdErr)
272            else:
273                Logger.Error("Parser",
274                         FORMAT_NOT_SUPPORTED,
275                         Cause,
276                         ContainerFile,
277                         LineNum)
278        elif Comment.startswith(TAB_PCD_VALIDLIST):
279            if ValidRangeNum > 0 or ExpressionNum > 0:
280                Logger.Error('Parser',
281                             FORMAT_NOT_SUPPORTED,
282                             ST.WRN_MULTI_PCD_RANGES,
283                             File = ContainerFile,
284                             Line = LineNum)
285            elif ValidValueNum > 0:
286                Logger.Error('Parser',
287                             FORMAT_NOT_SUPPORTED,
288                             ST.WRN_MULTI_PCD_VALIDVALUE,
289                             File = ContainerFile,
290                             Line = LineNum)
291            else:
292                PcdErr = PcdErrorObject()
293                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
294                PcdErr.SetCName(CName)
295                PcdErr.SetFileLine(Comment)
296                PcdErr.SetLineNum(LineNum)
297                ValidValueNum += 1
298                ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip()
299            Valid, Cause = _CheckListExpression(ValidValueExpr)
300            if Valid:
301                ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
302                ValueList = ValidValue.split(TAB_VALUE_SPLIT)
303                if len(ValueList) > 1:
304                    PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
305                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
306                else:
307                    PcdErr.SetValidValue(ValidValue)
308                PcdErrList.append(PcdErr)
309            else:
310                Logger.Error("Parser",
311                         FORMAT_NOT_SUPPORTED,
312                         Cause,
313                         ContainerFile,
314                         LineNum)
315        elif Comment.startswith(TAB_PCD_EXPRESSION):
316            if ValidRangeNum > 0 or ValidValueNum > 0:
317                Logger.Error('Parser',
318                             FORMAT_NOT_SUPPORTED,
319                             ST.WRN_MULTI_PCD_RANGES,
320                             File = ContainerFile,
321                             Line = LineNum)
322            else:
323                PcdErr = PcdErrorObject()
324                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
325                PcdErr.SetCName(CName)
326                PcdErr.SetFileLine(Comment)
327                PcdErr.SetLineNum(LineNum)
328                ExpressionNum += 1
329            Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip()
330            Valid, Cause = _CheckExpression(Expression)
331            if Valid:
332                ValueList = Expression.split(TAB_VALUE_SPLIT)
333                if len(ValueList) > 1:
334                    PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
335                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
336                else:
337                    PcdErr.SetExpression(Expression)
338                PcdErrList.append(PcdErr)
339            else:
340                Logger.Error("Parser",
341                         FORMAT_NOT_SUPPORTED,
342                         Cause,
343                         ContainerFile,
344                         LineNum)
345        elif Comment.startswith(TAB_PCD_PROMPT):
346            if PromptStr:
347                Logger.Error('Parser',
348                             FORMAT_NOT_SUPPORTED,
349                             ST.WRN_MULTI_PCD_PROMPT,
350                             File = ContainerFile,
351                             Line = LineNum)
352            PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip()
353        else:
354            if Comment:
355                HelpStr += Comment + '\n'
356
357    #
358    # remove the last EOL if the comment is of format 'FOO\n'
359    #
360    if HelpStr.endswith('\n'):
361        if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
362            HelpStr = HelpStr[:-1]
363
364    return HelpStr, PcdErrList, PromptStr
365
366## ParseDecPcdTailComment
367#
368# @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)
369# @param ContainerFile:      Input value for filename of Dec file
370# @retVal SupModuleList:  The supported module type list detected
371# @retVal HelpStr:  The generic help text string detected
372#
373def ParseDecPcdTailComment (TailCommentList, ContainerFile):
374    assert(len(TailCommentList) == 1)
375    TailComment = TailCommentList[0][0]
376    LineNum = TailCommentList[0][1]
377
378    Comment = TailComment.lstrip(" #")
379
380    ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
381
382    #
383    # get first word and compare with SUP_MODULE_LIST
384    #
385    MatchObject = ReFindFirstWordRe.match(Comment)
386    if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
387        return None, Comment
388
389    #
390    # parse line, it must have supported module type specified
391    #
392    if Comment.find(TAB_COMMENT_SPLIT) == -1:
393        Comment += TAB_COMMENT_SPLIT
394    SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
395    SupModuleList = []
396    for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
397        if not Mod:
398            continue
399        elif Mod not in SUP_MODULE_LIST:
400            Logger.Error("UPT",
401                         FORMAT_INVALID,
402                         ST.WRN_INVALID_MODULE_TYPE%Mod,
403                         ContainerFile,
404                         LineNum)
405        else:
406            SupModuleList.append(Mod)
407
408    return SupModuleList, HelpStr
409
410## _CheckListExpression
411#
412# @param Expression: Pcd value list expression
413#
414def _CheckListExpression(Expression):
415    ListExpr = ''
416    if TAB_VALUE_SPLIT in Expression:
417        ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
418    else:
419        ListExpr = Expression
420
421    return IsValidListExpr(ListExpr)
422
423## _CheckExpression
424#
425# @param Expression: Pcd value expression
426#
427def _CheckExpression(Expression):
428    Expr = ''
429    if TAB_VALUE_SPLIT in Expression:
430        Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
431    else:
432        Expr = Expression
433    return IsValidLogicalExpr(Expr, True)
434
435## _CheckRangeExpression
436#
437# @param Expression:    Pcd range expression
438#
439def _CheckRangeExpression(Expression):
440    RangeExpr = ''
441    if TAB_VALUE_SPLIT in Expression:
442        RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
443    else:
444        RangeExpr = Expression
445
446    return IsValidRangeExpr(RangeExpr)
447
448## ValidateCopyright
449#
450#
451#
452def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
453    if not Result:
454        Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)
455
456## _ValidateCopyright
457#
458# @param Line:    Line that contains copyright information, # stripped
459#
460# @retval Result: True if line is conformed to Spec format, False else
461# @retval ErrMsg: the detailed error description
462#
463def _ValidateCopyright(Line):
464    if Line:
465        pass
466    Result = True
467    ErrMsg = ''
468
469    return Result, ErrMsg
470
471def GenerateTokenList (Comment):
472    #
473    # Tokenize Comment using '#' and ' ' as token separators
474    #
475    ReplacedComment = None
476    while Comment != ReplacedComment:
477        ReplacedComment = Comment
478        Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')
479    return Comment.split('#')
480
481
482#
483# Comment       - Comment to parse
484# TypeTokens    - A dictionary of type token synonyms
485# RemoveTokens  - A list of tokens to remove from help text
486# ParseVariable - True for parsing [Guids].  Otherwise False
487#
488def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
489    #
490    # Initialize return values
491    #
492    Usage = None
493    Type = None
494    String = None
495
496    Comment = Comment[0]
497
498    NumTokens = 2
499    if ParseVariable:
500        #
501        # Remove white space around first instance of ':' from Comment if 'Variable'
502        # is in front of ':' and Variable is the 1st or 2nd token in Comment.
503        #
504        List = Comment.split(':', 1)
505        if len(List) > 1:
506            SubList = GenerateTokenList (List[0].strip())
507            if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
508                if List[1].strip().find('L"') == 0:
509                    Comment = List[0].strip() + ':' + List[1].strip()
510
511        #
512        # Remove first instance of L"<VariableName> from Comment and put into String
513        # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or
514        # L"<VariableName>" is the third token immediately following 'Variable:'.
515        #
516        End = -1
517        Start = Comment.find('Variable:L"')
518        if Start >= 0:
519            String = Comment[Start + 9:]
520            End = String[2:].find('"')
521        else:
522            Start = Comment.find('L"')
523            if Start >= 0:
524                String = Comment[Start:]
525                End = String[2:].find('"')
526        if End >= 0:
527            SubList = GenerateTokenList (Comment[:Start])
528            if len(SubList) < 2:
529                Comment = Comment[:Start] + String[End + 3:]
530                String = String[:End + 3]
531                Type = 'Variable'
532                NumTokens = 1
533
534    #
535    # Initialize HelpText to Comment.
536    # Content will be remove from HelpText as matching tokens are found
537    #
538    HelpText = Comment
539
540    #
541    # Tokenize Comment using '#' and ' ' as token separators
542    #
543    List = GenerateTokenList (Comment)
544
545    #
546    # Search first two tokens for Usage and Type and remove any matching tokens
547    # from HelpText
548    #
549    for Token in List[0:NumTokens]:
550        if Usage is None and Token in UsageTokens:
551            Usage = UsageTokens[Token]
552            HelpText = HelpText.replace(Token, '')
553    if Usage is not None or not ParseVariable:
554        for Token in List[0:NumTokens]:
555            if Type is None and Token in TypeTokens:
556                Type = TypeTokens[Token]
557                HelpText = HelpText.replace(Token, '')
558            if Usage is not None:
559                for Token in List[0:NumTokens]:
560                    if Token in RemoveTokens:
561                        HelpText = HelpText.replace(Token, '')
562
563    #
564    # If no Usage token is present and set Usage to UNDEFINED
565    #
566    if Usage is None:
567        Usage = 'UNDEFINED'
568
569    #
570    # If no Type token is present and set Type to UNDEFINED
571    #
572    if Type is None:
573        Type = 'UNDEFINED'
574
575    #
576    # If Type is not 'Variable:', then set String to None
577    #
578    if Type != 'Variable':
579        String = None
580
581    #
582    # Strip ' ' and '#' from the beginning of HelpText
583    # If HelpText is an empty string after all parsing is
584    # complete then set HelpText to None
585    #
586    HelpText = HelpText.lstrip('# ')
587    if HelpText == '':
588        HelpText = None
589
590    #
591    # Return parsing results
592    #
593    return Usage, Type, String, HelpText
594