1## @file 2# This file is used to define comment parsing interface 3# 4# Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR> 5# 6# SPDX-License-Identifier: BSD-2-Clause-Patent 7# 8 9''' 10CommentParsing 11''' 12 13## 14# Import Modules 15# 16import re 17 18from Library.StringUtils import GetSplitValueList 19from Library.StringUtils import CleanString2 20from Library.DataType import HEADER_COMMENT_NOT_STARTED 21from Library.DataType import TAB_COMMENT_SPLIT 22from Library.DataType import HEADER_COMMENT_LICENSE 23from Library.DataType import HEADER_COMMENT_ABSTRACT 24from Library.DataType import HEADER_COMMENT_COPYRIGHT 25from Library.DataType import HEADER_COMMENT_DESCRIPTION 26from Library.DataType import TAB_SPACE_SPLIT 27from Library.DataType import TAB_COMMA_SPLIT 28from Library.DataType import SUP_MODULE_LIST 29from Library.DataType import TAB_VALUE_SPLIT 30from Library.DataType import TAB_PCD_VALIDRANGE 31from Library.DataType import TAB_PCD_VALIDLIST 32from Library.DataType import TAB_PCD_EXPRESSION 33from Library.DataType import TAB_PCD_PROMPT 34from Library.DataType import TAB_CAPHEX_START 35from Library.DataType import TAB_HEX_START 36from Library.DataType import PCD_ERR_CODE_MAX_SIZE 37from Library.ExpressionValidate import IsValidRangeExpr 38from Library.ExpressionValidate import IsValidListExpr 39from Library.ExpressionValidate import IsValidLogicalExpr 40from Object.POM.CommonObject import TextObject 41from Object.POM.CommonObject import PcdErrorObject 42import Logger.Log as Logger 43from Logger.ToolError import FORMAT_INVALID 44from Logger.ToolError import FORMAT_NOT_SUPPORTED 45from Logger import StringTable as ST 46 47## ParseHeaderCommentSection 48# 49# Parse Header comment section lines, extract Abstract, Description, Copyright 50# , License lines 51# 52# @param CommentList: List of (Comment, LineNumber) 53# @param FileName: FileName of the comment 54# 55def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False): 56 Abstract = '' 57 Description = '' 58 Copyright = '' 59 License = '' 60 EndOfLine = "\n" 61 if IsBinaryHeader: 62 STR_HEADER_COMMENT_START = "@BinaryHeader" 63 else: 64 STR_HEADER_COMMENT_START = "@file" 65 HeaderCommentStage = HEADER_COMMENT_NOT_STARTED 66 67 # 68 # first find the last copyright line 69 # 70 Last = 0 71 for Index in range(len(CommentList)-1, 0, -1): 72 Line = CommentList[Index][0] 73 if _IsCopyrightLine(Line): 74 Last = Index 75 break 76 77 for Item in CommentList: 78 Line = Item[0] 79 LineNo = Item[1] 80 81 if not Line.startswith(TAB_COMMENT_SPLIT) and Line: 82 Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1]) 83 Comment = CleanString2(Line)[1] 84 Comment = Comment.strip() 85 # 86 # if there are blank lines between License or Description, keep them as they would be 87 # indication of different block; or in the position that Abstract should be, also keep it 88 # as it indicates that no abstract 89 # 90 if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \ 91 HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]: 92 continue 93 94 if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED: 95 if Comment.startswith(STR_HEADER_COMMENT_START): 96 HeaderCommentStage = HEADER_COMMENT_ABSTRACT 97 else: 98 License += Comment + EndOfLine 99 else: 100 if HeaderCommentStage == HEADER_COMMENT_ABSTRACT: 101 # 102 # in case there is no abstract and description 103 # 104 if not Comment: 105 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION 106 elif _IsCopyrightLine(Comment): 107 Result, ErrMsg = _ValidateCopyright(Comment) 108 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 109 Copyright += Comment + EndOfLine 110 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT 111 else: 112 Abstract += Comment + EndOfLine 113 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION 114 elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION: 115 # 116 # in case there is no description 117 # 118 if _IsCopyrightLine(Comment): 119 Result, ErrMsg = _ValidateCopyright(Comment) 120 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 121 Copyright += Comment + EndOfLine 122 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT 123 else: 124 Description += Comment + EndOfLine 125 elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT: 126 if _IsCopyrightLine(Comment): 127 Result, ErrMsg = _ValidateCopyright(Comment) 128 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 129 Copyright += Comment + EndOfLine 130 else: 131 # 132 # Contents after copyright line are license, those non-copyright lines in between 133 # copyright line will be discarded 134 # 135 if LineNo > Last: 136 if License: 137 License += EndOfLine 138 License += Comment + EndOfLine 139 HeaderCommentStage = HEADER_COMMENT_LICENSE 140 else: 141 if not Comment and not License: 142 continue 143 License += Comment + EndOfLine 144 145 return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip() 146 147## _IsCopyrightLine 148# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" 149# followed by zero or more white space characters followed by a "(" character 150# 151# @param LineContent: the line need to be checked 152# @return: True if current line is copyright line, False else 153# 154def _IsCopyrightLine (LineContent): 155 LineContent = LineContent.upper() 156 Result = False 157 158 ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL) 159 if ReIsCopyrightRe.search(LineContent): 160 Result = True 161 162 return Result 163 164## ParseGenericComment 165# 166# @param GenericComment: Generic comment list, element of 167# (CommentLine, LineNum) 168# @param ContainerFile: Input value for filename of Dec file 169# 170def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None): 171 if ContainerFile: 172 pass 173 HelpTxt = None 174 HelpStr = '' 175 176 for Item in GenericComment: 177 CommentLine = Item[0] 178 Comment = CleanString2(CommentLine)[1] 179 if SkipTag is not None and Comment.startswith(SkipTag): 180 Comment = Comment.replace(SkipTag, '', 1) 181 HelpStr += Comment + '\n' 182 183 if HelpStr: 184 HelpTxt = TextObject() 185 if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n': 186 HelpStr = HelpStr[:-1] 187 HelpTxt.SetString(HelpStr) 188 189 return HelpTxt 190 191## ParsePcdErrorCode 192# 193# @param Value: original ErrorCode value 194# @param ContainerFile: Input value for filename of Dec file 195# @param LineNum: Line Num 196# 197def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None): 198 try: 199 if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)): 200 Base = 16 201 else: 202 Base = 10 203 ErrorCode = int(Value, Base) 204 if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0: 205 Logger.Error('Parser', 206 FORMAT_NOT_SUPPORTED, 207 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value, 208 File = ContainerFile, 209 Line = LineNum) 210 ErrorCode = '0x%x' % ErrorCode 211 return ErrorCode 212 except ValueError as XStr: 213 if XStr: 214 pass 215 Logger.Error('Parser', 216 FORMAT_NOT_SUPPORTED, 217 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value, 218 File = ContainerFile, 219 Line = LineNum) 220 221## ParseDecPcdGenericComment 222# 223# @param GenericComment: Generic comment list, element of (CommentLine, 224# LineNum) 225# @param ContainerFile: Input value for filename of Dec file 226# 227def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict): 228 HelpStr = '' 229 PromptStr = '' 230 PcdErr = None 231 PcdErrList = [] 232 ValidValueNum = 0 233 ValidRangeNum = 0 234 ExpressionNum = 0 235 236 for (CommentLine, LineNum) in GenericComment: 237 Comment = CleanString2(CommentLine)[1] 238 # 239 # To replace Macro 240 # 241 MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)' 242 MatchedStrs = re.findall(MACRO_PATTERN, Comment) 243 for MatchedStr in MatchedStrs: 244 if MatchedStr: 245 Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip() 246 if Macro in MacroReplaceDict: 247 Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro]) 248 if Comment.startswith(TAB_PCD_VALIDRANGE): 249 if ValidValueNum > 0 or ExpressionNum > 0: 250 Logger.Error('Parser', 251 FORMAT_NOT_SUPPORTED, 252 ST.WRN_MULTI_PCD_RANGES, 253 File = ContainerFile, 254 Line = LineNum) 255 else: 256 PcdErr = PcdErrorObject() 257 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 258 PcdErr.SetCName(CName) 259 PcdErr.SetFileLine(Comment) 260 PcdErr.SetLineNum(LineNum) 261 ValidRangeNum += 1 262 ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip() 263 Valid, Cause = _CheckRangeExpression(ValidRange) 264 if Valid: 265 ValueList = ValidRange.split(TAB_VALUE_SPLIT) 266 if len(ValueList) > 1: 267 PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 268 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 269 else: 270 PcdErr.SetValidValueRange(ValidRange) 271 PcdErrList.append(PcdErr) 272 else: 273 Logger.Error("Parser", 274 FORMAT_NOT_SUPPORTED, 275 Cause, 276 ContainerFile, 277 LineNum) 278 elif Comment.startswith(TAB_PCD_VALIDLIST): 279 if ValidRangeNum > 0 or ExpressionNum > 0: 280 Logger.Error('Parser', 281 FORMAT_NOT_SUPPORTED, 282 ST.WRN_MULTI_PCD_RANGES, 283 File = ContainerFile, 284 Line = LineNum) 285 elif ValidValueNum > 0: 286 Logger.Error('Parser', 287 FORMAT_NOT_SUPPORTED, 288 ST.WRN_MULTI_PCD_VALIDVALUE, 289 File = ContainerFile, 290 Line = LineNum) 291 else: 292 PcdErr = PcdErrorObject() 293 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 294 PcdErr.SetCName(CName) 295 PcdErr.SetFileLine(Comment) 296 PcdErr.SetLineNum(LineNum) 297 ValidValueNum += 1 298 ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip() 299 Valid, Cause = _CheckListExpression(ValidValueExpr) 300 if Valid: 301 ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT) 302 ValueList = ValidValue.split(TAB_VALUE_SPLIT) 303 if len(ValueList) > 1: 304 PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 305 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 306 else: 307 PcdErr.SetValidValue(ValidValue) 308 PcdErrList.append(PcdErr) 309 else: 310 Logger.Error("Parser", 311 FORMAT_NOT_SUPPORTED, 312 Cause, 313 ContainerFile, 314 LineNum) 315 elif Comment.startswith(TAB_PCD_EXPRESSION): 316 if ValidRangeNum > 0 or ValidValueNum > 0: 317 Logger.Error('Parser', 318 FORMAT_NOT_SUPPORTED, 319 ST.WRN_MULTI_PCD_RANGES, 320 File = ContainerFile, 321 Line = LineNum) 322 else: 323 PcdErr = PcdErrorObject() 324 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 325 PcdErr.SetCName(CName) 326 PcdErr.SetFileLine(Comment) 327 PcdErr.SetLineNum(LineNum) 328 ExpressionNum += 1 329 Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip() 330 Valid, Cause = _CheckExpression(Expression) 331 if Valid: 332 ValueList = Expression.split(TAB_VALUE_SPLIT) 333 if len(ValueList) > 1: 334 PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 335 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 336 else: 337 PcdErr.SetExpression(Expression) 338 PcdErrList.append(PcdErr) 339 else: 340 Logger.Error("Parser", 341 FORMAT_NOT_SUPPORTED, 342 Cause, 343 ContainerFile, 344 LineNum) 345 elif Comment.startswith(TAB_PCD_PROMPT): 346 if PromptStr: 347 Logger.Error('Parser', 348 FORMAT_NOT_SUPPORTED, 349 ST.WRN_MULTI_PCD_PROMPT, 350 File = ContainerFile, 351 Line = LineNum) 352 PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip() 353 else: 354 if Comment: 355 HelpStr += Comment + '\n' 356 357 # 358 # remove the last EOL if the comment is of format 'FOO\n' 359 # 360 if HelpStr.endswith('\n'): 361 if HelpStr != '\n' and not HelpStr.endswith('\n\n'): 362 HelpStr = HelpStr[:-1] 363 364 return HelpStr, PcdErrList, PromptStr 365 366## ParseDecPcdTailComment 367# 368# @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum) 369# @param ContainerFile: Input value for filename of Dec file 370# @retVal SupModuleList: The supported module type list detected 371# @retVal HelpStr: The generic help text string detected 372# 373def ParseDecPcdTailComment (TailCommentList, ContainerFile): 374 assert(len(TailCommentList) == 1) 375 TailComment = TailCommentList[0][0] 376 LineNum = TailCommentList[0][1] 377 378 Comment = TailComment.lstrip(" #") 379 380 ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL) 381 382 # 383 # get first word and compare with SUP_MODULE_LIST 384 # 385 MatchObject = ReFindFirstWordRe.match(Comment) 386 if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST): 387 return None, Comment 388 389 # 390 # parse line, it must have supported module type specified 391 # 392 if Comment.find(TAB_COMMENT_SPLIT) == -1: 393 Comment += TAB_COMMENT_SPLIT 394 SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1) 395 SupModuleList = [] 396 for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT): 397 if not Mod: 398 continue 399 elif Mod not in SUP_MODULE_LIST: 400 Logger.Error("UPT", 401 FORMAT_INVALID, 402 ST.WRN_INVALID_MODULE_TYPE%Mod, 403 ContainerFile, 404 LineNum) 405 else: 406 SupModuleList.append(Mod) 407 408 return SupModuleList, HelpStr 409 410## _CheckListExpression 411# 412# @param Expression: Pcd value list expression 413# 414def _CheckListExpression(Expression): 415 ListExpr = '' 416 if TAB_VALUE_SPLIT in Expression: 417 ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 418 else: 419 ListExpr = Expression 420 421 return IsValidListExpr(ListExpr) 422 423## _CheckExpression 424# 425# @param Expression: Pcd value expression 426# 427def _CheckExpression(Expression): 428 Expr = '' 429 if TAB_VALUE_SPLIT in Expression: 430 Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 431 else: 432 Expr = Expression 433 return IsValidLogicalExpr(Expr, True) 434 435## _CheckRangeExpression 436# 437# @param Expression: Pcd range expression 438# 439def _CheckRangeExpression(Expression): 440 RangeExpr = '' 441 if TAB_VALUE_SPLIT in Expression: 442 RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 443 else: 444 RangeExpr = Expression 445 446 return IsValidRangeExpr(RangeExpr) 447 448## ValidateCopyright 449# 450# 451# 452def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg): 453 if not Result: 454 Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg) 455 456## _ValidateCopyright 457# 458# @param Line: Line that contains copyright information, # stripped 459# 460# @retval Result: True if line is conformed to Spec format, False else 461# @retval ErrMsg: the detailed error description 462# 463def _ValidateCopyright(Line): 464 if Line: 465 pass 466 Result = True 467 ErrMsg = '' 468 469 return Result, ErrMsg 470 471def GenerateTokenList (Comment): 472 # 473 # Tokenize Comment using '#' and ' ' as token separators 474 # 475 ReplacedComment = None 476 while Comment != ReplacedComment: 477 ReplacedComment = Comment 478 Comment = Comment.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ') 479 return Comment.split('#') 480 481 482# 483# Comment - Comment to parse 484# TypeTokens - A dictionary of type token synonyms 485# RemoveTokens - A list of tokens to remove from help text 486# ParseVariable - True for parsing [Guids]. Otherwise False 487# 488def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable): 489 # 490 # Initialize return values 491 # 492 Usage = None 493 Type = None 494 String = None 495 496 Comment = Comment[0] 497 498 NumTokens = 2 499 if ParseVariable: 500 # 501 # Remove white space around first instance of ':' from Comment if 'Variable' 502 # is in front of ':' and Variable is the 1st or 2nd token in Comment. 503 # 504 List = Comment.split(':', 1) 505 if len(List) > 1: 506 SubList = GenerateTokenList (List[0].strip()) 507 if len(SubList) in [1, 2] and SubList[-1] == 'Variable': 508 if List[1].strip().find('L"') == 0: 509 Comment = List[0].strip() + ':' + List[1].strip() 510 511 # 512 # Remove first instance of L"<VariableName> from Comment and put into String 513 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or 514 # L"<VariableName>" is the third token immediately following 'Variable:'. 515 # 516 End = -1 517 Start = Comment.find('Variable:L"') 518 if Start >= 0: 519 String = Comment[Start + 9:] 520 End = String[2:].find('"') 521 else: 522 Start = Comment.find('L"') 523 if Start >= 0: 524 String = Comment[Start:] 525 End = String[2:].find('"') 526 if End >= 0: 527 SubList = GenerateTokenList (Comment[:Start]) 528 if len(SubList) < 2: 529 Comment = Comment[:Start] + String[End + 3:] 530 String = String[:End + 3] 531 Type = 'Variable' 532 NumTokens = 1 533 534 # 535 # Initialize HelpText to Comment. 536 # Content will be remove from HelpText as matching tokens are found 537 # 538 HelpText = Comment 539 540 # 541 # Tokenize Comment using '#' and ' ' as token separators 542 # 543 List = GenerateTokenList (Comment) 544 545 # 546 # Search first two tokens for Usage and Type and remove any matching tokens 547 # from HelpText 548 # 549 for Token in List[0:NumTokens]: 550 if Usage is None and Token in UsageTokens: 551 Usage = UsageTokens[Token] 552 HelpText = HelpText.replace(Token, '') 553 if Usage is not None or not ParseVariable: 554 for Token in List[0:NumTokens]: 555 if Type is None and Token in TypeTokens: 556 Type = TypeTokens[Token] 557 HelpText = HelpText.replace(Token, '') 558 if Usage is not None: 559 for Token in List[0:NumTokens]: 560 if Token in RemoveTokens: 561 HelpText = HelpText.replace(Token, '') 562 563 # 564 # If no Usage token is present and set Usage to UNDEFINED 565 # 566 if Usage is None: 567 Usage = 'UNDEFINED' 568 569 # 570 # If no Type token is present and set Type to UNDEFINED 571 # 572 if Type is None: 573 Type = 'UNDEFINED' 574 575 # 576 # If Type is not 'Variable:', then set String to None 577 # 578 if Type != 'Variable': 579 String = None 580 581 # 582 # Strip ' ' and '#' from the beginning of HelpText 583 # If HelpText is an empty string after all parsing is 584 # complete then set HelpText to None 585 # 586 HelpText = HelpText.lstrip('# ') 587 if HelpText == '': 588 HelpText = None 589 590 # 591 # Return parsing results 592 # 593 return Usage, Type, String, HelpText 594