1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines the clang::Preprocessor interface. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H 15 #define LLVM_CLANG_LEX_PREPROCESSOR_H 16 17 #include "clang/Basic/Diagnostic.h" 18 #include "clang/Basic/DiagnosticIDs.h" 19 #include "clang/Basic/IdentifierTable.h" 20 #include "clang/Basic/LLVM.h" 21 #include "clang/Basic/LangOptions.h" 22 #include "clang/Basic/Module.h" 23 #include "clang/Basic/SourceLocation.h" 24 #include "clang/Basic/SourceManager.h" 25 #include "clang/Basic/TokenKinds.h" 26 #include "clang/Lex/HeaderSearch.h" 27 #include "clang/Lex/Lexer.h" 28 #include "clang/Lex/MacroInfo.h" 29 #include "clang/Lex/ModuleLoader.h" 30 #include "clang/Lex/ModuleMap.h" 31 #include "clang/Lex/PPCallbacks.h" 32 #include "clang/Lex/Token.h" 33 #include "clang/Lex/TokenLexer.h" 34 #include "llvm/ADT/ArrayRef.h" 35 #include "llvm/ADT/DenseMap.h" 36 #include "llvm/ADT/FoldingSet.h" 37 #include "llvm/ADT/FunctionExtras.h" 38 #include "llvm/ADT/None.h" 39 #include "llvm/ADT/Optional.h" 40 #include "llvm/ADT/PointerUnion.h" 41 #include "llvm/ADT/STLExtras.h" 42 #include "llvm/ADT/SmallPtrSet.h" 43 #include "llvm/ADT/SmallVector.h" 44 #include "llvm/ADT/StringRef.h" 45 #include "llvm/ADT/TinyPtrVector.h" 46 #include "llvm/ADT/iterator_range.h" 47 #include "llvm/Support/Allocator.h" 48 #include "llvm/Support/Casting.h" 49 #include "llvm/Support/Registry.h" 50 #include <cassert> 51 #include <cstddef> 52 #include <cstdint> 53 #include <map> 54 #include <memory> 55 #include <string> 56 #include <utility> 57 #include <vector> 58 59 namespace llvm { 60 61 template<unsigned InternalLen> class SmallString; 62 63 } // namespace llvm 64 65 namespace clang { 66 67 class CodeCompletionHandler; 68 class CommentHandler; 69 class DirectoryEntry; 70 class DirectoryLookup; 71 class EmptylineHandler; 72 class ExternalPreprocessorSource; 73 class FileEntry; 74 class FileManager; 75 class HeaderSearch; 76 class MacroArgs; 77 class PragmaHandler; 78 class PragmaNamespace; 79 class PreprocessingRecord; 80 class PreprocessorLexer; 81 class PreprocessorOptions; 82 class ScratchBuffer; 83 class TargetInfo; 84 85 namespace Builtin { 86 class Context; 87 } 88 89 /// Stores token information for comparing actual tokens with 90 /// predefined values. Only handles simple tokens and identifiers. 91 class TokenValue { 92 tok::TokenKind Kind; 93 IdentifierInfo *II; 94 95 public: 96 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) { 97 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported."); 98 assert(Kind != tok::identifier && 99 "Identifiers should be created by TokenValue(IdentifierInfo *)"); 100 assert(!tok::isLiteral(Kind) && "Literals are not supported."); 101 assert(!tok::isAnnotation(Kind) && "Annotations are not supported."); 102 } 103 104 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {} 105 106 bool operator==(const Token &Tok) const { 107 return Tok.getKind() == Kind && 108 (!II || II == Tok.getIdentifierInfo()); 109 } 110 }; 111 112 /// Context in which macro name is used. 113 enum MacroUse { 114 // other than #define or #undef 115 MU_Other = 0, 116 117 // macro name specified in #define 118 MU_Define = 1, 119 120 // macro name specified in #undef 121 MU_Undef = 2 122 }; 123 124 /// Engages in a tight little dance with the lexer to efficiently 125 /// preprocess tokens. 126 /// 127 /// Lexers know only about tokens within a single source file, and don't 128 /// know anything about preprocessor-level issues like the \#include stack, 129 /// token expansion, etc. 130 class Preprocessor { 131 friend class VAOptDefinitionContext; 132 friend class VariadicMacroScopeGuard; 133 134 llvm::unique_function<void(const clang::Token &)> OnToken; 135 std::shared_ptr<PreprocessorOptions> PPOpts; 136 DiagnosticsEngine *Diags; 137 LangOptions &LangOpts; 138 const TargetInfo *Target = nullptr; 139 const TargetInfo *AuxTarget = nullptr; 140 FileManager &FileMgr; 141 SourceManager &SourceMgr; 142 std::unique_ptr<ScratchBuffer> ScratchBuf; 143 HeaderSearch &HeaderInfo; 144 ModuleLoader &TheModuleLoader; 145 146 /// External source of macros. 147 ExternalPreprocessorSource *ExternalSource; 148 149 /// A BumpPtrAllocator object used to quickly allocate and release 150 /// objects internal to the Preprocessor. 151 llvm::BumpPtrAllocator BP; 152 153 /// Identifiers for builtin macros and other builtins. 154 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__ 155 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__ 156 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__ 157 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__ 158 IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__ 159 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__ 160 IdentifierInfo *Ident__COUNTER__; // __COUNTER__ 161 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma 162 IdentifierInfo *Ident__identifier; // __identifier 163 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__ 164 IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__ 165 IdentifierInfo *Ident__has_feature; // __has_feature 166 IdentifierInfo *Ident__has_extension; // __has_extension 167 IdentifierInfo *Ident__has_builtin; // __has_builtin 168 IdentifierInfo *Ident__has_attribute; // __has_attribute 169 IdentifierInfo *Ident__has_include; // __has_include 170 IdentifierInfo *Ident__has_include_next; // __has_include_next 171 IdentifierInfo *Ident__has_warning; // __has_warning 172 IdentifierInfo *Ident__is_identifier; // __is_identifier 173 IdentifierInfo *Ident__building_module; // __building_module 174 IdentifierInfo *Ident__MODULE__; // __MODULE__ 175 IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute 176 IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute 177 IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute 178 IdentifierInfo *Ident__is_target_arch; // __is_target_arch 179 IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor 180 IdentifierInfo *Ident__is_target_os; // __is_target_os 181 IdentifierInfo *Ident__is_target_environment; // __is_target_environment 182 IdentifierInfo *Ident__FLT_EVAL_METHOD__; // __FLT_EVAL_METHOD 183 184 // Weak, only valid (and set) while InMacroArgs is true. 185 Token* ArgMacro; 186 187 SourceLocation DATELoc, TIMELoc; 188 189 // FEM_UnsetOnCommandLine means that an explicit evaluation method was 190 // not specified on the command line. The target is queried to set the 191 // default evaluation method. 192 LangOptions::FPEvalMethodKind CurrentFPEvalMethod = 193 LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine; 194 195 // Keeps the value of the last evaluation method before a 196 // `pragma float_control (precise,off) is applied. 197 LangOptions::FPEvalMethodKind LastFPEvalMethod = 198 LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine; 199 200 // The most recent pragma location where the floating point evaluation 201 // method was modified. This is used to determine whether the 202 // 'pragma clang fp eval_method' was used whithin the current scope. 203 SourceLocation LastFPEvalPragmaLocation; 204 205 LangOptions::FPEvalMethodKind TUFPEvalMethod = 206 LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine; 207 208 // Next __COUNTER__ value, starts at 0. 209 unsigned CounterValue = 0; 210 211 enum { 212 /// Maximum depth of \#includes. 213 MaxAllowedIncludeStackDepth = 200 214 }; 215 216 // State that is set before the preprocessor begins. 217 bool KeepComments : 1; 218 bool KeepMacroComments : 1; 219 bool SuppressIncludeNotFoundError : 1; 220 221 // State that changes while the preprocessor runs: 222 bool InMacroArgs : 1; // True if parsing fn macro invocation args. 223 224 /// Whether the preprocessor owns the header search object. 225 bool OwnsHeaderSearch : 1; 226 227 /// True if macro expansion is disabled. 228 bool DisableMacroExpansion : 1; 229 230 /// Temporarily disables DisableMacroExpansion (i.e. enables expansion) 231 /// when parsing preprocessor directives. 232 bool MacroExpansionInDirectivesOverride : 1; 233 234 class ResetMacroExpansionHelper; 235 236 /// Whether we have already loaded macros from the external source. 237 mutable bool ReadMacrosFromExternalSource : 1; 238 239 /// True if pragmas are enabled. 240 bool PragmasEnabled : 1; 241 242 /// True if the current build action is a preprocessing action. 243 bool PreprocessedOutput : 1; 244 245 /// True if we are currently preprocessing a #if or #elif directive 246 bool ParsingIfOrElifDirective; 247 248 /// True if we are pre-expanding macro arguments. 249 bool InMacroArgPreExpansion; 250 251 /// Mapping/lookup information for all identifiers in 252 /// the program, including program keywords. 253 mutable IdentifierTable Identifiers; 254 255 /// This table contains all the selectors in the program. 256 /// 257 /// Unlike IdentifierTable above, this table *isn't* populated by the 258 /// preprocessor. It is declared/expanded here because its role/lifetime is 259 /// conceptually similar to the IdentifierTable. In addition, the current 260 /// control flow (in clang::ParseAST()), make it convenient to put here. 261 /// 262 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to 263 /// the lifetime of the preprocessor. 264 SelectorTable Selectors; 265 266 /// Information about builtins. 267 std::unique_ptr<Builtin::Context> BuiltinInfo; 268 269 /// Tracks all of the pragmas that the client registered 270 /// with this preprocessor. 271 std::unique_ptr<PragmaNamespace> PragmaHandlers; 272 273 /// Pragma handlers of the original source is stored here during the 274 /// parsing of a model file. 275 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup; 276 277 /// Tracks all of the comment handlers that the client registered 278 /// with this preprocessor. 279 std::vector<CommentHandler *> CommentHandlers; 280 281 /// Empty line handler. 282 EmptylineHandler *Emptyline = nullptr; 283 284 /// True if we want to ignore EOF token and continue later on (thus 285 /// avoid tearing the Lexer and etc. down). 286 bool IncrementalProcessing = false; 287 288 public: 289 /// The kind of translation unit we are processing. 290 const TranslationUnitKind TUKind; 291 292 private: 293 /// The code-completion handler. 294 CodeCompletionHandler *CodeComplete = nullptr; 295 296 /// The file that we're performing code-completion for, if any. 297 const FileEntry *CodeCompletionFile = nullptr; 298 299 /// The offset in file for the code-completion point. 300 unsigned CodeCompletionOffset = 0; 301 302 /// The location for the code-completion point. This gets instantiated 303 /// when the CodeCompletionFile gets \#include'ed for preprocessing. 304 SourceLocation CodeCompletionLoc; 305 306 /// The start location for the file of the code-completion point. 307 /// 308 /// This gets instantiated when the CodeCompletionFile gets \#include'ed 309 /// for preprocessing. 310 SourceLocation CodeCompletionFileLoc; 311 312 /// The source location of the \c import contextual keyword we just 313 /// lexed, if any. 314 SourceLocation ModuleImportLoc; 315 316 /// The module import path that we're currently processing. 317 SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath; 318 319 /// Whether the last token we lexed was an '@'. 320 bool LastTokenWasAt = false; 321 322 /// A position within a C++20 import-seq. 323 class ImportSeq { 324 public: 325 enum State : int { 326 // Positive values represent a number of unclosed brackets. 327 AtTopLevel = 0, 328 AfterTopLevelTokenSeq = -1, 329 AfterExport = -2, 330 AfterImportSeq = -3, 331 }; 332 333 ImportSeq(State S) : S(S) {} 334 335 /// Saw any kind of open bracket. 336 void handleOpenBracket() { 337 S = static_cast<State>(std::max<int>(S, 0) + 1); 338 } 339 /// Saw any kind of close bracket other than '}'. 340 void handleCloseBracket() { 341 S = static_cast<State>(std::max<int>(S, 1) - 1); 342 } 343 /// Saw a close brace. 344 void handleCloseBrace() { 345 handleCloseBracket(); 346 if (S == AtTopLevel && !AfterHeaderName) 347 S = AfterTopLevelTokenSeq; 348 } 349 /// Saw a semicolon. 350 void handleSemi() { 351 if (atTopLevel()) { 352 S = AfterTopLevelTokenSeq; 353 AfterHeaderName = false; 354 } 355 } 356 357 /// Saw an 'export' identifier. 358 void handleExport() { 359 if (S == AfterTopLevelTokenSeq) 360 S = AfterExport; 361 else if (S <= 0) 362 S = AtTopLevel; 363 } 364 /// Saw an 'import' identifier. 365 void handleImport() { 366 if (S == AfterTopLevelTokenSeq || S == AfterExport) 367 S = AfterImportSeq; 368 else if (S <= 0) 369 S = AtTopLevel; 370 } 371 372 /// Saw a 'header-name' token; do not recognize any more 'import' tokens 373 /// until we reach a top-level semicolon. 374 void handleHeaderName() { 375 if (S == AfterImportSeq) 376 AfterHeaderName = true; 377 handleMisc(); 378 } 379 380 /// Saw any other token. 381 void handleMisc() { 382 if (S <= 0) 383 S = AtTopLevel; 384 } 385 386 bool atTopLevel() { return S <= 0; } 387 bool afterImportSeq() { return S == AfterImportSeq; } 388 389 private: 390 State S; 391 /// Whether we're in the pp-import-suffix following the header-name in a 392 /// pp-import. If so, a close-brace is not sufficient to end the 393 /// top-level-token-seq of an import-seq. 394 bool AfterHeaderName = false; 395 }; 396 397 /// Our current position within a C++20 import-seq. 398 ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq; 399 400 /// Whether the module import expects an identifier next. Otherwise, 401 /// it expects a '.' or ';'. 402 bool ModuleImportExpectsIdentifier = false; 403 404 /// The identifier and source location of the currently-active 405 /// \#pragma clang arc_cf_code_audited begin. 406 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo; 407 408 /// The source location of the currently-active 409 /// \#pragma clang assume_nonnull begin. 410 SourceLocation PragmaAssumeNonNullLoc; 411 412 /// Set only for preambles which end with an active 413 /// \#pragma clang assume_nonnull begin. 414 /// 415 /// When the preamble is loaded into the main file, 416 /// `PragmaAssumeNonNullLoc` will be set to this to 417 /// replay the unterminated assume_nonnull. 418 SourceLocation PreambleRecordedPragmaAssumeNonNullLoc; 419 420 /// True if we hit the code-completion point. 421 bool CodeCompletionReached = false; 422 423 /// The code completion token containing the information 424 /// on the stem that is to be code completed. 425 IdentifierInfo *CodeCompletionII = nullptr; 426 427 /// Range for the code completion token. 428 SourceRange CodeCompletionTokenRange; 429 430 /// The directory that the main file should be considered to occupy, 431 /// if it does not correspond to a real file (as happens when building a 432 /// module). 433 const DirectoryEntry *MainFileDir = nullptr; 434 435 /// The number of bytes that we will initially skip when entering the 436 /// main file, along with a flag that indicates whether skipping this number 437 /// of bytes will place the lexer at the start of a line. 438 /// 439 /// This is used when loading a precompiled preamble. 440 std::pair<int, bool> SkipMainFilePreamble; 441 442 /// Whether we hit an error due to reaching max allowed include depth. Allows 443 /// to avoid hitting the same error over and over again. 444 bool HasReachedMaxIncludeDepth = false; 445 446 /// The number of currently-active calls to Lex. 447 /// 448 /// Lex is reentrant, and asking for an (end-of-phase-4) token can often 449 /// require asking for multiple additional tokens. This counter makes it 450 /// possible for Lex to detect whether it's producing a token for the end 451 /// of phase 4 of translation or for some other situation. 452 unsigned LexLevel = 0; 453 454 /// The number of (LexLevel 0) preprocessor tokens. 455 unsigned TokenCount = 0; 456 457 /// Preprocess every token regardless of LexLevel. 458 bool PreprocessToken = false; 459 460 /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens 461 /// warning, or zero for unlimited. 462 unsigned MaxTokens = 0; 463 SourceLocation MaxTokensOverrideLoc; 464 465 public: 466 struct PreambleSkipInfo { 467 SourceLocation HashTokenLoc; 468 SourceLocation IfTokenLoc; 469 bool FoundNonSkipPortion; 470 bool FoundElse; 471 SourceLocation ElseLoc; 472 473 PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, 474 bool FoundNonSkipPortion, bool FoundElse, 475 SourceLocation ElseLoc) 476 : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc), 477 FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse), 478 ElseLoc(ElseLoc) {} 479 }; 480 481 using IncludedFilesSet = llvm::DenseSet<const FileEntry *>; 482 483 private: 484 friend class ASTReader; 485 friend class MacroArgs; 486 487 class PreambleConditionalStackStore { 488 enum State { 489 Off = 0, 490 Recording = 1, 491 Replaying = 2, 492 }; 493 494 public: 495 PreambleConditionalStackStore() = default; 496 497 void startRecording() { ConditionalStackState = Recording; } 498 void startReplaying() { ConditionalStackState = Replaying; } 499 bool isRecording() const { return ConditionalStackState == Recording; } 500 bool isReplaying() const { return ConditionalStackState == Replaying; } 501 502 ArrayRef<PPConditionalInfo> getStack() const { 503 return ConditionalStack; 504 } 505 506 void doneReplaying() { 507 ConditionalStack.clear(); 508 ConditionalStackState = Off; 509 } 510 511 void setStack(ArrayRef<PPConditionalInfo> s) { 512 if (!isRecording() && !isReplaying()) 513 return; 514 ConditionalStack.clear(); 515 ConditionalStack.append(s.begin(), s.end()); 516 } 517 518 bool hasRecordedPreamble() const { return !ConditionalStack.empty(); } 519 520 bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); } 521 522 void clearSkipInfo() { SkipInfo.reset(); } 523 524 llvm::Optional<PreambleSkipInfo> SkipInfo; 525 526 private: 527 SmallVector<PPConditionalInfo, 4> ConditionalStack; 528 State ConditionalStackState = Off; 529 } PreambleConditionalStack; 530 531 /// The current top of the stack that we're lexing from if 532 /// not expanding a macro and we are lexing directly from source code. 533 /// 534 /// Only one of CurLexer, or CurTokenLexer will be non-null. 535 std::unique_ptr<Lexer> CurLexer; 536 537 /// The current top of the stack what we're lexing from 538 /// if not expanding a macro. 539 /// 540 /// This is an alias for CurLexer. 541 PreprocessorLexer *CurPPLexer = nullptr; 542 543 /// Used to find the current FileEntry, if CurLexer is non-null 544 /// and if applicable. 545 /// 546 /// This allows us to implement \#include_next and find directory-specific 547 /// properties. 548 ConstSearchDirIterator CurDirLookup = nullptr; 549 550 /// The current macro we are expanding, if we are expanding a macro. 551 /// 552 /// One of CurLexer and CurTokenLexer must be null. 553 std::unique_ptr<TokenLexer> CurTokenLexer; 554 555 /// The kind of lexer we're currently working with. 556 enum CurLexerKind { 557 CLK_Lexer, 558 CLK_TokenLexer, 559 CLK_CachingLexer, 560 CLK_DependencyDirectivesLexer, 561 CLK_LexAfterModuleImport 562 } CurLexerKind = CLK_Lexer; 563 564 /// If the current lexer is for a submodule that is being built, this 565 /// is that submodule. 566 Module *CurLexerSubmodule = nullptr; 567 568 /// Keeps track of the stack of files currently 569 /// \#included, and macros currently being expanded from, not counting 570 /// CurLexer/CurTokenLexer. 571 struct IncludeStackInfo { 572 enum CurLexerKind CurLexerKind; 573 Module *TheSubmodule; 574 std::unique_ptr<Lexer> TheLexer; 575 PreprocessorLexer *ThePPLexer; 576 std::unique_ptr<TokenLexer> TheTokenLexer; 577 ConstSearchDirIterator TheDirLookup; 578 579 // The following constructors are completely useless copies of the default 580 // versions, only needed to pacify MSVC. 581 IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule, 582 std::unique_ptr<Lexer> &&TheLexer, 583 PreprocessorLexer *ThePPLexer, 584 std::unique_ptr<TokenLexer> &&TheTokenLexer, 585 ConstSearchDirIterator TheDirLookup) 586 : CurLexerKind(std::move(CurLexerKind)), 587 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)), 588 ThePPLexer(std::move(ThePPLexer)), 589 TheTokenLexer(std::move(TheTokenLexer)), 590 TheDirLookup(std::move(TheDirLookup)) {} 591 }; 592 std::vector<IncludeStackInfo> IncludeMacroStack; 593 594 /// Actions invoked when some preprocessor activity is 595 /// encountered (e.g. a file is \#included, etc). 596 std::unique_ptr<PPCallbacks> Callbacks; 597 598 struct MacroExpandsInfo { 599 Token Tok; 600 MacroDefinition MD; 601 SourceRange Range; 602 603 MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range) 604 : Tok(Tok), MD(MD), Range(Range) {} 605 }; 606 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks; 607 608 /// Information about a name that has been used to define a module macro. 609 struct ModuleMacroInfo { 610 /// The most recent macro directive for this identifier. 611 MacroDirective *MD; 612 613 /// The active module macros for this identifier. 614 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros; 615 616 /// The generation number at which we last updated ActiveModuleMacros. 617 /// \see Preprocessor::VisibleModules. 618 unsigned ActiveModuleMacrosGeneration = 0; 619 620 /// Whether this macro name is ambiguous. 621 bool IsAmbiguous = false; 622 623 /// The module macros that are overridden by this macro. 624 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros; 625 626 ModuleMacroInfo(MacroDirective *MD) : MD(MD) {} 627 }; 628 629 /// The state of a macro for an identifier. 630 class MacroState { 631 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State; 632 633 ModuleMacroInfo *getModuleInfo(Preprocessor &PP, 634 const IdentifierInfo *II) const { 635 if (II->isOutOfDate()) 636 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II)); 637 // FIXME: Find a spare bit on IdentifierInfo and store a 638 // HasModuleMacros flag. 639 if (!II->hasMacroDefinition() || 640 (!PP.getLangOpts().Modules && 641 !PP.getLangOpts().ModulesLocalVisibility) || 642 !PP.CurSubmoduleState->VisibleModules.getGeneration()) 643 return nullptr; 644 645 auto *Info = State.dyn_cast<ModuleMacroInfo*>(); 646 if (!Info) { 647 Info = new (PP.getPreprocessorAllocator()) 648 ModuleMacroInfo(State.get<MacroDirective *>()); 649 State = Info; 650 } 651 652 if (PP.CurSubmoduleState->VisibleModules.getGeneration() != 653 Info->ActiveModuleMacrosGeneration) 654 PP.updateModuleMacroInfo(II, *Info); 655 return Info; 656 } 657 658 public: 659 MacroState() : MacroState(nullptr) {} 660 MacroState(MacroDirective *MD) : State(MD) {} 661 662 MacroState(MacroState &&O) noexcept : State(O.State) { 663 O.State = (MacroDirective *)nullptr; 664 } 665 666 MacroState &operator=(MacroState &&O) noexcept { 667 auto S = O.State; 668 O.State = (MacroDirective *)nullptr; 669 State = S; 670 return *this; 671 } 672 673 ~MacroState() { 674 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 675 Info->~ModuleMacroInfo(); 676 } 677 678 MacroDirective *getLatest() const { 679 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 680 return Info->MD; 681 return State.get<MacroDirective*>(); 682 } 683 684 void setLatest(MacroDirective *MD) { 685 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 686 Info->MD = MD; 687 else 688 State = MD; 689 } 690 691 bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const { 692 auto *Info = getModuleInfo(PP, II); 693 return Info ? Info->IsAmbiguous : false; 694 } 695 696 ArrayRef<ModuleMacro *> 697 getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const { 698 if (auto *Info = getModuleInfo(PP, II)) 699 return Info->ActiveModuleMacros; 700 return None; 701 } 702 703 MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc, 704 SourceManager &SourceMgr) const { 705 // FIXME: Incorporate module macros into the result of this. 706 if (auto *Latest = getLatest()) 707 return Latest->findDirectiveAtLoc(Loc, SourceMgr); 708 return {}; 709 } 710 711 void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) { 712 if (auto *Info = getModuleInfo(PP, II)) { 713 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(), 714 Info->ActiveModuleMacros.begin(), 715 Info->ActiveModuleMacros.end()); 716 Info->ActiveModuleMacros.clear(); 717 Info->IsAmbiguous = false; 718 } 719 } 720 721 ArrayRef<ModuleMacro*> getOverriddenMacros() const { 722 if (auto *Info = State.dyn_cast<ModuleMacroInfo*>()) 723 return Info->OverriddenMacros; 724 return None; 725 } 726 727 void setOverriddenMacros(Preprocessor &PP, 728 ArrayRef<ModuleMacro *> Overrides) { 729 auto *Info = State.dyn_cast<ModuleMacroInfo*>(); 730 if (!Info) { 731 if (Overrides.empty()) 732 return; 733 Info = new (PP.getPreprocessorAllocator()) 734 ModuleMacroInfo(State.get<MacroDirective *>()); 735 State = Info; 736 } 737 Info->OverriddenMacros.clear(); 738 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(), 739 Overrides.begin(), Overrides.end()); 740 Info->ActiveModuleMacrosGeneration = 0; 741 } 742 }; 743 744 /// For each IdentifierInfo that was associated with a macro, we 745 /// keep a mapping to the history of all macro definitions and #undefs in 746 /// the reverse order (the latest one is in the head of the list). 747 /// 748 /// This mapping lives within the \p CurSubmoduleState. 749 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>; 750 751 struct SubmoduleState; 752 753 /// Information about a submodule that we're currently building. 754 struct BuildingSubmoduleInfo { 755 /// The module that we are building. 756 Module *M; 757 758 /// The location at which the module was included. 759 SourceLocation ImportLoc; 760 761 /// Whether we entered this submodule via a pragma. 762 bool IsPragma; 763 764 /// The previous SubmoduleState. 765 SubmoduleState *OuterSubmoduleState; 766 767 /// The number of pending module macro names when we started building this. 768 unsigned OuterPendingModuleMacroNames; 769 770 BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma, 771 SubmoduleState *OuterSubmoduleState, 772 unsigned OuterPendingModuleMacroNames) 773 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma), 774 OuterSubmoduleState(OuterSubmoduleState), 775 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {} 776 }; 777 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack; 778 779 /// Information about a submodule's preprocessor state. 780 struct SubmoduleState { 781 /// The macros for the submodule. 782 MacroMap Macros; 783 784 /// The set of modules that are visible within the submodule. 785 VisibleModuleSet VisibleModules; 786 787 // FIXME: CounterValue? 788 // FIXME: PragmaPushMacroInfo? 789 }; 790 std::map<Module *, SubmoduleState> Submodules; 791 792 /// The preprocessor state for preprocessing outside of any submodule. 793 SubmoduleState NullSubmoduleState; 794 795 /// The current submodule state. Will be \p NullSubmoduleState if we're not 796 /// in a submodule. 797 SubmoduleState *CurSubmoduleState; 798 799 /// The files that have been included. 800 IncludedFilesSet IncludedFiles; 801 802 /// The set of known macros exported from modules. 803 llvm::FoldingSet<ModuleMacro> ModuleMacros; 804 805 /// The names of potential module macros that we've not yet processed. 806 llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames; 807 808 /// The list of module macros, for each identifier, that are not overridden by 809 /// any other module macro. 810 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>> 811 LeafModuleMacros; 812 813 /// Macros that we want to warn because they are not used at the end 814 /// of the translation unit. 815 /// 816 /// We store just their SourceLocations instead of 817 /// something like MacroInfo*. The benefit of this is that when we are 818 /// deserializing from PCH, we don't need to deserialize identifier & macros 819 /// just so that we can report that they are unused, we just warn using 820 /// the SourceLocations of this set (that will be filled by the ASTReader). 821 using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>; 822 WarnUnusedMacroLocsTy WarnUnusedMacroLocs; 823 824 /// This is a pair of an optional message and source location used for pragmas 825 /// that annotate macros like pragma clang restrict_expansion and pragma clang 826 /// deprecated. This pair stores the optional message and the location of the 827 /// annotation pragma for use producing diagnostics and notes. 828 using MsgLocationPair = std::pair<std::string, SourceLocation>; 829 830 struct MacroAnnotationInfo { 831 SourceLocation Location; 832 std::string Message; 833 }; 834 835 struct MacroAnnotations { 836 llvm::Optional<MacroAnnotationInfo> DeprecationInfo; 837 llvm::Optional<MacroAnnotationInfo> RestrictExpansionInfo; 838 llvm::Optional<SourceLocation> FinalAnnotationLoc; 839 840 static MacroAnnotations makeDeprecation(SourceLocation Loc, 841 std::string Msg) { 842 return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)}, 843 llvm::None, llvm::None}; 844 } 845 846 static MacroAnnotations makeRestrictExpansion(SourceLocation Loc, 847 std::string Msg) { 848 return MacroAnnotations{ 849 llvm::None, MacroAnnotationInfo{Loc, std::move(Msg)}, llvm::None}; 850 } 851 852 static MacroAnnotations makeFinal(SourceLocation Loc) { 853 return MacroAnnotations{llvm::None, llvm::None, Loc}; 854 } 855 }; 856 857 /// Warning information for macro annotations. 858 llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos; 859 860 /// A "freelist" of MacroArg objects that can be 861 /// reused for quick allocation. 862 MacroArgs *MacroArgCache = nullptr; 863 864 /// For each IdentifierInfo used in a \#pragma push_macro directive, 865 /// we keep a MacroInfo stack used to restore the previous macro value. 866 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>> 867 PragmaPushMacroInfo; 868 869 // Various statistics we track for performance analysis. 870 unsigned NumDirectives = 0; 871 unsigned NumDefined = 0; 872 unsigned NumUndefined = 0; 873 unsigned NumPragma = 0; 874 unsigned NumIf = 0; 875 unsigned NumElse = 0; 876 unsigned NumEndif = 0; 877 unsigned NumEnteredSourceFiles = 0; 878 unsigned MaxIncludeStackDepth = 0; 879 unsigned NumMacroExpanded = 0; 880 unsigned NumFnMacroExpanded = 0; 881 unsigned NumBuiltinMacroExpanded = 0; 882 unsigned NumFastMacroExpanded = 0; 883 unsigned NumTokenPaste = 0; 884 unsigned NumFastTokenPaste = 0; 885 unsigned NumSkipped = 0; 886 887 /// The predefined macros that preprocessor should use from the 888 /// command line etc. 889 std::string Predefines; 890 891 /// The file ID for the preprocessor predefines. 892 FileID PredefinesFileID; 893 894 /// The file ID for the PCH through header. 895 FileID PCHThroughHeaderFileID; 896 897 /// Whether tokens are being skipped until a #pragma hdrstop is seen. 898 bool SkippingUntilPragmaHdrStop = false; 899 900 /// Whether tokens are being skipped until the through header is seen. 901 bool SkippingUntilPCHThroughHeader = false; 902 903 /// \{ 904 /// Cache of macro expanders to reduce malloc traffic. 905 enum { TokenLexerCacheSize = 8 }; 906 unsigned NumCachedTokenLexers; 907 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize]; 908 /// \} 909 910 /// Keeps macro expanded tokens for TokenLexers. 911 // 912 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is 913 /// going to lex in the cache and when it finishes the tokens are removed 914 /// from the end of the cache. 915 SmallVector<Token, 16> MacroExpandedTokens; 916 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack; 917 918 /// A record of the macro definitions and expansions that 919 /// occurred during preprocessing. 920 /// 921 /// This is an optional side structure that can be enabled with 922 /// \c createPreprocessingRecord() prior to preprocessing. 923 PreprocessingRecord *Record = nullptr; 924 925 /// Cached tokens state. 926 using CachedTokensTy = SmallVector<Token, 1>; 927 928 /// Cached tokens are stored here when we do backtracking or 929 /// lookahead. They are "lexed" by the CachingLex() method. 930 CachedTokensTy CachedTokens; 931 932 /// The position of the cached token that CachingLex() should 933 /// "lex" next. 934 /// 935 /// If it points beyond the CachedTokens vector, it means that a normal 936 /// Lex() should be invoked. 937 CachedTokensTy::size_type CachedLexPos = 0; 938 939 /// Stack of backtrack positions, allowing nested backtracks. 940 /// 941 /// The EnableBacktrackAtThisPos() method pushes a position to 942 /// indicate where CachedLexPos should be set when the BackTrack() method is 943 /// invoked (at which point the last position is popped). 944 std::vector<CachedTokensTy::size_type> BacktrackPositions; 945 946 struct MacroInfoChain { 947 MacroInfo MI; 948 MacroInfoChain *Next; 949 }; 950 951 /// MacroInfos are managed as a chain for easy disposal. This is the head 952 /// of that list. 953 MacroInfoChain *MIChainHead = nullptr; 954 955 /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running. 956 /// This is used to guard against calling this function recursively. 957 /// 958 /// See comments at the use-site for more context about why it is needed. 959 bool SkippingExcludedConditionalBlock = false; 960 961 /// Keeps track of skipped range mappings that were recorded while skipping 962 /// excluded conditional directives. It maps the source buffer pointer at 963 /// the beginning of a skipped block, to the number of bytes that should be 964 /// skipped. 965 llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges; 966 967 void updateOutOfDateIdentifier(IdentifierInfo &II) const; 968 969 public: 970 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, 971 DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, 972 HeaderSearch &Headers, ModuleLoader &TheModuleLoader, 973 IdentifierInfoLookup *IILookup = nullptr, 974 bool OwnsHeaderSearch = false, 975 TranslationUnitKind TUKind = TU_Complete); 976 977 ~Preprocessor(); 978 979 /// Initialize the preprocessor using information about the target. 980 /// 981 /// \param Target is owned by the caller and must remain valid for the 982 /// lifetime of the preprocessor. 983 /// \param AuxTarget is owned by the caller and must remain valid for 984 /// the lifetime of the preprocessor. 985 void Initialize(const TargetInfo &Target, 986 const TargetInfo *AuxTarget = nullptr); 987 988 /// Initialize the preprocessor to parse a model file 989 /// 990 /// To parse model files the preprocessor of the original source is reused to 991 /// preserver the identifier table. However to avoid some duplicate 992 /// information in the preprocessor some cleanup is needed before it is used 993 /// to parse model files. This method does that cleanup. 994 void InitializeForModelFile(); 995 996 /// Cleanup after model file parsing 997 void FinalizeForModelFile(); 998 999 /// Retrieve the preprocessor options used to initialize this 1000 /// preprocessor. 1001 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; } 1002 1003 DiagnosticsEngine &getDiagnostics() const { return *Diags; } 1004 void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; } 1005 1006 const LangOptions &getLangOpts() const { return LangOpts; } 1007 const TargetInfo &getTargetInfo() const { return *Target; } 1008 const TargetInfo *getAuxTargetInfo() const { return AuxTarget; } 1009 FileManager &getFileManager() const { return FileMgr; } 1010 SourceManager &getSourceManager() const { return SourceMgr; } 1011 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; } 1012 1013 IdentifierTable &getIdentifierTable() { return Identifiers; } 1014 const IdentifierTable &getIdentifierTable() const { return Identifiers; } 1015 SelectorTable &getSelectorTable() { return Selectors; } 1016 Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; } 1017 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; } 1018 1019 void setExternalSource(ExternalPreprocessorSource *Source) { 1020 ExternalSource = Source; 1021 } 1022 1023 ExternalPreprocessorSource *getExternalSource() const { 1024 return ExternalSource; 1025 } 1026 1027 /// Retrieve the module loader associated with this preprocessor. 1028 ModuleLoader &getModuleLoader() const { return TheModuleLoader; } 1029 1030 bool hadModuleLoaderFatalFailure() const { 1031 return TheModuleLoader.HadFatalFailure; 1032 } 1033 1034 /// Retrieve the number of Directives that have been processed by the 1035 /// Preprocessor. 1036 unsigned getNumDirectives() const { 1037 return NumDirectives; 1038 } 1039 1040 /// True if we are currently preprocessing a #if or #elif directive 1041 bool isParsingIfOrElifDirective() const { 1042 return ParsingIfOrElifDirective; 1043 } 1044 1045 /// Control whether the preprocessor retains comments in output. 1046 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) { 1047 this->KeepComments = KeepComments | KeepMacroComments; 1048 this->KeepMacroComments = KeepMacroComments; 1049 } 1050 1051 bool getCommentRetentionState() const { return KeepComments; } 1052 1053 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; } 1054 bool getPragmasEnabled() const { return PragmasEnabled; } 1055 1056 void SetSuppressIncludeNotFoundError(bool Suppress) { 1057 SuppressIncludeNotFoundError = Suppress; 1058 } 1059 1060 bool GetSuppressIncludeNotFoundError() { 1061 return SuppressIncludeNotFoundError; 1062 } 1063 1064 /// Sets whether the preprocessor is responsible for producing output or if 1065 /// it is producing tokens to be consumed by Parse and Sema. 1066 void setPreprocessedOutput(bool IsPreprocessedOutput) { 1067 PreprocessedOutput = IsPreprocessedOutput; 1068 } 1069 1070 /// Returns true if the preprocessor is responsible for generating output, 1071 /// false if it is producing tokens to be consumed by Parse and Sema. 1072 bool isPreprocessedOutput() const { return PreprocessedOutput; } 1073 1074 /// Return true if we are lexing directly from the specified lexer. 1075 bool isCurrentLexer(const PreprocessorLexer *L) const { 1076 return CurPPLexer == L; 1077 } 1078 1079 /// Return the current lexer being lexed from. 1080 /// 1081 /// Note that this ignores any potentially active macro expansions and _Pragma 1082 /// expansions going on at the time. 1083 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; } 1084 1085 /// Return the current file lexer being lexed from. 1086 /// 1087 /// Note that this ignores any potentially active macro expansions and _Pragma 1088 /// expansions going on at the time. 1089 PreprocessorLexer *getCurrentFileLexer() const; 1090 1091 /// Return the submodule owning the file being lexed. This may not be 1092 /// the current module if we have changed modules since entering the file. 1093 Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; } 1094 1095 /// Returns the FileID for the preprocessor predefines. 1096 FileID getPredefinesFileID() const { return PredefinesFileID; } 1097 1098 /// \{ 1099 /// Accessors for preprocessor callbacks. 1100 /// 1101 /// Note that this class takes ownership of any PPCallbacks object given to 1102 /// it. 1103 PPCallbacks *getPPCallbacks() const { return Callbacks.get(); } 1104 void addPPCallbacks(std::unique_ptr<PPCallbacks> C) { 1105 if (Callbacks) 1106 C = std::make_unique<PPChainedCallbacks>(std::move(C), 1107 std::move(Callbacks)); 1108 Callbacks = std::move(C); 1109 } 1110 /// \} 1111 1112 /// Get the number of tokens processed so far. 1113 unsigned getTokenCount() const { return TokenCount; } 1114 1115 /// Get the max number of tokens before issuing a -Wmax-tokens warning. 1116 unsigned getMaxTokens() const { return MaxTokens; } 1117 1118 void overrideMaxTokens(unsigned Value, SourceLocation Loc) { 1119 MaxTokens = Value; 1120 MaxTokensOverrideLoc = Loc; 1121 }; 1122 1123 SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; } 1124 1125 /// Register a function that would be called on each token in the final 1126 /// expanded token stream. 1127 /// This also reports annotation tokens produced by the parser. 1128 void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) { 1129 OnToken = std::move(F); 1130 } 1131 1132 void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; } 1133 1134 bool isMacroDefined(StringRef Id) { 1135 return isMacroDefined(&Identifiers.get(Id)); 1136 } 1137 bool isMacroDefined(const IdentifierInfo *II) { 1138 return II->hasMacroDefinition() && 1139 (!getLangOpts().Modules || (bool)getMacroDefinition(II)); 1140 } 1141 1142 /// Determine whether II is defined as a macro within the module M, 1143 /// if that is a module that we've already preprocessed. Does not check for 1144 /// macros imported into M. 1145 bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) { 1146 if (!II->hasMacroDefinition()) 1147 return false; 1148 auto I = Submodules.find(M); 1149 if (I == Submodules.end()) 1150 return false; 1151 auto J = I->second.Macros.find(II); 1152 if (J == I->second.Macros.end()) 1153 return false; 1154 auto *MD = J->second.getLatest(); 1155 return MD && MD->isDefined(); 1156 } 1157 1158 MacroDefinition getMacroDefinition(const IdentifierInfo *II) { 1159 if (!II->hasMacroDefinition()) 1160 return {}; 1161 1162 MacroState &S = CurSubmoduleState->Macros[II]; 1163 auto *MD = S.getLatest(); 1164 while (MD && isa<VisibilityMacroDirective>(MD)) 1165 MD = MD->getPrevious(); 1166 return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD), 1167 S.getActiveModuleMacros(*this, II), 1168 S.isAmbiguous(*this, II)); 1169 } 1170 1171 MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, 1172 SourceLocation Loc) { 1173 if (!II->hadMacroDefinition()) 1174 return {}; 1175 1176 MacroState &S = CurSubmoduleState->Macros[II]; 1177 MacroDirective::DefInfo DI; 1178 if (auto *MD = S.getLatest()) 1179 DI = MD->findDirectiveAtLoc(Loc, getSourceManager()); 1180 // FIXME: Compute the set of active module macros at the specified location. 1181 return MacroDefinition(DI.getDirective(), 1182 S.getActiveModuleMacros(*this, II), 1183 S.isAmbiguous(*this, II)); 1184 } 1185 1186 /// Given an identifier, return its latest non-imported MacroDirective 1187 /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd. 1188 MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const { 1189 if (!II->hasMacroDefinition()) 1190 return nullptr; 1191 1192 auto *MD = getLocalMacroDirectiveHistory(II); 1193 if (!MD || MD->getDefinition().isUndefined()) 1194 return nullptr; 1195 1196 return MD; 1197 } 1198 1199 const MacroInfo *getMacroInfo(const IdentifierInfo *II) const { 1200 return const_cast<Preprocessor*>(this)->getMacroInfo(II); 1201 } 1202 1203 MacroInfo *getMacroInfo(const IdentifierInfo *II) { 1204 if (!II->hasMacroDefinition()) 1205 return nullptr; 1206 if (auto MD = getMacroDefinition(II)) 1207 return MD.getMacroInfo(); 1208 return nullptr; 1209 } 1210 1211 /// Given an identifier, return the latest non-imported macro 1212 /// directive for that identifier. 1213 /// 1214 /// One can iterate over all previous macro directives from the most recent 1215 /// one. 1216 MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const; 1217 1218 /// Add a directive to the macro directive history for this identifier. 1219 void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD); 1220 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, 1221 SourceLocation Loc) { 1222 DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc); 1223 appendMacroDirective(II, MD); 1224 return MD; 1225 } 1226 DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, 1227 MacroInfo *MI) { 1228 return appendDefMacroDirective(II, MI, MI->getDefinitionLoc()); 1229 } 1230 1231 /// Set a MacroDirective that was loaded from a PCH file. 1232 void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, 1233 MacroDirective *MD); 1234 1235 /// Register an exported macro for a module and identifier. 1236 ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, 1237 ArrayRef<ModuleMacro *> Overrides, bool &IsNew); 1238 ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II); 1239 1240 /// Get the list of leaf (non-overridden) module macros for a name. 1241 ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const { 1242 if (II->isOutOfDate()) 1243 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II)); 1244 auto I = LeafModuleMacros.find(II); 1245 if (I != LeafModuleMacros.end()) 1246 return I->second; 1247 return None; 1248 } 1249 1250 /// Get the list of submodules that we're currently building. 1251 ArrayRef<BuildingSubmoduleInfo> getBuildingSubmodules() const { 1252 return BuildingSubmoduleStack; 1253 } 1254 1255 /// \{ 1256 /// Iterators for the macro history table. Currently defined macros have 1257 /// IdentifierInfo::hasMacroDefinition() set and an empty 1258 /// MacroInfo::getUndefLoc() at the head of the list. 1259 using macro_iterator = MacroMap::const_iterator; 1260 1261 macro_iterator macro_begin(bool IncludeExternalMacros = true) const; 1262 macro_iterator macro_end(bool IncludeExternalMacros = true) const; 1263 1264 llvm::iterator_range<macro_iterator> 1265 macros(bool IncludeExternalMacros = true) const { 1266 macro_iterator begin = macro_begin(IncludeExternalMacros); 1267 macro_iterator end = macro_end(IncludeExternalMacros); 1268 return llvm::make_range(begin, end); 1269 } 1270 1271 /// \} 1272 1273 /// Mark the file as included. 1274 /// Returns true if this is the first time the file was included. 1275 bool markIncluded(const FileEntry *File) { 1276 HeaderInfo.getFileInfo(File); 1277 return IncludedFiles.insert(File).second; 1278 } 1279 1280 /// Return true if this header has already been included. 1281 bool alreadyIncluded(const FileEntry *File) const { 1282 return IncludedFiles.count(File); 1283 } 1284 1285 /// Get the set of included files. 1286 IncludedFilesSet &getIncludedFiles() { return IncludedFiles; } 1287 const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; } 1288 1289 /// Return the name of the macro defined before \p Loc that has 1290 /// spelling \p Tokens. If there are multiple macros with same spelling, 1291 /// return the last one defined. 1292 StringRef getLastMacroWithSpelling(SourceLocation Loc, 1293 ArrayRef<TokenValue> Tokens) const; 1294 1295 /// Set the predefines for this Preprocessor. 1296 /// 1297 /// These predefines are automatically injected when parsing the main file. 1298 void setPredefines(std::string P) { Predefines = std::move(P); } 1299 1300 /// Return information about the specified preprocessor 1301 /// identifier token. 1302 IdentifierInfo *getIdentifierInfo(StringRef Name) const { 1303 return &Identifiers.get(Name); 1304 } 1305 1306 /// Add the specified pragma handler to this preprocessor. 1307 /// 1308 /// If \p Namespace is non-null, then it is a token required to exist on the 1309 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC". 1310 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler); 1311 void AddPragmaHandler(PragmaHandler *Handler) { 1312 AddPragmaHandler(StringRef(), Handler); 1313 } 1314 1315 /// Remove the specific pragma handler from this preprocessor. 1316 /// 1317 /// If \p Namespace is non-null, then it should be the namespace that 1318 /// \p Handler was added to. It is an error to remove a handler that 1319 /// has not been registered. 1320 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler); 1321 void RemovePragmaHandler(PragmaHandler *Handler) { 1322 RemovePragmaHandler(StringRef(), Handler); 1323 } 1324 1325 /// Install empty handlers for all pragmas (making them ignored). 1326 void IgnorePragmas(); 1327 1328 /// Set empty line handler. 1329 void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; } 1330 1331 EmptylineHandler *getEmptylineHandler() const { return Emptyline; } 1332 1333 /// Add the specified comment handler to the preprocessor. 1334 void addCommentHandler(CommentHandler *Handler); 1335 1336 /// Remove the specified comment handler. 1337 /// 1338 /// It is an error to remove a handler that has not been registered. 1339 void removeCommentHandler(CommentHandler *Handler); 1340 1341 /// Set the code completion handler to the given object. 1342 void setCodeCompletionHandler(CodeCompletionHandler &Handler) { 1343 CodeComplete = &Handler; 1344 } 1345 1346 /// Retrieve the current code-completion handler. 1347 CodeCompletionHandler *getCodeCompletionHandler() const { 1348 return CodeComplete; 1349 } 1350 1351 /// Clear out the code completion handler. 1352 void clearCodeCompletionHandler() { 1353 CodeComplete = nullptr; 1354 } 1355 1356 /// Hook used by the lexer to invoke the "included file" code 1357 /// completion point. 1358 void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled); 1359 1360 /// Hook used by the lexer to invoke the "natural language" code 1361 /// completion point. 1362 void CodeCompleteNaturalLanguage(); 1363 1364 /// Set the code completion token for filtering purposes. 1365 void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) { 1366 CodeCompletionII = Filter; 1367 } 1368 1369 /// Set the code completion token range for detecting replacement range later 1370 /// on. 1371 void setCodeCompletionTokenRange(const SourceLocation Start, 1372 const SourceLocation End) { 1373 CodeCompletionTokenRange = {Start, End}; 1374 } 1375 SourceRange getCodeCompletionTokenRange() const { 1376 return CodeCompletionTokenRange; 1377 } 1378 1379 /// Get the code completion token for filtering purposes. 1380 StringRef getCodeCompletionFilter() { 1381 if (CodeCompletionII) 1382 return CodeCompletionII->getName(); 1383 return {}; 1384 } 1385 1386 /// Retrieve the preprocessing record, or NULL if there is no 1387 /// preprocessing record. 1388 PreprocessingRecord *getPreprocessingRecord() const { return Record; } 1389 1390 /// Create a new preprocessing record, which will keep track of 1391 /// all macro expansions, macro definitions, etc. 1392 void createPreprocessingRecord(); 1393 1394 /// Returns true if the FileEntry is the PCH through header. 1395 bool isPCHThroughHeader(const FileEntry *FE); 1396 1397 /// True if creating a PCH with a through header. 1398 bool creatingPCHWithThroughHeader(); 1399 1400 /// True if using a PCH with a through header. 1401 bool usingPCHWithThroughHeader(); 1402 1403 /// True if creating a PCH with a #pragma hdrstop. 1404 bool creatingPCHWithPragmaHdrStop(); 1405 1406 /// True if using a PCH with a #pragma hdrstop. 1407 bool usingPCHWithPragmaHdrStop(); 1408 1409 /// Skip tokens until after the #include of the through header or 1410 /// until after a #pragma hdrstop. 1411 void SkipTokensWhileUsingPCH(); 1412 1413 /// Process directives while skipping until the through header or 1414 /// #pragma hdrstop is found. 1415 void HandleSkippedDirectiveWhileUsingPCH(Token &Result, 1416 SourceLocation HashLoc); 1417 1418 /// Enter the specified FileID as the main source file, 1419 /// which implicitly adds the builtin defines etc. 1420 void EnterMainSourceFile(); 1421 1422 /// Inform the preprocessor callbacks that processing is complete. 1423 void EndSourceFile(); 1424 1425 /// Add a source file to the top of the include stack and 1426 /// start lexing tokens from it instead of the current buffer. 1427 /// 1428 /// Emits a diagnostic, doesn't enter the file, and returns true on error. 1429 bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, 1430 SourceLocation Loc, bool IsFirstIncludeOfFile = true); 1431 1432 /// Add a Macro to the top of the include stack and start lexing 1433 /// tokens from it instead of the current buffer. 1434 /// 1435 /// \param Args specifies the tokens input to a function-like macro. 1436 /// \param ILEnd specifies the location of the ')' for a function-like macro 1437 /// or the identifier for an object-like macro. 1438 void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, 1439 MacroArgs *Args); 1440 1441 private: 1442 /// Add a "macro" context to the top of the include stack, 1443 /// which will cause the lexer to start returning the specified tokens. 1444 /// 1445 /// If \p DisableMacroExpansion is true, tokens lexed from the token stream 1446 /// will not be subject to further macro expansion. Otherwise, these tokens 1447 /// will be re-macro-expanded when/if expansion is enabled. 1448 /// 1449 /// If \p OwnsTokens is false, this method assumes that the specified stream 1450 /// of tokens has a permanent owner somewhere, so they do not need to be 1451 /// copied. If it is true, it assumes the array of tokens is allocated with 1452 /// \c new[] and the Preprocessor will delete[] it. 1453 /// 1454 /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag 1455 /// set, see the flag documentation for details. 1456 void EnterTokenStream(const Token *Toks, unsigned NumToks, 1457 bool DisableMacroExpansion, bool OwnsTokens, 1458 bool IsReinject); 1459 1460 public: 1461 void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks, 1462 bool DisableMacroExpansion, bool IsReinject) { 1463 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true, 1464 IsReinject); 1465 } 1466 1467 void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion, 1468 bool IsReinject) { 1469 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false, 1470 IsReinject); 1471 } 1472 1473 /// Pop the current lexer/macro exp off the top of the lexer stack. 1474 /// 1475 /// This should only be used in situations where the current state of the 1476 /// top-of-stack lexer is known. 1477 void RemoveTopOfLexerStack(); 1478 1479 /// From the point that this method is called, and until 1480 /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor 1481 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will 1482 /// make the Preprocessor re-lex the same tokens. 1483 /// 1484 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can 1485 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will 1486 /// be combined with the EnableBacktrackAtThisPos calls in reverse order. 1487 /// 1488 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack 1489 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of 1490 /// tokens will continue indefinitely. 1491 /// 1492 void EnableBacktrackAtThisPos(); 1493 1494 /// Disable the last EnableBacktrackAtThisPos call. 1495 void CommitBacktrackedTokens(); 1496 1497 /// Make Preprocessor re-lex the tokens that were lexed since 1498 /// EnableBacktrackAtThisPos() was previously called. 1499 void Backtrack(); 1500 1501 /// True if EnableBacktrackAtThisPos() was called and 1502 /// caching of tokens is on. 1503 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); } 1504 1505 /// Lex the next token for this preprocessor. 1506 void Lex(Token &Result); 1507 1508 /// Lex a token, forming a header-name token if possible. 1509 bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true); 1510 1511 bool LexAfterModuleImport(Token &Result); 1512 void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks); 1513 1514 void makeModuleVisible(Module *M, SourceLocation Loc); 1515 1516 SourceLocation getModuleImportLoc(Module *M) const { 1517 return CurSubmoduleState->VisibleModules.getImportLoc(M); 1518 } 1519 1520 /// Lex a string literal, which may be the concatenation of multiple 1521 /// string literals and may even come from macro expansion. 1522 /// \returns true on success, false if a error diagnostic has been generated. 1523 bool LexStringLiteral(Token &Result, std::string &String, 1524 const char *DiagnosticTag, bool AllowMacroExpansion) { 1525 if (AllowMacroExpansion) 1526 Lex(Result); 1527 else 1528 LexUnexpandedToken(Result); 1529 return FinishLexStringLiteral(Result, String, DiagnosticTag, 1530 AllowMacroExpansion); 1531 } 1532 1533 /// Complete the lexing of a string literal where the first token has 1534 /// already been lexed (see LexStringLiteral). 1535 bool FinishLexStringLiteral(Token &Result, std::string &String, 1536 const char *DiagnosticTag, 1537 bool AllowMacroExpansion); 1538 1539 /// Lex a token. If it's a comment, keep lexing until we get 1540 /// something not a comment. 1541 /// 1542 /// This is useful in -E -C mode where comments would foul up preprocessor 1543 /// directive handling. 1544 void LexNonComment(Token &Result) { 1545 do 1546 Lex(Result); 1547 while (Result.getKind() == tok::comment); 1548 } 1549 1550 /// Just like Lex, but disables macro expansion of identifier tokens. 1551 void LexUnexpandedToken(Token &Result) { 1552 // Disable macro expansion. 1553 bool OldVal = DisableMacroExpansion; 1554 DisableMacroExpansion = true; 1555 // Lex the token. 1556 Lex(Result); 1557 1558 // Reenable it. 1559 DisableMacroExpansion = OldVal; 1560 } 1561 1562 /// Like LexNonComment, but this disables macro expansion of 1563 /// identifier tokens. 1564 void LexUnexpandedNonComment(Token &Result) { 1565 do 1566 LexUnexpandedToken(Result); 1567 while (Result.getKind() == tok::comment); 1568 } 1569 1570 /// Parses a simple integer literal to get its numeric value. Floating 1571 /// point literals and user defined literals are rejected. Used primarily to 1572 /// handle pragmas that accept integer arguments. 1573 bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value); 1574 1575 /// Disables macro expansion everywhere except for preprocessor directives. 1576 void SetMacroExpansionOnlyInDirectives() { 1577 DisableMacroExpansion = true; 1578 MacroExpansionInDirectivesOverride = true; 1579 } 1580 1581 /// Peeks ahead N tokens and returns that token without consuming any 1582 /// tokens. 1583 /// 1584 /// LookAhead(0) returns the next token that would be returned by Lex(), 1585 /// LookAhead(1) returns the token after it, etc. This returns normal 1586 /// tokens after phase 5. As such, it is equivalent to using 1587 /// 'Lex', not 'LexUnexpandedToken'. 1588 const Token &LookAhead(unsigned N) { 1589 assert(LexLevel == 0 && "cannot use lookahead while lexing"); 1590 if (CachedLexPos + N < CachedTokens.size()) 1591 return CachedTokens[CachedLexPos+N]; 1592 else 1593 return PeekAhead(N+1); 1594 } 1595 1596 /// When backtracking is enabled and tokens are cached, 1597 /// this allows to revert a specific number of tokens. 1598 /// 1599 /// Note that the number of tokens being reverted should be up to the last 1600 /// backtrack position, not more. 1601 void RevertCachedTokens(unsigned N) { 1602 assert(isBacktrackEnabled() && 1603 "Should only be called when tokens are cached for backtracking"); 1604 assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back()) 1605 && "Should revert tokens up to the last backtrack position, not more"); 1606 assert(signed(CachedLexPos) - signed(N) >= 0 && 1607 "Corrupted backtrack positions ?"); 1608 CachedLexPos -= N; 1609 } 1610 1611 /// Enters a token in the token stream to be lexed next. 1612 /// 1613 /// If BackTrack() is called afterwards, the token will remain at the 1614 /// insertion point. 1615 /// If \p IsReinject is true, resulting token will have Token::IsReinjected 1616 /// flag set. See the flag documentation for details. 1617 void EnterToken(const Token &Tok, bool IsReinject) { 1618 if (LexLevel) { 1619 // It's not correct in general to enter caching lex mode while in the 1620 // middle of a nested lexing action. 1621 auto TokCopy = std::make_unique<Token[]>(1); 1622 TokCopy[0] = Tok; 1623 EnterTokenStream(std::move(TokCopy), 1, true, IsReinject); 1624 } else { 1625 EnterCachingLexMode(); 1626 assert(IsReinject && "new tokens in the middle of cached stream"); 1627 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); 1628 } 1629 } 1630 1631 /// We notify the Preprocessor that if it is caching tokens (because 1632 /// backtrack is enabled) it should replace the most recent cached tokens 1633 /// with the given annotation token. This function has no effect if 1634 /// backtracking is not enabled. 1635 /// 1636 /// Note that the use of this function is just for optimization, so that the 1637 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is 1638 /// invoked. 1639 void AnnotateCachedTokens(const Token &Tok) { 1640 assert(Tok.isAnnotation() && "Expected annotation token"); 1641 if (CachedLexPos != 0 && isBacktrackEnabled()) 1642 AnnotatePreviousCachedTokens(Tok); 1643 } 1644 1645 /// Get the location of the last cached token, suitable for setting the end 1646 /// location of an annotation token. 1647 SourceLocation getLastCachedTokenLocation() const { 1648 assert(CachedLexPos != 0); 1649 return CachedTokens[CachedLexPos-1].getLastLoc(); 1650 } 1651 1652 /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in 1653 /// CachedTokens. 1654 bool IsPreviousCachedToken(const Token &Tok) const; 1655 1656 /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens 1657 /// in \p NewToks. 1658 /// 1659 /// Useful when a token needs to be split in smaller ones and CachedTokens 1660 /// most recent token must to be updated to reflect that. 1661 void ReplacePreviousCachedToken(ArrayRef<Token> NewToks); 1662 1663 /// Replace the last token with an annotation token. 1664 /// 1665 /// Like AnnotateCachedTokens(), this routine replaces an 1666 /// already-parsed (and resolved) token with an annotation 1667 /// token. However, this routine only replaces the last token with 1668 /// the annotation token; it does not affect any other cached 1669 /// tokens. This function has no effect if backtracking is not 1670 /// enabled. 1671 void ReplaceLastTokenWithAnnotation(const Token &Tok) { 1672 assert(Tok.isAnnotation() && "Expected annotation token"); 1673 if (CachedLexPos != 0 && isBacktrackEnabled()) 1674 CachedTokens[CachedLexPos-1] = Tok; 1675 } 1676 1677 /// Enter an annotation token into the token stream. 1678 void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, 1679 void *AnnotationVal); 1680 1681 /// Determine whether it's possible for a future call to Lex to produce an 1682 /// annotation token created by a previous call to EnterAnnotationToken. 1683 bool mightHavePendingAnnotationTokens() { 1684 return CurLexerKind != CLK_Lexer; 1685 } 1686 1687 /// Update the current token to represent the provided 1688 /// identifier, in order to cache an action performed by typo correction. 1689 void TypoCorrectToken(const Token &Tok) { 1690 assert(Tok.getIdentifierInfo() && "Expected identifier token"); 1691 if (CachedLexPos != 0 && isBacktrackEnabled()) 1692 CachedTokens[CachedLexPos-1] = Tok; 1693 } 1694 1695 /// Recompute the current lexer kind based on the CurLexer/ 1696 /// CurTokenLexer pointers. 1697 void recomputeCurLexerKind(); 1698 1699 /// Returns true if incremental processing is enabled 1700 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; } 1701 1702 /// Enables the incremental processing 1703 void enableIncrementalProcessing(bool value = true) { 1704 IncrementalProcessing = value; 1705 } 1706 1707 /// Specify the point at which code-completion will be performed. 1708 /// 1709 /// \param File the file in which code completion should occur. If 1710 /// this file is included multiple times, code-completion will 1711 /// perform completion the first time it is included. If NULL, this 1712 /// function clears out the code-completion point. 1713 /// 1714 /// \param Line the line at which code completion should occur 1715 /// (1-based). 1716 /// 1717 /// \param Column the column at which code completion should occur 1718 /// (1-based). 1719 /// 1720 /// \returns true if an error occurred, false otherwise. 1721 bool SetCodeCompletionPoint(const FileEntry *File, 1722 unsigned Line, unsigned Column); 1723 1724 /// Determine if we are performing code completion. 1725 bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; } 1726 1727 /// Returns the location of the code-completion point. 1728 /// 1729 /// Returns an invalid location if code-completion is not enabled or the file 1730 /// containing the code-completion point has not been lexed yet. 1731 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; } 1732 1733 /// Returns the start location of the file of code-completion point. 1734 /// 1735 /// Returns an invalid location if code-completion is not enabled or the file 1736 /// containing the code-completion point has not been lexed yet. 1737 SourceLocation getCodeCompletionFileLoc() const { 1738 return CodeCompletionFileLoc; 1739 } 1740 1741 /// Returns true if code-completion is enabled and we have hit the 1742 /// code-completion point. 1743 bool isCodeCompletionReached() const { return CodeCompletionReached; } 1744 1745 /// Note that we hit the code-completion point. 1746 void setCodeCompletionReached() { 1747 assert(isCodeCompletionEnabled() && "Code-completion not enabled!"); 1748 CodeCompletionReached = true; 1749 // Silence any diagnostics that occur after we hit the code-completion. 1750 getDiagnostics().setSuppressAllDiagnostics(true); 1751 } 1752 1753 /// The location of the currently-active \#pragma clang 1754 /// arc_cf_code_audited begin. 1755 /// 1756 /// Returns an invalid location if there is no such pragma active. 1757 std::pair<IdentifierInfo *, SourceLocation> 1758 getPragmaARCCFCodeAuditedInfo() const { 1759 return PragmaARCCFCodeAuditedInfo; 1760 } 1761 1762 /// Set the location of the currently-active \#pragma clang 1763 /// arc_cf_code_audited begin. An invalid location ends the pragma. 1764 void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, 1765 SourceLocation Loc) { 1766 PragmaARCCFCodeAuditedInfo = {Ident, Loc}; 1767 } 1768 1769 /// The location of the currently-active \#pragma clang 1770 /// assume_nonnull begin. 1771 /// 1772 /// Returns an invalid location if there is no such pragma active. 1773 SourceLocation getPragmaAssumeNonNullLoc() const { 1774 return PragmaAssumeNonNullLoc; 1775 } 1776 1777 /// Set the location of the currently-active \#pragma clang 1778 /// assume_nonnull begin. An invalid location ends the pragma. 1779 void setPragmaAssumeNonNullLoc(SourceLocation Loc) { 1780 PragmaAssumeNonNullLoc = Loc; 1781 } 1782 1783 /// Get the location of the recorded unterminated \#pragma clang 1784 /// assume_nonnull begin in the preamble, if one exists. 1785 /// 1786 /// Returns an invalid location if the premable did not end with 1787 /// such a pragma active or if there is no recorded preamble. 1788 SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const { 1789 return PreambleRecordedPragmaAssumeNonNullLoc; 1790 } 1791 1792 /// Record the location of the unterminated \#pragma clang 1793 /// assume_nonnull begin in the preamble. 1794 void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc) { 1795 PreambleRecordedPragmaAssumeNonNullLoc = Loc; 1796 } 1797 1798 /// Set the directory in which the main file should be considered 1799 /// to have been found, if it is not a real file. 1800 void setMainFileDir(const DirectoryEntry *Dir) { 1801 MainFileDir = Dir; 1802 } 1803 1804 /// Instruct the preprocessor to skip part of the main source file. 1805 /// 1806 /// \param Bytes The number of bytes in the preamble to skip. 1807 /// 1808 /// \param StartOfLine Whether skipping these bytes puts the lexer at the 1809 /// start of a line. 1810 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) { 1811 SkipMainFilePreamble.first = Bytes; 1812 SkipMainFilePreamble.second = StartOfLine; 1813 } 1814 1815 /// Forwarding function for diagnostics. This emits a diagnostic at 1816 /// the specified Token's location, translating the token's start 1817 /// position in the current buffer into a SourcePosition object for rendering. 1818 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const { 1819 return Diags->Report(Loc, DiagID); 1820 } 1821 1822 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const { 1823 return Diags->Report(Tok.getLocation(), DiagID); 1824 } 1825 1826 /// Return the 'spelling' of the token at the given 1827 /// location; does not go up to the spelling location or down to the 1828 /// expansion location. 1829 /// 1830 /// \param buffer A buffer which will be used only if the token requires 1831 /// "cleaning", e.g. if it contains trigraphs or escaped newlines 1832 /// \param invalid If non-null, will be set \c true if an error occurs. 1833 StringRef getSpelling(SourceLocation loc, 1834 SmallVectorImpl<char> &buffer, 1835 bool *invalid = nullptr) const { 1836 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid); 1837 } 1838 1839 /// Return the 'spelling' of the Tok token. 1840 /// 1841 /// The spelling of a token is the characters used to represent the token in 1842 /// the source file after trigraph expansion and escaped-newline folding. In 1843 /// particular, this wants to get the true, uncanonicalized, spelling of 1844 /// things like digraphs, UCNs, etc. 1845 /// 1846 /// \param Invalid If non-null, will be set \c true if an error occurs. 1847 std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const { 1848 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid); 1849 } 1850 1851 /// Get the spelling of a token into a preallocated buffer, instead 1852 /// of as an std::string. 1853 /// 1854 /// The caller is required to allocate enough space for the token, which is 1855 /// guaranteed to be at least Tok.getLength() bytes long. The length of the 1856 /// actual result is returned. 1857 /// 1858 /// Note that this method may do two possible things: it may either fill in 1859 /// the buffer specified with characters, or it may *change the input pointer* 1860 /// to point to a constant buffer with the data already in it (avoiding a 1861 /// copy). The caller is not allowed to modify the returned buffer pointer 1862 /// if an internal buffer is returned. 1863 unsigned getSpelling(const Token &Tok, const char *&Buffer, 1864 bool *Invalid = nullptr) const { 1865 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid); 1866 } 1867 1868 /// Get the spelling of a token into a SmallVector. 1869 /// 1870 /// Note that the returned StringRef may not point to the 1871 /// supplied buffer if a copy can be avoided. 1872 StringRef getSpelling(const Token &Tok, 1873 SmallVectorImpl<char> &Buffer, 1874 bool *Invalid = nullptr) const; 1875 1876 /// Relex the token at the specified location. 1877 /// \returns true if there was a failure, false on success. 1878 bool getRawToken(SourceLocation Loc, Token &Result, 1879 bool IgnoreWhiteSpace = false) { 1880 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace); 1881 } 1882 1883 /// Given a Token \p Tok that is a numeric constant with length 1, 1884 /// return the character. 1885 char 1886 getSpellingOfSingleCharacterNumericConstant(const Token &Tok, 1887 bool *Invalid = nullptr) const { 1888 assert(Tok.is(tok::numeric_constant) && 1889 Tok.getLength() == 1 && "Called on unsupported token"); 1890 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1"); 1891 1892 // If the token is carrying a literal data pointer, just use it. 1893 if (const char *D = Tok.getLiteralData()) 1894 return *D; 1895 1896 // Otherwise, fall back on getCharacterData, which is slower, but always 1897 // works. 1898 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid); 1899 } 1900 1901 /// Retrieve the name of the immediate macro expansion. 1902 /// 1903 /// This routine starts from a source location, and finds the name of the 1904 /// macro responsible for its immediate expansion. It looks through any 1905 /// intervening macro argument expansions to compute this. It returns a 1906 /// StringRef that refers to the SourceManager-owned buffer of the source 1907 /// where that macro name is spelled. Thus, the result shouldn't out-live 1908 /// the SourceManager. 1909 StringRef getImmediateMacroName(SourceLocation Loc) { 1910 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts()); 1911 } 1912 1913 /// Plop the specified string into a scratch buffer and set the 1914 /// specified token's location and length to it. 1915 /// 1916 /// If specified, the source location provides a location of the expansion 1917 /// point of the token. 1918 void CreateString(StringRef Str, Token &Tok, 1919 SourceLocation ExpansionLocStart = SourceLocation(), 1920 SourceLocation ExpansionLocEnd = SourceLocation()); 1921 1922 /// Split the first Length characters out of the token starting at TokLoc 1923 /// and return a location pointing to the split token. Re-lexing from the 1924 /// split token will return the split token rather than the original. 1925 SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length); 1926 1927 /// Computes the source location just past the end of the 1928 /// token at this source location. 1929 /// 1930 /// This routine can be used to produce a source location that 1931 /// points just past the end of the token referenced by \p Loc, and 1932 /// is generally used when a diagnostic needs to point just after a 1933 /// token where it expected something different that it received. If 1934 /// the returned source location would not be meaningful (e.g., if 1935 /// it points into a macro), this routine returns an invalid 1936 /// source location. 1937 /// 1938 /// \param Offset an offset from the end of the token, where the source 1939 /// location should refer to. The default offset (0) produces a source 1940 /// location pointing just past the end of the token; an offset of 1 produces 1941 /// a source location pointing to the last character in the token, etc. 1942 SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) { 1943 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts); 1944 } 1945 1946 /// Returns true if the given MacroID location points at the first 1947 /// token of the macro expansion. 1948 /// 1949 /// \param MacroBegin If non-null and function returns true, it is set to 1950 /// begin location of the macro. 1951 bool isAtStartOfMacroExpansion(SourceLocation loc, 1952 SourceLocation *MacroBegin = nullptr) const { 1953 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts, 1954 MacroBegin); 1955 } 1956 1957 /// Returns true if the given MacroID location points at the last 1958 /// token of the macro expansion. 1959 /// 1960 /// \param MacroEnd If non-null and function returns true, it is set to 1961 /// end location of the macro. 1962 bool isAtEndOfMacroExpansion(SourceLocation loc, 1963 SourceLocation *MacroEnd = nullptr) const { 1964 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd); 1965 } 1966 1967 /// Print the token to stderr, used for debugging. 1968 void DumpToken(const Token &Tok, bool DumpFlags = false) const; 1969 void DumpLocation(SourceLocation Loc) const; 1970 void DumpMacro(const MacroInfo &MI) const; 1971 void dumpMacroInfo(const IdentifierInfo *II); 1972 1973 /// Given a location that specifies the start of a 1974 /// token, return a new location that specifies a character within the token. 1975 SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, 1976 unsigned Char) const { 1977 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts); 1978 } 1979 1980 /// Increment the counters for the number of token paste operations 1981 /// performed. 1982 /// 1983 /// If fast was specified, this is a 'fast paste' case we handled. 1984 void IncrementPasteCounter(bool isFast) { 1985 if (isFast) 1986 ++NumFastTokenPaste; 1987 else 1988 ++NumTokenPaste; 1989 } 1990 1991 void PrintStats(); 1992 1993 size_t getTotalMemory() const; 1994 1995 /// When the macro expander pastes together a comment (/##/) in Microsoft 1996 /// mode, this method handles updating the current state, returning the 1997 /// token on the next source line. 1998 void HandleMicrosoftCommentPaste(Token &Tok); 1999 2000 //===--------------------------------------------------------------------===// 2001 // Preprocessor callback methods. These are invoked by a lexer as various 2002 // directives and events are found. 2003 2004 /// Given a tok::raw_identifier token, look up the 2005 /// identifier information for the token and install it into the token, 2006 /// updating the token kind accordingly. 2007 IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const; 2008 2009 private: 2010 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons; 2011 2012 public: 2013 /// Specifies the reason for poisoning an identifier. 2014 /// 2015 /// If that identifier is accessed while poisoned, then this reason will be 2016 /// used instead of the default "poisoned" diagnostic. 2017 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID); 2018 2019 /// Display reason for poisoned identifier. 2020 void HandlePoisonedIdentifier(Token & Identifier); 2021 2022 void MaybeHandlePoisonedIdentifier(Token & Identifier) { 2023 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) { 2024 if(II->isPoisoned()) { 2025 HandlePoisonedIdentifier(Identifier); 2026 } 2027 } 2028 } 2029 2030 private: 2031 /// Identifiers used for SEH handling in Borland. These are only 2032 /// allowed in particular circumstances 2033 // __except block 2034 IdentifierInfo *Ident__exception_code, 2035 *Ident___exception_code, 2036 *Ident_GetExceptionCode; 2037 // __except filter expression 2038 IdentifierInfo *Ident__exception_info, 2039 *Ident___exception_info, 2040 *Ident_GetExceptionInfo; 2041 // __finally 2042 IdentifierInfo *Ident__abnormal_termination, 2043 *Ident___abnormal_termination, 2044 *Ident_AbnormalTermination; 2045 2046 const char *getCurLexerEndPos(); 2047 void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod); 2048 2049 public: 2050 void PoisonSEHIdentifiers(bool Poison = true); // Borland 2051 2052 /// Callback invoked when the lexer reads an identifier and has 2053 /// filled in the tokens IdentifierInfo member. 2054 /// 2055 /// This callback potentially macro expands it or turns it into a named 2056 /// token (like 'for'). 2057 /// 2058 /// \returns true if we actually computed a token, false if we need to 2059 /// lex again. 2060 bool HandleIdentifier(Token &Identifier); 2061 2062 /// Callback invoked when the lexer hits the end of the current file. 2063 /// 2064 /// This either returns the EOF token and returns true, or 2065 /// pops a level off the include stack and returns false, at which point the 2066 /// client should call lex again. 2067 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false); 2068 2069 /// Callback invoked when the current TokenLexer hits the end of its 2070 /// token stream. 2071 bool HandleEndOfTokenLexer(Token &Result); 2072 2073 /// Callback invoked when the lexer sees a # token at the start of a 2074 /// line. 2075 /// 2076 /// This consumes the directive, modifies the lexer/preprocessor state, and 2077 /// advances the lexer(s) so that the next token read is the correct one. 2078 void HandleDirective(Token &Result); 2079 2080 /// Ensure that the next token is a tok::eod token. 2081 /// 2082 /// If not, emit a diagnostic and consume up until the eod. 2083 /// If \p EnableMacros is true, then we consider macros that expand to zero 2084 /// tokens as being ok. 2085 /// 2086 /// \return The location of the end of the directive (the terminating 2087 /// newline). 2088 SourceLocation CheckEndOfDirective(const char *DirType, 2089 bool EnableMacros = false); 2090 2091 /// Read and discard all tokens remaining on the current line until 2092 /// the tok::eod token is found. Returns the range of the skipped tokens. 2093 SourceRange DiscardUntilEndOfDirective(); 2094 2095 /// Returns true if the preprocessor has seen a use of 2096 /// __DATE__ or __TIME__ in the file so far. 2097 bool SawDateOrTime() const { 2098 return DATELoc != SourceLocation() || TIMELoc != SourceLocation(); 2099 } 2100 unsigned getCounterValue() const { return CounterValue; } 2101 void setCounterValue(unsigned V) { CounterValue = V; } 2102 2103 LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const { 2104 assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine && 2105 "FPEvalMethod should be set either from command line or from the " 2106 "target info"); 2107 return CurrentFPEvalMethod; 2108 } 2109 2110 LangOptions::FPEvalMethodKind getTUFPEvalMethod() const { 2111 return TUFPEvalMethod; 2112 } 2113 2114 SourceLocation getLastFPEvalPragmaLocation() const { 2115 return LastFPEvalPragmaLocation; 2116 } 2117 2118 LangOptions::FPEvalMethodKind getLastFPEvalMethod() const { 2119 return LastFPEvalMethod; 2120 } 2121 2122 void setLastFPEvalMethod(LangOptions::FPEvalMethodKind Val) { 2123 LastFPEvalMethod = Val; 2124 } 2125 2126 void setCurrentFPEvalMethod(SourceLocation PragmaLoc, 2127 LangOptions::FPEvalMethodKind Val) { 2128 assert(Val != LangOptions::FEM_UnsetOnCommandLine && 2129 "FPEvalMethod should never be set to FEM_UnsetOnCommandLine"); 2130 // This is the location of the '#pragma float_control" where the 2131 // execution state is modifed. 2132 LastFPEvalPragmaLocation = PragmaLoc; 2133 CurrentFPEvalMethod = Val; 2134 TUFPEvalMethod = Val; 2135 } 2136 2137 void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val) { 2138 assert(Val != LangOptions::FEM_UnsetOnCommandLine && 2139 "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine"); 2140 TUFPEvalMethod = Val; 2141 } 2142 2143 /// Retrieves the module that we're currently building, if any. 2144 Module *getCurrentModule(); 2145 2146 /// Allocate a new MacroInfo object with the provided SourceLocation. 2147 MacroInfo *AllocateMacroInfo(SourceLocation L); 2148 2149 /// Turn the specified lexer token into a fully checked and spelled 2150 /// filename, e.g. as an operand of \#include. 2151 /// 2152 /// The caller is expected to provide a buffer that is large enough to hold 2153 /// the spelling of the filename, but is also expected to handle the case 2154 /// when this method decides to use a different buffer. 2155 /// 2156 /// \returns true if the input filename was in <>'s or false if it was 2157 /// in ""'s. 2158 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer); 2159 2160 /// Given a "foo" or \<foo> reference, look up the indicated file. 2161 /// 2162 /// Returns None on failure. \p isAngled indicates whether the file 2163 /// reference is for system \#include's or not (i.e. using <> instead of ""). 2164 Optional<FileEntryRef> 2165 LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, 2166 ConstSearchDirIterator FromDir, const FileEntry *FromFile, 2167 ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath, 2168 SmallVectorImpl<char> *RelativePath, 2169 ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, 2170 bool *IsFrameworkFound, bool SkipCache = false); 2171 2172 /// Return true if we're in the top-level file, not in a \#include. 2173 bool isInPrimaryFile() const; 2174 2175 /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is 2176 /// followed by EOD. Return true if the token is not a valid on-off-switch. 2177 bool LexOnOffSwitch(tok::OnOffSwitch &Result); 2178 2179 bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, 2180 bool *ShadowFlag = nullptr); 2181 2182 void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma); 2183 Module *LeaveSubmodule(bool ForPragma); 2184 2185 private: 2186 friend void TokenLexer::ExpandFunctionArguments(); 2187 2188 void PushIncludeMacroStack() { 2189 assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer"); 2190 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule, 2191 std::move(CurLexer), CurPPLexer, 2192 std::move(CurTokenLexer), CurDirLookup); 2193 CurPPLexer = nullptr; 2194 } 2195 2196 void PopIncludeMacroStack() { 2197 CurLexer = std::move(IncludeMacroStack.back().TheLexer); 2198 CurPPLexer = IncludeMacroStack.back().ThePPLexer; 2199 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); 2200 CurDirLookup = IncludeMacroStack.back().TheDirLookup; 2201 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule; 2202 CurLexerKind = IncludeMacroStack.back().CurLexerKind; 2203 IncludeMacroStack.pop_back(); 2204 } 2205 2206 void PropagateLineStartLeadingSpaceInfo(Token &Result); 2207 2208 /// Determine whether we need to create module macros for #defines in the 2209 /// current context. 2210 bool needModuleMacros() const; 2211 2212 /// Update the set of active module macros and ambiguity flag for a module 2213 /// macro name. 2214 void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info); 2215 2216 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI, 2217 SourceLocation Loc); 2218 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc); 2219 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc, 2220 bool isPublic); 2221 2222 /// Lex and validate a macro name, which occurs after a 2223 /// \#define or \#undef. 2224 /// 2225 /// \param MacroNameTok Token that represents the name defined or undefined. 2226 /// \param IsDefineUndef Kind if preprocessor directive. 2227 /// \param ShadowFlag Points to flag that is set if macro name shadows 2228 /// a keyword. 2229 /// 2230 /// This emits a diagnostic, sets the token kind to eod, 2231 /// and discards the rest of the macro line if the macro name is invalid. 2232 void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other, 2233 bool *ShadowFlag = nullptr); 2234 2235 /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the 2236 /// entire line) of the macro's tokens and adds them to MacroInfo, and while 2237 /// doing so performs certain validity checks including (but not limited to): 2238 /// - # (stringization) is followed by a macro parameter 2239 /// \param MacroNameTok - Token that represents the macro name 2240 /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard 2241 /// 2242 /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and 2243 /// returns a nullptr if an invalid sequence of tokens is encountered. 2244 MacroInfo *ReadOptionalMacroParameterListAndBody( 2245 const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard); 2246 2247 /// The ( starting an argument list of a macro definition has just been read. 2248 /// Lex the rest of the parameters and the closing ), updating \p MI with 2249 /// what we learn and saving in \p LastTok the last token read. 2250 /// Return true if an error occurs parsing the arg list. 2251 bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok); 2252 2253 /// Provide a suggestion for a typoed directive. If there is no typo, then 2254 /// just skip suggesting. 2255 /// 2256 /// \param Tok - Token that represents the directive 2257 /// \param Directive - String reference for the directive name 2258 void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const; 2259 2260 /// We just read a \#if or related directive and decided that the 2261 /// subsequent tokens are in the \#if'd out portion of the 2262 /// file. Lex the rest of the file, until we see an \#endif. If \p 2263 /// FoundNonSkipPortion is true, then we have already emitted code for part of 2264 /// this \#if directive, so \#else/\#elif blocks should never be entered. If 2265 /// \p FoundElse is false, then \#else directives are ok, if not, then we have 2266 /// already seen one so a \#else directive is a duplicate. When this returns, 2267 /// the caller can lex the first valid token. 2268 void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, 2269 SourceLocation IfTokenLoc, 2270 bool FoundNonSkipPortion, bool FoundElse, 2271 SourceLocation ElseLoc = SourceLocation()); 2272 2273 /// Information about the result for evaluating an expression for a 2274 /// preprocessor directive. 2275 struct DirectiveEvalResult { 2276 /// Whether the expression was evaluated as true or not. 2277 bool Conditional; 2278 2279 /// True if the expression contained identifiers that were undefined. 2280 bool IncludedUndefinedIds; 2281 2282 /// The source range for the expression. 2283 SourceRange ExprRange; 2284 }; 2285 2286 /// Evaluate an integer constant expression that may occur after a 2287 /// \#if or \#elif directive and return a \p DirectiveEvalResult object. 2288 /// 2289 /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro. 2290 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro); 2291 2292 /// Process a '__has_include("path")' expression. 2293 /// 2294 /// Returns true if successful. 2295 bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II); 2296 2297 /// Process '__has_include_next("path")' expression. 2298 /// 2299 /// Returns true if successful. 2300 bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II); 2301 2302 /// Get the directory and file from which to start \#include_next lookup. 2303 std::pair<ConstSearchDirIterator, const FileEntry *> 2304 getIncludeNextStart(const Token &IncludeNextTok) const; 2305 2306 /// Install the standard preprocessor pragmas: 2307 /// \#pragma GCC poison/system_header/dependency and \#pragma once. 2308 void RegisterBuiltinPragmas(); 2309 2310 /// Register builtin macros such as __LINE__ with the identifier table. 2311 void RegisterBuiltinMacros(); 2312 2313 /// If an identifier token is read that is to be expanded as a macro, handle 2314 /// it and return the next token as 'Tok'. If we lexed a token, return true; 2315 /// otherwise the caller should lex again. 2316 bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD); 2317 2318 /// Cache macro expanded tokens for TokenLexers. 2319 // 2320 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is 2321 /// going to lex in the cache and when it finishes the tokens are removed 2322 /// from the end of the cache. 2323 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer, 2324 ArrayRef<Token> tokens); 2325 2326 void removeCachedMacroExpandedTokensOfLastLexer(); 2327 2328 /// Determine whether the next preprocessor token to be 2329 /// lexed is a '('. If so, consume the token and return true, if not, this 2330 /// method should have no observable side-effect on the lexed tokens. 2331 bool isNextPPTokenLParen(); 2332 2333 /// After reading "MACRO(", this method is invoked to read all of the formal 2334 /// arguments specified for the macro invocation. Returns null on error. 2335 MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI, 2336 SourceLocation &MacroEnd); 2337 2338 /// If an identifier token is read that is to be expanded 2339 /// as a builtin macro, handle it and return the next token as 'Tok'. 2340 void ExpandBuiltinMacro(Token &Tok); 2341 2342 /// Read a \c _Pragma directive, slice it up, process it, then 2343 /// return the first token after the directive. 2344 /// This assumes that the \c _Pragma token has just been read into \p Tok. 2345 void Handle_Pragma(Token &Tok); 2346 2347 /// Like Handle_Pragma except the pragma text is not enclosed within 2348 /// a string literal. 2349 void HandleMicrosoft__pragma(Token &Tok); 2350 2351 /// Add a lexer to the top of the include stack and 2352 /// start lexing tokens from it instead of the current buffer. 2353 void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir); 2354 2355 /// Set the FileID for the preprocessor predefines. 2356 void setPredefinesFileID(FileID FID) { 2357 assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!"); 2358 PredefinesFileID = FID; 2359 } 2360 2361 /// Set the FileID for the PCH through header. 2362 void setPCHThroughHeaderFileID(FileID FID); 2363 2364 /// Returns true if we are lexing from a file and not a 2365 /// pragma or a macro. 2366 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) { 2367 return L ? !L->isPragmaLexer() : P != nullptr; 2368 } 2369 2370 static bool IsFileLexer(const IncludeStackInfo& I) { 2371 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer); 2372 } 2373 2374 bool IsFileLexer() const { 2375 return IsFileLexer(CurLexer.get(), CurPPLexer); 2376 } 2377 2378 //===--------------------------------------------------------------------===// 2379 // Caching stuff. 2380 void CachingLex(Token &Result); 2381 2382 bool InCachingLexMode() const { 2383 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means 2384 // that we are past EOF, not that we are in CachingLex mode. 2385 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty(); 2386 } 2387 2388 void EnterCachingLexMode(); 2389 void EnterCachingLexModeUnchecked(); 2390 2391 void ExitCachingLexMode() { 2392 if (InCachingLexMode()) 2393 RemoveTopOfLexerStack(); 2394 } 2395 2396 const Token &PeekAhead(unsigned N); 2397 void AnnotatePreviousCachedTokens(const Token &Tok); 2398 2399 //===--------------------------------------------------------------------===// 2400 /// Handle*Directive - implement the various preprocessor directives. These 2401 /// should side-effect the current preprocessor object so that the next call 2402 /// to Lex() will return the appropriate token next. 2403 void HandleLineDirective(); 2404 void HandleDigitDirective(Token &Tok); 2405 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning); 2406 void HandleIdentSCCSDirective(Token &Tok); 2407 void HandleMacroPublicDirective(Token &Tok); 2408 void HandleMacroPrivateDirective(); 2409 2410 /// An additional notification that can be produced by a header inclusion or 2411 /// import to tell the parser what happened. 2412 struct ImportAction { 2413 enum ActionKind { 2414 None, 2415 ModuleBegin, 2416 ModuleImport, 2417 SkippedModuleImport, 2418 Failure, 2419 } Kind; 2420 Module *ModuleForHeader = nullptr; 2421 2422 ImportAction(ActionKind AK, Module *Mod = nullptr) 2423 : Kind(AK), ModuleForHeader(Mod) { 2424 assert((AK == None || Mod || AK == Failure) && 2425 "no module for module action"); 2426 } 2427 }; 2428 2429 Optional<FileEntryRef> LookupHeaderIncludeOrImport( 2430 ConstSearchDirIterator *CurDir, StringRef &Filename, 2431 SourceLocation FilenameLoc, CharSourceRange FilenameRange, 2432 const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl, 2433 bool &IsMapped, ConstSearchDirIterator LookupFrom, 2434 const FileEntry *LookupFromFile, StringRef &LookupFilename, 2435 SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath, 2436 ModuleMap::KnownHeader &SuggestedModule, bool isAngled); 2437 2438 // File inclusion. 2439 void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok, 2440 ConstSearchDirIterator LookupFrom = nullptr, 2441 const FileEntry *LookupFromFile = nullptr); 2442 ImportAction 2443 HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok, 2444 Token &FilenameTok, SourceLocation EndLoc, 2445 ConstSearchDirIterator LookupFrom = nullptr, 2446 const FileEntry *LookupFromFile = nullptr); 2447 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); 2448 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); 2449 void HandleImportDirective(SourceLocation HashLoc, Token &Tok); 2450 void HandleMicrosoftImportDirective(Token &Tok); 2451 2452 public: 2453 /// Check that the given module is available, producing a diagnostic if not. 2454 /// \return \c true if the check failed (because the module is not available). 2455 /// \c false if the module appears to be usable. 2456 static bool checkModuleIsAvailable(const LangOptions &LangOpts, 2457 const TargetInfo &TargetInfo, 2458 DiagnosticsEngine &Diags, Module *M); 2459 2460 // Module inclusion testing. 2461 /// Find the module that owns the source or header file that 2462 /// \p Loc points to. If the location is in a file that was included 2463 /// into a module, or is outside any module, returns nullptr. 2464 Module *getModuleForLocation(SourceLocation Loc); 2465 2466 /// We want to produce a diagnostic at location IncLoc concerning an 2467 /// unreachable effect at location MLoc (eg, where a desired entity was 2468 /// declared or defined). Determine whether the right way to make MLoc 2469 /// reachable is by #include, and if so, what header should be included. 2470 /// 2471 /// This is not necessarily fast, and might load unexpected module maps, so 2472 /// should only be called by code that intends to produce an error. 2473 /// 2474 /// \param IncLoc The location at which the missing effect was detected. 2475 /// \param MLoc A location within an unimported module at which the desired 2476 /// effect occurred. 2477 /// \return A file that can be #included to provide the desired effect. Null 2478 /// if no such file could be determined or if a #include is not 2479 /// appropriate (eg, if a module should be imported instead). 2480 const FileEntry *getHeaderToIncludeForDiagnostics(SourceLocation IncLoc, 2481 SourceLocation MLoc); 2482 2483 bool isRecordingPreamble() const { 2484 return PreambleConditionalStack.isRecording(); 2485 } 2486 2487 bool hasRecordedPreamble() const { 2488 return PreambleConditionalStack.hasRecordedPreamble(); 2489 } 2490 2491 ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const { 2492 return PreambleConditionalStack.getStack(); 2493 } 2494 2495 void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) { 2496 PreambleConditionalStack.setStack(s); 2497 } 2498 2499 void setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s, 2500 llvm::Optional<PreambleSkipInfo> SkipInfo) { 2501 PreambleConditionalStack.startReplaying(); 2502 PreambleConditionalStack.setStack(s); 2503 PreambleConditionalStack.SkipInfo = SkipInfo; 2504 } 2505 2506 llvm::Optional<PreambleSkipInfo> getPreambleSkipInfo() const { 2507 return PreambleConditionalStack.SkipInfo; 2508 } 2509 2510 private: 2511 /// After processing predefined file, initialize the conditional stack from 2512 /// the preamble. 2513 void replayPreambleConditionalStack(); 2514 2515 // Macro handling. 2516 void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard); 2517 void HandleUndefDirective(); 2518 2519 // Conditional Inclusion. 2520 void HandleIfdefDirective(Token &Result, const Token &HashToken, 2521 bool isIfndef, bool ReadAnyTokensBeforeDirective); 2522 void HandleIfDirective(Token &IfToken, const Token &HashToken, 2523 bool ReadAnyTokensBeforeDirective); 2524 void HandleEndifDirective(Token &EndifToken); 2525 void HandleElseDirective(Token &Result, const Token &HashToken); 2526 void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken, 2527 tok::PPKeywordKind Kind); 2528 2529 // Pragmas. 2530 void HandlePragmaDirective(PragmaIntroducer Introducer); 2531 2532 public: 2533 void HandlePragmaOnce(Token &OnceTok); 2534 void HandlePragmaMark(Token &MarkTok); 2535 void HandlePragmaPoison(); 2536 void HandlePragmaSystemHeader(Token &SysHeaderTok); 2537 void HandlePragmaDependency(Token &DependencyTok); 2538 void HandlePragmaPushMacro(Token &Tok); 2539 void HandlePragmaPopMacro(Token &Tok); 2540 void HandlePragmaIncludeAlias(Token &Tok); 2541 void HandlePragmaModuleBuild(Token &Tok); 2542 void HandlePragmaHdrstop(Token &Tok); 2543 IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok); 2544 2545 // Return true and store the first token only if any CommentHandler 2546 // has inserted some tokens and getCommentRetentionState() is false. 2547 bool HandleComment(Token &result, SourceRange Comment); 2548 2549 /// A macro is used, update information about macros that need unused 2550 /// warnings. 2551 void markMacroAsUsed(MacroInfo *MI); 2552 2553 void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg, 2554 SourceLocation AnnotationLoc) { 2555 auto Annotations = AnnotationInfos.find(II); 2556 if (Annotations == AnnotationInfos.end()) 2557 AnnotationInfos.insert(std::make_pair( 2558 II, 2559 MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg)))); 2560 else 2561 Annotations->second.DeprecationInfo = 2562 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)}; 2563 } 2564 2565 void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg, 2566 SourceLocation AnnotationLoc) { 2567 auto Annotations = AnnotationInfos.find(II); 2568 if (Annotations == AnnotationInfos.end()) 2569 AnnotationInfos.insert( 2570 std::make_pair(II, MacroAnnotations::makeRestrictExpansion( 2571 AnnotationLoc, std::move(Msg)))); 2572 else 2573 Annotations->second.RestrictExpansionInfo = 2574 MacroAnnotationInfo{AnnotationLoc, std::move(Msg)}; 2575 } 2576 2577 void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) { 2578 auto Annotations = AnnotationInfos.find(II); 2579 if (Annotations == AnnotationInfos.end()) 2580 AnnotationInfos.insert( 2581 std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc))); 2582 else 2583 Annotations->second.FinalAnnotationLoc = AnnotationLoc; 2584 } 2585 2586 const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const { 2587 return AnnotationInfos.find(II)->second; 2588 } 2589 2590 void emitMacroExpansionWarnings(const Token &Identifier) const { 2591 if (Identifier.getIdentifierInfo()->isDeprecatedMacro()) 2592 emitMacroDeprecationWarning(Identifier); 2593 2594 if (Identifier.getIdentifierInfo()->isRestrictExpansion() && 2595 !SourceMgr.isInMainFile(Identifier.getLocation())) 2596 emitRestrictExpansionWarning(Identifier); 2597 } 2598 2599 static void processPathForFileMacro(SmallVectorImpl<char> &Path, 2600 const LangOptions &LangOpts, 2601 const TargetInfo &TI); 2602 2603 private: 2604 void emitMacroDeprecationWarning(const Token &Identifier) const; 2605 void emitRestrictExpansionWarning(const Token &Identifier) const; 2606 void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const; 2607 }; 2608 2609 /// Abstract base class that describes a handler that will receive 2610 /// source ranges for each of the comments encountered in the source file. 2611 class CommentHandler { 2612 public: 2613 virtual ~CommentHandler(); 2614 2615 // The handler shall return true if it has pushed any tokens 2616 // to be read using e.g. EnterToken or EnterTokenStream. 2617 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0; 2618 }; 2619 2620 /// Abstract base class that describes a handler that will receive 2621 /// source ranges for empty lines encountered in the source file. 2622 class EmptylineHandler { 2623 public: 2624 virtual ~EmptylineHandler(); 2625 2626 // The handler handles empty lines. 2627 virtual void HandleEmptyline(SourceRange Range) = 0; 2628 }; 2629 2630 /// Registry of pragma handlers added by plugins 2631 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>; 2632 2633 } // namespace clang 2634 2635 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H 2636