1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Diagnostic.h"
18 #include "clang/Basic/DiagnosticIDs.h"
19 #include "clang/Basic/IdentifierTable.h"
20 #include "clang/Basic/LLVM.h"
21 #include "clang/Basic/LangOptions.h"
22 #include "clang/Basic/Module.h"
23 #include "clang/Basic/SourceLocation.h"
24 #include "clang/Basic/SourceManager.h"
25 #include "clang/Basic/TokenKinds.h"
26 #include "clang/Lex/HeaderSearch.h"
27 #include "clang/Lex/Lexer.h"
28 #include "clang/Lex/MacroInfo.h"
29 #include "clang/Lex/ModuleLoader.h"
30 #include "clang/Lex/ModuleMap.h"
31 #include "clang/Lex/PPCallbacks.h"
32 #include "clang/Lex/Token.h"
33 #include "clang/Lex/TokenLexer.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseMap.h"
36 #include "llvm/ADT/FoldingSet.h"
37 #include "llvm/ADT/FunctionExtras.h"
38 #include "llvm/ADT/PointerUnion.h"
39 #include "llvm/ADT/STLExtras.h"
40 #include "llvm/ADT/SmallPtrSet.h"
41 #include "llvm/ADT/SmallVector.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/TinyPtrVector.h"
44 #include "llvm/ADT/iterator_range.h"
45 #include "llvm/Support/Allocator.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Registry.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <map>
52 #include <memory>
53 #include <optional>
54 #include <string>
55 #include <utility>
56 #include <vector>
57 
58 namespace llvm {
59 
60 template<unsigned InternalLen> class SmallString;
61 
62 } // namespace llvm
63 
64 namespace clang {
65 
66 class CodeCompletionHandler;
67 class CommentHandler;
68 class DirectoryEntry;
69 class EmptylineHandler;
70 class ExternalPreprocessorSource;
71 class FileEntry;
72 class FileManager;
73 class HeaderSearch;
74 class MacroArgs;
75 class PragmaHandler;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
80 class ScratchBuffer;
81 class TargetInfo;
82 
83 namespace Builtin {
84 class Context;
85 }
86 
87 /// Stores token information for comparing actual tokens with
88 /// predefined values.  Only handles simple tokens and identifiers.
89 class TokenValue {
90   tok::TokenKind Kind;
91   IdentifierInfo *II;
92 
93 public:
TokenValue(tok::TokenKind Kind)94   TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
95     assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
96     assert(Kind != tok::identifier &&
97            "Identifiers should be created by TokenValue(IdentifierInfo *)");
98     assert(!tok::isLiteral(Kind) && "Literals are not supported.");
99     assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
100   }
101 
TokenValue(IdentifierInfo * II)102   TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
103 
104   bool operator==(const Token &Tok) const {
105     return Tok.getKind() == Kind &&
106         (!II || II == Tok.getIdentifierInfo());
107   }
108 };
109 
110 /// Context in which macro name is used.
111 enum MacroUse {
112   // other than #define or #undef
113   MU_Other  = 0,
114 
115   // macro name specified in #define
116   MU_Define = 1,
117 
118   // macro name specified in #undef
119   MU_Undef  = 2
120 };
121 
122 /// Engages in a tight little dance with the lexer to efficiently
123 /// preprocess tokens.
124 ///
125 /// Lexers know only about tokens within a single source file, and don't
126 /// know anything about preprocessor-level issues like the \#include stack,
127 /// token expansion, etc.
128 class Preprocessor {
129   friend class VAOptDefinitionContext;
130   friend class VariadicMacroScopeGuard;
131 
132   llvm::unique_function<void(const clang::Token &)> OnToken;
133   std::shared_ptr<PreprocessorOptions> PPOpts;
134   DiagnosticsEngine        *Diags;
135   LangOptions       &LangOpts;
136   const TargetInfo *Target = nullptr;
137   const TargetInfo *AuxTarget = nullptr;
138   FileManager       &FileMgr;
139   SourceManager     &SourceMgr;
140   std::unique_ptr<ScratchBuffer> ScratchBuf;
141   HeaderSearch      &HeaderInfo;
142   ModuleLoader      &TheModuleLoader;
143 
144   /// External source of macros.
145   ExternalPreprocessorSource *ExternalSource;
146 
147   /// A BumpPtrAllocator object used to quickly allocate and release
148   /// objects internal to the Preprocessor.
149   llvm::BumpPtrAllocator BP;
150 
151   /// Identifiers for builtin macros and other builtins.
152   IdentifierInfo *Ident__LINE__, *Ident__FILE__;   // __LINE__, __FILE__
153   IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
154   IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
155   IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
156   IdentifierInfo *Ident__FILE_NAME__;              // __FILE_NAME__
157   IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
158   IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
159   IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
160   IdentifierInfo *Ident__identifier;               // __identifier
161   IdentifierInfo *Ident__VA_ARGS__;                // __VA_ARGS__
162   IdentifierInfo *Ident__VA_OPT__;                 // __VA_OPT__
163   IdentifierInfo *Ident__has_feature;              // __has_feature
164   IdentifierInfo *Ident__has_extension;            // __has_extension
165   IdentifierInfo *Ident__has_builtin;              // __has_builtin
166   IdentifierInfo *Ident__has_constexpr_builtin;    // __has_constexpr_builtin
167   IdentifierInfo *Ident__has_attribute;            // __has_attribute
168   IdentifierInfo *Ident__has_include;              // __has_include
169   IdentifierInfo *Ident__has_include_next;         // __has_include_next
170   IdentifierInfo *Ident__has_warning;              // __has_warning
171   IdentifierInfo *Ident__is_identifier;            // __is_identifier
172   IdentifierInfo *Ident__building_module;          // __building_module
173   IdentifierInfo *Ident__MODULE__;                 // __MODULE__
174   IdentifierInfo *Ident__has_cpp_attribute;        // __has_cpp_attribute
175   IdentifierInfo *Ident__has_c_attribute;          // __has_c_attribute
176   IdentifierInfo *Ident__has_declspec;             // __has_declspec_attribute
177   IdentifierInfo *Ident__is_target_arch;           // __is_target_arch
178   IdentifierInfo *Ident__is_target_vendor;         // __is_target_vendor
179   IdentifierInfo *Ident__is_target_os;             // __is_target_os
180   IdentifierInfo *Ident__is_target_environment;    // __is_target_environment
181   IdentifierInfo *Ident__is_target_variant_os;
182   IdentifierInfo *Ident__is_target_variant_environment;
183   IdentifierInfo *Ident__FLT_EVAL_METHOD__;        // __FLT_EVAL_METHOD
184 
185   // Weak, only valid (and set) while InMacroArgs is true.
186   Token* ArgMacro;
187 
188   SourceLocation DATELoc, TIMELoc;
189 
190   // FEM_UnsetOnCommandLine means that an explicit evaluation method was
191   // not specified on the command line. The target is queried to set the
192   // default evaluation method.
193   LangOptions::FPEvalMethodKind CurrentFPEvalMethod =
194       LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
195 
196   // The most recent pragma location where the floating point evaluation
197   // method was modified. This is used to determine whether the
198   // 'pragma clang fp eval_method' was used whithin the current scope.
199   SourceLocation LastFPEvalPragmaLocation;
200 
201   LangOptions::FPEvalMethodKind TUFPEvalMethod =
202       LangOptions::FPEvalMethodKind::FEM_UnsetOnCommandLine;
203 
204   // Next __COUNTER__ value, starts at 0.
205   unsigned CounterValue = 0;
206 
207   enum {
208     /// Maximum depth of \#includes.
209     MaxAllowedIncludeStackDepth = 200
210   };
211 
212   // State that is set before the preprocessor begins.
213   bool KeepComments : 1;
214   bool KeepMacroComments : 1;
215   bool SuppressIncludeNotFoundError : 1;
216 
217   // State that changes while the preprocessor runs:
218   bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
219 
220   /// Whether the preprocessor owns the header search object.
221   bool OwnsHeaderSearch : 1;
222 
223   /// True if macro expansion is disabled.
224   bool DisableMacroExpansion : 1;
225 
226   /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
227   /// when parsing preprocessor directives.
228   bool MacroExpansionInDirectivesOverride : 1;
229 
230   class ResetMacroExpansionHelper;
231 
232   /// Whether we have already loaded macros from the external source.
233   mutable bool ReadMacrosFromExternalSource : 1;
234 
235   /// True if pragmas are enabled.
236   bool PragmasEnabled : 1;
237 
238   /// True if the current build action is a preprocessing action.
239   bool PreprocessedOutput : 1;
240 
241   /// True if we are currently preprocessing a #if or #elif directive
242   bool ParsingIfOrElifDirective;
243 
244   /// True if we are pre-expanding macro arguments.
245   bool InMacroArgPreExpansion;
246 
247   /// Mapping/lookup information for all identifiers in
248   /// the program, including program keywords.
249   mutable IdentifierTable Identifiers;
250 
251   /// This table contains all the selectors in the program.
252   ///
253   /// Unlike IdentifierTable above, this table *isn't* populated by the
254   /// preprocessor. It is declared/expanded here because its role/lifetime is
255   /// conceptually similar to the IdentifierTable. In addition, the current
256   /// control flow (in clang::ParseAST()), make it convenient to put here.
257   ///
258   /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
259   /// the lifetime of the preprocessor.
260   SelectorTable Selectors;
261 
262   /// Information about builtins.
263   std::unique_ptr<Builtin::Context> BuiltinInfo;
264 
265   /// Tracks all of the pragmas that the client registered
266   /// with this preprocessor.
267   std::unique_ptr<PragmaNamespace> PragmaHandlers;
268 
269   /// Pragma handlers of the original source is stored here during the
270   /// parsing of a model file.
271   std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
272 
273   /// Tracks all of the comment handlers that the client registered
274   /// with this preprocessor.
275   std::vector<CommentHandler *> CommentHandlers;
276 
277   /// Empty line handler.
278   EmptylineHandler *Emptyline = nullptr;
279 
280 public:
281   /// The kind of translation unit we are processing.
282   const TranslationUnitKind TUKind;
283 
284 private:
285   /// The code-completion handler.
286   CodeCompletionHandler *CodeComplete = nullptr;
287 
288   /// The file that we're performing code-completion for, if any.
289   const FileEntry *CodeCompletionFile = nullptr;
290 
291   /// The offset in file for the code-completion point.
292   unsigned CodeCompletionOffset = 0;
293 
294   /// The location for the code-completion point. This gets instantiated
295   /// when the CodeCompletionFile gets \#include'ed for preprocessing.
296   SourceLocation CodeCompletionLoc;
297 
298   /// The start location for the file of the code-completion point.
299   ///
300   /// This gets instantiated when the CodeCompletionFile gets \#include'ed
301   /// for preprocessing.
302   SourceLocation CodeCompletionFileLoc;
303 
304   /// The source location of the \c import contextual keyword we just
305   /// lexed, if any.
306   SourceLocation ModuleImportLoc;
307 
308   /// The import path for named module that we're currently processing.
309   SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> NamedModuleImportPath;
310 
311   /// Whether the import is an `@import` or a standard c++ modules import.
312   bool IsAtImport = false;
313 
314   /// Whether the last token we lexed was an '@'.
315   bool LastTokenWasAt = false;
316 
317   /// A position within a C++20 import-seq.
318   class StdCXXImportSeq {
319   public:
320     enum State : int {
321       // Positive values represent a number of unclosed brackets.
322       AtTopLevel = 0,
323       AfterTopLevelTokenSeq = -1,
324       AfterExport = -2,
325       AfterImportSeq = -3,
326     };
327 
StdCXXImportSeq(State S)328     StdCXXImportSeq(State S) : S(S) {}
329 
330     /// Saw any kind of open bracket.
handleOpenBracket()331     void handleOpenBracket() {
332       S = static_cast<State>(std::max<int>(S, 0) + 1);
333     }
334     /// Saw any kind of close bracket other than '}'.
handleCloseBracket()335     void handleCloseBracket() {
336       S = static_cast<State>(std::max<int>(S, 1) - 1);
337     }
338     /// Saw a close brace.
handleCloseBrace()339     void handleCloseBrace() {
340       handleCloseBracket();
341       if (S == AtTopLevel && !AfterHeaderName)
342         S = AfterTopLevelTokenSeq;
343     }
344     /// Saw a semicolon.
handleSemi()345     void handleSemi() {
346       if (atTopLevel()) {
347         S = AfterTopLevelTokenSeq;
348         AfterHeaderName = false;
349       }
350     }
351 
352     /// Saw an 'export' identifier.
handleExport()353     void handleExport() {
354       if (S == AfterTopLevelTokenSeq)
355         S = AfterExport;
356       else if (S <= 0)
357         S = AtTopLevel;
358     }
359     /// Saw an 'import' identifier.
handleImport()360     void handleImport() {
361       if (S == AfterTopLevelTokenSeq || S == AfterExport)
362         S = AfterImportSeq;
363       else if (S <= 0)
364         S = AtTopLevel;
365     }
366 
367     /// Saw a 'header-name' token; do not recognize any more 'import' tokens
368     /// until we reach a top-level semicolon.
handleHeaderName()369     void handleHeaderName() {
370       if (S == AfterImportSeq)
371         AfterHeaderName = true;
372       handleMisc();
373     }
374 
375     /// Saw any other token.
handleMisc()376     void handleMisc() {
377       if (S <= 0)
378         S = AtTopLevel;
379     }
380 
atTopLevel()381     bool atTopLevel() { return S <= 0; }
afterImportSeq()382     bool afterImportSeq() { return S == AfterImportSeq; }
afterTopLevelSeq()383     bool afterTopLevelSeq() { return S == AfterTopLevelTokenSeq; }
384 
385   private:
386     State S;
387     /// Whether we're in the pp-import-suffix following the header-name in a
388     /// pp-import. If so, a close-brace is not sufficient to end the
389     /// top-level-token-seq of an import-seq.
390     bool AfterHeaderName = false;
391   };
392 
393   /// Our current position within a C++20 import-seq.
394   StdCXXImportSeq StdCXXImportSeqState = StdCXXImportSeq::AfterTopLevelTokenSeq;
395 
396   /// Track whether we are in a Global Module Fragment
397   class TrackGMF {
398   public:
399     enum GMFState : int {
400       GMFActive = 1,
401       MaybeGMF = 0,
402       BeforeGMFIntroducer = -1,
403       GMFAbsentOrEnded = -2,
404     };
405 
TrackGMF(GMFState S)406     TrackGMF(GMFState S) : S(S) {}
407 
408     /// Saw a semicolon.
handleSemi()409     void handleSemi() {
410       // If it is immediately after the first instance of the module keyword,
411       // then that introduces the GMF.
412       if (S == MaybeGMF)
413         S = GMFActive;
414     }
415 
416     /// Saw an 'export' identifier.
handleExport()417     void handleExport() {
418       // The presence of an 'export' keyword always ends or excludes a GMF.
419       S = GMFAbsentOrEnded;
420     }
421 
422     /// Saw an 'import' identifier.
handleImport(bool AfterTopLevelTokenSeq)423     void handleImport(bool AfterTopLevelTokenSeq) {
424       // If we see this before any 'module' kw, then we have no GMF.
425       if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
426         S = GMFAbsentOrEnded;
427     }
428 
429     /// Saw a 'module' identifier.
handleModule(bool AfterTopLevelTokenSeq)430     void handleModule(bool AfterTopLevelTokenSeq) {
431       // This was the first module identifier and not preceded by any token
432       // that would exclude a GMF.  It could begin a GMF, but only if directly
433       // followed by a semicolon.
434       if (AfterTopLevelTokenSeq && S == BeforeGMFIntroducer)
435         S = MaybeGMF;
436       else
437         S = GMFAbsentOrEnded;
438     }
439 
440     /// Saw any other token.
handleMisc()441     void handleMisc() {
442       // We saw something other than ; after the 'module' kw, so not a GMF.
443       if (S == MaybeGMF)
444         S = GMFAbsentOrEnded;
445     }
446 
inGMF()447     bool inGMF() { return S == GMFActive; }
448 
449   private:
450     /// Track the transitions into and out of a Global Module Fragment,
451     /// if one is present.
452     GMFState S;
453   };
454 
455   TrackGMF TrackGMFState = TrackGMF::BeforeGMFIntroducer;
456 
457   /// Track the status of the c++20 module decl.
458   ///
459   ///   module-declaration:
460   ///     'export'[opt] 'module' module-name module-partition[opt]
461   ///     attribute-specifier-seq[opt] ';'
462   ///
463   ///   module-name:
464   ///     module-name-qualifier[opt] identifier
465   ///
466   ///   module-partition:
467   ///     ':' module-name-qualifier[opt] identifier
468   ///
469   ///   module-name-qualifier:
470   ///     identifier '.'
471   ///     module-name-qualifier identifier '.'
472   ///
473   /// Transition state:
474   ///
475   ///   NotAModuleDecl --- export ---> FoundExport
476   ///   NotAModuleDecl --- module ---> ImplementationCandidate
477   ///   FoundExport --- module ---> InterfaceCandidate
478   ///   ImplementationCandidate --- Identifier ---> ImplementationCandidate
479   ///   ImplementationCandidate --- period ---> ImplementationCandidate
480   ///   ImplementationCandidate --- colon ---> ImplementationCandidate
481   ///   InterfaceCandidate --- Identifier ---> InterfaceCandidate
482   ///   InterfaceCandidate --- period ---> InterfaceCandidate
483   ///   InterfaceCandidate --- colon ---> InterfaceCandidate
484   ///   ImplementationCandidate --- Semi ---> NamedModuleImplementation
485   ///   NamedModuleInterface --- Semi ---> NamedModuleInterface
486   ///   NamedModuleImplementation --- Anything ---> NamedModuleImplementation
487   ///   NamedModuleInterface --- Anything ---> NamedModuleInterface
488   ///
489   /// FIXME: We haven't handle attribute-specifier-seq here. It may not be bad
490   /// soon since we don't support any module attributes yet.
491   class ModuleDeclSeq {
492     enum ModuleDeclState : int {
493       NotAModuleDecl,
494       FoundExport,
495       InterfaceCandidate,
496       ImplementationCandidate,
497       NamedModuleInterface,
498       NamedModuleImplementation,
499     };
500 
501   public:
ModuleDeclSeq()502     ModuleDeclSeq() : State(NotAModuleDecl) {}
503 
handleExport()504     void handleExport() {
505       if (State == NotAModuleDecl)
506         State = FoundExport;
507       else if (!isNamedModule())
508         reset();
509     }
510 
handleModule()511     void handleModule() {
512       if (State == FoundExport)
513         State = InterfaceCandidate;
514       else if (State == NotAModuleDecl)
515         State = ImplementationCandidate;
516       else if (!isNamedModule())
517         reset();
518     }
519 
handleIdentifier(IdentifierInfo * Identifier)520     void handleIdentifier(IdentifierInfo *Identifier) {
521       if (isModuleCandidate() && Identifier)
522         Name += Identifier->getName().str();
523       else if (!isNamedModule())
524         reset();
525     }
526 
handleColon()527     void handleColon() {
528       if (isModuleCandidate())
529         Name += ":";
530       else if (!isNamedModule())
531         reset();
532     }
533 
handlePeriod()534     void handlePeriod() {
535       if (isModuleCandidate())
536         Name += ".";
537       else if (!isNamedModule())
538         reset();
539     }
540 
handleSemi()541     void handleSemi() {
542       if (!Name.empty() && isModuleCandidate()) {
543         if (State == InterfaceCandidate)
544           State = NamedModuleInterface;
545         else if (State == ImplementationCandidate)
546           State = NamedModuleImplementation;
547         else
548           llvm_unreachable("Unimaged ModuleDeclState.");
549       } else if (!isNamedModule())
550         reset();
551     }
552 
handleMisc()553     void handleMisc() {
554       if (!isNamedModule())
555         reset();
556     }
557 
isModuleCandidate()558     bool isModuleCandidate() const {
559       return State == InterfaceCandidate || State == ImplementationCandidate;
560     }
561 
isNamedModule()562     bool isNamedModule() const {
563       return State == NamedModuleInterface ||
564              State == NamedModuleImplementation;
565     }
566 
isNamedInterface()567     bool isNamedInterface() const { return State == NamedModuleInterface; }
568 
isImplementationUnit()569     bool isImplementationUnit() const {
570       return State == NamedModuleImplementation && !getName().contains(':');
571     }
572 
getName()573     StringRef getName() const {
574       assert(isNamedModule() && "Can't get name from a non named module");
575       return Name;
576     }
577 
getPrimaryName()578     StringRef getPrimaryName() const {
579       assert(isNamedModule() && "Can't get name from a non named module");
580       return getName().split(':').first;
581     }
582 
reset()583     void reset() {
584       Name.clear();
585       State = NotAModuleDecl;
586     }
587 
588   private:
589     ModuleDeclState State;
590     std::string Name;
591   };
592 
593   ModuleDeclSeq ModuleDeclState;
594 
595   /// Whether the module import expects an identifier next. Otherwise,
596   /// it expects a '.' or ';'.
597   bool ModuleImportExpectsIdentifier = false;
598 
599   /// The identifier and source location of the currently-active
600   /// \#pragma clang arc_cf_code_audited begin.
601   std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
602 
603   /// The source location of the currently-active
604   /// \#pragma clang assume_nonnull begin.
605   SourceLocation PragmaAssumeNonNullLoc;
606 
607   /// Set only for preambles which end with an active
608   /// \#pragma clang assume_nonnull begin.
609   ///
610   /// When the preamble is loaded into the main file,
611   /// `PragmaAssumeNonNullLoc` will be set to this to
612   /// replay the unterminated assume_nonnull.
613   SourceLocation PreambleRecordedPragmaAssumeNonNullLoc;
614 
615   /// True if we hit the code-completion point.
616   bool CodeCompletionReached = false;
617 
618   /// The code completion token containing the information
619   /// on the stem that is to be code completed.
620   IdentifierInfo *CodeCompletionII = nullptr;
621 
622   /// Range for the code completion token.
623   SourceRange CodeCompletionTokenRange;
624 
625   /// The directory that the main file should be considered to occupy,
626   /// if it does not correspond to a real file (as happens when building a
627   /// module).
628   const DirectoryEntry *MainFileDir = nullptr;
629 
630   /// The number of bytes that we will initially skip when entering the
631   /// main file, along with a flag that indicates whether skipping this number
632   /// of bytes will place the lexer at the start of a line.
633   ///
634   /// This is used when loading a precompiled preamble.
635   std::pair<int, bool> SkipMainFilePreamble;
636 
637   /// Whether we hit an error due to reaching max allowed include depth. Allows
638   /// to avoid hitting the same error over and over again.
639   bool HasReachedMaxIncludeDepth = false;
640 
641   /// The number of currently-active calls to Lex.
642   ///
643   /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
644   /// require asking for multiple additional tokens. This counter makes it
645   /// possible for Lex to detect whether it's producing a token for the end
646   /// of phase 4 of translation or for some other situation.
647   unsigned LexLevel = 0;
648 
649   /// The number of (LexLevel 0) preprocessor tokens.
650   unsigned TokenCount = 0;
651 
652   /// Preprocess every token regardless of LexLevel.
653   bool PreprocessToken = false;
654 
655   /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
656   /// warning, or zero for unlimited.
657   unsigned MaxTokens = 0;
658   SourceLocation MaxTokensOverrideLoc;
659 
660 public:
661   struct PreambleSkipInfo {
662     SourceLocation HashTokenLoc;
663     SourceLocation IfTokenLoc;
664     bool FoundNonSkipPortion;
665     bool FoundElse;
666     SourceLocation ElseLoc;
667 
PreambleSkipInfoPreambleSkipInfo668     PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
669                      bool FoundNonSkipPortion, bool FoundElse,
670                      SourceLocation ElseLoc)
671         : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
672           FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
673           ElseLoc(ElseLoc) {}
674   };
675 
676   using IncludedFilesSet = llvm::DenseSet<const FileEntry *>;
677 
678 private:
679   friend class ASTReader;
680   friend class MacroArgs;
681 
682   class PreambleConditionalStackStore {
683     enum State {
684       Off = 0,
685       Recording = 1,
686       Replaying = 2,
687     };
688 
689   public:
690     PreambleConditionalStackStore() = default;
691 
startRecording()692     void startRecording() { ConditionalStackState = Recording; }
startReplaying()693     void startReplaying() { ConditionalStackState = Replaying; }
isRecording()694     bool isRecording() const { return ConditionalStackState == Recording; }
isReplaying()695     bool isReplaying() const { return ConditionalStackState == Replaying; }
696 
getStack()697     ArrayRef<PPConditionalInfo> getStack() const {
698       return ConditionalStack;
699     }
700 
doneReplaying()701     void doneReplaying() {
702       ConditionalStack.clear();
703       ConditionalStackState = Off;
704     }
705 
setStack(ArrayRef<PPConditionalInfo> s)706     void setStack(ArrayRef<PPConditionalInfo> s) {
707       if (!isRecording() && !isReplaying())
708         return;
709       ConditionalStack.clear();
710       ConditionalStack.append(s.begin(), s.end());
711     }
712 
hasRecordedPreamble()713     bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
714 
reachedEOFWhileSkipping()715     bool reachedEOFWhileSkipping() const { return SkipInfo.has_value(); }
716 
clearSkipInfo()717     void clearSkipInfo() { SkipInfo.reset(); }
718 
719     std::optional<PreambleSkipInfo> SkipInfo;
720 
721   private:
722     SmallVector<PPConditionalInfo, 4> ConditionalStack;
723     State ConditionalStackState = Off;
724   } PreambleConditionalStack;
725 
726   /// The current top of the stack that we're lexing from if
727   /// not expanding a macro and we are lexing directly from source code.
728   ///
729   /// Only one of CurLexer, or CurTokenLexer will be non-null.
730   std::unique_ptr<Lexer> CurLexer;
731 
732   /// The current top of the stack what we're lexing from
733   /// if not expanding a macro.
734   ///
735   /// This is an alias for CurLexer.
736   PreprocessorLexer *CurPPLexer = nullptr;
737 
738   /// Used to find the current FileEntry, if CurLexer is non-null
739   /// and if applicable.
740   ///
741   /// This allows us to implement \#include_next and find directory-specific
742   /// properties.
743   ConstSearchDirIterator CurDirLookup = nullptr;
744 
745   /// The current macro we are expanding, if we are expanding a macro.
746   ///
747   /// One of CurLexer and CurTokenLexer must be null.
748   std::unique_ptr<TokenLexer> CurTokenLexer;
749 
750   /// The kind of lexer we're currently working with.
751   enum CurLexerKind {
752     CLK_Lexer,
753     CLK_TokenLexer,
754     CLK_CachingLexer,
755     CLK_DependencyDirectivesLexer,
756     CLK_LexAfterModuleImport
757   } CurLexerKind = CLK_Lexer;
758 
759   /// If the current lexer is for a submodule that is being built, this
760   /// is that submodule.
761   Module *CurLexerSubmodule = nullptr;
762 
763   /// Keeps track of the stack of files currently
764   /// \#included, and macros currently being expanded from, not counting
765   /// CurLexer/CurTokenLexer.
766   struct IncludeStackInfo {
767     enum CurLexerKind           CurLexerKind;
768     Module                     *TheSubmodule;
769     std::unique_ptr<Lexer>      TheLexer;
770     PreprocessorLexer          *ThePPLexer;
771     std::unique_ptr<TokenLexer> TheTokenLexer;
772     ConstSearchDirIterator      TheDirLookup;
773 
774     // The following constructors are completely useless copies of the default
775     // versions, only needed to pacify MSVC.
IncludeStackInfoIncludeStackInfo776     IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
777                      std::unique_ptr<Lexer> &&TheLexer,
778                      PreprocessorLexer *ThePPLexer,
779                      std::unique_ptr<TokenLexer> &&TheTokenLexer,
780                      ConstSearchDirIterator TheDirLookup)
781         : CurLexerKind(std::move(CurLexerKind)),
782           TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
783           ThePPLexer(std::move(ThePPLexer)),
784           TheTokenLexer(std::move(TheTokenLexer)),
785           TheDirLookup(std::move(TheDirLookup)) {}
786   };
787   std::vector<IncludeStackInfo> IncludeMacroStack;
788 
789   /// Actions invoked when some preprocessor activity is
790   /// encountered (e.g. a file is \#included, etc).
791   std::unique_ptr<PPCallbacks> Callbacks;
792 
793   struct MacroExpandsInfo {
794     Token Tok;
795     MacroDefinition MD;
796     SourceRange Range;
797 
MacroExpandsInfoMacroExpandsInfo798     MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
799         : Tok(Tok), MD(MD), Range(Range) {}
800   };
801   SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
802 
803   /// Information about a name that has been used to define a module macro.
804   struct ModuleMacroInfo {
805     /// The most recent macro directive for this identifier.
806     MacroDirective *MD;
807 
808     /// The active module macros for this identifier.
809     llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
810 
811     /// The generation number at which we last updated ActiveModuleMacros.
812     /// \see Preprocessor::VisibleModules.
813     unsigned ActiveModuleMacrosGeneration = 0;
814 
815     /// Whether this macro name is ambiguous.
816     bool IsAmbiguous = false;
817 
818     /// The module macros that are overridden by this macro.
819     llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
820 
ModuleMacroInfoModuleMacroInfo821     ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
822   };
823 
824   /// The state of a macro for an identifier.
825   class MacroState {
826     mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
827 
getModuleInfo(Preprocessor & PP,const IdentifierInfo * II)828     ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
829                                    const IdentifierInfo *II) const {
830       if (II->isOutOfDate())
831         PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
832       // FIXME: Find a spare bit on IdentifierInfo and store a
833       //        HasModuleMacros flag.
834       if (!II->hasMacroDefinition() ||
835           (!PP.getLangOpts().Modules &&
836            !PP.getLangOpts().ModulesLocalVisibility) ||
837           !PP.CurSubmoduleState->VisibleModules.getGeneration())
838         return nullptr;
839 
840       auto *Info = State.dyn_cast<ModuleMacroInfo*>();
841       if (!Info) {
842         Info = new (PP.getPreprocessorAllocator())
843             ModuleMacroInfo(State.get<MacroDirective *>());
844         State = Info;
845       }
846 
847       if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
848           Info->ActiveModuleMacrosGeneration)
849         PP.updateModuleMacroInfo(II, *Info);
850       return Info;
851     }
852 
853   public:
MacroState()854     MacroState() : MacroState(nullptr) {}
MacroState(MacroDirective * MD)855     MacroState(MacroDirective *MD) : State(MD) {}
856 
MacroState(MacroState && O)857     MacroState(MacroState &&O) noexcept : State(O.State) {
858       O.State = (MacroDirective *)nullptr;
859     }
860 
861     MacroState &operator=(MacroState &&O) noexcept {
862       auto S = O.State;
863       O.State = (MacroDirective *)nullptr;
864       State = S;
865       return *this;
866     }
867 
~MacroState()868     ~MacroState() {
869       if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
870         Info->~ModuleMacroInfo();
871     }
872 
getLatest()873     MacroDirective *getLatest() const {
874       if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
875         return Info->MD;
876       return State.get<MacroDirective*>();
877     }
878 
setLatest(MacroDirective * MD)879     void setLatest(MacroDirective *MD) {
880       if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
881         Info->MD = MD;
882       else
883         State = MD;
884     }
885 
isAmbiguous(Preprocessor & PP,const IdentifierInfo * II)886     bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
887       auto *Info = getModuleInfo(PP, II);
888       return Info ? Info->IsAmbiguous : false;
889     }
890 
891     ArrayRef<ModuleMacro *>
getActiveModuleMacros(Preprocessor & PP,const IdentifierInfo * II)892     getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
893       if (auto *Info = getModuleInfo(PP, II))
894         return Info->ActiveModuleMacros;
895       return std::nullopt;
896     }
897 
findDirectiveAtLoc(SourceLocation Loc,SourceManager & SourceMgr)898     MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
899                                                SourceManager &SourceMgr) const {
900       // FIXME: Incorporate module macros into the result of this.
901       if (auto *Latest = getLatest())
902         return Latest->findDirectiveAtLoc(Loc, SourceMgr);
903       return {};
904     }
905 
overrideActiveModuleMacros(Preprocessor & PP,IdentifierInfo * II)906     void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
907       if (auto *Info = getModuleInfo(PP, II)) {
908         Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
909                                       Info->ActiveModuleMacros.begin(),
910                                       Info->ActiveModuleMacros.end());
911         Info->ActiveModuleMacros.clear();
912         Info->IsAmbiguous = false;
913       }
914     }
915 
getOverriddenMacros()916     ArrayRef<ModuleMacro*> getOverriddenMacros() const {
917       if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
918         return Info->OverriddenMacros;
919       return std::nullopt;
920     }
921 
setOverriddenMacros(Preprocessor & PP,ArrayRef<ModuleMacro * > Overrides)922     void setOverriddenMacros(Preprocessor &PP,
923                              ArrayRef<ModuleMacro *> Overrides) {
924       auto *Info = State.dyn_cast<ModuleMacroInfo*>();
925       if (!Info) {
926         if (Overrides.empty())
927           return;
928         Info = new (PP.getPreprocessorAllocator())
929             ModuleMacroInfo(State.get<MacroDirective *>());
930         State = Info;
931       }
932       Info->OverriddenMacros.clear();
933       Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
934                                     Overrides.begin(), Overrides.end());
935       Info->ActiveModuleMacrosGeneration = 0;
936     }
937   };
938 
939   /// For each IdentifierInfo that was associated with a macro, we
940   /// keep a mapping to the history of all macro definitions and #undefs in
941   /// the reverse order (the latest one is in the head of the list).
942   ///
943   /// This mapping lives within the \p CurSubmoduleState.
944   using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
945 
946   struct SubmoduleState;
947 
948   /// Information about a submodule that we're currently building.
949   struct BuildingSubmoduleInfo {
950     /// The module that we are building.
951     Module *M;
952 
953     /// The location at which the module was included.
954     SourceLocation ImportLoc;
955 
956     /// Whether we entered this submodule via a pragma.
957     bool IsPragma;
958 
959     /// The previous SubmoduleState.
960     SubmoduleState *OuterSubmoduleState;
961 
962     /// The number of pending module macro names when we started building this.
963     unsigned OuterPendingModuleMacroNames;
964 
BuildingSubmoduleInfoBuildingSubmoduleInfo965     BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
966                           SubmoduleState *OuterSubmoduleState,
967                           unsigned OuterPendingModuleMacroNames)
968         : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
969           OuterSubmoduleState(OuterSubmoduleState),
970           OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
971   };
972   SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
973 
974   /// Information about a submodule's preprocessor state.
975   struct SubmoduleState {
976     /// The macros for the submodule.
977     MacroMap Macros;
978 
979     /// The set of modules that are visible within the submodule.
980     VisibleModuleSet VisibleModules;
981 
982     // FIXME: CounterValue?
983     // FIXME: PragmaPushMacroInfo?
984   };
985   std::map<Module *, SubmoduleState> Submodules;
986 
987   /// The preprocessor state for preprocessing outside of any submodule.
988   SubmoduleState NullSubmoduleState;
989 
990   /// The current submodule state. Will be \p NullSubmoduleState if we're not
991   /// in a submodule.
992   SubmoduleState *CurSubmoduleState;
993 
994   /// The files that have been included.
995   IncludedFilesSet IncludedFiles;
996 
997   /// The set of top-level modules that affected preprocessing, but were not
998   /// imported.
999   llvm::SmallSetVector<Module *, 2> AffectingClangModules;
1000 
1001   /// The set of known macros exported from modules.
1002   llvm::FoldingSet<ModuleMacro> ModuleMacros;
1003 
1004   /// The names of potential module macros that we've not yet processed.
1005   llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
1006 
1007   /// The list of module macros, for each identifier, that are not overridden by
1008   /// any other module macro.
1009   llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
1010       LeafModuleMacros;
1011 
1012   /// Macros that we want to warn because they are not used at the end
1013   /// of the translation unit.
1014   ///
1015   /// We store just their SourceLocations instead of
1016   /// something like MacroInfo*. The benefit of this is that when we are
1017   /// deserializing from PCH, we don't need to deserialize identifier & macros
1018   /// just so that we can report that they are unused, we just warn using
1019   /// the SourceLocations of this set (that will be filled by the ASTReader).
1020   using WarnUnusedMacroLocsTy = llvm::SmallDenseSet<SourceLocation, 32>;
1021   WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
1022 
1023   /// This is a pair of an optional message and source location used for pragmas
1024   /// that annotate macros like pragma clang restrict_expansion and pragma clang
1025   /// deprecated. This pair stores the optional message and the location of the
1026   /// annotation pragma for use producing diagnostics and notes.
1027   using MsgLocationPair = std::pair<std::string, SourceLocation>;
1028 
1029   struct MacroAnnotationInfo {
1030     SourceLocation Location;
1031     std::string Message;
1032   };
1033 
1034   struct MacroAnnotations {
1035     std::optional<MacroAnnotationInfo> DeprecationInfo;
1036     std::optional<MacroAnnotationInfo> RestrictExpansionInfo;
1037     std::optional<SourceLocation> FinalAnnotationLoc;
1038 
makeDeprecationMacroAnnotations1039     static MacroAnnotations makeDeprecation(SourceLocation Loc,
1040                                             std::string Msg) {
1041       return MacroAnnotations{MacroAnnotationInfo{Loc, std::move(Msg)},
1042                               std::nullopt, std::nullopt};
1043     }
1044 
makeRestrictExpansionMacroAnnotations1045     static MacroAnnotations makeRestrictExpansion(SourceLocation Loc,
1046                                                   std::string Msg) {
1047       return MacroAnnotations{
1048           std::nullopt, MacroAnnotationInfo{Loc, std::move(Msg)}, std::nullopt};
1049     }
1050 
makeFinalMacroAnnotations1051     static MacroAnnotations makeFinal(SourceLocation Loc) {
1052       return MacroAnnotations{std::nullopt, std::nullopt, Loc};
1053     }
1054   };
1055 
1056   /// Warning information for macro annotations.
1057   llvm::DenseMap<const IdentifierInfo *, MacroAnnotations> AnnotationInfos;
1058 
1059   /// A "freelist" of MacroArg objects that can be
1060   /// reused for quick allocation.
1061   MacroArgs *MacroArgCache = nullptr;
1062 
1063   /// For each IdentifierInfo used in a \#pragma push_macro directive,
1064   /// we keep a MacroInfo stack used to restore the previous macro value.
1065   llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
1066       PragmaPushMacroInfo;
1067 
1068   // Various statistics we track for performance analysis.
1069   unsigned NumDirectives = 0;
1070   unsigned NumDefined = 0;
1071   unsigned NumUndefined = 0;
1072   unsigned NumPragma = 0;
1073   unsigned NumIf = 0;
1074   unsigned NumElse = 0;
1075   unsigned NumEndif = 0;
1076   unsigned NumEnteredSourceFiles = 0;
1077   unsigned MaxIncludeStackDepth = 0;
1078   unsigned NumMacroExpanded = 0;
1079   unsigned NumFnMacroExpanded = 0;
1080   unsigned NumBuiltinMacroExpanded = 0;
1081   unsigned NumFastMacroExpanded = 0;
1082   unsigned NumTokenPaste = 0;
1083   unsigned NumFastTokenPaste = 0;
1084   unsigned NumSkipped = 0;
1085 
1086   /// The predefined macros that preprocessor should use from the
1087   /// command line etc.
1088   std::string Predefines;
1089 
1090   /// The file ID for the preprocessor predefines.
1091   FileID PredefinesFileID;
1092 
1093   /// The file ID for the PCH through header.
1094   FileID PCHThroughHeaderFileID;
1095 
1096   /// Whether tokens are being skipped until a #pragma hdrstop is seen.
1097   bool SkippingUntilPragmaHdrStop = false;
1098 
1099   /// Whether tokens are being skipped until the through header is seen.
1100   bool SkippingUntilPCHThroughHeader = false;
1101 
1102   /// \{
1103   /// Cache of macro expanders to reduce malloc traffic.
1104   enum { TokenLexerCacheSize = 8 };
1105   unsigned NumCachedTokenLexers;
1106   std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
1107   /// \}
1108 
1109   /// Keeps macro expanded tokens for TokenLexers.
1110   //
1111   /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1112   /// going to lex in the cache and when it finishes the tokens are removed
1113   /// from the end of the cache.
1114   SmallVector<Token, 16> MacroExpandedTokens;
1115   std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
1116 
1117   /// A record of the macro definitions and expansions that
1118   /// occurred during preprocessing.
1119   ///
1120   /// This is an optional side structure that can be enabled with
1121   /// \c createPreprocessingRecord() prior to preprocessing.
1122   PreprocessingRecord *Record = nullptr;
1123 
1124   /// Cached tokens state.
1125   using CachedTokensTy = SmallVector<Token, 1>;
1126 
1127   /// Cached tokens are stored here when we do backtracking or
1128   /// lookahead. They are "lexed" by the CachingLex() method.
1129   CachedTokensTy CachedTokens;
1130 
1131   /// The position of the cached token that CachingLex() should
1132   /// "lex" next.
1133   ///
1134   /// If it points beyond the CachedTokens vector, it means that a normal
1135   /// Lex() should be invoked.
1136   CachedTokensTy::size_type CachedLexPos = 0;
1137 
1138   /// Stack of backtrack positions, allowing nested backtracks.
1139   ///
1140   /// The EnableBacktrackAtThisPos() method pushes a position to
1141   /// indicate where CachedLexPos should be set when the BackTrack() method is
1142   /// invoked (at which point the last position is popped).
1143   std::vector<CachedTokensTy::size_type> BacktrackPositions;
1144 
1145   /// True if \p Preprocessor::SkipExcludedConditionalBlock() is running.
1146   /// This is used to guard against calling this function recursively.
1147   ///
1148   /// See comments at the use-site for more context about why it is needed.
1149   bool SkippingExcludedConditionalBlock = false;
1150 
1151   /// Keeps track of skipped range mappings that were recorded while skipping
1152   /// excluded conditional directives. It maps the source buffer pointer at
1153   /// the beginning of a skipped block, to the number of bytes that should be
1154   /// skipped.
1155   llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
1156 
1157   void updateOutOfDateIdentifier(IdentifierInfo &II) const;
1158 
1159 public:
1160   Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
1161                DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
1162                HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
1163                IdentifierInfoLookup *IILookup = nullptr,
1164                bool OwnsHeaderSearch = false,
1165                TranslationUnitKind TUKind = TU_Complete);
1166 
1167   ~Preprocessor();
1168 
1169   /// Initialize the preprocessor using information about the target.
1170   ///
1171   /// \param Target is owned by the caller and must remain valid for the
1172   /// lifetime of the preprocessor.
1173   /// \param AuxTarget is owned by the caller and must remain valid for
1174   /// the lifetime of the preprocessor.
1175   void Initialize(const TargetInfo &Target,
1176                   const TargetInfo *AuxTarget = nullptr);
1177 
1178   /// Initialize the preprocessor to parse a model file
1179   ///
1180   /// To parse model files the preprocessor of the original source is reused to
1181   /// preserver the identifier table. However to avoid some duplicate
1182   /// information in the preprocessor some cleanup is needed before it is used
1183   /// to parse model files. This method does that cleanup.
1184   void InitializeForModelFile();
1185 
1186   /// Cleanup after model file parsing
1187   void FinalizeForModelFile();
1188 
1189   /// Retrieve the preprocessor options used to initialize this
1190   /// preprocessor.
getPreprocessorOpts()1191   PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
1192 
getDiagnostics()1193   DiagnosticsEngine &getDiagnostics() const { return *Diags; }
setDiagnostics(DiagnosticsEngine & D)1194   void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
1195 
getLangOpts()1196   const LangOptions &getLangOpts() const { return LangOpts; }
getTargetInfo()1197   const TargetInfo &getTargetInfo() const { return *Target; }
getAuxTargetInfo()1198   const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
getFileManager()1199   FileManager &getFileManager() const { return FileMgr; }
getSourceManager()1200   SourceManager &getSourceManager() const { return SourceMgr; }
getHeaderSearchInfo()1201   HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
1202 
getIdentifierTable()1203   IdentifierTable &getIdentifierTable() { return Identifiers; }
getIdentifierTable()1204   const IdentifierTable &getIdentifierTable() const { return Identifiers; }
getSelectorTable()1205   SelectorTable &getSelectorTable() { return Selectors; }
getBuiltinInfo()1206   Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
getPreprocessorAllocator()1207   llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
1208 
setExternalSource(ExternalPreprocessorSource * Source)1209   void setExternalSource(ExternalPreprocessorSource *Source) {
1210     ExternalSource = Source;
1211   }
1212 
getExternalSource()1213   ExternalPreprocessorSource *getExternalSource() const {
1214     return ExternalSource;
1215   }
1216 
1217   /// Retrieve the module loader associated with this preprocessor.
getModuleLoader()1218   ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
1219 
hadModuleLoaderFatalFailure()1220   bool hadModuleLoaderFatalFailure() const {
1221     return TheModuleLoader.HadFatalFailure;
1222   }
1223 
1224   /// Retrieve the number of Directives that have been processed by the
1225   /// Preprocessor.
getNumDirectives()1226   unsigned getNumDirectives() const {
1227     return NumDirectives;
1228   }
1229 
1230   /// True if we are currently preprocessing a #if or #elif directive
isParsingIfOrElifDirective()1231   bool isParsingIfOrElifDirective() const {
1232     return ParsingIfOrElifDirective;
1233   }
1234 
1235   /// Control whether the preprocessor retains comments in output.
SetCommentRetentionState(bool KeepComments,bool KeepMacroComments)1236   void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
1237     this->KeepComments = KeepComments | KeepMacroComments;
1238     this->KeepMacroComments = KeepMacroComments;
1239   }
1240 
getCommentRetentionState()1241   bool getCommentRetentionState() const { return KeepComments; }
1242 
setPragmasEnabled(bool Enabled)1243   void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
getPragmasEnabled()1244   bool getPragmasEnabled() const { return PragmasEnabled; }
1245 
SetSuppressIncludeNotFoundError(bool Suppress)1246   void SetSuppressIncludeNotFoundError(bool Suppress) {
1247     SuppressIncludeNotFoundError = Suppress;
1248   }
1249 
GetSuppressIncludeNotFoundError()1250   bool GetSuppressIncludeNotFoundError() {
1251     return SuppressIncludeNotFoundError;
1252   }
1253 
1254   /// Sets whether the preprocessor is responsible for producing output or if
1255   /// it is producing tokens to be consumed by Parse and Sema.
setPreprocessedOutput(bool IsPreprocessedOutput)1256   void setPreprocessedOutput(bool IsPreprocessedOutput) {
1257     PreprocessedOutput = IsPreprocessedOutput;
1258   }
1259 
1260   /// Returns true if the preprocessor is responsible for generating output,
1261   /// false if it is producing tokens to be consumed by Parse and Sema.
isPreprocessedOutput()1262   bool isPreprocessedOutput() const { return PreprocessedOutput; }
1263 
1264   /// Return true if we are lexing directly from the specified lexer.
isCurrentLexer(const PreprocessorLexer * L)1265   bool isCurrentLexer(const PreprocessorLexer *L) const {
1266     return CurPPLexer == L;
1267   }
1268 
1269   /// Return the current lexer being lexed from.
1270   ///
1271   /// Note that this ignores any potentially active macro expansions and _Pragma
1272   /// expansions going on at the time.
getCurrentLexer()1273   PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
1274 
1275   /// Return the current file lexer being lexed from.
1276   ///
1277   /// Note that this ignores any potentially active macro expansions and _Pragma
1278   /// expansions going on at the time.
1279   PreprocessorLexer *getCurrentFileLexer() const;
1280 
1281   /// Return the submodule owning the file being lexed. This may not be
1282   /// the current module if we have changed modules since entering the file.
getCurrentLexerSubmodule()1283   Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1284 
1285   /// Returns the FileID for the preprocessor predefines.
getPredefinesFileID()1286   FileID getPredefinesFileID() const { return PredefinesFileID; }
1287 
1288   /// \{
1289   /// Accessors for preprocessor callbacks.
1290   ///
1291   /// Note that this class takes ownership of any PPCallbacks object given to
1292   /// it.
getPPCallbacks()1293   PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
addPPCallbacks(std::unique_ptr<PPCallbacks> C)1294   void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1295     if (Callbacks)
1296       C = std::make_unique<PPChainedCallbacks>(std::move(C),
1297                                                 std::move(Callbacks));
1298     Callbacks = std::move(C);
1299   }
1300   /// \}
1301 
1302   /// Get the number of tokens processed so far.
getTokenCount()1303   unsigned getTokenCount() const { return TokenCount; }
1304 
1305   /// Get the max number of tokens before issuing a -Wmax-tokens warning.
getMaxTokens()1306   unsigned getMaxTokens() const { return MaxTokens; }
1307 
overrideMaxTokens(unsigned Value,SourceLocation Loc)1308   void overrideMaxTokens(unsigned Value, SourceLocation Loc) {
1309     MaxTokens = Value;
1310     MaxTokensOverrideLoc = Loc;
1311   };
1312 
getMaxTokensOverrideLoc()1313   SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1314 
1315   /// Register a function that would be called on each token in the final
1316   /// expanded token stream.
1317   /// This also reports annotation tokens produced by the parser.
setTokenWatcher(llvm::unique_function<void (const clang::Token &)> F)1318   void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1319     OnToken = std::move(F);
1320   }
1321 
setPreprocessToken(bool Preprocess)1322   void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; }
1323 
isMacroDefined(StringRef Id)1324   bool isMacroDefined(StringRef Id) {
1325     return isMacroDefined(&Identifiers.get(Id));
1326   }
isMacroDefined(const IdentifierInfo * II)1327   bool isMacroDefined(const IdentifierInfo *II) {
1328     return II->hasMacroDefinition() &&
1329            (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1330   }
1331 
1332   /// Determine whether II is defined as a macro within the module M,
1333   /// if that is a module that we've already preprocessed. Does not check for
1334   /// macros imported into M.
isMacroDefinedInLocalModule(const IdentifierInfo * II,Module * M)1335   bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) {
1336     if (!II->hasMacroDefinition())
1337       return false;
1338     auto I = Submodules.find(M);
1339     if (I == Submodules.end())
1340       return false;
1341     auto J = I->second.Macros.find(II);
1342     if (J == I->second.Macros.end())
1343       return false;
1344     auto *MD = J->second.getLatest();
1345     return MD && MD->isDefined();
1346   }
1347 
getMacroDefinition(const IdentifierInfo * II)1348   MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
1349     if (!II->hasMacroDefinition())
1350       return {};
1351 
1352     MacroState &S = CurSubmoduleState->Macros[II];
1353     auto *MD = S.getLatest();
1354     while (MD && isa<VisibilityMacroDirective>(MD))
1355       MD = MD->getPrevious();
1356     return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1357                            S.getActiveModuleMacros(*this, II),
1358                            S.isAmbiguous(*this, II));
1359   }
1360 
getMacroDefinitionAtLoc(const IdentifierInfo * II,SourceLocation Loc)1361   MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
1362                                           SourceLocation Loc) {
1363     if (!II->hadMacroDefinition())
1364       return {};
1365 
1366     MacroState &S = CurSubmoduleState->Macros[II];
1367     MacroDirective::DefInfo DI;
1368     if (auto *MD = S.getLatest())
1369       DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1370     // FIXME: Compute the set of active module macros at the specified location.
1371     return MacroDefinition(DI.getDirective(),
1372                            S.getActiveModuleMacros(*this, II),
1373                            S.isAmbiguous(*this, II));
1374   }
1375 
1376   /// Given an identifier, return its latest non-imported MacroDirective
1377   /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
getLocalMacroDirective(const IdentifierInfo * II)1378   MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const {
1379     if (!II->hasMacroDefinition())
1380       return nullptr;
1381 
1382     auto *MD = getLocalMacroDirectiveHistory(II);
1383     if (!MD || MD->getDefinition().isUndefined())
1384       return nullptr;
1385 
1386     return MD;
1387   }
1388 
getMacroInfo(const IdentifierInfo * II)1389   const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1390     return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1391   }
1392 
getMacroInfo(const IdentifierInfo * II)1393   MacroInfo *getMacroInfo(const IdentifierInfo *II) {
1394     if (!II->hasMacroDefinition())
1395       return nullptr;
1396     if (auto MD = getMacroDefinition(II))
1397       return MD.getMacroInfo();
1398     return nullptr;
1399   }
1400 
1401   /// Given an identifier, return the latest non-imported macro
1402   /// directive for that identifier.
1403   ///
1404   /// One can iterate over all previous macro directives from the most recent
1405   /// one.
1406   MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1407 
1408   /// Add a directive to the macro directive history for this identifier.
1409   void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
appendDefMacroDirective(IdentifierInfo * II,MacroInfo * MI,SourceLocation Loc)1410   DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
1411                                              SourceLocation Loc) {
1412     DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1413     appendMacroDirective(II, MD);
1414     return MD;
1415   }
appendDefMacroDirective(IdentifierInfo * II,MacroInfo * MI)1416   DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II,
1417                                              MacroInfo *MI) {
1418     return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1419   }
1420 
1421   /// Set a MacroDirective that was loaded from a PCH file.
1422   void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1423                                MacroDirective *MD);
1424 
1425   /// Register an exported macro for a module and identifier.
1426   ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1427                               ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1428   ModuleMacro *getModuleMacro(Module *Mod, const IdentifierInfo *II);
1429 
1430   /// Get the list of leaf (non-overridden) module macros for a name.
getLeafModuleMacros(const IdentifierInfo * II)1431   ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
1432     if (II->isOutOfDate())
1433       updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1434     auto I = LeafModuleMacros.find(II);
1435     if (I != LeafModuleMacros.end())
1436       return I->second;
1437     return std::nullopt;
1438   }
1439 
1440   /// Get the list of submodules that we're currently building.
getBuildingSubmodules()1441   ArrayRef<BuildingSubmoduleInfo> getBuildingSubmodules() const {
1442     return BuildingSubmoduleStack;
1443   }
1444 
1445   /// \{
1446   /// Iterators for the macro history table. Currently defined macros have
1447   /// IdentifierInfo::hasMacroDefinition() set and an empty
1448   /// MacroInfo::getUndefLoc() at the head of the list.
1449   using macro_iterator = MacroMap::const_iterator;
1450 
1451   macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1452   macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1453 
1454   llvm::iterator_range<macro_iterator>
1455   macros(bool IncludeExternalMacros = true) const {
1456     macro_iterator begin = macro_begin(IncludeExternalMacros);
1457     macro_iterator end = macro_end(IncludeExternalMacros);
1458     return llvm::make_range(begin, end);
1459   }
1460 
1461   /// \}
1462 
1463   /// Mark the given clang module as affecting the current clang module or translation unit.
markClangModuleAsAffecting(Module * M)1464   void markClangModuleAsAffecting(Module *M) {
1465     assert(M->isModuleMapModule());
1466     if (!BuildingSubmoduleStack.empty()) {
1467       if (M != BuildingSubmoduleStack.back().M)
1468         BuildingSubmoduleStack.back().M->AffectingClangModules.insert(M);
1469     } else {
1470       AffectingClangModules.insert(M);
1471     }
1472   }
1473 
1474   /// Get the set of top-level clang modules that affected preprocessing, but were not
1475   /// imported.
getAffectingClangModules()1476   const llvm::SmallSetVector<Module *, 2> &getAffectingClangModules() const {
1477     return AffectingClangModules;
1478   }
1479 
1480   /// Mark the file as included.
1481   /// Returns true if this is the first time the file was included.
markIncluded(const FileEntry * File)1482   bool markIncluded(const FileEntry *File) {
1483     HeaderInfo.getFileInfo(File);
1484     return IncludedFiles.insert(File).second;
1485   }
1486 
1487   /// Return true if this header has already been included.
alreadyIncluded(const FileEntry * File)1488   bool alreadyIncluded(const FileEntry *File) const {
1489     return IncludedFiles.count(File);
1490   }
1491 
1492   /// Get the set of included files.
getIncludedFiles()1493   IncludedFilesSet &getIncludedFiles() { return IncludedFiles; }
getIncludedFiles()1494   const IncludedFilesSet &getIncludedFiles() const { return IncludedFiles; }
1495 
1496   /// Return the name of the macro defined before \p Loc that has
1497   /// spelling \p Tokens.  If there are multiple macros with same spelling,
1498   /// return the last one defined.
1499   StringRef getLastMacroWithSpelling(SourceLocation Loc,
1500                                      ArrayRef<TokenValue> Tokens) const;
1501 
1502   /// Get the predefines for this processor.
1503   /// Used by some third-party tools to inspect and add predefines (see
1504   /// https://github.com/llvm/llvm-project/issues/57483).
getPredefines()1505   const std::string &getPredefines() const { return Predefines; }
1506 
1507   /// Set the predefines for this Preprocessor.
1508   ///
1509   /// These predefines are automatically injected when parsing the main file.
setPredefines(std::string P)1510   void setPredefines(std::string P) { Predefines = std::move(P); }
1511 
1512   /// Return information about the specified preprocessor
1513   /// identifier token.
getIdentifierInfo(StringRef Name)1514   IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1515     return &Identifiers.get(Name);
1516   }
1517 
1518   /// Add the specified pragma handler to this preprocessor.
1519   ///
1520   /// If \p Namespace is non-null, then it is a token required to exist on the
1521   /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1522   void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
AddPragmaHandler(PragmaHandler * Handler)1523   void AddPragmaHandler(PragmaHandler *Handler) {
1524     AddPragmaHandler(StringRef(), Handler);
1525   }
1526 
1527   /// Remove the specific pragma handler from this preprocessor.
1528   ///
1529   /// If \p Namespace is non-null, then it should be the namespace that
1530   /// \p Handler was added to. It is an error to remove a handler that
1531   /// has not been registered.
1532   void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
RemovePragmaHandler(PragmaHandler * Handler)1533   void RemovePragmaHandler(PragmaHandler *Handler) {
1534     RemovePragmaHandler(StringRef(), Handler);
1535   }
1536 
1537   /// Install empty handlers for all pragmas (making them ignored).
1538   void IgnorePragmas();
1539 
1540   /// Set empty line handler.
setEmptylineHandler(EmptylineHandler * Handler)1541   void setEmptylineHandler(EmptylineHandler *Handler) { Emptyline = Handler; }
1542 
getEmptylineHandler()1543   EmptylineHandler *getEmptylineHandler() const { return Emptyline; }
1544 
1545   /// Add the specified comment handler to the preprocessor.
1546   void addCommentHandler(CommentHandler *Handler);
1547 
1548   /// Remove the specified comment handler.
1549   ///
1550   /// It is an error to remove a handler that has not been registered.
1551   void removeCommentHandler(CommentHandler *Handler);
1552 
1553   /// Set the code completion handler to the given object.
setCodeCompletionHandler(CodeCompletionHandler & Handler)1554   void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1555     CodeComplete = &Handler;
1556   }
1557 
1558   /// Retrieve the current code-completion handler.
getCodeCompletionHandler()1559   CodeCompletionHandler *getCodeCompletionHandler() const {
1560     return CodeComplete;
1561   }
1562 
1563   /// Clear out the code completion handler.
clearCodeCompletionHandler()1564   void clearCodeCompletionHandler() {
1565     CodeComplete = nullptr;
1566   }
1567 
1568   /// Hook used by the lexer to invoke the "included file" code
1569   /// completion point.
1570   void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1571 
1572   /// Hook used by the lexer to invoke the "natural language" code
1573   /// completion point.
1574   void CodeCompleteNaturalLanguage();
1575 
1576   /// Set the code completion token for filtering purposes.
setCodeCompletionIdentifierInfo(IdentifierInfo * Filter)1577   void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1578     CodeCompletionII = Filter;
1579   }
1580 
1581   /// Set the code completion token range for detecting replacement range later
1582   /// on.
setCodeCompletionTokenRange(const SourceLocation Start,const SourceLocation End)1583   void setCodeCompletionTokenRange(const SourceLocation Start,
1584                                    const SourceLocation End) {
1585     CodeCompletionTokenRange = {Start, End};
1586   }
getCodeCompletionTokenRange()1587   SourceRange getCodeCompletionTokenRange() const {
1588     return CodeCompletionTokenRange;
1589   }
1590 
1591   /// Get the code completion token for filtering purposes.
getCodeCompletionFilter()1592   StringRef getCodeCompletionFilter() {
1593     if (CodeCompletionII)
1594       return CodeCompletionII->getName();
1595     return {};
1596   }
1597 
1598   /// Retrieve the preprocessing record, or NULL if there is no
1599   /// preprocessing record.
getPreprocessingRecord()1600   PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1601 
1602   /// Create a new preprocessing record, which will keep track of
1603   /// all macro expansions, macro definitions, etc.
1604   void createPreprocessingRecord();
1605 
1606   /// Returns true if the FileEntry is the PCH through header.
1607   bool isPCHThroughHeader(const FileEntry *FE);
1608 
1609   /// True if creating a PCH with a through header.
1610   bool creatingPCHWithThroughHeader();
1611 
1612   /// True if using a PCH with a through header.
1613   bool usingPCHWithThroughHeader();
1614 
1615   /// True if creating a PCH with a #pragma hdrstop.
1616   bool creatingPCHWithPragmaHdrStop();
1617 
1618   /// True if using a PCH with a #pragma hdrstop.
1619   bool usingPCHWithPragmaHdrStop();
1620 
1621   /// Skip tokens until after the #include of the through header or
1622   /// until after a #pragma hdrstop.
1623   void SkipTokensWhileUsingPCH();
1624 
1625   /// Process directives while skipping until the through header or
1626   /// #pragma hdrstop is found.
1627   void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1628                                            SourceLocation HashLoc);
1629 
1630   /// Enter the specified FileID as the main source file,
1631   /// which implicitly adds the builtin defines etc.
1632   void EnterMainSourceFile();
1633 
1634   /// Inform the preprocessor callbacks that processing is complete.
1635   void EndSourceFile();
1636 
1637   /// Add a source file to the top of the include stack and
1638   /// start lexing tokens from it instead of the current buffer.
1639   ///
1640   /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1641   bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir,
1642                        SourceLocation Loc, bool IsFirstIncludeOfFile = true);
1643 
1644   /// Add a Macro to the top of the include stack and start lexing
1645   /// tokens from it instead of the current buffer.
1646   ///
1647   /// \param Args specifies the tokens input to a function-like macro.
1648   /// \param ILEnd specifies the location of the ')' for a function-like macro
1649   /// or the identifier for an object-like macro.
1650   void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1651                   MacroArgs *Args);
1652 
1653 private:
1654   /// Add a "macro" context to the top of the include stack,
1655   /// which will cause the lexer to start returning the specified tokens.
1656   ///
1657   /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1658   /// will not be subject to further macro expansion. Otherwise, these tokens
1659   /// will be re-macro-expanded when/if expansion is enabled.
1660   ///
1661   /// If \p OwnsTokens is false, this method assumes that the specified stream
1662   /// of tokens has a permanent owner somewhere, so they do not need to be
1663   /// copied. If it is true, it assumes the array of tokens is allocated with
1664   /// \c new[] and the Preprocessor will delete[] it.
1665   ///
1666   /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1667   /// set, see the flag documentation for details.
1668   void EnterTokenStream(const Token *Toks, unsigned NumToks,
1669                         bool DisableMacroExpansion, bool OwnsTokens,
1670                         bool IsReinject);
1671 
1672 public:
EnterTokenStream(std::unique_ptr<Token[]> Toks,unsigned NumToks,bool DisableMacroExpansion,bool IsReinject)1673   void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1674                         bool DisableMacroExpansion, bool IsReinject) {
1675     EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1676                      IsReinject);
1677   }
1678 
EnterTokenStream(ArrayRef<Token> Toks,bool DisableMacroExpansion,bool IsReinject)1679   void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1680                         bool IsReinject) {
1681     EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1682                      IsReinject);
1683   }
1684 
1685   /// Pop the current lexer/macro exp off the top of the lexer stack.
1686   ///
1687   /// This should only be used in situations where the current state of the
1688   /// top-of-stack lexer is known.
1689   void RemoveTopOfLexerStack();
1690 
1691   /// From the point that this method is called, and until
1692   /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1693   /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1694   /// make the Preprocessor re-lex the same tokens.
1695   ///
1696   /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1697   /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1698   /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1699   ///
1700   /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1701   /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1702   /// tokens will continue indefinitely.
1703   ///
1704   void EnableBacktrackAtThisPos();
1705 
1706   /// Disable the last EnableBacktrackAtThisPos call.
1707   void CommitBacktrackedTokens();
1708 
1709   /// Make Preprocessor re-lex the tokens that were lexed since
1710   /// EnableBacktrackAtThisPos() was previously called.
1711   void Backtrack();
1712 
1713   /// True if EnableBacktrackAtThisPos() was called and
1714   /// caching of tokens is on.
isBacktrackEnabled()1715   bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1716 
1717   /// Lex the next token for this preprocessor.
1718   void Lex(Token &Result);
1719 
1720   /// Lex a token, forming a header-name token if possible.
1721   bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1722 
1723   bool LexAfterModuleImport(Token &Result);
1724   void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1725 
1726   void makeModuleVisible(Module *M, SourceLocation Loc);
1727 
getModuleImportLoc(Module * M)1728   SourceLocation getModuleImportLoc(Module *M) const {
1729     return CurSubmoduleState->VisibleModules.getImportLoc(M);
1730   }
1731 
1732   /// Lex a string literal, which may be the concatenation of multiple
1733   /// string literals and may even come from macro expansion.
1734   /// \returns true on success, false if a error diagnostic has been generated.
LexStringLiteral(Token & Result,std::string & String,const char * DiagnosticTag,bool AllowMacroExpansion)1735   bool LexStringLiteral(Token &Result, std::string &String,
1736                         const char *DiagnosticTag, bool AllowMacroExpansion) {
1737     if (AllowMacroExpansion)
1738       Lex(Result);
1739     else
1740       LexUnexpandedToken(Result);
1741     return FinishLexStringLiteral(Result, String, DiagnosticTag,
1742                                   AllowMacroExpansion);
1743   }
1744 
1745   /// Complete the lexing of a string literal where the first token has
1746   /// already been lexed (see LexStringLiteral).
1747   bool FinishLexStringLiteral(Token &Result, std::string &String,
1748                               const char *DiagnosticTag,
1749                               bool AllowMacroExpansion);
1750 
1751   /// Lex a token.  If it's a comment, keep lexing until we get
1752   /// something not a comment.
1753   ///
1754   /// This is useful in -E -C mode where comments would foul up preprocessor
1755   /// directive handling.
LexNonComment(Token & Result)1756   void LexNonComment(Token &Result) {
1757     do
1758       Lex(Result);
1759     while (Result.getKind() == tok::comment);
1760   }
1761 
1762   /// Just like Lex, but disables macro expansion of identifier tokens.
LexUnexpandedToken(Token & Result)1763   void LexUnexpandedToken(Token &Result) {
1764     // Disable macro expansion.
1765     bool OldVal = DisableMacroExpansion;
1766     DisableMacroExpansion = true;
1767     // Lex the token.
1768     Lex(Result);
1769 
1770     // Reenable it.
1771     DisableMacroExpansion = OldVal;
1772   }
1773 
1774   /// Like LexNonComment, but this disables macro expansion of
1775   /// identifier tokens.
LexUnexpandedNonComment(Token & Result)1776   void LexUnexpandedNonComment(Token &Result) {
1777     do
1778       LexUnexpandedToken(Result);
1779     while (Result.getKind() == tok::comment);
1780   }
1781 
1782   /// Parses a simple integer literal to get its numeric value.  Floating
1783   /// point literals and user defined literals are rejected.  Used primarily to
1784   /// handle pragmas that accept integer arguments.
1785   bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1786 
1787   /// Disables macro expansion everywhere except for preprocessor directives.
SetMacroExpansionOnlyInDirectives()1788   void SetMacroExpansionOnlyInDirectives() {
1789     DisableMacroExpansion = true;
1790     MacroExpansionInDirectivesOverride = true;
1791   }
1792 
1793   /// Peeks ahead N tokens and returns that token without consuming any
1794   /// tokens.
1795   ///
1796   /// LookAhead(0) returns the next token that would be returned by Lex(),
1797   /// LookAhead(1) returns the token after it, etc.  This returns normal
1798   /// tokens after phase 5.  As such, it is equivalent to using
1799   /// 'Lex', not 'LexUnexpandedToken'.
LookAhead(unsigned N)1800   const Token &LookAhead(unsigned N) {
1801     assert(LexLevel == 0 && "cannot use lookahead while lexing");
1802     if (CachedLexPos + N < CachedTokens.size())
1803       return CachedTokens[CachedLexPos+N];
1804     else
1805       return PeekAhead(N+1);
1806   }
1807 
1808   /// When backtracking is enabled and tokens are cached,
1809   /// this allows to revert a specific number of tokens.
1810   ///
1811   /// Note that the number of tokens being reverted should be up to the last
1812   /// backtrack position, not more.
RevertCachedTokens(unsigned N)1813   void RevertCachedTokens(unsigned N) {
1814     assert(isBacktrackEnabled() &&
1815            "Should only be called when tokens are cached for backtracking");
1816     assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1817          && "Should revert tokens up to the last backtrack position, not more");
1818     assert(signed(CachedLexPos) - signed(N) >= 0 &&
1819            "Corrupted backtrack positions ?");
1820     CachedLexPos -= N;
1821   }
1822 
1823   /// Enters a token in the token stream to be lexed next.
1824   ///
1825   /// If BackTrack() is called afterwards, the token will remain at the
1826   /// insertion point.
1827   /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1828   /// flag set. See the flag documentation for details.
EnterToken(const Token & Tok,bool IsReinject)1829   void EnterToken(const Token &Tok, bool IsReinject) {
1830     if (LexLevel) {
1831       // It's not correct in general to enter caching lex mode while in the
1832       // middle of a nested lexing action.
1833       auto TokCopy = std::make_unique<Token[]>(1);
1834       TokCopy[0] = Tok;
1835       EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1836     } else {
1837       EnterCachingLexMode();
1838       assert(IsReinject && "new tokens in the middle of cached stream");
1839       CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1840     }
1841   }
1842 
1843   /// We notify the Preprocessor that if it is caching tokens (because
1844   /// backtrack is enabled) it should replace the most recent cached tokens
1845   /// with the given annotation token. This function has no effect if
1846   /// backtracking is not enabled.
1847   ///
1848   /// Note that the use of this function is just for optimization, so that the
1849   /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1850   /// invoked.
AnnotateCachedTokens(const Token & Tok)1851   void AnnotateCachedTokens(const Token &Tok) {
1852     assert(Tok.isAnnotation() && "Expected annotation token");
1853     if (CachedLexPos != 0 && isBacktrackEnabled())
1854       AnnotatePreviousCachedTokens(Tok);
1855   }
1856 
1857   /// Get the location of the last cached token, suitable for setting the end
1858   /// location of an annotation token.
getLastCachedTokenLocation()1859   SourceLocation getLastCachedTokenLocation() const {
1860     assert(CachedLexPos != 0);
1861     return CachedTokens[CachedLexPos-1].getLastLoc();
1862   }
1863 
1864   /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1865   /// CachedTokens.
1866   bool IsPreviousCachedToken(const Token &Tok) const;
1867 
1868   /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1869   /// in \p NewToks.
1870   ///
1871   /// Useful when a token needs to be split in smaller ones and CachedTokens
1872   /// most recent token must to be updated to reflect that.
1873   void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1874 
1875   /// Replace the last token with an annotation token.
1876   ///
1877   /// Like AnnotateCachedTokens(), this routine replaces an
1878   /// already-parsed (and resolved) token with an annotation
1879   /// token. However, this routine only replaces the last token with
1880   /// the annotation token; it does not affect any other cached
1881   /// tokens. This function has no effect if backtracking is not
1882   /// enabled.
ReplaceLastTokenWithAnnotation(const Token & Tok)1883   void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1884     assert(Tok.isAnnotation() && "Expected annotation token");
1885     if (CachedLexPos != 0 && isBacktrackEnabled())
1886       CachedTokens[CachedLexPos-1] = Tok;
1887   }
1888 
1889   /// Enter an annotation token into the token stream.
1890   void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1891                             void *AnnotationVal);
1892 
1893   /// Determine whether it's possible for a future call to Lex to produce an
1894   /// annotation token created by a previous call to EnterAnnotationToken.
mightHavePendingAnnotationTokens()1895   bool mightHavePendingAnnotationTokens() {
1896     return CurLexerKind != CLK_Lexer;
1897   }
1898 
1899   /// Update the current token to represent the provided
1900   /// identifier, in order to cache an action performed by typo correction.
TypoCorrectToken(const Token & Tok)1901   void TypoCorrectToken(const Token &Tok) {
1902     assert(Tok.getIdentifierInfo() && "Expected identifier token");
1903     if (CachedLexPos != 0 && isBacktrackEnabled())
1904       CachedTokens[CachedLexPos-1] = Tok;
1905   }
1906 
1907   /// Recompute the current lexer kind based on the CurLexer/
1908   /// CurTokenLexer pointers.
1909   void recomputeCurLexerKind();
1910 
1911   /// Returns true if incremental processing is enabled
isIncrementalProcessingEnabled()1912   bool isIncrementalProcessingEnabled() const {
1913     return getLangOpts().IncrementalExtensions;
1914   }
1915 
1916   /// Enables the incremental processing
1917   void enableIncrementalProcessing(bool value = true) {
1918     // FIXME: Drop this interface.
1919     const_cast<LangOptions &>(getLangOpts()).IncrementalExtensions = value;
1920   }
1921 
1922   /// Specify the point at which code-completion will be performed.
1923   ///
1924   /// \param File the file in which code completion should occur. If
1925   /// this file is included multiple times, code-completion will
1926   /// perform completion the first time it is included. If NULL, this
1927   /// function clears out the code-completion point.
1928   ///
1929   /// \param Line the line at which code completion should occur
1930   /// (1-based).
1931   ///
1932   /// \param Column the column at which code completion should occur
1933   /// (1-based).
1934   ///
1935   /// \returns true if an error occurred, false otherwise.
1936   bool SetCodeCompletionPoint(const FileEntry *File,
1937                               unsigned Line, unsigned Column);
1938 
1939   /// Determine if we are performing code completion.
isCodeCompletionEnabled()1940   bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1941 
1942   /// Returns the location of the code-completion point.
1943   ///
1944   /// Returns an invalid location if code-completion is not enabled or the file
1945   /// containing the code-completion point has not been lexed yet.
getCodeCompletionLoc()1946   SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1947 
1948   /// Returns the start location of the file of code-completion point.
1949   ///
1950   /// Returns an invalid location if code-completion is not enabled or the file
1951   /// containing the code-completion point has not been lexed yet.
getCodeCompletionFileLoc()1952   SourceLocation getCodeCompletionFileLoc() const {
1953     return CodeCompletionFileLoc;
1954   }
1955 
1956   /// Returns true if code-completion is enabled and we have hit the
1957   /// code-completion point.
isCodeCompletionReached()1958   bool isCodeCompletionReached() const { return CodeCompletionReached; }
1959 
1960   /// Note that we hit the code-completion point.
setCodeCompletionReached()1961   void setCodeCompletionReached() {
1962     assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1963     CodeCompletionReached = true;
1964     // Silence any diagnostics that occur after we hit the code-completion.
1965     getDiagnostics().setSuppressAllDiagnostics(true);
1966   }
1967 
1968   /// The location of the currently-active \#pragma clang
1969   /// arc_cf_code_audited begin.
1970   ///
1971   /// Returns an invalid location if there is no such pragma active.
1972   std::pair<IdentifierInfo *, SourceLocation>
getPragmaARCCFCodeAuditedInfo()1973   getPragmaARCCFCodeAuditedInfo() const {
1974     return PragmaARCCFCodeAuditedInfo;
1975   }
1976 
1977   /// Set the location of the currently-active \#pragma clang
1978   /// arc_cf_code_audited begin.  An invalid location ends the pragma.
setPragmaARCCFCodeAuditedInfo(IdentifierInfo * Ident,SourceLocation Loc)1979   void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
1980                                      SourceLocation Loc) {
1981     PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1982   }
1983 
1984   /// The location of the currently-active \#pragma clang
1985   /// assume_nonnull begin.
1986   ///
1987   /// Returns an invalid location if there is no such pragma active.
getPragmaAssumeNonNullLoc()1988   SourceLocation getPragmaAssumeNonNullLoc() const {
1989     return PragmaAssumeNonNullLoc;
1990   }
1991 
1992   /// Set the location of the currently-active \#pragma clang
1993   /// assume_nonnull begin.  An invalid location ends the pragma.
setPragmaAssumeNonNullLoc(SourceLocation Loc)1994   void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
1995     PragmaAssumeNonNullLoc = Loc;
1996   }
1997 
1998   /// Get the location of the recorded unterminated \#pragma clang
1999   /// assume_nonnull begin in the preamble, if one exists.
2000   ///
2001   /// Returns an invalid location if the premable did not end with
2002   /// such a pragma active or if there is no recorded preamble.
getPreambleRecordedPragmaAssumeNonNullLoc()2003   SourceLocation getPreambleRecordedPragmaAssumeNonNullLoc() const {
2004     return PreambleRecordedPragmaAssumeNonNullLoc;
2005   }
2006 
2007   /// Record the location of the unterminated \#pragma clang
2008   /// assume_nonnull begin in the preamble.
setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc)2009   void setPreambleRecordedPragmaAssumeNonNullLoc(SourceLocation Loc) {
2010     PreambleRecordedPragmaAssumeNonNullLoc = Loc;
2011   }
2012 
2013   /// Set the directory in which the main file should be considered
2014   /// to have been found, if it is not a real file.
setMainFileDir(const DirectoryEntry * Dir)2015   void setMainFileDir(const DirectoryEntry *Dir) {
2016     MainFileDir = Dir;
2017   }
2018 
2019   /// Instruct the preprocessor to skip part of the main source file.
2020   ///
2021   /// \param Bytes The number of bytes in the preamble to skip.
2022   ///
2023   /// \param StartOfLine Whether skipping these bytes puts the lexer at the
2024   /// start of a line.
setSkipMainFilePreamble(unsigned Bytes,bool StartOfLine)2025   void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
2026     SkipMainFilePreamble.first = Bytes;
2027     SkipMainFilePreamble.second = StartOfLine;
2028   }
2029 
2030   /// Forwarding function for diagnostics.  This emits a diagnostic at
2031   /// the specified Token's location, translating the token's start
2032   /// position in the current buffer into a SourcePosition object for rendering.
Diag(SourceLocation Loc,unsigned DiagID)2033   DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
2034     return Diags->Report(Loc, DiagID);
2035   }
2036 
Diag(const Token & Tok,unsigned DiagID)2037   DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
2038     return Diags->Report(Tok.getLocation(), DiagID);
2039   }
2040 
2041   /// Return the 'spelling' of the token at the given
2042   /// location; does not go up to the spelling location or down to the
2043   /// expansion location.
2044   ///
2045   /// \param buffer A buffer which will be used only if the token requires
2046   ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
2047   /// \param invalid If non-null, will be set \c true if an error occurs.
2048   StringRef getSpelling(SourceLocation loc,
2049                         SmallVectorImpl<char> &buffer,
2050                         bool *invalid = nullptr) const {
2051     return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
2052   }
2053 
2054   /// Return the 'spelling' of the Tok token.
2055   ///
2056   /// The spelling of a token is the characters used to represent the token in
2057   /// the source file after trigraph expansion and escaped-newline folding.  In
2058   /// particular, this wants to get the true, uncanonicalized, spelling of
2059   /// things like digraphs, UCNs, etc.
2060   ///
2061   /// \param Invalid If non-null, will be set \c true if an error occurs.
2062   std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
2063     return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
2064   }
2065 
2066   /// Get the spelling of a token into a preallocated buffer, instead
2067   /// of as an std::string.
2068   ///
2069   /// The caller is required to allocate enough space for the token, which is
2070   /// guaranteed to be at least Tok.getLength() bytes long. The length of the
2071   /// actual result is returned.
2072   ///
2073   /// Note that this method may do two possible things: it may either fill in
2074   /// the buffer specified with characters, or it may *change the input pointer*
2075   /// to point to a constant buffer with the data already in it (avoiding a
2076   /// copy).  The caller is not allowed to modify the returned buffer pointer
2077   /// if an internal buffer is returned.
2078   unsigned getSpelling(const Token &Tok, const char *&Buffer,
2079                        bool *Invalid = nullptr) const {
2080     return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
2081   }
2082 
2083   /// Get the spelling of a token into a SmallVector.
2084   ///
2085   /// Note that the returned StringRef may not point to the
2086   /// supplied buffer if a copy can be avoided.
2087   StringRef getSpelling(const Token &Tok,
2088                         SmallVectorImpl<char> &Buffer,
2089                         bool *Invalid = nullptr) const;
2090 
2091   /// Relex the token at the specified location.
2092   /// \returns true if there was a failure, false on success.
2093   bool getRawToken(SourceLocation Loc, Token &Result,
2094                    bool IgnoreWhiteSpace = false) {
2095     return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
2096   }
2097 
2098   /// Given a Token \p Tok that is a numeric constant with length 1,
2099   /// return the character.
2100   char
2101   getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
2102                                               bool *Invalid = nullptr) const {
2103     assert(Tok.is(tok::numeric_constant) &&
2104            Tok.getLength() == 1 && "Called on unsupported token");
2105     assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
2106 
2107     // If the token is carrying a literal data pointer, just use it.
2108     if (const char *D = Tok.getLiteralData())
2109       return *D;
2110 
2111     // Otherwise, fall back on getCharacterData, which is slower, but always
2112     // works.
2113     return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
2114   }
2115 
2116   /// Retrieve the name of the immediate macro expansion.
2117   ///
2118   /// This routine starts from a source location, and finds the name of the
2119   /// macro responsible for its immediate expansion. It looks through any
2120   /// intervening macro argument expansions to compute this. It returns a
2121   /// StringRef that refers to the SourceManager-owned buffer of the source
2122   /// where that macro name is spelled. Thus, the result shouldn't out-live
2123   /// the SourceManager.
getImmediateMacroName(SourceLocation Loc)2124   StringRef getImmediateMacroName(SourceLocation Loc) {
2125     return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
2126   }
2127 
2128   /// Plop the specified string into a scratch buffer and set the
2129   /// specified token's location and length to it.
2130   ///
2131   /// If specified, the source location provides a location of the expansion
2132   /// point of the token.
2133   void CreateString(StringRef Str, Token &Tok,
2134                     SourceLocation ExpansionLocStart = SourceLocation(),
2135                     SourceLocation ExpansionLocEnd = SourceLocation());
2136 
2137   /// Split the first Length characters out of the token starting at TokLoc
2138   /// and return a location pointing to the split token. Re-lexing from the
2139   /// split token will return the split token rather than the original.
2140   SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
2141 
2142   /// Computes the source location just past the end of the
2143   /// token at this source location.
2144   ///
2145   /// This routine can be used to produce a source location that
2146   /// points just past the end of the token referenced by \p Loc, and
2147   /// is generally used when a diagnostic needs to point just after a
2148   /// token where it expected something different that it received. If
2149   /// the returned source location would not be meaningful (e.g., if
2150   /// it points into a macro), this routine returns an invalid
2151   /// source location.
2152   ///
2153   /// \param Offset an offset from the end of the token, where the source
2154   /// location should refer to. The default offset (0) produces a source
2155   /// location pointing just past the end of the token; an offset of 1 produces
2156   /// a source location pointing to the last character in the token, etc.
2157   SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
2158     return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
2159   }
2160 
2161   /// Returns true if the given MacroID location points at the first
2162   /// token of the macro expansion.
2163   ///
2164   /// \param MacroBegin If non-null and function returns true, it is set to
2165   /// begin location of the macro.
2166   bool isAtStartOfMacroExpansion(SourceLocation loc,
2167                                  SourceLocation *MacroBegin = nullptr) const {
2168     return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
2169                                             MacroBegin);
2170   }
2171 
2172   /// Returns true if the given MacroID location points at the last
2173   /// token of the macro expansion.
2174   ///
2175   /// \param MacroEnd If non-null and function returns true, it is set to
2176   /// end location of the macro.
2177   bool isAtEndOfMacroExpansion(SourceLocation loc,
2178                                SourceLocation *MacroEnd = nullptr) const {
2179     return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
2180   }
2181 
2182   /// Print the token to stderr, used for debugging.
2183   void DumpToken(const Token &Tok, bool DumpFlags = false) const;
2184   void DumpLocation(SourceLocation Loc) const;
2185   void DumpMacro(const MacroInfo &MI) const;
2186   void dumpMacroInfo(const IdentifierInfo *II);
2187 
2188   /// Given a location that specifies the start of a
2189   /// token, return a new location that specifies a character within the token.
AdvanceToTokenCharacter(SourceLocation TokStart,unsigned Char)2190   SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
2191                                          unsigned Char) const {
2192     return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
2193   }
2194 
2195   /// Increment the counters for the number of token paste operations
2196   /// performed.
2197   ///
2198   /// If fast was specified, this is a 'fast paste' case we handled.
IncrementPasteCounter(bool isFast)2199   void IncrementPasteCounter(bool isFast) {
2200     if (isFast)
2201       ++NumFastTokenPaste;
2202     else
2203       ++NumTokenPaste;
2204   }
2205 
2206   void PrintStats();
2207 
2208   size_t getTotalMemory() const;
2209 
2210   /// When the macro expander pastes together a comment (/##/) in Microsoft
2211   /// mode, this method handles updating the current state, returning the
2212   /// token on the next source line.
2213   void HandleMicrosoftCommentPaste(Token &Tok);
2214 
2215   //===--------------------------------------------------------------------===//
2216   // Preprocessor callback methods.  These are invoked by a lexer as various
2217   // directives and events are found.
2218 
2219   /// Given a tok::raw_identifier token, look up the
2220   /// identifier information for the token and install it into the token,
2221   /// updating the token kind accordingly.
2222   IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
2223 
2224 private:
2225   llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
2226 
2227 public:
2228   /// Specifies the reason for poisoning an identifier.
2229   ///
2230   /// If that identifier is accessed while poisoned, then this reason will be
2231   /// used instead of the default "poisoned" diagnostic.
2232   void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
2233 
2234   /// Display reason for poisoned identifier.
2235   void HandlePoisonedIdentifier(Token & Identifier);
2236 
MaybeHandlePoisonedIdentifier(Token & Identifier)2237   void MaybeHandlePoisonedIdentifier(Token & Identifier) {
2238     if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
2239       if(II->isPoisoned()) {
2240         HandlePoisonedIdentifier(Identifier);
2241       }
2242     }
2243   }
2244 
2245 private:
2246   /// Identifiers used for SEH handling in Borland. These are only
2247   /// allowed in particular circumstances
2248   // __except block
2249   IdentifierInfo *Ident__exception_code,
2250                  *Ident___exception_code,
2251                  *Ident_GetExceptionCode;
2252   // __except filter expression
2253   IdentifierInfo *Ident__exception_info,
2254                  *Ident___exception_info,
2255                  *Ident_GetExceptionInfo;
2256   // __finally
2257   IdentifierInfo *Ident__abnormal_termination,
2258                  *Ident___abnormal_termination,
2259                  *Ident_AbnormalTermination;
2260 
2261   const char *getCurLexerEndPos();
2262   void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
2263 
2264 public:
2265   void PoisonSEHIdentifiers(bool Poison = true); // Borland
2266 
2267   /// Callback invoked when the lexer reads an identifier and has
2268   /// filled in the tokens IdentifierInfo member.
2269   ///
2270   /// This callback potentially macro expands it or turns it into a named
2271   /// token (like 'for').
2272   ///
2273   /// \returns true if we actually computed a token, false if we need to
2274   /// lex again.
2275   bool HandleIdentifier(Token &Identifier);
2276 
2277   /// Callback invoked when the lexer hits the end of the current file.
2278   ///
2279   /// This either returns the EOF token and returns true, or
2280   /// pops a level off the include stack and returns false, at which point the
2281   /// client should call lex again.
2282   bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
2283 
2284   /// Callback invoked when the current TokenLexer hits the end of its
2285   /// token stream.
2286   bool HandleEndOfTokenLexer(Token &Result);
2287 
2288   /// Callback invoked when the lexer sees a # token at the start of a
2289   /// line.
2290   ///
2291   /// This consumes the directive, modifies the lexer/preprocessor state, and
2292   /// advances the lexer(s) so that the next token read is the correct one.
2293   void HandleDirective(Token &Result);
2294 
2295   /// Ensure that the next token is a tok::eod token.
2296   ///
2297   /// If not, emit a diagnostic and consume up until the eod.
2298   /// If \p EnableMacros is true, then we consider macros that expand to zero
2299   /// tokens as being ok.
2300   ///
2301   /// \return The location of the end of the directive (the terminating
2302   /// newline).
2303   SourceLocation CheckEndOfDirective(const char *DirType,
2304                                      bool EnableMacros = false);
2305 
2306   /// Read and discard all tokens remaining on the current line until
2307   /// the tok::eod token is found. Returns the range of the skipped tokens.
2308   SourceRange DiscardUntilEndOfDirective();
2309 
2310   /// Returns true if the preprocessor has seen a use of
2311   /// __DATE__ or __TIME__ in the file so far.
SawDateOrTime()2312   bool SawDateOrTime() const {
2313     return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
2314   }
getCounterValue()2315   unsigned getCounterValue() const { return CounterValue; }
setCounterValue(unsigned V)2316   void setCounterValue(unsigned V) { CounterValue = V; }
2317 
getCurrentFPEvalMethod()2318   LangOptions::FPEvalMethodKind getCurrentFPEvalMethod() const {
2319     assert(CurrentFPEvalMethod != LangOptions::FEM_UnsetOnCommandLine &&
2320            "FPEvalMethod should be set either from command line or from the "
2321            "target info");
2322     return CurrentFPEvalMethod;
2323   }
2324 
getTUFPEvalMethod()2325   LangOptions::FPEvalMethodKind getTUFPEvalMethod() const {
2326     return TUFPEvalMethod;
2327   }
2328 
getLastFPEvalPragmaLocation()2329   SourceLocation getLastFPEvalPragmaLocation() const {
2330     return LastFPEvalPragmaLocation;
2331   }
2332 
setCurrentFPEvalMethod(SourceLocation PragmaLoc,LangOptions::FPEvalMethodKind Val)2333   void setCurrentFPEvalMethod(SourceLocation PragmaLoc,
2334                               LangOptions::FPEvalMethodKind Val) {
2335     assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2336            "FPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2337     // This is the location of the '#pragma float_control" where the
2338     // execution state is modifed.
2339     LastFPEvalPragmaLocation = PragmaLoc;
2340     CurrentFPEvalMethod = Val;
2341     TUFPEvalMethod = Val;
2342   }
2343 
setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)2344   void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val) {
2345     assert(Val != LangOptions::FEM_UnsetOnCommandLine &&
2346            "TUPEvalMethod should never be set to FEM_UnsetOnCommandLine");
2347     TUFPEvalMethod = Val;
2348   }
2349 
2350   /// Retrieves the module that we're currently building, if any.
2351   Module *getCurrentModule();
2352 
2353   /// Retrieves the module whose implementation we're current compiling, if any.
2354   Module *getCurrentModuleImplementation();
2355 
2356   /// If we are preprocessing a named module.
isInNamedModule()2357   bool isInNamedModule() const { return ModuleDeclState.isNamedModule(); }
2358 
2359   /// If we are proprocessing a named interface unit.
2360   /// Note that a module implementation partition is not considered as an
2361   /// named interface unit here although it is importable
2362   /// to ease the parsing.
isInNamedInterfaceUnit()2363   bool isInNamedInterfaceUnit() const {
2364     return ModuleDeclState.isNamedInterface();
2365   }
2366 
2367   /// Get the named module name we're preprocessing.
2368   /// Requires we're preprocessing a named module.
getNamedModuleName()2369   StringRef getNamedModuleName() const { return ModuleDeclState.getName(); }
2370 
2371   /// If we are implementing an implementation module unit.
2372   /// Note that the module implementation partition is not considered as an
2373   /// implementation unit.
isInImplementationUnit()2374   bool isInImplementationUnit() const {
2375     return ModuleDeclState.isImplementationUnit();
2376   }
2377 
2378   /// If we're importing a standard C++20 Named Modules.
isInImportingCXXNamedModules()2379   bool isInImportingCXXNamedModules() const {
2380     // NamedModuleImportPath will be non-empty only if we're importing
2381     // Standard C++ named modules.
2382     return !NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules &&
2383            !IsAtImport;
2384   }
2385 
2386   /// Allocate a new MacroInfo object with the provided SourceLocation.
2387   MacroInfo *AllocateMacroInfo(SourceLocation L);
2388 
2389   /// Turn the specified lexer token into a fully checked and spelled
2390   /// filename, e.g. as an operand of \#include.
2391   ///
2392   /// The caller is expected to provide a buffer that is large enough to hold
2393   /// the spelling of the filename, but is also expected to handle the case
2394   /// when this method decides to use a different buffer.
2395   ///
2396   /// \returns true if the input filename was in <>'s or false if it was
2397   /// in ""'s.
2398   bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
2399 
2400   /// Given a "foo" or \<foo> reference, look up the indicated file.
2401   ///
2402   /// Returns std::nullopt on failure.  \p isAngled indicates whether the file
2403   /// reference is for system \#include's or not (i.e. using <> instead of "").
2404   OptionalFileEntryRef
2405   LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
2406              ConstSearchDirIterator FromDir, const FileEntry *FromFile,
2407              ConstSearchDirIterator *CurDir, SmallVectorImpl<char> *SearchPath,
2408              SmallVectorImpl<char> *RelativePath,
2409              ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
2410              bool *IsFrameworkFound, bool SkipCache = false,
2411              bool OpenFile = true, bool CacheFailures = true);
2412 
2413   /// Return true if we're in the top-level file, not in a \#include.
2414   bool isInPrimaryFile() const;
2415 
2416   /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2417   /// followed by EOD.  Return true if the token is not a valid on-off-switch.
2418   bool LexOnOffSwitch(tok::OnOffSwitch &Result);
2419 
2420   bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2421                       bool *ShadowFlag = nullptr);
2422 
2423   void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2424   Module *LeaveSubmodule(bool ForPragma);
2425 
2426 private:
2427   friend void TokenLexer::ExpandFunctionArguments();
2428 
PushIncludeMacroStack()2429   void PushIncludeMacroStack() {
2430     assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
2431     IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2432                                    std::move(CurLexer), CurPPLexer,
2433                                    std::move(CurTokenLexer), CurDirLookup);
2434     CurPPLexer = nullptr;
2435   }
2436 
PopIncludeMacroStack()2437   void PopIncludeMacroStack() {
2438     CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2439     CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2440     CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2441     CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
2442     CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2443     CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2444     IncludeMacroStack.pop_back();
2445   }
2446 
2447   void PropagateLineStartLeadingSpaceInfo(Token &Result);
2448 
2449   /// Determine whether we need to create module macros for #defines in the
2450   /// current context.
2451   bool needModuleMacros() const;
2452 
2453   /// Update the set of active module macros and ambiguity flag for a module
2454   /// macro name.
2455   void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2456 
2457   DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2458                                                SourceLocation Loc);
2459   UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2460   VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2461                                                              bool isPublic);
2462 
2463   /// Lex and validate a macro name, which occurs after a
2464   /// \#define or \#undef.
2465   ///
2466   /// \param MacroNameTok Token that represents the name defined or undefined.
2467   /// \param IsDefineUndef Kind if preprocessor directive.
2468   /// \param ShadowFlag Points to flag that is set if macro name shadows
2469   ///                   a keyword.
2470   ///
2471   /// This emits a diagnostic, sets the token kind to eod,
2472   /// and discards the rest of the macro line if the macro name is invalid.
2473   void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2474                      bool *ShadowFlag = nullptr);
2475 
2476   /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2477   /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2478   /// doing so performs certain validity checks including (but not limited to):
2479   ///   - # (stringization) is followed by a macro parameter
2480   /// \param MacroNameTok - Token that represents the macro name
2481   /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2482   ///
2483   ///  Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2484   ///  returns a nullptr if an invalid sequence of tokens is encountered.
2485   MacroInfo *ReadOptionalMacroParameterListAndBody(
2486       const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2487 
2488   /// The ( starting an argument list of a macro definition has just been read.
2489   /// Lex the rest of the parameters and the closing ), updating \p MI with
2490   /// what we learn and saving in \p LastTok the last token read.
2491   /// Return true if an error occurs parsing the arg list.
2492   bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2493 
2494   /// Provide a suggestion for a typoed directive. If there is no typo, then
2495   /// just skip suggesting.
2496   ///
2497   /// \param Tok - Token that represents the directive
2498   /// \param Directive - String reference for the directive name
2499   void SuggestTypoedDirective(const Token &Tok, StringRef Directive) const;
2500 
2501   /// We just read a \#if or related directive and decided that the
2502   /// subsequent tokens are in the \#if'd out portion of the
2503   /// file.  Lex the rest of the file, until we see an \#endif.  If \p
2504   /// FoundNonSkipPortion is true, then we have already emitted code for part of
2505   /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2506   /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2507   /// already seen one so a \#else directive is a duplicate.  When this returns,
2508   /// the caller can lex the first valid token.
2509   void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2510                                     SourceLocation IfTokenLoc,
2511                                     bool FoundNonSkipPortion, bool FoundElse,
2512                                     SourceLocation ElseLoc = SourceLocation());
2513 
2514   /// Information about the result for evaluating an expression for a
2515   /// preprocessor directive.
2516   struct DirectiveEvalResult {
2517     /// Whether the expression was evaluated as true or not.
2518     bool Conditional;
2519 
2520     /// True if the expression contained identifiers that were undefined.
2521     bool IncludedUndefinedIds;
2522 
2523     /// The source range for the expression.
2524     SourceRange ExprRange;
2525   };
2526 
2527   /// Evaluate an integer constant expression that may occur after a
2528   /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2529   ///
2530   /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2531   DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2532 
2533   /// Process a '__has_include("path")' expression.
2534   ///
2535   /// Returns true if successful.
2536   bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II);
2537 
2538   /// Process '__has_include_next("path")' expression.
2539   ///
2540   /// Returns true if successful.
2541   bool EvaluateHasIncludeNext(Token &Tok, IdentifierInfo *II);
2542 
2543   /// Get the directory and file from which to start \#include_next lookup.
2544   std::pair<ConstSearchDirIterator, const FileEntry *>
2545   getIncludeNextStart(const Token &IncludeNextTok) const;
2546 
2547   /// Install the standard preprocessor pragmas:
2548   /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2549   void RegisterBuiltinPragmas();
2550 
2551   /// Register builtin macros such as __LINE__ with the identifier table.
2552   void RegisterBuiltinMacros();
2553 
2554   /// If an identifier token is read that is to be expanded as a macro, handle
2555   /// it and return the next token as 'Tok'.  If we lexed a token, return true;
2556   /// otherwise the caller should lex again.
2557   bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2558 
2559   /// Cache macro expanded tokens for TokenLexers.
2560   //
2561   /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2562   /// going to lex in the cache and when it finishes the tokens are removed
2563   /// from the end of the cache.
2564   Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2565                                   ArrayRef<Token> tokens);
2566 
2567   void removeCachedMacroExpandedTokensOfLastLexer();
2568 
2569   /// Determine whether the next preprocessor token to be
2570   /// lexed is a '('.  If so, consume the token and return true, if not, this
2571   /// method should have no observable side-effect on the lexed tokens.
2572   bool isNextPPTokenLParen();
2573 
2574   /// After reading "MACRO(", this method is invoked to read all of the formal
2575   /// arguments specified for the macro invocation.  Returns null on error.
2576   MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2577                                        SourceLocation &MacroEnd);
2578 
2579   /// If an identifier token is read that is to be expanded
2580   /// as a builtin macro, handle it and return the next token as 'Tok'.
2581   void ExpandBuiltinMacro(Token &Tok);
2582 
2583   /// Read a \c _Pragma directive, slice it up, process it, then
2584   /// return the first token after the directive.
2585   /// This assumes that the \c _Pragma token has just been read into \p Tok.
2586   void Handle_Pragma(Token &Tok);
2587 
2588   /// Like Handle_Pragma except the pragma text is not enclosed within
2589   /// a string literal.
2590   void HandleMicrosoft__pragma(Token &Tok);
2591 
2592   /// Add a lexer to the top of the include stack and
2593   /// start lexing tokens from it instead of the current buffer.
2594   void EnterSourceFileWithLexer(Lexer *TheLexer, ConstSearchDirIterator Dir);
2595 
2596   /// Set the FileID for the preprocessor predefines.
setPredefinesFileID(FileID FID)2597   void setPredefinesFileID(FileID FID) {
2598     assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2599     PredefinesFileID = FID;
2600   }
2601 
2602   /// Set the FileID for the PCH through header.
2603   void setPCHThroughHeaderFileID(FileID FID);
2604 
2605   /// Returns true if we are lexing from a file and not a
2606   /// pragma or a macro.
IsFileLexer(const Lexer * L,const PreprocessorLexer * P)2607   static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2608     return L ? !L->isPragmaLexer() : P != nullptr;
2609   }
2610 
IsFileLexer(const IncludeStackInfo & I)2611   static bool IsFileLexer(const IncludeStackInfo& I) {
2612     return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2613   }
2614 
IsFileLexer()2615   bool IsFileLexer() const {
2616     return IsFileLexer(CurLexer.get(), CurPPLexer);
2617   }
2618 
2619   //===--------------------------------------------------------------------===//
2620   // Caching stuff.
2621   void CachingLex(Token &Result);
2622 
InCachingLexMode()2623   bool InCachingLexMode() const {
2624     // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2625     // that we are past EOF, not that we are in CachingLex mode.
2626     return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2627   }
2628 
2629   void EnterCachingLexMode();
2630   void EnterCachingLexModeUnchecked();
2631 
ExitCachingLexMode()2632   void ExitCachingLexMode() {
2633     if (InCachingLexMode())
2634       RemoveTopOfLexerStack();
2635   }
2636 
2637   const Token &PeekAhead(unsigned N);
2638   void AnnotatePreviousCachedTokens(const Token &Tok);
2639 
2640   //===--------------------------------------------------------------------===//
2641   /// Handle*Directive - implement the various preprocessor directives.  These
2642   /// should side-effect the current preprocessor object so that the next call
2643   /// to Lex() will return the appropriate token next.
2644   void HandleLineDirective();
2645   void HandleDigitDirective(Token &Tok);
2646   void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2647   void HandleIdentSCCSDirective(Token &Tok);
2648   void HandleMacroPublicDirective(Token &Tok);
2649   void HandleMacroPrivateDirective();
2650 
2651   /// An additional notification that can be produced by a header inclusion or
2652   /// import to tell the parser what happened.
2653   struct ImportAction {
2654     enum ActionKind {
2655       None,
2656       ModuleBegin,
2657       ModuleImport,
2658       HeaderUnitImport,
2659       SkippedModuleImport,
2660       Failure,
2661     } Kind;
2662     Module *ModuleForHeader = nullptr;
2663 
2664     ImportAction(ActionKind AK, Module *Mod = nullptr)
KindImportAction2665         : Kind(AK), ModuleForHeader(Mod) {
2666       assert((AK == None || Mod || AK == Failure) &&
2667              "no module for module action");
2668     }
2669   };
2670 
2671   OptionalFileEntryRef LookupHeaderIncludeOrImport(
2672       ConstSearchDirIterator *CurDir, StringRef &Filename,
2673       SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2674       const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2675       bool &IsMapped, ConstSearchDirIterator LookupFrom,
2676       const FileEntry *LookupFromFile, StringRef &LookupFilename,
2677       SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2678       ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2679 
2680   // File inclusion.
2681   void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2682                               ConstSearchDirIterator LookupFrom = nullptr,
2683                               const FileEntry *LookupFromFile = nullptr);
2684   ImportAction
2685   HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2686                               Token &FilenameTok, SourceLocation EndLoc,
2687                               ConstSearchDirIterator LookupFrom = nullptr,
2688                               const FileEntry *LookupFromFile = nullptr);
2689   void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2690   void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2691   void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2692   void HandleMicrosoftImportDirective(Token &Tok);
2693 
2694 public:
2695   /// Check that the given module is available, producing a diagnostic if not.
2696   /// \return \c true if the check failed (because the module is not available).
2697   ///         \c false if the module appears to be usable.
2698   static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2699                                      const TargetInfo &TargetInfo,
2700                                      DiagnosticsEngine &Diags, Module *M);
2701 
2702   // Module inclusion testing.
2703   /// Find the module that owns the source or header file that
2704   /// \p Loc points to. If the location is in a file that was included
2705   /// into a module, or is outside any module, returns nullptr.
2706   Module *getModuleForLocation(SourceLocation Loc, bool AllowTextual);
2707 
2708   /// We want to produce a diagnostic at location IncLoc concerning an
2709   /// unreachable effect at location MLoc (eg, where a desired entity was
2710   /// declared or defined). Determine whether the right way to make MLoc
2711   /// reachable is by #include, and if so, what header should be included.
2712   ///
2713   /// This is not necessarily fast, and might load unexpected module maps, so
2714   /// should only be called by code that intends to produce an error.
2715   ///
2716   /// \param IncLoc The location at which the missing effect was detected.
2717   /// \param MLoc A location within an unimported module at which the desired
2718   ///        effect occurred.
2719   /// \return A file that can be #included to provide the desired effect. Null
2720   ///         if no such file could be determined or if a #include is not
2721   ///         appropriate (eg, if a module should be imported instead).
2722   const FileEntry *getHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2723                                                     SourceLocation MLoc);
2724 
isRecordingPreamble()2725   bool isRecordingPreamble() const {
2726     return PreambleConditionalStack.isRecording();
2727   }
2728 
hasRecordedPreamble()2729   bool hasRecordedPreamble() const {
2730     return PreambleConditionalStack.hasRecordedPreamble();
2731   }
2732 
getPreambleConditionalStack()2733   ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2734       return PreambleConditionalStack.getStack();
2735   }
2736 
setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s)2737   void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2738     PreambleConditionalStack.setStack(s);
2739   }
2740 
setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s,std::optional<PreambleSkipInfo> SkipInfo)2741   void setReplayablePreambleConditionalStack(
2742       ArrayRef<PPConditionalInfo> s, std::optional<PreambleSkipInfo> SkipInfo) {
2743     PreambleConditionalStack.startReplaying();
2744     PreambleConditionalStack.setStack(s);
2745     PreambleConditionalStack.SkipInfo = SkipInfo;
2746   }
2747 
getPreambleSkipInfo()2748   std::optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2749     return PreambleConditionalStack.SkipInfo;
2750   }
2751 
2752 private:
2753   /// After processing predefined file, initialize the conditional stack from
2754   /// the preamble.
2755   void replayPreambleConditionalStack();
2756 
2757   // Macro handling.
2758   void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2759   void HandleUndefDirective();
2760 
2761   // Conditional Inclusion.
2762   void HandleIfdefDirective(Token &Result, const Token &HashToken,
2763                             bool isIfndef, bool ReadAnyTokensBeforeDirective);
2764   void HandleIfDirective(Token &IfToken, const Token &HashToken,
2765                          bool ReadAnyTokensBeforeDirective);
2766   void HandleEndifDirective(Token &EndifToken);
2767   void HandleElseDirective(Token &Result, const Token &HashToken);
2768   void HandleElifFamilyDirective(Token &ElifToken, const Token &HashToken,
2769                                  tok::PPKeywordKind Kind);
2770 
2771   // Pragmas.
2772   void HandlePragmaDirective(PragmaIntroducer Introducer);
2773 
2774 public:
2775   void HandlePragmaOnce(Token &OnceTok);
2776   void HandlePragmaMark(Token &MarkTok);
2777   void HandlePragmaPoison();
2778   void HandlePragmaSystemHeader(Token &SysHeaderTok);
2779   void HandlePragmaDependency(Token &DependencyTok);
2780   void HandlePragmaPushMacro(Token &Tok);
2781   void HandlePragmaPopMacro(Token &Tok);
2782   void HandlePragmaIncludeAlias(Token &Tok);
2783   void HandlePragmaModuleBuild(Token &Tok);
2784   void HandlePragmaHdrstop(Token &Tok);
2785   IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2786 
2787   // Return true and store the first token only if any CommentHandler
2788   // has inserted some tokens and getCommentRetentionState() is false.
2789   bool HandleComment(Token &result, SourceRange Comment);
2790 
2791   /// A macro is used, update information about macros that need unused
2792   /// warnings.
2793   void markMacroAsUsed(MacroInfo *MI);
2794 
addMacroDeprecationMsg(const IdentifierInfo * II,std::string Msg,SourceLocation AnnotationLoc)2795   void addMacroDeprecationMsg(const IdentifierInfo *II, std::string Msg,
2796                               SourceLocation AnnotationLoc) {
2797     auto Annotations = AnnotationInfos.find(II);
2798     if (Annotations == AnnotationInfos.end())
2799       AnnotationInfos.insert(std::make_pair(
2800           II,
2801           MacroAnnotations::makeDeprecation(AnnotationLoc, std::move(Msg))));
2802     else
2803       Annotations->second.DeprecationInfo =
2804           MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2805   }
2806 
addRestrictExpansionMsg(const IdentifierInfo * II,std::string Msg,SourceLocation AnnotationLoc)2807   void addRestrictExpansionMsg(const IdentifierInfo *II, std::string Msg,
2808                                SourceLocation AnnotationLoc) {
2809     auto Annotations = AnnotationInfos.find(II);
2810     if (Annotations == AnnotationInfos.end())
2811       AnnotationInfos.insert(
2812           std::make_pair(II, MacroAnnotations::makeRestrictExpansion(
2813                                  AnnotationLoc, std::move(Msg))));
2814     else
2815       Annotations->second.RestrictExpansionInfo =
2816           MacroAnnotationInfo{AnnotationLoc, std::move(Msg)};
2817   }
2818 
addFinalLoc(const IdentifierInfo * II,SourceLocation AnnotationLoc)2819   void addFinalLoc(const IdentifierInfo *II, SourceLocation AnnotationLoc) {
2820     auto Annotations = AnnotationInfos.find(II);
2821     if (Annotations == AnnotationInfos.end())
2822       AnnotationInfos.insert(
2823           std::make_pair(II, MacroAnnotations::makeFinal(AnnotationLoc)));
2824     else
2825       Annotations->second.FinalAnnotationLoc = AnnotationLoc;
2826   }
2827 
getMacroAnnotations(const IdentifierInfo * II)2828   const MacroAnnotations &getMacroAnnotations(const IdentifierInfo *II) const {
2829     return AnnotationInfos.find(II)->second;
2830   }
2831 
emitMacroExpansionWarnings(const Token & Identifier)2832   void emitMacroExpansionWarnings(const Token &Identifier) const {
2833     if (Identifier.getIdentifierInfo()->isDeprecatedMacro())
2834       emitMacroDeprecationWarning(Identifier);
2835 
2836     if (Identifier.getIdentifierInfo()->isRestrictExpansion() &&
2837         !SourceMgr.isInMainFile(Identifier.getLocation()))
2838       emitRestrictExpansionWarning(Identifier);
2839   }
2840 
2841   static void processPathForFileMacro(SmallVectorImpl<char> &Path,
2842                                       const LangOptions &LangOpts,
2843                                       const TargetInfo &TI);
2844 
2845 private:
2846   void emitMacroDeprecationWarning(const Token &Identifier) const;
2847   void emitRestrictExpansionWarning(const Token &Identifier) const;
2848   void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
2849 };
2850 
2851 /// Abstract base class that describes a handler that will receive
2852 /// source ranges for each of the comments encountered in the source file.
2853 class CommentHandler {
2854 public:
2855   virtual ~CommentHandler();
2856 
2857   // The handler shall return true if it has pushed any tokens
2858   // to be read using e.g. EnterToken or EnterTokenStream.
2859   virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2860 };
2861 
2862 /// Abstract base class that describes a handler that will receive
2863 /// source ranges for empty lines encountered in the source file.
2864 class EmptylineHandler {
2865 public:
2866   virtual ~EmptylineHandler();
2867 
2868   // The handler handles empty lines.
2869   virtual void HandleEmptyline(SourceRange Range) = 0;
2870 };
2871 
2872 /// Registry of pragma handlers added by plugins
2873 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2874 
2875 } // namespace clang
2876 
2877 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
2878