10b57cec5SDimitry Andric //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file implements pieces of the Preprocessor interface that manage the
100b57cec5SDimitry Andric // current lexer stack.
110b57cec5SDimitry Andric //
120b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
130b57cec5SDimitry Andric
140b57cec5SDimitry Andric #include "clang/Basic/FileManager.h"
150b57cec5SDimitry Andric #include "clang/Basic/SourceManager.h"
160b57cec5SDimitry Andric #include "clang/Lex/HeaderSearch.h"
170b57cec5SDimitry Andric #include "clang/Lex/LexDiagnostic.h"
180b57cec5SDimitry Andric #include "clang/Lex/MacroInfo.h"
19e8d8bef9SDimitry Andric #include "clang/Lex/Preprocessor.h"
20e8d8bef9SDimitry Andric #include "clang/Lex/PreprocessorOptions.h"
210b57cec5SDimitry Andric #include "llvm/ADT/StringSwitch.h"
220b57cec5SDimitry Andric #include "llvm/Support/FileSystem.h"
23e8d8bef9SDimitry Andric #include "llvm/Support/MemoryBufferRef.h"
240b57cec5SDimitry Andric #include "llvm/Support/Path.h"
25bdd1243dSDimitry Andric #include <optional>
266e75b2fbSDimitry Andric
270b57cec5SDimitry Andric using namespace clang;
280b57cec5SDimitry Andric
290b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
300b57cec5SDimitry Andric // Miscellaneous Methods.
310b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
320b57cec5SDimitry Andric
330b57cec5SDimitry Andric /// isInPrimaryFile - Return true if we're in the top-level file, not in a
340b57cec5SDimitry Andric /// \#include. This looks through macro expansions and active _Pragma lexers.
isInPrimaryFile() const350b57cec5SDimitry Andric bool Preprocessor::isInPrimaryFile() const {
360b57cec5SDimitry Andric if (IsFileLexer())
370b57cec5SDimitry Andric return IncludeMacroStack.empty();
380b57cec5SDimitry Andric
390b57cec5SDimitry Andric // If there are any stacked lexers, we're in a #include.
400b57cec5SDimitry Andric assert(IsFileLexer(IncludeMacroStack[0]) &&
410b57cec5SDimitry Andric "Top level include stack isn't our primary lexer?");
42349cc55cSDimitry Andric return llvm::none_of(
43349cc55cSDimitry Andric llvm::drop_begin(IncludeMacroStack),
440b57cec5SDimitry Andric [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); });
450b57cec5SDimitry Andric }
460b57cec5SDimitry Andric
470b57cec5SDimitry Andric /// getCurrentLexer - Return the current file lexer being lexed from. Note
480b57cec5SDimitry Andric /// that this ignores any potentially active macro expansions and _Pragma
490b57cec5SDimitry Andric /// expansions going on at the time.
getCurrentFileLexer() const500b57cec5SDimitry Andric PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
510b57cec5SDimitry Andric if (IsFileLexer())
520b57cec5SDimitry Andric return CurPPLexer;
530b57cec5SDimitry Andric
540b57cec5SDimitry Andric // Look for a stacked lexer.
550b57cec5SDimitry Andric for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
560b57cec5SDimitry Andric if (IsFileLexer(ISI))
570b57cec5SDimitry Andric return ISI.ThePPLexer;
580b57cec5SDimitry Andric }
590b57cec5SDimitry Andric return nullptr;
600b57cec5SDimitry Andric }
610b57cec5SDimitry Andric
620b57cec5SDimitry Andric
630b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
640b57cec5SDimitry Andric // Methods for Entering and Callbacks for leaving various contexts
650b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
660b57cec5SDimitry Andric
670b57cec5SDimitry Andric /// EnterSourceFile - Add a source file to the top of the include stack and
680b57cec5SDimitry Andric /// start lexing tokens from it instead of the current buffer.
EnterSourceFile(FileID FID,ConstSearchDirIterator CurDir,SourceLocation Loc,bool IsFirstIncludeOfFile)6981ad6265SDimitry Andric bool Preprocessor::EnterSourceFile(FileID FID, ConstSearchDirIterator CurDir,
70349cc55cSDimitry Andric SourceLocation Loc,
71349cc55cSDimitry Andric bool IsFirstIncludeOfFile) {
720b57cec5SDimitry Andric assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
730b57cec5SDimitry Andric ++NumEnteredSourceFiles;
740b57cec5SDimitry Andric
750b57cec5SDimitry Andric if (MaxIncludeStackDepth < IncludeMacroStack.size())
760b57cec5SDimitry Andric MaxIncludeStackDepth = IncludeMacroStack.size();
770b57cec5SDimitry Andric
780b57cec5SDimitry Andric // Get the MemoryBuffer for this FID, if it fails, we fail.
79bdd1243dSDimitry Andric std::optional<llvm::MemoryBufferRef> InputFile =
80e8d8bef9SDimitry Andric getSourceManager().getBufferOrNone(FID, Loc);
81e8d8bef9SDimitry Andric if (!InputFile) {
820b57cec5SDimitry Andric SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
830b57cec5SDimitry Andric Diag(Loc, diag::err_pp_error_opening_file)
840b57cec5SDimitry Andric << std::string(SourceMgr.getBufferName(FileStart)) << "";
850b57cec5SDimitry Andric return true;
860b57cec5SDimitry Andric }
870b57cec5SDimitry Andric
880b57cec5SDimitry Andric if (isCodeCompletionEnabled() &&
890b57cec5SDimitry Andric SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
900b57cec5SDimitry Andric CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
910b57cec5SDimitry Andric CodeCompletionLoc =
920b57cec5SDimitry Andric CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
930b57cec5SDimitry Andric }
940b57cec5SDimitry Andric
9581ad6265SDimitry Andric Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
9681ad6265SDimitry Andric if (getPreprocessorOpts().DependencyDirectivesForFile &&
9781ad6265SDimitry Andric FID != PredefinesFileID) {
98bdd1243dSDimitry Andric if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) {
99bdd1243dSDimitry Andric if (std::optional<ArrayRef<dependency_directives_scan::Directive>>
10081ad6265SDimitry Andric DepDirectives =
10181ad6265SDimitry Andric getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
10281ad6265SDimitry Andric TheLexer->DepDirectives = *DepDirectives;
10381ad6265SDimitry Andric }
10481ad6265SDimitry Andric }
10581ad6265SDimitry Andric }
10681ad6265SDimitry Andric
10781ad6265SDimitry Andric EnterSourceFileWithLexer(TheLexer, CurDir);
1080b57cec5SDimitry Andric return false;
1090b57cec5SDimitry Andric }
1100b57cec5SDimitry Andric
1110b57cec5SDimitry Andric /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
1120b57cec5SDimitry Andric /// and start lexing tokens from it instead of the current buffer.
EnterSourceFileWithLexer(Lexer * TheLexer,ConstSearchDirIterator CurDir)1130b57cec5SDimitry Andric void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
11481ad6265SDimitry Andric ConstSearchDirIterator CurDir) {
11581ad6265SDimitry Andric PreprocessorLexer *PrevPPLexer = CurPPLexer;
1160b57cec5SDimitry Andric
1170b57cec5SDimitry Andric // Add the current lexer to the include stack.
1180b57cec5SDimitry Andric if (CurPPLexer || CurTokenLexer)
1190b57cec5SDimitry Andric PushIncludeMacroStack();
1200b57cec5SDimitry Andric
1210b57cec5SDimitry Andric CurLexer.reset(TheLexer);
1220b57cec5SDimitry Andric CurPPLexer = TheLexer;
1230b57cec5SDimitry Andric CurDirLookup = CurDir;
1240b57cec5SDimitry Andric CurLexerSubmodule = nullptr;
1255f757f3fSDimitry Andric if (CurLexerCallback != CLK_LexAfterModuleImport)
1265f757f3fSDimitry Andric CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
12781ad6265SDimitry Andric ? CLK_DependencyDirectivesLexer
12881ad6265SDimitry Andric : CLK_Lexer;
1290b57cec5SDimitry Andric
1300b57cec5SDimitry Andric // Notify the client, if desired, that we are in a new source file.
1310b57cec5SDimitry Andric if (Callbacks && !CurLexer->Is_PragmaLexer) {
1320b57cec5SDimitry Andric SrcMgr::CharacteristicKind FileType =
1330b57cec5SDimitry Andric SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
1340b57cec5SDimitry Andric
13581ad6265SDimitry Andric FileID PrevFID;
13681ad6265SDimitry Andric SourceLocation EnterLoc;
13781ad6265SDimitry Andric if (PrevPPLexer) {
13881ad6265SDimitry Andric PrevFID = PrevPPLexer->getFileID();
13981ad6265SDimitry Andric EnterLoc = PrevPPLexer->getSourceLocation();
14081ad6265SDimitry Andric }
14181ad6265SDimitry Andric Callbacks->FileChanged(CurLexer->getFileLoc(), PPCallbacks::EnterFile,
14281ad6265SDimitry Andric FileType, PrevFID);
14381ad6265SDimitry Andric Callbacks->LexedFileChanged(CurLexer->getFileID(),
14481ad6265SDimitry Andric PPCallbacks::LexedFileChangeReason::EnterFile,
14581ad6265SDimitry Andric FileType, PrevFID, EnterLoc);
1460b57cec5SDimitry Andric }
1470b57cec5SDimitry Andric }
1480b57cec5SDimitry Andric
1490b57cec5SDimitry Andric /// EnterMacro - Add a Macro to the top of the include stack and start lexing
1500b57cec5SDimitry Andric /// tokens from it instead of the current buffer.
EnterMacro(Token & Tok,SourceLocation ILEnd,MacroInfo * Macro,MacroArgs * Args)1510b57cec5SDimitry Andric void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
1520b57cec5SDimitry Andric MacroInfo *Macro, MacroArgs *Args) {
1530b57cec5SDimitry Andric std::unique_ptr<TokenLexer> TokLexer;
1540b57cec5SDimitry Andric if (NumCachedTokenLexers == 0) {
155a7dea167SDimitry Andric TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
1560b57cec5SDimitry Andric } else {
1570b57cec5SDimitry Andric TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
1580b57cec5SDimitry Andric TokLexer->Init(Tok, ILEnd, Macro, Args);
1590b57cec5SDimitry Andric }
1600b57cec5SDimitry Andric
1610b57cec5SDimitry Andric PushIncludeMacroStack();
1620b57cec5SDimitry Andric CurDirLookup = nullptr;
1630b57cec5SDimitry Andric CurTokenLexer = std::move(TokLexer);
1645f757f3fSDimitry Andric if (CurLexerCallback != CLK_LexAfterModuleImport)
1655f757f3fSDimitry Andric CurLexerCallback = CLK_TokenLexer;
1660b57cec5SDimitry Andric }
1670b57cec5SDimitry Andric
1680b57cec5SDimitry Andric /// EnterTokenStream - Add a "macro" context to the top of the include stack,
1690b57cec5SDimitry Andric /// which will cause the lexer to start returning the specified tokens.
1700b57cec5SDimitry Andric ///
1710b57cec5SDimitry Andric /// If DisableMacroExpansion is true, tokens lexed from the token stream will
1720b57cec5SDimitry Andric /// not be subject to further macro expansion. Otherwise, these tokens will
1730b57cec5SDimitry Andric /// be re-macro-expanded when/if expansion is enabled.
1740b57cec5SDimitry Andric ///
1750b57cec5SDimitry Andric /// If OwnsTokens is false, this method assumes that the specified stream of
1760b57cec5SDimitry Andric /// tokens has a permanent owner somewhere, so they do not need to be copied.
1770b57cec5SDimitry Andric /// If it is true, it assumes the array of tokens is allocated with new[] and
1780b57cec5SDimitry Andric /// must be freed.
1790b57cec5SDimitry Andric ///
EnterTokenStream(const Token * Toks,unsigned NumToks,bool DisableMacroExpansion,bool OwnsTokens,bool IsReinject)1800b57cec5SDimitry Andric void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
1810b57cec5SDimitry Andric bool DisableMacroExpansion, bool OwnsTokens,
1820b57cec5SDimitry Andric bool IsReinject) {
1835f757f3fSDimitry Andric if (CurLexerCallback == CLK_CachingLexer) {
1840b57cec5SDimitry Andric if (CachedLexPos < CachedTokens.size()) {
1850b57cec5SDimitry Andric assert(IsReinject && "new tokens in the middle of cached stream");
1860b57cec5SDimitry Andric // We're entering tokens into the middle of our cached token stream. We
1870b57cec5SDimitry Andric // can't represent that, so just insert the tokens into the buffer.
1880b57cec5SDimitry Andric CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
1890b57cec5SDimitry Andric Toks, Toks + NumToks);
1900b57cec5SDimitry Andric if (OwnsTokens)
1910b57cec5SDimitry Andric delete [] Toks;
1920b57cec5SDimitry Andric return;
1930b57cec5SDimitry Andric }
1940b57cec5SDimitry Andric
1950b57cec5SDimitry Andric // New tokens are at the end of the cached token sequnece; insert the
1960b57cec5SDimitry Andric // token stream underneath the caching lexer.
1970b57cec5SDimitry Andric ExitCachingLexMode();
1980b57cec5SDimitry Andric EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
1990b57cec5SDimitry Andric IsReinject);
2000b57cec5SDimitry Andric EnterCachingLexMode();
2010b57cec5SDimitry Andric return;
2020b57cec5SDimitry Andric }
2030b57cec5SDimitry Andric
2040b57cec5SDimitry Andric // Create a macro expander to expand from the specified token stream.
2050b57cec5SDimitry Andric std::unique_ptr<TokenLexer> TokLexer;
2060b57cec5SDimitry Andric if (NumCachedTokenLexers == 0) {
207a7dea167SDimitry Andric TokLexer = std::make_unique<TokenLexer>(
2080b57cec5SDimitry Andric Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
2090b57cec5SDimitry Andric } else {
2100b57cec5SDimitry Andric TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
2110b57cec5SDimitry Andric TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
2120b57cec5SDimitry Andric IsReinject);
2130b57cec5SDimitry Andric }
2140b57cec5SDimitry Andric
2150b57cec5SDimitry Andric // Save our current state.
2160b57cec5SDimitry Andric PushIncludeMacroStack();
2170b57cec5SDimitry Andric CurDirLookup = nullptr;
2180b57cec5SDimitry Andric CurTokenLexer = std::move(TokLexer);
2195f757f3fSDimitry Andric if (CurLexerCallback != CLK_LexAfterModuleImport)
2205f757f3fSDimitry Andric CurLexerCallback = CLK_TokenLexer;
2210b57cec5SDimitry Andric }
2220b57cec5SDimitry Andric
2230b57cec5SDimitry Andric /// Compute the relative path that names the given file relative to
2240b57cec5SDimitry Andric /// the given directory.
computeRelativePath(FileManager & FM,const DirectoryEntry * Dir,FileEntryRef File,SmallString<128> & Result)2250b57cec5SDimitry Andric static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
22606c3fb27SDimitry Andric FileEntryRef File, SmallString<128> &Result) {
2270b57cec5SDimitry Andric Result.clear();
2280b57cec5SDimitry Andric
22906c3fb27SDimitry Andric StringRef FilePath = File.getDir().getName();
2300b57cec5SDimitry Andric StringRef Path = FilePath;
2310b57cec5SDimitry Andric while (!Path.empty()) {
232a7dea167SDimitry Andric if (auto CurDir = FM.getDirectory(Path)) {
233a7dea167SDimitry Andric if (*CurDir == Dir) {
2340b57cec5SDimitry Andric Result = FilePath.substr(Path.size());
2350b57cec5SDimitry Andric llvm::sys::path::append(Result,
23606c3fb27SDimitry Andric llvm::sys::path::filename(File.getName()));
2370b57cec5SDimitry Andric return;
2380b57cec5SDimitry Andric }
2390b57cec5SDimitry Andric }
2400b57cec5SDimitry Andric
2410b57cec5SDimitry Andric Path = llvm::sys::path::parent_path(Path);
2420b57cec5SDimitry Andric }
2430b57cec5SDimitry Andric
24406c3fb27SDimitry Andric Result = File.getName();
2450b57cec5SDimitry Andric }
2460b57cec5SDimitry Andric
PropagateLineStartLeadingSpaceInfo(Token & Result)2470b57cec5SDimitry Andric void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
2480b57cec5SDimitry Andric if (CurTokenLexer) {
2490b57cec5SDimitry Andric CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
2500b57cec5SDimitry Andric return;
2510b57cec5SDimitry Andric }
2520b57cec5SDimitry Andric if (CurLexer) {
2530b57cec5SDimitry Andric CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
2540b57cec5SDimitry Andric return;
2550b57cec5SDimitry Andric }
2560b57cec5SDimitry Andric // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
2570b57cec5SDimitry Andric // but it might if they're empty?
2580b57cec5SDimitry Andric }
2590b57cec5SDimitry Andric
2600b57cec5SDimitry Andric /// Determine the location to use as the end of the buffer for a lexer.
2610b57cec5SDimitry Andric ///
2620b57cec5SDimitry Andric /// If the file ends with a newline, form the EOF token on the newline itself,
2630b57cec5SDimitry Andric /// rather than "on the line following it", which doesn't exist. This makes
2640b57cec5SDimitry Andric /// diagnostics relating to the end of file include the last file that the user
2650b57cec5SDimitry Andric /// actually typed, which is goodness.
getCurLexerEndPos()2660b57cec5SDimitry Andric const char *Preprocessor::getCurLexerEndPos() {
2670b57cec5SDimitry Andric const char *EndPos = CurLexer->BufferEnd;
2680b57cec5SDimitry Andric if (EndPos != CurLexer->BufferStart &&
2690b57cec5SDimitry Andric (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
2700b57cec5SDimitry Andric --EndPos;
2710b57cec5SDimitry Andric
2720b57cec5SDimitry Andric // Handle \n\r and \r\n:
2730b57cec5SDimitry Andric if (EndPos != CurLexer->BufferStart &&
2740b57cec5SDimitry Andric (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
2750b57cec5SDimitry Andric EndPos[-1] != EndPos[0])
2760b57cec5SDimitry Andric --EndPos;
2770b57cec5SDimitry Andric }
2780b57cec5SDimitry Andric
2790b57cec5SDimitry Andric return EndPos;
2800b57cec5SDimitry Andric }
2810b57cec5SDimitry Andric
collectAllSubModulesWithUmbrellaHeader(const Module & Mod,SmallVectorImpl<const Module * > & SubMods)2820b57cec5SDimitry Andric static void collectAllSubModulesWithUmbrellaHeader(
2830b57cec5SDimitry Andric const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
28406c3fb27SDimitry Andric if (Mod.getUmbrellaHeaderAsWritten())
2850b57cec5SDimitry Andric SubMods.push_back(&Mod);
2860b57cec5SDimitry Andric for (auto *M : Mod.submodules())
2870b57cec5SDimitry Andric collectAllSubModulesWithUmbrellaHeader(*M, SubMods);
2880b57cec5SDimitry Andric }
2890b57cec5SDimitry Andric
diagnoseMissingHeaderInUmbrellaDir(const Module & Mod)2900b57cec5SDimitry Andric void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
29106c3fb27SDimitry Andric std::optional<Module::Header> UmbrellaHeader =
29206c3fb27SDimitry Andric Mod.getUmbrellaHeaderAsWritten();
29306c3fb27SDimitry Andric assert(UmbrellaHeader && "Module must use umbrella header");
29406c3fb27SDimitry Andric const FileID &File = SourceMgr.translateFile(UmbrellaHeader->Entry);
295e8d8bef9SDimitry Andric SourceLocation ExpectedHeadersLoc = SourceMgr.getLocForEndOfFile(File);
296e8d8bef9SDimitry Andric if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
297e8d8bef9SDimitry Andric ExpectedHeadersLoc))
2980b57cec5SDimitry Andric return;
2990b57cec5SDimitry Andric
3000b57cec5SDimitry Andric ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
30106c3fb27SDimitry Andric OptionalDirectoryEntryRef Dir = Mod.getEffectiveUmbrellaDir();
3020b57cec5SDimitry Andric llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
3030b57cec5SDimitry Andric std::error_code EC;
3040b57cec5SDimitry Andric for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC),
3050b57cec5SDimitry Andric End;
3060b57cec5SDimitry Andric Entry != End && !EC; Entry.increment(EC)) {
3070b57cec5SDimitry Andric using llvm::StringSwitch;
3080b57cec5SDimitry Andric
3090b57cec5SDimitry Andric // Check whether this entry has an extension typically associated with
3100b57cec5SDimitry Andric // headers.
3110b57cec5SDimitry Andric if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
3120b57cec5SDimitry Andric .Cases(".h", ".H", ".hh", ".hpp", true)
3130b57cec5SDimitry Andric .Default(false))
3140b57cec5SDimitry Andric continue;
3150b57cec5SDimitry Andric
31606c3fb27SDimitry Andric if (auto Header = getFileManager().getOptionalFileRef(Entry->path()))
317a7dea167SDimitry Andric if (!getSourceManager().hasFileInfo(*Header)) {
318a7dea167SDimitry Andric if (!ModMap.isHeaderInUnavailableModule(*Header)) {
3190b57cec5SDimitry Andric // Find the relative path that would access this header.
3200b57cec5SDimitry Andric SmallString<128> RelativePath;
32106c3fb27SDimitry Andric computeRelativePath(FileMgr, *Dir, *Header, RelativePath);
322e8d8bef9SDimitry Andric Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header)
3230b57cec5SDimitry Andric << Mod.getFullModuleName() << RelativePath;
3240b57cec5SDimitry Andric }
3250b57cec5SDimitry Andric }
3260b57cec5SDimitry Andric }
3270b57cec5SDimitry Andric }
3280b57cec5SDimitry Andric
3290b57cec5SDimitry Andric /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
3300b57cec5SDimitry Andric /// the current file. This either returns the EOF token or pops a level off
3310b57cec5SDimitry Andric /// the include stack and keeps going.
HandleEndOfFile(Token & Result,bool isEndOfMacro)33281ad6265SDimitry Andric bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
3330b57cec5SDimitry Andric assert(!CurTokenLexer &&
3340b57cec5SDimitry Andric "Ending a file when currently in a macro!");
3350b57cec5SDimitry Andric
33606c3fb27SDimitry Andric SourceLocation UnclosedSafeBufferOptOutLoc;
33706c3fb27SDimitry Andric
33806c3fb27SDimitry Andric if (IncludeMacroStack.empty() &&
33906c3fb27SDimitry Andric isPPInSafeBufferOptOutRegion(UnclosedSafeBufferOptOutLoc)) {
34006c3fb27SDimitry Andric // To warn if a "-Wunsafe-buffer-usage" opt-out region is still open by the
34106c3fb27SDimitry Andric // end of a file.
34206c3fb27SDimitry Andric Diag(UnclosedSafeBufferOptOutLoc,
34306c3fb27SDimitry Andric diag::err_pp_unclosed_pragma_unsafe_buffer_usage);
34406c3fb27SDimitry Andric }
3450b57cec5SDimitry Andric // If we have an unclosed module region from a pragma at the end of a
3460b57cec5SDimitry Andric // module, complain and close it now.
3470b57cec5SDimitry Andric const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
3480b57cec5SDimitry Andric if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
3490b57cec5SDimitry Andric !BuildingSubmoduleStack.empty() &&
3500b57cec5SDimitry Andric BuildingSubmoduleStack.back().IsPragma) {
3510b57cec5SDimitry Andric Diag(BuildingSubmoduleStack.back().ImportLoc,
3520b57cec5SDimitry Andric diag::err_pp_module_begin_without_module_end);
3530b57cec5SDimitry Andric Module *M = LeaveSubmodule(/*ForPragma*/true);
3540b57cec5SDimitry Andric
3550b57cec5SDimitry Andric Result.startToken();
3560b57cec5SDimitry Andric const char *EndPos = getCurLexerEndPos();
3570b57cec5SDimitry Andric CurLexer->BufferPtr = EndPos;
3580b57cec5SDimitry Andric CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
3590b57cec5SDimitry Andric Result.setAnnotationEndLoc(Result.getLocation());
3600b57cec5SDimitry Andric Result.setAnnotationValue(M);
3610b57cec5SDimitry Andric return true;
3620b57cec5SDimitry Andric }
3630b57cec5SDimitry Andric
3640b57cec5SDimitry Andric // See if this file had a controlling macro.
3650b57cec5SDimitry Andric if (CurPPLexer) { // Not ending a macro, ignore it.
3660b57cec5SDimitry Andric if (const IdentifierInfo *ControllingMacro =
3670b57cec5SDimitry Andric CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
3680b57cec5SDimitry Andric // Okay, this has a controlling macro, remember in HeaderFileInfo.
3695f757f3fSDimitry Andric if (OptionalFileEntryRef FE = CurPPLexer->getFileEntry()) {
3705f757f3fSDimitry Andric HeaderInfo.SetFileControllingMacro(*FE, ControllingMacro);
3710b57cec5SDimitry Andric if (MacroInfo *MI =
3720b57cec5SDimitry Andric getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
3730b57cec5SDimitry Andric MI->setUsedForHeaderGuard(true);
3740b57cec5SDimitry Andric if (const IdentifierInfo *DefinedMacro =
3750b57cec5SDimitry Andric CurPPLexer->MIOpt.GetDefinedMacro()) {
3760b57cec5SDimitry Andric if (!isMacroDefined(ControllingMacro) &&
3770b57cec5SDimitry Andric DefinedMacro != ControllingMacro &&
378349cc55cSDimitry Andric CurLexer->isFirstTimeLexingFile()) {
3790b57cec5SDimitry Andric
3800b57cec5SDimitry Andric // If the edit distance between the two macros is more than 50%,
3810b57cec5SDimitry Andric // DefinedMacro may not be header guard, or can be header guard of
3820b57cec5SDimitry Andric // another header file. Therefore, it maybe defining something
3830b57cec5SDimitry Andric // completely different. This can be observed in the wild when
3840b57cec5SDimitry Andric // handling feature macros or header guards in different files.
3850b57cec5SDimitry Andric
3860b57cec5SDimitry Andric const StringRef ControllingMacroName = ControllingMacro->getName();
3870b57cec5SDimitry Andric const StringRef DefinedMacroName = DefinedMacro->getName();
3880b57cec5SDimitry Andric const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
3890b57cec5SDimitry Andric DefinedMacroName.size()) / 2;
3900b57cec5SDimitry Andric const unsigned ED = ControllingMacroName.edit_distance(
3910b57cec5SDimitry Andric DefinedMacroName, true, MaxHalfLength);
3920b57cec5SDimitry Andric if (ED <= MaxHalfLength) {
3930b57cec5SDimitry Andric // Emit a warning for a bad header guard.
3940b57cec5SDimitry Andric Diag(CurPPLexer->MIOpt.GetMacroLocation(),
3950b57cec5SDimitry Andric diag::warn_header_guard)
3960b57cec5SDimitry Andric << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
3970b57cec5SDimitry Andric Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
3980b57cec5SDimitry Andric diag::note_header_guard)
3990b57cec5SDimitry Andric << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
4000b57cec5SDimitry Andric << ControllingMacro
4010b57cec5SDimitry Andric << FixItHint::CreateReplacement(
4020b57cec5SDimitry Andric CurPPLexer->MIOpt.GetDefinedLocation(),
4030b57cec5SDimitry Andric ControllingMacro->getName());
4040b57cec5SDimitry Andric }
4050b57cec5SDimitry Andric }
4060b57cec5SDimitry Andric }
4070b57cec5SDimitry Andric }
4080b57cec5SDimitry Andric }
4090b57cec5SDimitry Andric }
4100b57cec5SDimitry Andric
4110b57cec5SDimitry Andric // Complain about reaching a true EOF within arc_cf_code_audited.
4120b57cec5SDimitry Andric // We don't want to complain about reaching the end of a macro
4130b57cec5SDimitry Andric // instantiation or a _Pragma.
414a7dea167SDimitry Andric if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro &&
415a7dea167SDimitry Andric !(CurLexer && CurLexer->Is_PragmaLexer)) {
416a7dea167SDimitry Andric Diag(PragmaARCCFCodeAuditedInfo.second,
417a7dea167SDimitry Andric diag::err_pp_eof_in_arc_cf_code_audited);
4180b57cec5SDimitry Andric
4190b57cec5SDimitry Andric // Recover by leaving immediately.
420a7dea167SDimitry Andric PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
4210b57cec5SDimitry Andric }
4220b57cec5SDimitry Andric
4230b57cec5SDimitry Andric // Complain about reaching a true EOF within assume_nonnull.
4240b57cec5SDimitry Andric // We don't want to complain about reaching the end of a macro
4250b57cec5SDimitry Andric // instantiation or a _Pragma.
4260b57cec5SDimitry Andric if (PragmaAssumeNonNullLoc.isValid() &&
4270b57cec5SDimitry Andric !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
42881ad6265SDimitry Andric // If we're at the end of generating a preamble, we should record the
42981ad6265SDimitry Andric // unterminated \#pragma clang assume_nonnull so we can restore it later
43081ad6265SDimitry Andric // when the preamble is loaded into the main file.
43181ad6265SDimitry Andric if (isRecordingPreamble() && isInPrimaryFile())
43281ad6265SDimitry Andric PreambleRecordedPragmaAssumeNonNullLoc = PragmaAssumeNonNullLoc;
43381ad6265SDimitry Andric else
4340b57cec5SDimitry Andric Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
4350b57cec5SDimitry Andric // Recover by leaving immediately.
4360b57cec5SDimitry Andric PragmaAssumeNonNullLoc = SourceLocation();
4370b57cec5SDimitry Andric }
4380b57cec5SDimitry Andric
4390b57cec5SDimitry Andric bool LeavingPCHThroughHeader = false;
4400b57cec5SDimitry Andric
4410b57cec5SDimitry Andric // If this is a #include'd file, pop it off the include stack and continue
4420b57cec5SDimitry Andric // lexing the #includer file.
4430b57cec5SDimitry Andric if (!IncludeMacroStack.empty()) {
4440b57cec5SDimitry Andric
4450b57cec5SDimitry Andric // If we lexed the code-completion file, act as if we reached EOF.
4460b57cec5SDimitry Andric if (isCodeCompletionEnabled() && CurPPLexer &&
4470b57cec5SDimitry Andric SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
4480b57cec5SDimitry Andric CodeCompletionFileLoc) {
4490b57cec5SDimitry Andric assert(CurLexer && "Got EOF but no current lexer set!");
4500b57cec5SDimitry Andric Result.startToken();
4510b57cec5SDimitry Andric CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
4520b57cec5SDimitry Andric CurLexer.reset();
4530b57cec5SDimitry Andric
4540b57cec5SDimitry Andric CurPPLexer = nullptr;
4550b57cec5SDimitry Andric recomputeCurLexerKind();
4560b57cec5SDimitry Andric return true;
4570b57cec5SDimitry Andric }
4580b57cec5SDimitry Andric
4590b57cec5SDimitry Andric if (!isEndOfMacro && CurPPLexer &&
4605ffd83dbSDimitry Andric (SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid() ||
4615ffd83dbSDimitry Andric // Predefines file doesn't have a valid include location.
4625ffd83dbSDimitry Andric (PredefinesFileID.isValid() &&
4635ffd83dbSDimitry Andric CurPPLexer->getFileID() == PredefinesFileID))) {
4640b57cec5SDimitry Andric // Notify SourceManager to record the number of FileIDs that were created
4650b57cec5SDimitry Andric // during lexing of the #include'd file.
4660b57cec5SDimitry Andric unsigned NumFIDs =
4670b57cec5SDimitry Andric SourceMgr.local_sloc_entry_size() -
4680b57cec5SDimitry Andric CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
4690b57cec5SDimitry Andric SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
4700b57cec5SDimitry Andric }
4710b57cec5SDimitry Andric
4720b57cec5SDimitry Andric bool ExitedFromPredefinesFile = false;
4730b57cec5SDimitry Andric FileID ExitedFID;
4740b57cec5SDimitry Andric if (!isEndOfMacro && CurPPLexer) {
4750b57cec5SDimitry Andric ExitedFID = CurPPLexer->getFileID();
4760b57cec5SDimitry Andric
4770b57cec5SDimitry Andric assert(PredefinesFileID.isValid() &&
4780b57cec5SDimitry Andric "HandleEndOfFile is called before PredefinesFileId is set");
4790b57cec5SDimitry Andric ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
4800b57cec5SDimitry Andric }
4810b57cec5SDimitry Andric
4820b57cec5SDimitry Andric if (LeavingSubmodule) {
4830b57cec5SDimitry Andric // We're done with this submodule.
4840b57cec5SDimitry Andric Module *M = LeaveSubmodule(/*ForPragma*/false);
4850b57cec5SDimitry Andric
4860b57cec5SDimitry Andric // Notify the parser that we've left the module.
4870b57cec5SDimitry Andric const char *EndPos = getCurLexerEndPos();
4880b57cec5SDimitry Andric Result.startToken();
4890b57cec5SDimitry Andric CurLexer->BufferPtr = EndPos;
4900b57cec5SDimitry Andric CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
4910b57cec5SDimitry Andric Result.setAnnotationEndLoc(Result.getLocation());
4920b57cec5SDimitry Andric Result.setAnnotationValue(M);
4930b57cec5SDimitry Andric }
4940b57cec5SDimitry Andric
4950b57cec5SDimitry Andric bool FoundPCHThroughHeader = false;
4960b57cec5SDimitry Andric if (CurPPLexer && creatingPCHWithThroughHeader() &&
4970b57cec5SDimitry Andric isPCHThroughHeader(
4980b57cec5SDimitry Andric SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
4990b57cec5SDimitry Andric FoundPCHThroughHeader = true;
5000b57cec5SDimitry Andric
5010b57cec5SDimitry Andric // We're done with the #included file.
5020b57cec5SDimitry Andric RemoveTopOfLexerStack();
5030b57cec5SDimitry Andric
5040b57cec5SDimitry Andric // Propagate info about start-of-line/leading white-space/etc.
5050b57cec5SDimitry Andric PropagateLineStartLeadingSpaceInfo(Result);
5060b57cec5SDimitry Andric
5070b57cec5SDimitry Andric // Notify the client, if desired, that we are in a new source file.
5080b57cec5SDimitry Andric if (Callbacks && !isEndOfMacro && CurPPLexer) {
50981ad6265SDimitry Andric SourceLocation Loc = CurPPLexer->getSourceLocation();
5100b57cec5SDimitry Andric SrcMgr::CharacteristicKind FileType =
51181ad6265SDimitry Andric SourceMgr.getFileCharacteristic(Loc);
51281ad6265SDimitry Andric Callbacks->FileChanged(Loc, PPCallbacks::ExitFile, FileType, ExitedFID);
51381ad6265SDimitry Andric Callbacks->LexedFileChanged(CurPPLexer->getFileID(),
51481ad6265SDimitry Andric PPCallbacks::LexedFileChangeReason::ExitFile,
51581ad6265SDimitry Andric FileType, ExitedFID, Loc);
5160b57cec5SDimitry Andric }
5170b57cec5SDimitry Andric
51881ad6265SDimitry Andric // Restore conditional stack as well as the recorded
51981ad6265SDimitry Andric // \#pragma clang assume_nonnull from the preamble right after exiting
52081ad6265SDimitry Andric // from the predefines file.
52181ad6265SDimitry Andric if (ExitedFromPredefinesFile) {
5220b57cec5SDimitry Andric replayPreambleConditionalStack();
52381ad6265SDimitry Andric if (PreambleRecordedPragmaAssumeNonNullLoc.isValid())
52481ad6265SDimitry Andric PragmaAssumeNonNullLoc = PreambleRecordedPragmaAssumeNonNullLoc;
52581ad6265SDimitry Andric }
5260b57cec5SDimitry Andric
5270b57cec5SDimitry Andric if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
5280b57cec5SDimitry Andric (isInPrimaryFile() ||
5290b57cec5SDimitry Andric CurPPLexer->getFileID() == getPredefinesFileID())) {
5300b57cec5SDimitry Andric // Leaving the through header. Continue directly to end of main file
5310b57cec5SDimitry Andric // processing.
5320b57cec5SDimitry Andric LeavingPCHThroughHeader = true;
5330b57cec5SDimitry Andric } else {
5340b57cec5SDimitry Andric // Client should lex another token unless we generated an EOM.
5350b57cec5SDimitry Andric return LeavingSubmodule;
5360b57cec5SDimitry Andric }
5370b57cec5SDimitry Andric }
5380b57cec5SDimitry Andric // If this is the end of the main file, form an EOF token.
5390b57cec5SDimitry Andric assert(CurLexer && "Got EOF but no current lexer set!");
5400b57cec5SDimitry Andric const char *EndPos = getCurLexerEndPos();
5410b57cec5SDimitry Andric Result.startToken();
5420b57cec5SDimitry Andric CurLexer->BufferPtr = EndPos;
54306c3fb27SDimitry Andric
5445f757f3fSDimitry Andric if (getLangOpts().IncrementalExtensions) {
54506c3fb27SDimitry Andric CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
54606c3fb27SDimitry Andric Result.setAnnotationEndLoc(Result.getLocation());
54706c3fb27SDimitry Andric Result.setAnnotationValue(nullptr);
54806c3fb27SDimitry Andric } else {
5490b57cec5SDimitry Andric CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
55006c3fb27SDimitry Andric }
5510b57cec5SDimitry Andric
5520b57cec5SDimitry Andric if (isCodeCompletionEnabled()) {
5530b57cec5SDimitry Andric // Inserting the code-completion point increases the source buffer by 1,
5540b57cec5SDimitry Andric // but the main FileID was created before inserting the point.
5550b57cec5SDimitry Andric // Compensate by reducing the EOF location by 1, otherwise the location
5560b57cec5SDimitry Andric // will point to the next FileID.
5570b57cec5SDimitry Andric // FIXME: This is hacky, the code-completion point should probably be
5580b57cec5SDimitry Andric // inserted before the main FileID is created.
5590b57cec5SDimitry Andric if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
5600b57cec5SDimitry Andric Result.setLocation(Result.getLocation().getLocWithOffset(-1));
5610b57cec5SDimitry Andric }
5620b57cec5SDimitry Andric
5630b57cec5SDimitry Andric if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
5640b57cec5SDimitry Andric // Reached the end of the compilation without finding the through header.
5650b57cec5SDimitry Andric Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
5660b57cec5SDimitry Andric << PPOpts->PCHThroughHeader << 0;
5670b57cec5SDimitry Andric }
5680b57cec5SDimitry Andric
5690b57cec5SDimitry Andric if (!isIncrementalProcessingEnabled())
5700b57cec5SDimitry Andric // We're done with lexing.
5710b57cec5SDimitry Andric CurLexer.reset();
5720b57cec5SDimitry Andric
5730b57cec5SDimitry Andric if (!isIncrementalProcessingEnabled())
5740b57cec5SDimitry Andric CurPPLexer = nullptr;
5750b57cec5SDimitry Andric
5760b57cec5SDimitry Andric if (TUKind == TU_Complete) {
5770b57cec5SDimitry Andric // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
5780b57cec5SDimitry Andric // collected all macro locations that we need to warn because they are not
5790b57cec5SDimitry Andric // used.
5800b57cec5SDimitry Andric for (WarnUnusedMacroLocsTy::iterator
5810b57cec5SDimitry Andric I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
5820b57cec5SDimitry Andric I!=E; ++I)
5830b57cec5SDimitry Andric Diag(*I, diag::pp_macro_not_used);
5840b57cec5SDimitry Andric }
5850b57cec5SDimitry Andric
5860b57cec5SDimitry Andric // If we are building a module that has an umbrella header, make sure that
5870b57cec5SDimitry Andric // each of the headers within the directory, including all submodules, is
5880b57cec5SDimitry Andric // covered by the umbrella header was actually included by the umbrella
5890b57cec5SDimitry Andric // header.
5900b57cec5SDimitry Andric if (Module *Mod = getCurrentModule()) {
5910b57cec5SDimitry Andric llvm::SmallVector<const Module *, 4> AllMods;
5920b57cec5SDimitry Andric collectAllSubModulesWithUmbrellaHeader(*Mod, AllMods);
5930b57cec5SDimitry Andric for (auto *M : AllMods)
5940b57cec5SDimitry Andric diagnoseMissingHeaderInUmbrellaDir(*M);
5950b57cec5SDimitry Andric }
5960b57cec5SDimitry Andric
5970b57cec5SDimitry Andric return true;
5980b57cec5SDimitry Andric }
5990b57cec5SDimitry Andric
6000b57cec5SDimitry Andric /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
6010b57cec5SDimitry Andric /// hits the end of its token stream.
HandleEndOfTokenLexer(Token & Result)6020b57cec5SDimitry Andric bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
6030b57cec5SDimitry Andric assert(CurTokenLexer && !CurPPLexer &&
6040b57cec5SDimitry Andric "Ending a macro when currently in a #include file!");
6050b57cec5SDimitry Andric
6060b57cec5SDimitry Andric if (!MacroExpandingLexersStack.empty() &&
6070b57cec5SDimitry Andric MacroExpandingLexersStack.back().first == CurTokenLexer.get())
6080b57cec5SDimitry Andric removeCachedMacroExpandedTokensOfLastLexer();
6090b57cec5SDimitry Andric
6100b57cec5SDimitry Andric // Delete or cache the now-dead macro expander.
6110b57cec5SDimitry Andric if (NumCachedTokenLexers == TokenLexerCacheSize)
6120b57cec5SDimitry Andric CurTokenLexer.reset();
6130b57cec5SDimitry Andric else
6140b57cec5SDimitry Andric TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
6150b57cec5SDimitry Andric
6160b57cec5SDimitry Andric // Handle this like a #include file being popped off the stack.
61781ad6265SDimitry Andric return HandleEndOfFile(Result, true);
6180b57cec5SDimitry Andric }
6190b57cec5SDimitry Andric
6200b57cec5SDimitry Andric /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
6210b57cec5SDimitry Andric /// lexer stack. This should only be used in situations where the current
6220b57cec5SDimitry Andric /// state of the top-of-stack lexer is unknown.
RemoveTopOfLexerStack()6230b57cec5SDimitry Andric void Preprocessor::RemoveTopOfLexerStack() {
6240b57cec5SDimitry Andric assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
6250b57cec5SDimitry Andric
6260b57cec5SDimitry Andric if (CurTokenLexer) {
6270b57cec5SDimitry Andric // Delete or cache the now-dead macro expander.
6280b57cec5SDimitry Andric if (NumCachedTokenLexers == TokenLexerCacheSize)
6290b57cec5SDimitry Andric CurTokenLexer.reset();
6300b57cec5SDimitry Andric else
6310b57cec5SDimitry Andric TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
6320b57cec5SDimitry Andric }
6330b57cec5SDimitry Andric
6340b57cec5SDimitry Andric PopIncludeMacroStack();
6350b57cec5SDimitry Andric }
6360b57cec5SDimitry Andric
6370b57cec5SDimitry Andric /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
6380b57cec5SDimitry Andric /// comment (/##/) in microsoft mode, this method handles updating the current
6390b57cec5SDimitry Andric /// state, returning the token on the next source line.
HandleMicrosoftCommentPaste(Token & Tok)6400b57cec5SDimitry Andric void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) {
6410b57cec5SDimitry Andric assert(CurTokenLexer && !CurPPLexer &&
6420b57cec5SDimitry Andric "Pasted comment can only be formed from macro");
6430b57cec5SDimitry Andric // We handle this by scanning for the closest real lexer, switching it to
6440b57cec5SDimitry Andric // raw mode and preprocessor mode. This will cause it to return \n as an
6450b57cec5SDimitry Andric // explicit EOD token.
6460b57cec5SDimitry Andric PreprocessorLexer *FoundLexer = nullptr;
6470b57cec5SDimitry Andric bool LexerWasInPPMode = false;
6480b57cec5SDimitry Andric for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
6490b57cec5SDimitry Andric if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
6500b57cec5SDimitry Andric
6510b57cec5SDimitry Andric // Once we find a real lexer, mark it as raw mode (disabling macro
6520b57cec5SDimitry Andric // expansions) and preprocessor mode (return EOD). We know that the lexer
6530b57cec5SDimitry Andric // was *not* in raw mode before, because the macro that the comment came
6540b57cec5SDimitry Andric // from was expanded. However, it could have already been in preprocessor
6550b57cec5SDimitry Andric // mode (#if COMMENT) in which case we have to return it to that mode and
6560b57cec5SDimitry Andric // return EOD.
6570b57cec5SDimitry Andric FoundLexer = ISI.ThePPLexer;
6580b57cec5SDimitry Andric FoundLexer->LexingRawMode = true;
6590b57cec5SDimitry Andric LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
6600b57cec5SDimitry Andric FoundLexer->ParsingPreprocessorDirective = true;
6610b57cec5SDimitry Andric break;
6620b57cec5SDimitry Andric }
6630b57cec5SDimitry Andric
6640b57cec5SDimitry Andric // Okay, we either found and switched over the lexer, or we didn't find a
6650b57cec5SDimitry Andric // lexer. In either case, finish off the macro the comment came from, getting
6660b57cec5SDimitry Andric // the next token.
6670b57cec5SDimitry Andric if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
6680b57cec5SDimitry Andric
6690b57cec5SDimitry Andric // Discarding comments as long as we don't have EOF or EOD. This 'comments
6700b57cec5SDimitry Andric // out' the rest of the line, including any tokens that came from other macros
6710b57cec5SDimitry Andric // that were active, as in:
6720b57cec5SDimitry Andric // #define submacro a COMMENT b
6730b57cec5SDimitry Andric // submacro c
6740b57cec5SDimitry Andric // which should lex to 'a' only: 'b' and 'c' should be removed.
6750b57cec5SDimitry Andric while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
6760b57cec5SDimitry Andric Lex(Tok);
6770b57cec5SDimitry Andric
6780b57cec5SDimitry Andric // If we got an eod token, then we successfully found the end of the line.
6790b57cec5SDimitry Andric if (Tok.is(tok::eod)) {
6800b57cec5SDimitry Andric assert(FoundLexer && "Can't get end of line without an active lexer");
6810b57cec5SDimitry Andric // Restore the lexer back to normal mode instead of raw mode.
6820b57cec5SDimitry Andric FoundLexer->LexingRawMode = false;
6830b57cec5SDimitry Andric
6840b57cec5SDimitry Andric // If the lexer was already in preprocessor mode, just return the EOD token
6850b57cec5SDimitry Andric // to finish the preprocessor line.
6860b57cec5SDimitry Andric if (LexerWasInPPMode) return;
6870b57cec5SDimitry Andric
6880b57cec5SDimitry Andric // Otherwise, switch out of PP mode and return the next lexed token.
6890b57cec5SDimitry Andric FoundLexer->ParsingPreprocessorDirective = false;
6900b57cec5SDimitry Andric return Lex(Tok);
6910b57cec5SDimitry Andric }
6920b57cec5SDimitry Andric
6930b57cec5SDimitry Andric // If we got an EOF token, then we reached the end of the token stream but
6940b57cec5SDimitry Andric // didn't find an explicit \n. This can only happen if there was no lexer
6950b57cec5SDimitry Andric // active (an active lexer would return EOD at EOF if there was no \n in
6960b57cec5SDimitry Andric // preprocessor directive mode), so just return EOF as our token.
6970b57cec5SDimitry Andric assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
6980b57cec5SDimitry Andric }
6990b57cec5SDimitry Andric
EnterSubmodule(Module * M,SourceLocation ImportLoc,bool ForPragma)7000b57cec5SDimitry Andric void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc,
7010b57cec5SDimitry Andric bool ForPragma) {
7020b57cec5SDimitry Andric if (!getLangOpts().ModulesLocalVisibility) {
7030b57cec5SDimitry Andric // Just track that we entered this submodule.
7040b57cec5SDimitry Andric BuildingSubmoduleStack.push_back(
7050b57cec5SDimitry Andric BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
7060b57cec5SDimitry Andric PendingModuleMacroNames.size()));
7070b57cec5SDimitry Andric if (Callbacks)
7080b57cec5SDimitry Andric Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
7090b57cec5SDimitry Andric return;
7100b57cec5SDimitry Andric }
7110b57cec5SDimitry Andric
7120b57cec5SDimitry Andric // Resolve as much of the module definition as we can now, before we enter
7130b57cec5SDimitry Andric // one of its headers.
7140b57cec5SDimitry Andric // FIXME: Can we enable Complain here?
7150b57cec5SDimitry Andric // FIXME: Can we do this when local visibility is disabled?
7160b57cec5SDimitry Andric ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
7170b57cec5SDimitry Andric ModMap.resolveExports(M, /*Complain=*/false);
7180b57cec5SDimitry Andric ModMap.resolveUses(M, /*Complain=*/false);
7190b57cec5SDimitry Andric ModMap.resolveConflicts(M, /*Complain=*/false);
7200b57cec5SDimitry Andric
7210b57cec5SDimitry Andric // If this is the first time we've entered this module, set up its state.
7220b57cec5SDimitry Andric auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
7230b57cec5SDimitry Andric auto &State = R.first->second;
7240b57cec5SDimitry Andric bool FirstTime = R.second;
7250b57cec5SDimitry Andric if (FirstTime) {
7260b57cec5SDimitry Andric // Determine the set of starting macros for this submodule; take these
7270b57cec5SDimitry Andric // from the "null" module (the predefines buffer).
7280b57cec5SDimitry Andric //
7290b57cec5SDimitry Andric // FIXME: If we have local visibility but not modules enabled, the
7300b57cec5SDimitry Andric // NullSubmoduleState is polluted by #defines in the top-level source
7310b57cec5SDimitry Andric // file.
7320b57cec5SDimitry Andric auto &StartingMacros = NullSubmoduleState.Macros;
7330b57cec5SDimitry Andric
7340b57cec5SDimitry Andric // Restore to the starting state.
7350b57cec5SDimitry Andric // FIXME: Do this lazily, when each macro name is first referenced.
7360b57cec5SDimitry Andric for (auto &Macro : StartingMacros) {
7370b57cec5SDimitry Andric // Skip uninteresting macros.
7380b57cec5SDimitry Andric if (!Macro.second.getLatest() &&
7390b57cec5SDimitry Andric Macro.second.getOverriddenMacros().empty())
7400b57cec5SDimitry Andric continue;
7410b57cec5SDimitry Andric
7420b57cec5SDimitry Andric MacroState MS(Macro.second.getLatest());
7430b57cec5SDimitry Andric MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
7440b57cec5SDimitry Andric State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
7450b57cec5SDimitry Andric }
7460b57cec5SDimitry Andric }
7470b57cec5SDimitry Andric
7480b57cec5SDimitry Andric // Track that we entered this module.
7490b57cec5SDimitry Andric BuildingSubmoduleStack.push_back(
7500b57cec5SDimitry Andric BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
7510b57cec5SDimitry Andric PendingModuleMacroNames.size()));
7520b57cec5SDimitry Andric
7530b57cec5SDimitry Andric if (Callbacks)
7540b57cec5SDimitry Andric Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
7550b57cec5SDimitry Andric
7560b57cec5SDimitry Andric // Switch to this submodule as the current submodule.
7570b57cec5SDimitry Andric CurSubmoduleState = &State;
7580b57cec5SDimitry Andric
7590b57cec5SDimitry Andric // This module is visible to itself.
7600b57cec5SDimitry Andric if (FirstTime)
7610b57cec5SDimitry Andric makeModuleVisible(M, ImportLoc);
7620b57cec5SDimitry Andric }
7630b57cec5SDimitry Andric
needModuleMacros() const7640b57cec5SDimitry Andric bool Preprocessor::needModuleMacros() const {
7650b57cec5SDimitry Andric // If we're not within a submodule, we never need to create ModuleMacros.
7660b57cec5SDimitry Andric if (BuildingSubmoduleStack.empty())
7670b57cec5SDimitry Andric return false;
7680b57cec5SDimitry Andric // If we are tracking module macro visibility even for textually-included
7690b57cec5SDimitry Andric // headers, we need ModuleMacros.
7700b57cec5SDimitry Andric if (getLangOpts().ModulesLocalVisibility)
7710b57cec5SDimitry Andric return true;
7720b57cec5SDimitry Andric // Otherwise, we only need module macros if we're actually compiling a module
7730b57cec5SDimitry Andric // interface.
7740b57cec5SDimitry Andric return getLangOpts().isCompilingModule();
7750b57cec5SDimitry Andric }
7760b57cec5SDimitry Andric
LeaveSubmodule(bool ForPragma)7770b57cec5SDimitry Andric Module *Preprocessor::LeaveSubmodule(bool ForPragma) {
7780b57cec5SDimitry Andric if (BuildingSubmoduleStack.empty() ||
7790b57cec5SDimitry Andric BuildingSubmoduleStack.back().IsPragma != ForPragma) {
7800b57cec5SDimitry Andric assert(ForPragma && "non-pragma module enter/leave mismatch");
7810b57cec5SDimitry Andric return nullptr;
7820b57cec5SDimitry Andric }
7830b57cec5SDimitry Andric
7840b57cec5SDimitry Andric auto &Info = BuildingSubmoduleStack.back();
7850b57cec5SDimitry Andric
7860b57cec5SDimitry Andric Module *LeavingMod = Info.M;
7870b57cec5SDimitry Andric SourceLocation ImportLoc = Info.ImportLoc;
7880b57cec5SDimitry Andric
7890b57cec5SDimitry Andric if (!needModuleMacros() ||
7900b57cec5SDimitry Andric (!getLangOpts().ModulesLocalVisibility &&
7910b57cec5SDimitry Andric LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
7920b57cec5SDimitry Andric // If we don't need module macros, or this is not a module for which we
7930b57cec5SDimitry Andric // are tracking macro visibility, don't build any, and preserve the list
7940b57cec5SDimitry Andric // of pending names for the surrounding submodule.
7950b57cec5SDimitry Andric BuildingSubmoduleStack.pop_back();
7960b57cec5SDimitry Andric
7970b57cec5SDimitry Andric if (Callbacks)
7980b57cec5SDimitry Andric Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
7990b57cec5SDimitry Andric
8000b57cec5SDimitry Andric makeModuleVisible(LeavingMod, ImportLoc);
8010b57cec5SDimitry Andric return LeavingMod;
8020b57cec5SDimitry Andric }
8030b57cec5SDimitry Andric
8040b57cec5SDimitry Andric // Create ModuleMacros for any macros defined in this submodule.
8050b57cec5SDimitry Andric llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
8060b57cec5SDimitry Andric for (unsigned I = Info.OuterPendingModuleMacroNames;
8070b57cec5SDimitry Andric I != PendingModuleMacroNames.size(); ++I) {
8080b57cec5SDimitry Andric auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
8090b57cec5SDimitry Andric if (!VisitedMacros.insert(II).second)
8100b57cec5SDimitry Andric continue;
8110b57cec5SDimitry Andric
8120b57cec5SDimitry Andric auto MacroIt = CurSubmoduleState->Macros.find(II);
8130b57cec5SDimitry Andric if (MacroIt == CurSubmoduleState->Macros.end())
8140b57cec5SDimitry Andric continue;
8150b57cec5SDimitry Andric auto &Macro = MacroIt->second;
8160b57cec5SDimitry Andric
8170b57cec5SDimitry Andric // Find the starting point for the MacroDirective chain in this submodule.
8180b57cec5SDimitry Andric MacroDirective *OldMD = nullptr;
8190b57cec5SDimitry Andric auto *OldState = Info.OuterSubmoduleState;
8200b57cec5SDimitry Andric if (getLangOpts().ModulesLocalVisibility)
8210b57cec5SDimitry Andric OldState = &NullSubmoduleState;
8220b57cec5SDimitry Andric if (OldState && OldState != CurSubmoduleState) {
8230b57cec5SDimitry Andric // FIXME: It'd be better to start at the state from when we most recently
8240b57cec5SDimitry Andric // entered this submodule, but it doesn't really matter.
8250b57cec5SDimitry Andric auto &OldMacros = OldState->Macros;
8260b57cec5SDimitry Andric auto OldMacroIt = OldMacros.find(II);
8270b57cec5SDimitry Andric if (OldMacroIt == OldMacros.end())
8280b57cec5SDimitry Andric OldMD = nullptr;
8290b57cec5SDimitry Andric else
8300b57cec5SDimitry Andric OldMD = OldMacroIt->second.getLatest();
8310b57cec5SDimitry Andric }
8320b57cec5SDimitry Andric
8330b57cec5SDimitry Andric // This module may have exported a new macro. If so, create a ModuleMacro
8340b57cec5SDimitry Andric // representing that fact.
8350b57cec5SDimitry Andric bool ExplicitlyPublic = false;
8360b57cec5SDimitry Andric for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
8370b57cec5SDimitry Andric assert(MD && "broken macro directive chain");
8380b57cec5SDimitry Andric
8390b57cec5SDimitry Andric if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
8400b57cec5SDimitry Andric // The latest visibility directive for a name in a submodule affects
8410b57cec5SDimitry Andric // all the directives that come before it.
8420b57cec5SDimitry Andric if (VisMD->isPublic())
8430b57cec5SDimitry Andric ExplicitlyPublic = true;
8440b57cec5SDimitry Andric else if (!ExplicitlyPublic)
8450b57cec5SDimitry Andric // Private with no following public directive: not exported.
8460b57cec5SDimitry Andric break;
8470b57cec5SDimitry Andric } else {
8480b57cec5SDimitry Andric MacroInfo *Def = nullptr;
8490b57cec5SDimitry Andric if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
8500b57cec5SDimitry Andric Def = DefMD->getInfo();
8510b57cec5SDimitry Andric
8520b57cec5SDimitry Andric // FIXME: Issue a warning if multiple headers for the same submodule
8530b57cec5SDimitry Andric // define a macro, rather than silently ignoring all but the first.
8540b57cec5SDimitry Andric bool IsNew;
8550b57cec5SDimitry Andric // Don't bother creating a module macro if it would represent a #undef
8560b57cec5SDimitry Andric // that doesn't override anything.
8570b57cec5SDimitry Andric if (Def || !Macro.getOverriddenMacros().empty())
8580b57cec5SDimitry Andric addModuleMacro(LeavingMod, II, Def,
8590b57cec5SDimitry Andric Macro.getOverriddenMacros(), IsNew);
8600b57cec5SDimitry Andric
8610b57cec5SDimitry Andric if (!getLangOpts().ModulesLocalVisibility) {
8620b57cec5SDimitry Andric // This macro is exposed to the rest of this compilation as a
8630b57cec5SDimitry Andric // ModuleMacro; we don't need to track its MacroDirective any more.
8640b57cec5SDimitry Andric Macro.setLatest(nullptr);
8650b57cec5SDimitry Andric Macro.setOverriddenMacros(*this, {});
8660b57cec5SDimitry Andric }
8670b57cec5SDimitry Andric break;
8680b57cec5SDimitry Andric }
8690b57cec5SDimitry Andric }
8700b57cec5SDimitry Andric }
8710b57cec5SDimitry Andric PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
8720b57cec5SDimitry Andric
8730b57cec5SDimitry Andric // FIXME: Before we leave this submodule, we should parse all the other
8740b57cec5SDimitry Andric // headers within it. Otherwise, we're left with an inconsistent state
8750b57cec5SDimitry Andric // where we've made the module visible but don't yet have its complete
8760b57cec5SDimitry Andric // contents.
8770b57cec5SDimitry Andric
8780b57cec5SDimitry Andric // Put back the outer module's state, if we're tracking it.
8790b57cec5SDimitry Andric if (getLangOpts().ModulesLocalVisibility)
8800b57cec5SDimitry Andric CurSubmoduleState = Info.OuterSubmoduleState;
8810b57cec5SDimitry Andric
8820b57cec5SDimitry Andric BuildingSubmoduleStack.pop_back();
8830b57cec5SDimitry Andric
8840b57cec5SDimitry Andric if (Callbacks)
8850b57cec5SDimitry Andric Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
8860b57cec5SDimitry Andric
8870b57cec5SDimitry Andric // A nested #include makes the included submodule visible.
8880b57cec5SDimitry Andric makeModuleVisible(LeavingMod, ImportLoc);
8890b57cec5SDimitry Andric return LeavingMod;
8900b57cec5SDimitry Andric }
891