1 //===- VirtualFileSystem.cpp - Virtual File System Layer ------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the VirtualFileSystem interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Basic/VirtualFileSystem.h"
15 #include "clang/Basic/LLVM.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/IntrusiveRefCntPtr.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSet.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/ADT/iterator_range.h"
27 #include "llvm/Config/llvm-config.h"
28 #include "llvm/Support/Compiler.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Support/Chrono.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/Errc.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/ErrorOr.h"
35 #include "llvm/Support/FileSystem.h"
36 #include "llvm/Support/MemoryBuffer.h"
37 #include "llvm/Support/Path.h"
38 #include "llvm/Support/Process.h"
39 #include "llvm/Support/SMLoc.h"
40 #include "llvm/Support/SourceMgr.h"
41 #include "llvm/Support/YAMLParser.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include <algorithm>
44 #include <atomic>
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <limits>
49 #include <map>
50 #include <memory>
51 #include <string>
52 #include <system_error>
53 #include <utility>
54 #include <vector>
55 
56 using namespace clang;
57 using namespace vfs;
58 using namespace llvm;
59 
60 using llvm::sys::fs::file_status;
61 using llvm::sys::fs::file_type;
62 using llvm::sys::fs::perms;
63 using llvm::sys::fs::UniqueID;
64 
Status(const file_status & Status)65 Status::Status(const file_status &Status)
66     : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
67       User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
68       Type(Status.type()), Perms(Status.permissions())  {}
69 
Status(StringRef Name,UniqueID UID,sys::TimePoint<> MTime,uint32_t User,uint32_t Group,uint64_t Size,file_type Type,perms Perms)70 Status::Status(StringRef Name, UniqueID UID, sys::TimePoint<> MTime,
71                uint32_t User, uint32_t Group, uint64_t Size, file_type Type,
72                perms Perms)
73     : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size),
74       Type(Type), Perms(Perms) {}
75 
copyWithNewName(const Status & In,StringRef NewName)76 Status Status::copyWithNewName(const Status &In, StringRef NewName) {
77   return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
78                 In.getUser(), In.getGroup(), In.getSize(), In.getType(),
79                 In.getPermissions());
80 }
81 
copyWithNewName(const file_status & In,StringRef NewName)82 Status Status::copyWithNewName(const file_status &In, StringRef NewName) {
83   return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
84                 In.getUser(), In.getGroup(), In.getSize(), In.type(),
85                 In.permissions());
86 }
87 
equivalent(const Status & Other) const88 bool Status::equivalent(const Status &Other) const {
89   assert(isStatusKnown() && Other.isStatusKnown());
90   return getUniqueID() == Other.getUniqueID();
91 }
92 
isDirectory() const93 bool Status::isDirectory() const {
94   return Type == file_type::directory_file;
95 }
96 
isRegularFile() const97 bool Status::isRegularFile() const {
98   return Type == file_type::regular_file;
99 }
100 
isOther() const101 bool Status::isOther() const {
102   return exists() && !isRegularFile() && !isDirectory() && !isSymlink();
103 }
104 
isSymlink() const105 bool Status::isSymlink() const {
106   return Type == file_type::symlink_file;
107 }
108 
isStatusKnown() const109 bool Status::isStatusKnown() const {
110   return Type != file_type::status_error;
111 }
112 
exists() const113 bool Status::exists() const {
114   return isStatusKnown() && Type != file_type::file_not_found;
115 }
116 
117 File::~File() = default;
118 
119 FileSystem::~FileSystem() = default;
120 
121 ErrorOr<std::unique_ptr<MemoryBuffer>>
getBufferForFile(const llvm::Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)122 FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
123                              bool RequiresNullTerminator, bool IsVolatile) {
124   auto F = openFileForRead(Name);
125   if (!F)
126     return F.getError();
127 
128   return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
129 }
130 
makeAbsolute(SmallVectorImpl<char> & Path) const131 std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
132   if (llvm::sys::path::is_absolute(Path))
133     return {};
134 
135   auto WorkingDir = getCurrentWorkingDirectory();
136   if (!WorkingDir)
137     return WorkingDir.getError();
138 
139   return llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
140 }
141 
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const142 std::error_code FileSystem::getRealPath(const Twine &Path,
143                                         SmallVectorImpl<char> &Output) const {
144   return errc::operation_not_permitted;
145 }
146 
exists(const Twine & Path)147 bool FileSystem::exists(const Twine &Path) {
148   auto Status = status(Path);
149   return Status && Status->exists();
150 }
151 
152 #ifndef NDEBUG
isTraversalComponent(StringRef Component)153 static bool isTraversalComponent(StringRef Component) {
154   return Component.equals("..") || Component.equals(".");
155 }
156 
pathHasTraversal(StringRef Path)157 static bool pathHasTraversal(StringRef Path) {
158   using namespace llvm::sys;
159 
160   for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path)))
161     if (isTraversalComponent(Comp))
162       return true;
163   return false;
164 }
165 #endif
166 
167 //===-----------------------------------------------------------------------===/
168 // RealFileSystem implementation
169 //===-----------------------------------------------------------------------===/
170 
171 namespace {
172 
173 /// Wrapper around a raw file descriptor.
174 class RealFile : public File {
175   friend class RealFileSystem;
176 
177   int FD;
178   Status S;
179   std::string RealName;
180 
RealFile(int FD,StringRef NewName,StringRef NewRealPathName)181   RealFile(int FD, StringRef NewName, StringRef NewRealPathName)
182       : FD(FD), S(NewName, {}, {}, {}, {}, {},
183                   llvm::sys::fs::file_type::status_error, {}),
184         RealName(NewRealPathName.str()) {
185     assert(FD >= 0 && "Invalid or inactive file descriptor");
186   }
187 
188 public:
189   ~RealFile() override;
190 
191   ErrorOr<Status> status() override;
192   ErrorOr<std::string> getName() override;
193   ErrorOr<std::unique_ptr<MemoryBuffer>> getBuffer(const Twine &Name,
194                                                    int64_t FileSize,
195                                                    bool RequiresNullTerminator,
196                                                    bool IsVolatile) override;
197   std::error_code close() override;
198 };
199 
200 } // namespace
201 
~RealFile()202 RealFile::~RealFile() { close(); }
203 
status()204 ErrorOr<Status> RealFile::status() {
205   assert(FD != -1 && "cannot stat closed file");
206   if (!S.isStatusKnown()) {
207     file_status RealStatus;
208     if (std::error_code EC = sys::fs::status(FD, RealStatus))
209       return EC;
210     S = Status::copyWithNewName(RealStatus, S.getName());
211   }
212   return S;
213 }
214 
getName()215 ErrorOr<std::string> RealFile::getName() {
216   return RealName.empty() ? S.getName().str() : RealName;
217 }
218 
219 ErrorOr<std::unique_ptr<MemoryBuffer>>
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)220 RealFile::getBuffer(const Twine &Name, int64_t FileSize,
221                     bool RequiresNullTerminator, bool IsVolatile) {
222   assert(FD != -1 && "cannot get buffer for closed file");
223   return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
224                                    IsVolatile);
225 }
226 
close()227 std::error_code RealFile::close() {
228   std::error_code EC = sys::Process::SafelyCloseFileDescriptor(FD);
229   FD = -1;
230   return EC;
231 }
232 
233 namespace {
234 
235 /// The file system according to your operating system.
236 class RealFileSystem : public FileSystem {
237 public:
238   ErrorOr<Status> status(const Twine &Path) override;
239   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
240   directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
241 
242   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
243   std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
244   std::error_code getRealPath(const Twine &Path,
245                               SmallVectorImpl<char> &Output) const override;
246 };
247 
248 } // namespace
249 
status(const Twine & Path)250 ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
251   sys::fs::file_status RealStatus;
252   if (std::error_code EC = sys::fs::status(Path, RealStatus))
253     return EC;
254   return Status::copyWithNewName(RealStatus, Path.str());
255 }
256 
257 ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Name)258 RealFileSystem::openFileForRead(const Twine &Name) {
259   int FD;
260   SmallString<256> RealName;
261   if (std::error_code EC =
262           sys::fs::openFileForRead(Name, FD, sys::fs::OF_None, &RealName))
263     return EC;
264   return std::unique_ptr<File>(new RealFile(FD, Name.str(), RealName.str()));
265 }
266 
getCurrentWorkingDirectory() const267 llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
268   SmallString<256> Dir;
269   if (std::error_code EC = llvm::sys::fs::current_path(Dir))
270     return EC;
271   return Dir.str().str();
272 }
273 
setCurrentWorkingDirectory(const Twine & Path)274 std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
275   // FIXME: chdir is thread hostile; on the other hand, creating the same
276   // behavior as chdir is complex: chdir resolves the path once, thus
277   // guaranteeing that all subsequent relative path operations work
278   // on the same path the original chdir resulted in. This makes a
279   // difference for example on network filesystems, where symlinks might be
280   // switched during runtime of the tool. Fixing this depends on having a
281   // file system abstraction that allows openat() style interactions.
282   return llvm::sys::fs::set_current_path(Path);
283 }
284 
285 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const286 RealFileSystem::getRealPath(const Twine &Path,
287                             SmallVectorImpl<char> &Output) const {
288   return llvm::sys::fs::real_path(Path, Output);
289 }
290 
getRealFileSystem()291 IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() {
292   static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
293   return FS;
294 }
295 
296 namespace {
297 
298 class RealFSDirIter : public clang::vfs::detail::DirIterImpl {
299   llvm::sys::fs::directory_iterator Iter;
300 
301 public:
RealFSDirIter(const Twine & Path,std::error_code & EC)302   RealFSDirIter(const Twine &Path, std::error_code &EC) : Iter(Path, EC) {
303     if (Iter != llvm::sys::fs::directory_iterator()) {
304       llvm::sys::fs::file_status S;
305       std::error_code ErrorCode = llvm::sys::fs::status(Iter->path(), S, true);
306       CurrentEntry = Status::copyWithNewName(S, Iter->path());
307       if (!EC)
308         EC = ErrorCode;
309     }
310   }
311 
increment()312   std::error_code increment() override {
313     std::error_code EC;
314     Iter.increment(EC);
315     if (Iter == llvm::sys::fs::directory_iterator()) {
316       CurrentEntry = Status();
317     } else {
318       llvm::sys::fs::file_status S;
319       std::error_code ErrorCode = llvm::sys::fs::status(Iter->path(), S, true);
320       CurrentEntry = Status::copyWithNewName(S, Iter->path());
321       if (!EC)
322         EC = ErrorCode;
323     }
324     return EC;
325   }
326 };
327 
328 } // namespace
329 
dir_begin(const Twine & Dir,std::error_code & EC)330 directory_iterator RealFileSystem::dir_begin(const Twine &Dir,
331                                              std::error_code &EC) {
332   return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC));
333 }
334 
335 //===-----------------------------------------------------------------------===/
336 // OverlayFileSystem implementation
337 //===-----------------------------------------------------------------------===/
338 
OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS)339 OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) {
340   FSList.push_back(std::move(BaseFS));
341 }
342 
pushOverlay(IntrusiveRefCntPtr<FileSystem> FS)343 void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) {
344   FSList.push_back(FS);
345   // Synchronize added file systems by duplicating the working directory from
346   // the first one in the list.
347   FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get());
348 }
349 
status(const Twine & Path)350 ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
351   // FIXME: handle symlinks that cross file systems
352   for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
353     ErrorOr<Status> Status = (*I)->status(Path);
354     if (Status || Status.getError() != llvm::errc::no_such_file_or_directory)
355       return Status;
356   }
357   return make_error_code(llvm::errc::no_such_file_or_directory);
358 }
359 
360 ErrorOr<std::unique_ptr<File>>
openFileForRead(const llvm::Twine & Path)361 OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
362   // FIXME: handle symlinks that cross file systems
363   for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
364     auto Result = (*I)->openFileForRead(Path);
365     if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
366       return Result;
367   }
368   return make_error_code(llvm::errc::no_such_file_or_directory);
369 }
370 
371 llvm::ErrorOr<std::string>
getCurrentWorkingDirectory() const372 OverlayFileSystem::getCurrentWorkingDirectory() const {
373   // All file systems are synchronized, just take the first working directory.
374   return FSList.front()->getCurrentWorkingDirectory();
375 }
376 
377 std::error_code
setCurrentWorkingDirectory(const Twine & Path)378 OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
379   for (auto &FS : FSList)
380     if (std::error_code EC = FS->setCurrentWorkingDirectory(Path))
381       return EC;
382   return {};
383 }
384 
385 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const386 OverlayFileSystem::getRealPath(const Twine &Path,
387                                SmallVectorImpl<char> &Output) const {
388   for (auto &FS : FSList)
389     if (FS->exists(Path))
390       return FS->getRealPath(Path, Output);
391   return errc::no_such_file_or_directory;
392 }
393 
394 clang::vfs::detail::DirIterImpl::~DirIterImpl() = default;
395 
396 namespace {
397 
398 class OverlayFSDirIterImpl : public clang::vfs::detail::DirIterImpl {
399   OverlayFileSystem &Overlays;
400   std::string Path;
401   OverlayFileSystem::iterator CurrentFS;
402   directory_iterator CurrentDirIter;
403   llvm::StringSet<> SeenNames;
404 
incrementFS()405   std::error_code incrementFS() {
406     assert(CurrentFS != Overlays.overlays_end() && "incrementing past end");
407     ++CurrentFS;
408     for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) {
409       std::error_code EC;
410       CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
411       if (EC && EC != errc::no_such_file_or_directory)
412         return EC;
413       if (CurrentDirIter != directory_iterator())
414         break; // found
415     }
416     return {};
417   }
418 
incrementDirIter(bool IsFirstTime)419   std::error_code incrementDirIter(bool IsFirstTime) {
420     assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
421            "incrementing past end");
422     std::error_code EC;
423     if (!IsFirstTime)
424       CurrentDirIter.increment(EC);
425     if (!EC && CurrentDirIter == directory_iterator())
426       EC = incrementFS();
427     return EC;
428   }
429 
incrementImpl(bool IsFirstTime)430   std::error_code incrementImpl(bool IsFirstTime) {
431     while (true) {
432       std::error_code EC = incrementDirIter(IsFirstTime);
433       if (EC || CurrentDirIter == directory_iterator()) {
434         CurrentEntry = Status();
435         return EC;
436       }
437       CurrentEntry = *CurrentDirIter;
438       StringRef Name = llvm::sys::path::filename(CurrentEntry.getName());
439       if (SeenNames.insert(Name).second)
440         return EC; // name not seen before
441     }
442     llvm_unreachable("returned above");
443   }
444 
445 public:
OverlayFSDirIterImpl(const Twine & Path,OverlayFileSystem & FS,std::error_code & EC)446   OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS,
447                        std::error_code &EC)
448       : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) {
449     CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
450     EC = incrementImpl(true);
451   }
452 
increment()453   std::error_code increment() override { return incrementImpl(false); }
454 };
455 
456 } // namespace
457 
dir_begin(const Twine & Dir,std::error_code & EC)458 directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
459                                                 std::error_code &EC) {
460   return directory_iterator(
461       std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
462 }
463 
464 namespace clang {
465 namespace vfs {
466 
467 namespace detail {
468 
469 enum InMemoryNodeKind { IME_File, IME_Directory };
470 
471 /// The in memory file system is a tree of Nodes. Every node can either be a
472 /// file or a directory.
473 class InMemoryNode {
474   Status Stat;
475   InMemoryNodeKind Kind;
476 
477 public:
InMemoryNode(Status Stat,InMemoryNodeKind Kind)478   InMemoryNode(Status Stat, InMemoryNodeKind Kind)
479       : Stat(std::move(Stat)), Kind(Kind) {}
480   virtual ~InMemoryNode() = default;
481 
getStatus() const482   const Status &getStatus() const { return Stat; }
getKind() const483   InMemoryNodeKind getKind() const { return Kind; }
484   virtual std::string toString(unsigned Indent) const = 0;
485 };
486 
487 namespace {
488 
489 class InMemoryFile : public InMemoryNode {
490   std::unique_ptr<llvm::MemoryBuffer> Buffer;
491 
492 public:
InMemoryFile(Status Stat,std::unique_ptr<llvm::MemoryBuffer> Buffer)493   InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
494       : InMemoryNode(std::move(Stat), IME_File), Buffer(std::move(Buffer)) {}
495 
getBuffer()496   llvm::MemoryBuffer *getBuffer() { return Buffer.get(); }
497 
toString(unsigned Indent) const498   std::string toString(unsigned Indent) const override {
499     return (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
500   }
501 
classof(const InMemoryNode * N)502   static bool classof(const InMemoryNode *N) {
503     return N->getKind() == IME_File;
504   }
505 };
506 
507 /// Adapt a InMemoryFile for VFS' File interface.
508 class InMemoryFileAdaptor : public File {
509   InMemoryFile &Node;
510 
511 public:
InMemoryFileAdaptor(InMemoryFile & Node)512   explicit InMemoryFileAdaptor(InMemoryFile &Node) : Node(Node) {}
513 
status()514   llvm::ErrorOr<Status> status() override { return Node.getStatus(); }
515 
516   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)517   getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
518             bool IsVolatile) override {
519     llvm::MemoryBuffer *Buf = Node.getBuffer();
520     return llvm::MemoryBuffer::getMemBuffer(
521         Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
522   }
523 
close()524   std::error_code close() override { return {}; }
525 };
526 
527 } // namespace
528 
529 class InMemoryDirectory : public InMemoryNode {
530   std::map<std::string, std::unique_ptr<InMemoryNode>> Entries;
531 
532 public:
InMemoryDirectory(Status Stat)533   InMemoryDirectory(Status Stat)
534       : InMemoryNode(std::move(Stat), IME_Directory) {}
535 
getChild(StringRef Name)536   InMemoryNode *getChild(StringRef Name) {
537     auto I = Entries.find(Name);
538     if (I != Entries.end())
539       return I->second.get();
540     return nullptr;
541   }
542 
addChild(StringRef Name,std::unique_ptr<InMemoryNode> Child)543   InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
544     return Entries.insert(make_pair(Name, std::move(Child)))
545         .first->second.get();
546   }
547 
548   using const_iterator = decltype(Entries)::const_iterator;
549 
begin() const550   const_iterator begin() const { return Entries.begin(); }
end() const551   const_iterator end() const { return Entries.end(); }
552 
toString(unsigned Indent) const553   std::string toString(unsigned Indent) const override {
554     std::string Result =
555         (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
556     for (const auto &Entry : Entries)
557       Result += Entry.second->toString(Indent + 2);
558     return Result;
559   }
560 
classof(const InMemoryNode * N)561   static bool classof(const InMemoryNode *N) {
562     return N->getKind() == IME_Directory;
563   }
564 };
565 
566 } // namespace detail
567 
InMemoryFileSystem(bool UseNormalizedPaths)568 InMemoryFileSystem::InMemoryFileSystem(bool UseNormalizedPaths)
569     : Root(new detail::InMemoryDirectory(
570           Status("", getNextVirtualUniqueID(), llvm::sys::TimePoint<>(), 0, 0,
571                  0, llvm::sys::fs::file_type::directory_file,
572                  llvm::sys::fs::perms::all_all))),
573       UseNormalizedPaths(UseNormalizedPaths) {}
574 
575 InMemoryFileSystem::~InMemoryFileSystem() = default;
576 
toString() const577 std::string InMemoryFileSystem::toString() const {
578   return Root->toString(/*Indent=*/0);
579 }
580 
addFile(const Twine & P,time_t ModificationTime,std::unique_ptr<llvm::MemoryBuffer> Buffer,Optional<uint32_t> User,Optional<uint32_t> Group,Optional<llvm::sys::fs::file_type> Type,Optional<llvm::sys::fs::perms> Perms)581 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
582                                  std::unique_ptr<llvm::MemoryBuffer> Buffer,
583                                  Optional<uint32_t> User,
584                                  Optional<uint32_t> Group,
585                                  Optional<llvm::sys::fs::file_type> Type,
586                                  Optional<llvm::sys::fs::perms> Perms) {
587   SmallString<128> Path;
588   P.toVector(Path);
589 
590   // Fix up relative paths. This just prepends the current working directory.
591   std::error_code EC = makeAbsolute(Path);
592   assert(!EC);
593   (void)EC;
594 
595   if (useNormalizedPaths())
596     llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
597 
598   if (Path.empty())
599     return false;
600 
601   detail::InMemoryDirectory *Dir = Root.get();
602   auto I = llvm::sys::path::begin(Path), E = sys::path::end(Path);
603   const auto ResolvedUser = User.getValueOr(0);
604   const auto ResolvedGroup = Group.getValueOr(0);
605   const auto ResolvedType = Type.getValueOr(sys::fs::file_type::regular_file);
606   const auto ResolvedPerms = Perms.getValueOr(sys::fs::all_all);
607   // Any intermediate directories we create should be accessible by
608   // the owner, even if Perms says otherwise for the final path.
609   const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all;
610   while (true) {
611     StringRef Name = *I;
612     detail::InMemoryNode *Node = Dir->getChild(Name);
613     ++I;
614     if (!Node) {
615       if (I == E) {
616         // End of the path, create a new file or directory.
617         Status Stat(P.str(), getNextVirtualUniqueID(),
618                     llvm::sys::toTimePoint(ModificationTime), ResolvedUser,
619                     ResolvedGroup, Buffer->getBufferSize(), ResolvedType,
620                     ResolvedPerms);
621         std::unique_ptr<detail::InMemoryNode> Child;
622         if (ResolvedType == sys::fs::file_type::directory_file) {
623           Child.reset(new detail::InMemoryDirectory(std::move(Stat)));
624         } else {
625           Child.reset(new detail::InMemoryFile(std::move(Stat),
626                                                std::move(Buffer)));
627         }
628         Dir->addChild(Name, std::move(Child));
629         return true;
630       }
631 
632       // Create a new directory. Use the path up to here.
633       Status Stat(
634           StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
635           getNextVirtualUniqueID(), llvm::sys::toTimePoint(ModificationTime),
636           ResolvedUser, ResolvedGroup, Buffer->getBufferSize(),
637           sys::fs::file_type::directory_file, NewDirectoryPerms);
638       Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
639           Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
640       continue;
641     }
642 
643     if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node)) {
644       Dir = NewDir;
645     } else {
646       assert(isa<detail::InMemoryFile>(Node) &&
647              "Must be either file or directory!");
648 
649       // Trying to insert a directory in place of a file.
650       if (I != E)
651         return false;
652 
653       // Return false only if the new file is different from the existing one.
654       return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
655              Buffer->getBuffer();
656     }
657   }
658 }
659 
addFileNoOwn(const Twine & P,time_t ModificationTime,llvm::MemoryBuffer * Buffer,Optional<uint32_t> User,Optional<uint32_t> Group,Optional<llvm::sys::fs::file_type> Type,Optional<llvm::sys::fs::perms> Perms)660 bool InMemoryFileSystem::addFileNoOwn(const Twine &P, time_t ModificationTime,
661                                       llvm::MemoryBuffer *Buffer,
662                                       Optional<uint32_t> User,
663                                       Optional<uint32_t> Group,
664                                       Optional<llvm::sys::fs::file_type> Type,
665                                       Optional<llvm::sys::fs::perms> Perms) {
666   return addFile(P, ModificationTime,
667                  llvm::MemoryBuffer::getMemBuffer(
668                      Buffer->getBuffer(), Buffer->getBufferIdentifier()),
669                  std::move(User), std::move(Group), std::move(Type),
670                  std::move(Perms));
671 }
672 
673 static ErrorOr<detail::InMemoryNode *>
lookupInMemoryNode(const InMemoryFileSystem & FS,detail::InMemoryDirectory * Dir,const Twine & P)674 lookupInMemoryNode(const InMemoryFileSystem &FS, detail::InMemoryDirectory *Dir,
675                    const Twine &P) {
676   SmallString<128> Path;
677   P.toVector(Path);
678 
679   // Fix up relative paths. This just prepends the current working directory.
680   std::error_code EC = FS.makeAbsolute(Path);
681   assert(!EC);
682   (void)EC;
683 
684   if (FS.useNormalizedPaths())
685     llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
686 
687   if (Path.empty())
688     return Dir;
689 
690   auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
691   while (true) {
692     detail::InMemoryNode *Node = Dir->getChild(*I);
693     ++I;
694     if (!Node)
695       return errc::no_such_file_or_directory;
696 
697     // Return the file if it's at the end of the path.
698     if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
699       if (I == E)
700         return File;
701       return errc::no_such_file_or_directory;
702     }
703 
704     // Traverse directories.
705     Dir = cast<detail::InMemoryDirectory>(Node);
706     if (I == E)
707       return Dir;
708   }
709 }
710 
status(const Twine & Path)711 llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
712   auto Node = lookupInMemoryNode(*this, Root.get(), Path);
713   if (Node)
714     return (*Node)->getStatus();
715   return Node.getError();
716 }
717 
718 llvm::ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Path)719 InMemoryFileSystem::openFileForRead(const Twine &Path) {
720   auto Node = lookupInMemoryNode(*this, Root.get(), Path);
721   if (!Node)
722     return Node.getError();
723 
724   // When we have a file provide a heap-allocated wrapper for the memory buffer
725   // to match the ownership semantics for File.
726   if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
727     return std::unique_ptr<File>(new detail::InMemoryFileAdaptor(*F));
728 
729   // FIXME: errc::not_a_file?
730   return make_error_code(llvm::errc::invalid_argument);
731 }
732 
733 namespace {
734 
735 /// Adaptor from InMemoryDir::iterator to directory_iterator.
736 class InMemoryDirIterator : public clang::vfs::detail::DirIterImpl {
737   detail::InMemoryDirectory::const_iterator I;
738   detail::InMemoryDirectory::const_iterator E;
739 
740 public:
741   InMemoryDirIterator() = default;
742 
InMemoryDirIterator(detail::InMemoryDirectory & Dir)743   explicit InMemoryDirIterator(detail::InMemoryDirectory &Dir)
744       : I(Dir.begin()), E(Dir.end()) {
745     if (I != E)
746       CurrentEntry = I->second->getStatus();
747   }
748 
increment()749   std::error_code increment() override {
750     ++I;
751     // When we're at the end, make CurrentEntry invalid and DirIterImpl will do
752     // the rest.
753     CurrentEntry = I != E ? I->second->getStatus() : Status();
754     return {};
755   }
756 };
757 
758 } // namespace
759 
dir_begin(const Twine & Dir,std::error_code & EC)760 directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
761                                                  std::error_code &EC) {
762   auto Node = lookupInMemoryNode(*this, Root.get(), Dir);
763   if (!Node) {
764     EC = Node.getError();
765     return directory_iterator(std::make_shared<InMemoryDirIterator>());
766   }
767 
768   if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
769     return directory_iterator(std::make_shared<InMemoryDirIterator>(*DirNode));
770 
771   EC = make_error_code(llvm::errc::not_a_directory);
772   return directory_iterator(std::make_shared<InMemoryDirIterator>());
773 }
774 
setCurrentWorkingDirectory(const Twine & P)775 std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
776   SmallString<128> Path;
777   P.toVector(Path);
778 
779   // Fix up relative paths. This just prepends the current working directory.
780   std::error_code EC = makeAbsolute(Path);
781   assert(!EC);
782   (void)EC;
783 
784   if (useNormalizedPaths())
785     llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
786 
787   if (!Path.empty())
788     WorkingDirectory = Path.str();
789   return {};
790 }
791 
792 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const793 InMemoryFileSystem::getRealPath(const Twine &Path,
794                                 SmallVectorImpl<char> &Output) const {
795   auto CWD = getCurrentWorkingDirectory();
796   if (!CWD || CWD->empty())
797     return errc::operation_not_permitted;
798   Path.toVector(Output);
799   if (auto EC = makeAbsolute(Output))
800     return EC;
801   llvm::sys::path::remove_dots(Output, /*remove_dot_dot=*/true);
802   return {};
803 }
804 
805 } // namespace vfs
806 } // namespace clang
807 
808 //===-----------------------------------------------------------------------===/
809 // RedirectingFileSystem implementation
810 //===-----------------------------------------------------------------------===/
811 
812 namespace {
813 
814 enum EntryKind {
815   EK_Directory,
816   EK_File
817 };
818 
819 /// A single file or directory in the VFS.
820 class Entry {
821   EntryKind Kind;
822   std::string Name;
823 
824 public:
Entry(EntryKind K,StringRef Name)825   Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
826   virtual ~Entry() = default;
827 
getName() const828   StringRef getName() const { return Name; }
getKind() const829   EntryKind getKind() const { return Kind; }
830 };
831 
832 class RedirectingDirectoryEntry : public Entry {
833   std::vector<std::unique_ptr<Entry>> Contents;
834   Status S;
835 
836 public:
RedirectingDirectoryEntry(StringRef Name,std::vector<std::unique_ptr<Entry>> Contents,Status S)837   RedirectingDirectoryEntry(StringRef Name,
838                             std::vector<std::unique_ptr<Entry>> Contents,
839                             Status S)
840       : Entry(EK_Directory, Name), Contents(std::move(Contents)),
841         S(std::move(S)) {}
RedirectingDirectoryEntry(StringRef Name,Status S)842   RedirectingDirectoryEntry(StringRef Name, Status S)
843       : Entry(EK_Directory, Name), S(std::move(S)) {}
844 
getStatus()845   Status getStatus() { return S; }
846 
addContent(std::unique_ptr<Entry> Content)847   void addContent(std::unique_ptr<Entry> Content) {
848     Contents.push_back(std::move(Content));
849   }
850 
getLastContent() const851   Entry *getLastContent() const { return Contents.back().get(); }
852 
853   using iterator = decltype(Contents)::iterator;
854 
contents_begin()855   iterator contents_begin() { return Contents.begin(); }
contents_end()856   iterator contents_end() { return Contents.end(); }
857 
classof(const Entry * E)858   static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
859 };
860 
861 class RedirectingFileEntry : public Entry {
862 public:
863   enum NameKind {
864     NK_NotSet,
865     NK_External,
866     NK_Virtual
867   };
868 
869 private:
870   std::string ExternalContentsPath;
871   NameKind UseName;
872 
873 public:
RedirectingFileEntry(StringRef Name,StringRef ExternalContentsPath,NameKind UseName)874   RedirectingFileEntry(StringRef Name, StringRef ExternalContentsPath,
875                        NameKind UseName)
876       : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath),
877         UseName(UseName) {}
878 
getExternalContentsPath() const879   StringRef getExternalContentsPath() const { return ExternalContentsPath; }
880 
881   /// whether to use the external path as the name for this file.
useExternalName(bool GlobalUseExternalName) const882   bool useExternalName(bool GlobalUseExternalName) const {
883     return UseName == NK_NotSet ? GlobalUseExternalName
884                                 : (UseName == NK_External);
885   }
886 
getUseName() const887   NameKind getUseName() const { return UseName; }
888 
classof(const Entry * E)889   static bool classof(const Entry *E) { return E->getKind() == EK_File; }
890 };
891 
892 class RedirectingFileSystem;
893 
894 class VFSFromYamlDirIterImpl : public clang::vfs::detail::DirIterImpl {
895   std::string Dir;
896   RedirectingFileSystem &FS;
897   RedirectingDirectoryEntry::iterator Current, End;
898 
899 public:
900   VFSFromYamlDirIterImpl(const Twine &Path, RedirectingFileSystem &FS,
901                          RedirectingDirectoryEntry::iterator Begin,
902                          RedirectingDirectoryEntry::iterator End,
903                          std::error_code &EC);
904 
905   std::error_code increment() override;
906 };
907 
908 /// A virtual file system parsed from a YAML file.
909 ///
910 /// Currently, this class allows creating virtual directories and mapping
911 /// virtual file paths to existing external files, available in \c ExternalFS.
912 ///
913 /// The basic structure of the parsed file is:
914 /// \verbatim
915 /// {
916 ///   'version': <version number>,
917 ///   <optional configuration>
918 ///   'roots': [
919 ///              <directory entries>
920 ///            ]
921 /// }
922 /// \endverbatim
923 ///
924 /// All configuration options are optional.
925 ///   'case-sensitive': <boolean, default=true>
926 ///   'use-external-names': <boolean, default=true>
927 ///   'overlay-relative': <boolean, default=false>
928 ///   'ignore-non-existent-contents': <boolean, default=true>
929 ///
930 /// Virtual directories are represented as
931 /// \verbatim
932 /// {
933 ///   'type': 'directory',
934 ///   'name': <string>,
935 ///   'contents': [ <file or directory entries> ]
936 /// }
937 /// \endverbatim
938 ///
939 /// The default attributes for virtual directories are:
940 /// \verbatim
941 /// MTime = now() when created
942 /// Perms = 0777
943 /// User = Group = 0
944 /// Size = 0
945 /// UniqueID = unspecified unique value
946 /// \endverbatim
947 ///
948 /// Re-mapped files are represented as
949 /// \verbatim
950 /// {
951 ///   'type': 'file',
952 ///   'name': <string>,
953 ///   'use-external-name': <boolean> # Optional
954 ///   'external-contents': <path to external file>)
955 /// }
956 /// \endverbatim
957 ///
958 /// and inherit their attributes from the external contents.
959 ///
960 /// In both cases, the 'name' field may contain multiple path components (e.g.
961 /// /path/to/file). However, any directory that contains more than one child
962 /// must be uniquely represented by a directory entry.
963 class RedirectingFileSystem : public vfs::FileSystem {
964   friend class RedirectingFileSystemParser;
965 
966   /// The root(s) of the virtual file system.
967   std::vector<std::unique_ptr<Entry>> Roots;
968 
969   /// The file system to use for external references.
970   IntrusiveRefCntPtr<FileSystem> ExternalFS;
971 
972   /// If IsRelativeOverlay is set, this represents the directory
973   /// path that should be prefixed to each 'external-contents' entry
974   /// when reading from YAML files.
975   std::string ExternalContentsPrefixDir;
976 
977   /// @name Configuration
978   /// @{
979 
980   /// Whether to perform case-sensitive comparisons.
981   ///
982   /// Currently, case-insensitive matching only works correctly with ASCII.
983   bool CaseSensitive = true;
984 
985   /// IsRelativeOverlay marks whether a IsExternalContentsPrefixDir path must
986   /// be prefixed in every 'external-contents' when reading from YAML files.
987   bool IsRelativeOverlay = false;
988 
989   /// Whether to use to use the value of 'external-contents' for the
990   /// names of files.  This global value is overridable on a per-file basis.
991   bool UseExternalNames = true;
992 
993   /// Whether an invalid path obtained via 'external-contents' should
994   /// cause iteration on the VFS to stop. If 'true', the VFS should ignore
995   /// the entry and continue with the next. Allows YAML files to be shared
996   /// across multiple compiler invocations regardless of prior existent
997   /// paths in 'external-contents'. This global value is overridable on a
998   /// per-file basis.
999   bool IgnoreNonExistentContents = true;
1000   /// @}
1001 
1002   /// Virtual file paths and external files could be canonicalized without "..",
1003   /// "." and "./" in their paths. FIXME: some unittests currently fail on
1004   /// win32 when using remove_dots and remove_leading_dotslash on paths.
1005   bool UseCanonicalizedPaths =
1006 #ifdef _WIN32
1007       false;
1008 #else
1009       true;
1010 #endif
1011 
1012 private:
RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)1013   RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)
1014       : ExternalFS(std::move(ExternalFS)) {}
1015 
1016   /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly
1017   /// recursing into the contents of \p From if it is a directory.
1018   ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
1019                               sys::path::const_iterator End, Entry *From);
1020 
1021   /// Get the status of a given an \c Entry.
1022   ErrorOr<Status> status(const Twine &Path, Entry *E);
1023 
1024 public:
1025   /// Looks up \p Path in \c Roots.
1026   ErrorOr<Entry *> lookupPath(const Twine &Path);
1027 
1028   /// Parses \p Buffer, which is expected to be in YAML format and
1029   /// returns a virtual file system representing its contents.
1030   static RedirectingFileSystem *
1031   create(std::unique_ptr<MemoryBuffer> Buffer,
1032          SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
1033          void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
1034 
1035   ErrorOr<Status> status(const Twine &Path) override;
1036   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
1037 
getCurrentWorkingDirectory() const1038   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
1039     return ExternalFS->getCurrentWorkingDirectory();
1040   }
1041 
setCurrentWorkingDirectory(const Twine & Path)1042   std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
1043     return ExternalFS->setCurrentWorkingDirectory(Path);
1044   }
1045 
dir_begin(const Twine & Dir,std::error_code & EC)1046   directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override{
1047     ErrorOr<Entry *> E = lookupPath(Dir);
1048     if (!E) {
1049       EC = E.getError();
1050       return {};
1051     }
1052     ErrorOr<Status> S = status(Dir, *E);
1053     if (!S) {
1054       EC = S.getError();
1055       return {};
1056     }
1057     if (!S->isDirectory()) {
1058       EC = std::error_code(static_cast<int>(errc::not_a_directory),
1059                            std::system_category());
1060       return {};
1061     }
1062 
1063     auto *D = cast<RedirectingDirectoryEntry>(*E);
1064     return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
1065         *this, D->contents_begin(), D->contents_end(), EC));
1066   }
1067 
setExternalContentsPrefixDir(StringRef PrefixDir)1068   void setExternalContentsPrefixDir(StringRef PrefixDir) {
1069     ExternalContentsPrefixDir = PrefixDir.str();
1070   }
1071 
getExternalContentsPrefixDir() const1072   StringRef getExternalContentsPrefixDir() const {
1073     return ExternalContentsPrefixDir;
1074   }
1075 
ignoreNonExistentContents() const1076   bool ignoreNonExistentContents() const {
1077     return IgnoreNonExistentContents;
1078   }
1079 
1080 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const1081 LLVM_DUMP_METHOD void dump() const {
1082     for (const auto &Root : Roots)
1083       dumpEntry(Root.get());
1084   }
1085 
dumpEntry(Entry * E,int NumSpaces=0) const1086 LLVM_DUMP_METHOD void dumpEntry(Entry *E, int NumSpaces = 0) const {
1087     StringRef Name = E->getName();
1088     for (int i = 0, e = NumSpaces; i < e; ++i)
1089       dbgs() << " ";
1090     dbgs() << "'" << Name.str().c_str() << "'" << "\n";
1091 
1092     if (E->getKind() == EK_Directory) {
1093       auto *DE = dyn_cast<RedirectingDirectoryEntry>(E);
1094       assert(DE && "Should be a directory");
1095 
1096       for (std::unique_ptr<Entry> &SubEntry :
1097            llvm::make_range(DE->contents_begin(), DE->contents_end()))
1098         dumpEntry(SubEntry.get(), NumSpaces+2);
1099     }
1100   }
1101 #endif
1102 };
1103 
1104 /// A helper class to hold the common YAML parsing state.
1105 class RedirectingFileSystemParser {
1106   yaml::Stream &Stream;
1107 
error(yaml::Node * N,const Twine & Msg)1108   void error(yaml::Node *N, const Twine &Msg) {
1109     Stream.printError(N, Msg);
1110   }
1111 
1112   // false on error
parseScalarString(yaml::Node * N,StringRef & Result,SmallVectorImpl<char> & Storage)1113   bool parseScalarString(yaml::Node *N, StringRef &Result,
1114                          SmallVectorImpl<char> &Storage) {
1115     const auto *S = dyn_cast<yaml::ScalarNode>(N);
1116 
1117     if (!S) {
1118       error(N, "expected string");
1119       return false;
1120     }
1121     Result = S->getValue(Storage);
1122     return true;
1123   }
1124 
1125   // false on error
parseScalarBool(yaml::Node * N,bool & Result)1126   bool parseScalarBool(yaml::Node *N, bool &Result) {
1127     SmallString<5> Storage;
1128     StringRef Value;
1129     if (!parseScalarString(N, Value, Storage))
1130       return false;
1131 
1132     if (Value.equals_lower("true") || Value.equals_lower("on") ||
1133         Value.equals_lower("yes") || Value == "1") {
1134       Result = true;
1135       return true;
1136     } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
1137                Value.equals_lower("no") || Value == "0") {
1138       Result = false;
1139       return true;
1140     }
1141 
1142     error(N, "expected boolean value");
1143     return false;
1144   }
1145 
1146   struct KeyStatus {
1147     bool Required;
1148     bool Seen = false;
1149 
KeyStatus__anond7e2b9a30711::RedirectingFileSystemParser::KeyStatus1150     KeyStatus(bool Required = false) : Required(Required) {}
1151   };
1152 
1153   using KeyStatusPair = std::pair<StringRef, KeyStatus>;
1154 
1155   // false on error
checkDuplicateOrUnknownKey(yaml::Node * KeyNode,StringRef Key,DenseMap<StringRef,KeyStatus> & Keys)1156   bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
1157                                   DenseMap<StringRef, KeyStatus> &Keys) {
1158     if (!Keys.count(Key)) {
1159       error(KeyNode, "unknown key");
1160       return false;
1161     }
1162     KeyStatus &S = Keys[Key];
1163     if (S.Seen) {
1164       error(KeyNode, Twine("duplicate key '") + Key + "'");
1165       return false;
1166     }
1167     S.Seen = true;
1168     return true;
1169   }
1170 
1171   // false on error
checkMissingKeys(yaml::Node * Obj,DenseMap<StringRef,KeyStatus> & Keys)1172   bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
1173     for (const auto &I : Keys) {
1174       if (I.second.Required && !I.second.Seen) {
1175         error(Obj, Twine("missing key '") + I.first + "'");
1176         return false;
1177       }
1178     }
1179     return true;
1180   }
1181 
lookupOrCreateEntry(RedirectingFileSystem * FS,StringRef Name,Entry * ParentEntry=nullptr)1182   Entry *lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
1183                              Entry *ParentEntry = nullptr) {
1184     if (!ParentEntry) { // Look for a existent root
1185       for (const auto &Root : FS->Roots) {
1186         if (Name.equals(Root->getName())) {
1187           ParentEntry = Root.get();
1188           return ParentEntry;
1189         }
1190       }
1191     } else { // Advance to the next component
1192       auto *DE = dyn_cast<RedirectingDirectoryEntry>(ParentEntry);
1193       for (std::unique_ptr<Entry> &Content :
1194            llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1195         auto *DirContent = dyn_cast<RedirectingDirectoryEntry>(Content.get());
1196         if (DirContent && Name.equals(Content->getName()))
1197           return DirContent;
1198       }
1199     }
1200 
1201     // ... or create a new one
1202     std::unique_ptr<Entry> E = llvm::make_unique<RedirectingDirectoryEntry>(
1203         Name,
1204         Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
1205                0, 0, 0, file_type::directory_file, sys::fs::all_all));
1206 
1207     if (!ParentEntry) { // Add a new root to the overlay
1208       FS->Roots.push_back(std::move(E));
1209       ParentEntry = FS->Roots.back().get();
1210       return ParentEntry;
1211     }
1212 
1213     auto *DE = dyn_cast<RedirectingDirectoryEntry>(ParentEntry);
1214     DE->addContent(std::move(E));
1215     return DE->getLastContent();
1216   }
1217 
uniqueOverlayTree(RedirectingFileSystem * FS,Entry * SrcE,Entry * NewParentE=nullptr)1218   void uniqueOverlayTree(RedirectingFileSystem *FS, Entry *SrcE,
1219                          Entry *NewParentE = nullptr) {
1220     StringRef Name = SrcE->getName();
1221     switch (SrcE->getKind()) {
1222     case EK_Directory: {
1223       auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE);
1224       assert(DE && "Must be a directory");
1225       // Empty directories could be present in the YAML as a way to
1226       // describe a file for a current directory after some of its subdir
1227       // is parsed. This only leads to redundant walks, ignore it.
1228       if (!Name.empty())
1229         NewParentE = lookupOrCreateEntry(FS, Name, NewParentE);
1230       for (std::unique_ptr<Entry> &SubEntry :
1231            llvm::make_range(DE->contents_begin(), DE->contents_end()))
1232         uniqueOverlayTree(FS, SubEntry.get(), NewParentE);
1233       break;
1234     }
1235     case EK_File: {
1236       auto *FE = dyn_cast<RedirectingFileEntry>(SrcE);
1237       assert(FE && "Must be a file");
1238       assert(NewParentE && "Parent entry must exist");
1239       auto *DE = dyn_cast<RedirectingDirectoryEntry>(NewParentE);
1240       DE->addContent(llvm::make_unique<RedirectingFileEntry>(
1241           Name, FE->getExternalContentsPath(), FE->getUseName()));
1242       break;
1243     }
1244     }
1245   }
1246 
parseEntry(yaml::Node * N,RedirectingFileSystem * FS)1247   std::unique_ptr<Entry> parseEntry(yaml::Node *N, RedirectingFileSystem *FS) {
1248     auto *M = dyn_cast<yaml::MappingNode>(N);
1249     if (!M) {
1250       error(N, "expected mapping node for file or directory entry");
1251       return nullptr;
1252     }
1253 
1254     KeyStatusPair Fields[] = {
1255       KeyStatusPair("name", true),
1256       KeyStatusPair("type", true),
1257       KeyStatusPair("contents", false),
1258       KeyStatusPair("external-contents", false),
1259       KeyStatusPair("use-external-name", false),
1260     };
1261 
1262     DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1263 
1264     bool HasContents = false; // external or otherwise
1265     std::vector<std::unique_ptr<Entry>> EntryArrayContents;
1266     std::string ExternalContentsPath;
1267     std::string Name;
1268     auto UseExternalName = RedirectingFileEntry::NK_NotSet;
1269     EntryKind Kind;
1270 
1271     for (auto &I : *M) {
1272       StringRef Key;
1273       // Reuse the buffer for key and value, since we don't look at key after
1274       // parsing value.
1275       SmallString<256> Buffer;
1276       if (!parseScalarString(I.getKey(), Key, Buffer))
1277         return nullptr;
1278 
1279       if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1280         return nullptr;
1281 
1282       StringRef Value;
1283       if (Key == "name") {
1284         if (!parseScalarString(I.getValue(), Value, Buffer))
1285           return nullptr;
1286 
1287         if (FS->UseCanonicalizedPaths) {
1288           SmallString<256> Path(Value);
1289           // Guarantee that old YAML files containing paths with ".." and "."
1290           // are properly canonicalized before read into the VFS.
1291           Path = sys::path::remove_leading_dotslash(Path);
1292           sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1293           Name = Path.str();
1294         } else {
1295           Name = Value;
1296         }
1297       } else if (Key == "type") {
1298         if (!parseScalarString(I.getValue(), Value, Buffer))
1299           return nullptr;
1300         if (Value == "file")
1301           Kind = EK_File;
1302         else if (Value == "directory")
1303           Kind = EK_Directory;
1304         else {
1305           error(I.getValue(), "unknown value for 'type'");
1306           return nullptr;
1307         }
1308       } else if (Key == "contents") {
1309         if (HasContents) {
1310           error(I.getKey(),
1311                 "entry already has 'contents' or 'external-contents'");
1312           return nullptr;
1313         }
1314         HasContents = true;
1315         auto *Contents = dyn_cast<yaml::SequenceNode>(I.getValue());
1316         if (!Contents) {
1317           // FIXME: this is only for directories, what about files?
1318           error(I.getValue(), "expected array");
1319           return nullptr;
1320         }
1321 
1322         for (auto &I : *Contents) {
1323           if (std::unique_ptr<Entry> E = parseEntry(&I, FS))
1324             EntryArrayContents.push_back(std::move(E));
1325           else
1326             return nullptr;
1327         }
1328       } else if (Key == "external-contents") {
1329         if (HasContents) {
1330           error(I.getKey(),
1331                 "entry already has 'contents' or 'external-contents'");
1332           return nullptr;
1333         }
1334         HasContents = true;
1335         if (!parseScalarString(I.getValue(), Value, Buffer))
1336           return nullptr;
1337 
1338         SmallString<256> FullPath;
1339         if (FS->IsRelativeOverlay) {
1340           FullPath = FS->getExternalContentsPrefixDir();
1341           assert(!FullPath.empty() &&
1342                  "External contents prefix directory must exist");
1343           llvm::sys::path::append(FullPath, Value);
1344         } else {
1345           FullPath = Value;
1346         }
1347 
1348         if (FS->UseCanonicalizedPaths) {
1349           // Guarantee that old YAML files containing paths with ".." and "."
1350           // are properly canonicalized before read into the VFS.
1351           FullPath = sys::path::remove_leading_dotslash(FullPath);
1352           sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
1353         }
1354         ExternalContentsPath = FullPath.str();
1355       } else if (Key == "use-external-name") {
1356         bool Val;
1357         if (!parseScalarBool(I.getValue(), Val))
1358           return nullptr;
1359         UseExternalName = Val ? RedirectingFileEntry::NK_External
1360                               : RedirectingFileEntry::NK_Virtual;
1361       } else {
1362         llvm_unreachable("key missing from Keys");
1363       }
1364     }
1365 
1366     if (Stream.failed())
1367       return nullptr;
1368 
1369     // check for missing keys
1370     if (!HasContents) {
1371       error(N, "missing key 'contents' or 'external-contents'");
1372       return nullptr;
1373     }
1374     if (!checkMissingKeys(N, Keys))
1375       return nullptr;
1376 
1377     // check invalid configuration
1378     if (Kind == EK_Directory &&
1379         UseExternalName != RedirectingFileEntry::NK_NotSet) {
1380       error(N, "'use-external-name' is not supported for directories");
1381       return nullptr;
1382     }
1383 
1384     // Remove trailing slash(es), being careful not to remove the root path
1385     StringRef Trimmed(Name);
1386     size_t RootPathLen = sys::path::root_path(Trimmed).size();
1387     while (Trimmed.size() > RootPathLen &&
1388            sys::path::is_separator(Trimmed.back()))
1389       Trimmed = Trimmed.slice(0, Trimmed.size()-1);
1390     // Get the last component
1391     StringRef LastComponent = sys::path::filename(Trimmed);
1392 
1393     std::unique_ptr<Entry> Result;
1394     switch (Kind) {
1395     case EK_File:
1396       Result = llvm::make_unique<RedirectingFileEntry>(
1397           LastComponent, std::move(ExternalContentsPath), UseExternalName);
1398       break;
1399     case EK_Directory:
1400       Result = llvm::make_unique<RedirectingDirectoryEntry>(
1401           LastComponent, std::move(EntryArrayContents),
1402           Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
1403                  0, 0, 0, file_type::directory_file, sys::fs::all_all));
1404       break;
1405     }
1406 
1407     StringRef Parent = sys::path::parent_path(Trimmed);
1408     if (Parent.empty())
1409       return Result;
1410 
1411     // if 'name' contains multiple components, create implicit directory entries
1412     for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
1413                                      E = sys::path::rend(Parent);
1414          I != E; ++I) {
1415       std::vector<std::unique_ptr<Entry>> Entries;
1416       Entries.push_back(std::move(Result));
1417       Result = llvm::make_unique<RedirectingDirectoryEntry>(
1418           *I, std::move(Entries),
1419           Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
1420                  0, 0, 0, file_type::directory_file, sys::fs::all_all));
1421     }
1422     return Result;
1423   }
1424 
1425 public:
RedirectingFileSystemParser(yaml::Stream & S)1426   RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {}
1427 
1428   // false on error
parse(yaml::Node * Root,RedirectingFileSystem * FS)1429   bool parse(yaml::Node *Root, RedirectingFileSystem *FS) {
1430     auto *Top = dyn_cast<yaml::MappingNode>(Root);
1431     if (!Top) {
1432       error(Root, "expected mapping node");
1433       return false;
1434     }
1435 
1436     KeyStatusPair Fields[] = {
1437       KeyStatusPair("version", true),
1438       KeyStatusPair("case-sensitive", false),
1439       KeyStatusPair("use-external-names", false),
1440       KeyStatusPair("overlay-relative", false),
1441       KeyStatusPair("ignore-non-existent-contents", false),
1442       KeyStatusPair("roots", true),
1443     };
1444 
1445     DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1446     std::vector<std::unique_ptr<Entry>> RootEntries;
1447 
1448     // Parse configuration and 'roots'
1449     for (auto &I : *Top) {
1450       SmallString<10> KeyBuffer;
1451       StringRef Key;
1452       if (!parseScalarString(I.getKey(), Key, KeyBuffer))
1453         return false;
1454 
1455       if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1456         return false;
1457 
1458       if (Key == "roots") {
1459         auto *Roots = dyn_cast<yaml::SequenceNode>(I.getValue());
1460         if (!Roots) {
1461           error(I.getValue(), "expected array");
1462           return false;
1463         }
1464 
1465         for (auto &I : *Roots) {
1466           if (std::unique_ptr<Entry> E = parseEntry(&I, FS))
1467             RootEntries.push_back(std::move(E));
1468           else
1469             return false;
1470         }
1471       } else if (Key == "version") {
1472         StringRef VersionString;
1473         SmallString<4> Storage;
1474         if (!parseScalarString(I.getValue(), VersionString, Storage))
1475           return false;
1476         int Version;
1477         if (VersionString.getAsInteger<int>(10, Version)) {
1478           error(I.getValue(), "expected integer");
1479           return false;
1480         }
1481         if (Version < 0) {
1482           error(I.getValue(), "invalid version number");
1483           return false;
1484         }
1485         if (Version != 0) {
1486           error(I.getValue(), "version mismatch, expected 0");
1487           return false;
1488         }
1489       } else if (Key == "case-sensitive") {
1490         if (!parseScalarBool(I.getValue(), FS->CaseSensitive))
1491           return false;
1492       } else if (Key == "overlay-relative") {
1493         if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay))
1494           return false;
1495       } else if (Key == "use-external-names") {
1496         if (!parseScalarBool(I.getValue(), FS->UseExternalNames))
1497           return false;
1498       } else if (Key == "ignore-non-existent-contents") {
1499         if (!parseScalarBool(I.getValue(), FS->IgnoreNonExistentContents))
1500           return false;
1501       } else {
1502         llvm_unreachable("key missing from Keys");
1503       }
1504     }
1505 
1506     if (Stream.failed())
1507       return false;
1508 
1509     if (!checkMissingKeys(Top, Keys))
1510       return false;
1511 
1512     // Now that we sucessefully parsed the YAML file, canonicalize the internal
1513     // representation to a proper directory tree so that we can search faster
1514     // inside the VFS.
1515     for (auto &E : RootEntries)
1516       uniqueOverlayTree(FS, E.get());
1517 
1518     return true;
1519   }
1520 };
1521 
1522 } // namespace
1523 
1524 RedirectingFileSystem *
create(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1525 RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
1526                               SourceMgr::DiagHandlerTy DiagHandler,
1527                               StringRef YAMLFilePath, void *DiagContext,
1528                               IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1529   SourceMgr SM;
1530   yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
1531 
1532   SM.setDiagHandler(DiagHandler, DiagContext);
1533   yaml::document_iterator DI = Stream.begin();
1534   yaml::Node *Root = DI->getRoot();
1535   if (DI == Stream.end() || !Root) {
1536     SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
1537     return nullptr;
1538   }
1539 
1540   RedirectingFileSystemParser P(Stream);
1541 
1542   std::unique_ptr<RedirectingFileSystem> FS(
1543       new RedirectingFileSystem(std::move(ExternalFS)));
1544 
1545   if (!YAMLFilePath.empty()) {
1546     // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
1547     // to each 'external-contents' path.
1548     //
1549     // Example:
1550     //    -ivfsoverlay dummy.cache/vfs/vfs.yaml
1551     // yields:
1552     //  FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
1553     //
1554     SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
1555     std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
1556     assert(!EC && "Overlay dir final path must be absolute");
1557     (void)EC;
1558     FS->setExternalContentsPrefixDir(OverlayAbsDir);
1559   }
1560 
1561   if (!P.parse(Root, FS.get()))
1562     return nullptr;
1563 
1564   return FS.release();
1565 }
1566 
lookupPath(const Twine & Path_)1567 ErrorOr<Entry *> RedirectingFileSystem::lookupPath(const Twine &Path_) {
1568   SmallString<256> Path;
1569   Path_.toVector(Path);
1570 
1571   // Handle relative paths
1572   if (std::error_code EC = makeAbsolute(Path))
1573     return EC;
1574 
1575   // Canonicalize path by removing ".", "..", "./", etc components. This is
1576   // a VFS request, do bot bother about symlinks in the path components
1577   // but canonicalize in order to perform the correct entry search.
1578   if (UseCanonicalizedPaths) {
1579     Path = sys::path::remove_leading_dotslash(Path);
1580     sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1581   }
1582 
1583   if (Path.empty())
1584     return make_error_code(llvm::errc::invalid_argument);
1585 
1586   sys::path::const_iterator Start = sys::path::begin(Path);
1587   sys::path::const_iterator End = sys::path::end(Path);
1588   for (const auto &Root : Roots) {
1589     ErrorOr<Entry *> Result = lookupPath(Start, End, Root.get());
1590     if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1591       return Result;
1592   }
1593   return make_error_code(llvm::errc::no_such_file_or_directory);
1594 }
1595 
1596 ErrorOr<Entry *>
lookupPath(sys::path::const_iterator Start,sys::path::const_iterator End,Entry * From)1597 RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
1598                                   sys::path::const_iterator End, Entry *From) {
1599 #ifndef _WIN32
1600   assert(!isTraversalComponent(*Start) &&
1601          !isTraversalComponent(From->getName()) &&
1602          "Paths should not contain traversal components");
1603 #else
1604   // FIXME: this is here to support windows, remove it once canonicalized
1605   // paths become globally default.
1606   if (Start->equals("."))
1607     ++Start;
1608 #endif
1609 
1610   StringRef FromName = From->getName();
1611 
1612   // Forward the search to the next component in case this is an empty one.
1613   if (!FromName.empty()) {
1614     if (CaseSensitive ? !Start->equals(FromName)
1615                       : !Start->equals_lower(FromName))
1616       // failure to match
1617       return make_error_code(llvm::errc::no_such_file_or_directory);
1618 
1619     ++Start;
1620 
1621     if (Start == End) {
1622       // Match!
1623       return From;
1624     }
1625   }
1626 
1627   auto *DE = dyn_cast<RedirectingDirectoryEntry>(From);
1628   if (!DE)
1629     return make_error_code(llvm::errc::not_a_directory);
1630 
1631   for (const std::unique_ptr<Entry> &DirEntry :
1632        llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1633     ErrorOr<Entry *> Result = lookupPath(Start, End, DirEntry.get());
1634     if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1635       return Result;
1636   }
1637   return make_error_code(llvm::errc::no_such_file_or_directory);
1638 }
1639 
getRedirectedFileStatus(const Twine & Path,bool UseExternalNames,Status ExternalStatus)1640 static Status getRedirectedFileStatus(const Twine &Path, bool UseExternalNames,
1641                                       Status ExternalStatus) {
1642   Status S = ExternalStatus;
1643   if (!UseExternalNames)
1644     S = Status::copyWithNewName(S, Path.str());
1645   S.IsVFSMapped = true;
1646   return S;
1647 }
1648 
status(const Twine & Path,Entry * E)1649 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path, Entry *E) {
1650   assert(E != nullptr);
1651   if (auto *F = dyn_cast<RedirectingFileEntry>(E)) {
1652     ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
1653     assert(!S || S->getName() == F->getExternalContentsPath());
1654     if (S)
1655       return getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1656                                      *S);
1657     return S;
1658   } else { // directory
1659     auto *DE = cast<RedirectingDirectoryEntry>(E);
1660     return Status::copyWithNewName(DE->getStatus(), Path.str());
1661   }
1662 }
1663 
status(const Twine & Path)1664 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
1665   ErrorOr<Entry *> Result = lookupPath(Path);
1666   if (!Result)
1667     return Result.getError();
1668   return status(Path, *Result);
1669 }
1670 
1671 namespace {
1672 
1673 /// Provide a file wrapper with an overriden status.
1674 class FileWithFixedStatus : public File {
1675   std::unique_ptr<File> InnerFile;
1676   Status S;
1677 
1678 public:
FileWithFixedStatus(std::unique_ptr<File> InnerFile,Status S)1679   FileWithFixedStatus(std::unique_ptr<File> InnerFile, Status S)
1680       : InnerFile(std::move(InnerFile)), S(std::move(S)) {}
1681 
status()1682   ErrorOr<Status> status() override { return S; }
1683   ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
1684 
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)1685   getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
1686             bool IsVolatile) override {
1687     return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
1688                                 IsVolatile);
1689   }
1690 
close()1691   std::error_code close() override { return InnerFile->close(); }
1692 };
1693 
1694 } // namespace
1695 
1696 ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Path)1697 RedirectingFileSystem::openFileForRead(const Twine &Path) {
1698   ErrorOr<Entry *> E = lookupPath(Path);
1699   if (!E)
1700     return E.getError();
1701 
1702   auto *F = dyn_cast<RedirectingFileEntry>(*E);
1703   if (!F) // FIXME: errc::not_a_file?
1704     return make_error_code(llvm::errc::invalid_argument);
1705 
1706   auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
1707   if (!Result)
1708     return Result;
1709 
1710   auto ExternalStatus = (*Result)->status();
1711   if (!ExternalStatus)
1712     return ExternalStatus.getError();
1713 
1714   // FIXME: Update the status with the name and VFSMapped.
1715   Status S = getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1716                                      *ExternalStatus);
1717   return std::unique_ptr<File>(
1718       llvm::make_unique<FileWithFixedStatus>(std::move(*Result), S));
1719 }
1720 
1721 IntrusiveRefCntPtr<FileSystem>
getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1722 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1723                     SourceMgr::DiagHandlerTy DiagHandler,
1724                     StringRef YAMLFilePath,
1725                     void *DiagContext,
1726                     IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1727   return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
1728                                        YAMLFilePath, DiagContext,
1729                                        std::move(ExternalFS));
1730 }
1731 
getVFSEntries(Entry * SrcE,SmallVectorImpl<StringRef> & Path,SmallVectorImpl<YAMLVFSEntry> & Entries)1732 static void getVFSEntries(Entry *SrcE, SmallVectorImpl<StringRef> &Path,
1733                           SmallVectorImpl<YAMLVFSEntry> &Entries) {
1734   auto Kind = SrcE->getKind();
1735   if (Kind == EK_Directory) {
1736     auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE);
1737     assert(DE && "Must be a directory");
1738     for (std::unique_ptr<Entry> &SubEntry :
1739          llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1740       Path.push_back(SubEntry->getName());
1741       getVFSEntries(SubEntry.get(), Path, Entries);
1742       Path.pop_back();
1743     }
1744     return;
1745   }
1746 
1747   assert(Kind == EK_File && "Must be a EK_File");
1748   auto *FE = dyn_cast<RedirectingFileEntry>(SrcE);
1749   assert(FE && "Must be a file");
1750   SmallString<128> VPath;
1751   for (auto &Comp : Path)
1752     llvm::sys::path::append(VPath, Comp);
1753   Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath()));
1754 }
1755 
collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,SmallVectorImpl<YAMLVFSEntry> & CollectedEntries,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1756 void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1757                              SourceMgr::DiagHandlerTy DiagHandler,
1758                              StringRef YAMLFilePath,
1759                              SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
1760                              void *DiagContext,
1761                              IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1762   RedirectingFileSystem *VFS = RedirectingFileSystem::create(
1763       std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
1764       std::move(ExternalFS));
1765   ErrorOr<Entry *> RootE = VFS->lookupPath("/");
1766   if (!RootE)
1767     return;
1768   SmallVector<StringRef, 8> Components;
1769   Components.push_back("/");
1770   getVFSEntries(*RootE, Components, CollectedEntries);
1771 }
1772 
getNextVirtualUniqueID()1773 UniqueID vfs::getNextVirtualUniqueID() {
1774   static std::atomic<unsigned> UID;
1775   unsigned ID = ++UID;
1776   // The following assumes that uint64_t max will never collide with a real
1777   // dev_t value from the OS.
1778   return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
1779 }
1780 
addFileMapping(StringRef VirtualPath,StringRef RealPath)1781 void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
1782   assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
1783   assert(sys::path::is_absolute(RealPath) && "real path not absolute");
1784   assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
1785   Mappings.emplace_back(VirtualPath, RealPath);
1786 }
1787 
1788 namespace {
1789 
1790 class JSONWriter {
1791   llvm::raw_ostream &OS;
1792   SmallVector<StringRef, 16> DirStack;
1793 
getDirIndent()1794   unsigned getDirIndent() { return 4 * DirStack.size(); }
getFileIndent()1795   unsigned getFileIndent() { return 4 * (DirStack.size() + 1); }
1796   bool containedIn(StringRef Parent, StringRef Path);
1797   StringRef containedPart(StringRef Parent, StringRef Path);
1798   void startDirectory(StringRef Path);
1799   void endDirectory();
1800   void writeEntry(StringRef VPath, StringRef RPath);
1801 
1802 public:
JSONWriter(llvm::raw_ostream & OS)1803   JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
1804 
1805   void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> UseExternalNames,
1806              Optional<bool> IsCaseSensitive, Optional<bool> IsOverlayRelative,
1807              Optional<bool> IgnoreNonExistentContents, StringRef OverlayDir);
1808 };
1809 
1810 } // namespace
1811 
containedIn(StringRef Parent,StringRef Path)1812 bool JSONWriter::containedIn(StringRef Parent, StringRef Path) {
1813   using namespace llvm::sys;
1814 
1815   // Compare each path component.
1816   auto IParent = path::begin(Parent), EParent = path::end(Parent);
1817   for (auto IChild = path::begin(Path), EChild = path::end(Path);
1818        IParent != EParent && IChild != EChild; ++IParent, ++IChild) {
1819     if (*IParent != *IChild)
1820       return false;
1821   }
1822   // Have we exhausted the parent path?
1823   return IParent == EParent;
1824 }
1825 
containedPart(StringRef Parent,StringRef Path)1826 StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) {
1827   assert(!Parent.empty());
1828   assert(containedIn(Parent, Path));
1829   return Path.slice(Parent.size() + 1, StringRef::npos);
1830 }
1831 
startDirectory(StringRef Path)1832 void JSONWriter::startDirectory(StringRef Path) {
1833   StringRef Name =
1834       DirStack.empty() ? Path : containedPart(DirStack.back(), Path);
1835   DirStack.push_back(Path);
1836   unsigned Indent = getDirIndent();
1837   OS.indent(Indent) << "{\n";
1838   OS.indent(Indent + 2) << "'type': 'directory',\n";
1839   OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n";
1840   OS.indent(Indent + 2) << "'contents': [\n";
1841 }
1842 
endDirectory()1843 void JSONWriter::endDirectory() {
1844   unsigned Indent = getDirIndent();
1845   OS.indent(Indent + 2) << "]\n";
1846   OS.indent(Indent) << "}";
1847 
1848   DirStack.pop_back();
1849 }
1850 
writeEntry(StringRef VPath,StringRef RPath)1851 void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
1852   unsigned Indent = getFileIndent();
1853   OS.indent(Indent) << "{\n";
1854   OS.indent(Indent + 2) << "'type': 'file',\n";
1855   OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n";
1856   OS.indent(Indent + 2) << "'external-contents': \""
1857                         << llvm::yaml::escape(RPath) << "\"\n";
1858   OS.indent(Indent) << "}";
1859 }
1860 
write(ArrayRef<YAMLVFSEntry> Entries,Optional<bool> UseExternalNames,Optional<bool> IsCaseSensitive,Optional<bool> IsOverlayRelative,Optional<bool> IgnoreNonExistentContents,StringRef OverlayDir)1861 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
1862                        Optional<bool> UseExternalNames,
1863                        Optional<bool> IsCaseSensitive,
1864                        Optional<bool> IsOverlayRelative,
1865                        Optional<bool> IgnoreNonExistentContents,
1866                        StringRef OverlayDir) {
1867   using namespace llvm::sys;
1868 
1869   OS << "{\n"
1870         "  'version': 0,\n";
1871   if (IsCaseSensitive.hasValue())
1872     OS << "  'case-sensitive': '"
1873        << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
1874   if (UseExternalNames.hasValue())
1875     OS << "  'use-external-names': '"
1876        << (UseExternalNames.getValue() ? "true" : "false") << "',\n";
1877   bool UseOverlayRelative = false;
1878   if (IsOverlayRelative.hasValue()) {
1879     UseOverlayRelative = IsOverlayRelative.getValue();
1880     OS << "  'overlay-relative': '"
1881        << (UseOverlayRelative ? "true" : "false") << "',\n";
1882   }
1883   if (IgnoreNonExistentContents.hasValue())
1884     OS << "  'ignore-non-existent-contents': '"
1885        << (IgnoreNonExistentContents.getValue() ? "true" : "false") << "',\n";
1886   OS << "  'roots': [\n";
1887 
1888   if (!Entries.empty()) {
1889     const YAMLVFSEntry &Entry = Entries.front();
1890     startDirectory(path::parent_path(Entry.VPath));
1891 
1892     StringRef RPath = Entry.RPath;
1893     if (UseOverlayRelative) {
1894       unsigned OverlayDirLen = OverlayDir.size();
1895       assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1896              "Overlay dir must be contained in RPath");
1897       RPath = RPath.slice(OverlayDirLen, RPath.size());
1898     }
1899 
1900     writeEntry(path::filename(Entry.VPath), RPath);
1901 
1902     for (const auto &Entry : Entries.slice(1)) {
1903       StringRef Dir = path::parent_path(Entry.VPath);
1904       if (Dir == DirStack.back())
1905         OS << ",\n";
1906       else {
1907         while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
1908           OS << "\n";
1909           endDirectory();
1910         }
1911         OS << ",\n";
1912         startDirectory(Dir);
1913       }
1914       StringRef RPath = Entry.RPath;
1915       if (UseOverlayRelative) {
1916         unsigned OverlayDirLen = OverlayDir.size();
1917         assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1918                "Overlay dir must be contained in RPath");
1919         RPath = RPath.slice(OverlayDirLen, RPath.size());
1920       }
1921       writeEntry(path::filename(Entry.VPath), RPath);
1922     }
1923 
1924     while (!DirStack.empty()) {
1925       OS << "\n";
1926       endDirectory();
1927     }
1928     OS << "\n";
1929   }
1930 
1931   OS << "  ]\n"
1932      << "}\n";
1933 }
1934 
write(llvm::raw_ostream & OS)1935 void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
1936   llvm::sort(Mappings.begin(), Mappings.end(),
1937              [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) {
1938     return LHS.VPath < RHS.VPath;
1939   });
1940 
1941   JSONWriter(OS).write(Mappings, UseExternalNames, IsCaseSensitive,
1942                        IsOverlayRelative, IgnoreNonExistentContents,
1943                        OverlayDir);
1944 }
1945 
VFSFromYamlDirIterImpl(const Twine & _Path,RedirectingFileSystem & FS,RedirectingDirectoryEntry::iterator Begin,RedirectingDirectoryEntry::iterator End,std::error_code & EC)1946 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
1947     const Twine &_Path, RedirectingFileSystem &FS,
1948     RedirectingDirectoryEntry::iterator Begin,
1949     RedirectingDirectoryEntry::iterator End, std::error_code &EC)
1950     : Dir(_Path.str()), FS(FS), Current(Begin), End(End) {
1951   while (Current != End) {
1952     SmallString<128> PathStr(Dir);
1953     llvm::sys::path::append(PathStr, (*Current)->getName());
1954     llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
1955     if (S) {
1956       CurrentEntry = *S;
1957       return;
1958     }
1959     // Skip entries which do not map to a reliable external content.
1960     if (FS.ignoreNonExistentContents() &&
1961         S.getError() == llvm::errc::no_such_file_or_directory) {
1962       ++Current;
1963       continue;
1964     } else {
1965       EC = S.getError();
1966       break;
1967     }
1968   }
1969 }
1970 
increment()1971 std::error_code VFSFromYamlDirIterImpl::increment() {
1972   assert(Current != End && "cannot iterate past end");
1973   while (++Current != End) {
1974     SmallString<128> PathStr(Dir);
1975     llvm::sys::path::append(PathStr, (*Current)->getName());
1976     llvm::ErrorOr<vfs::Status> S = FS.status(PathStr);
1977     if (!S) {
1978       // Skip entries which do not map to a reliable external content.
1979       if (FS.ignoreNonExistentContents() &&
1980           S.getError() == llvm::errc::no_such_file_or_directory) {
1981         continue;
1982       } else {
1983         return S.getError();
1984       }
1985     }
1986     CurrentEntry = *S;
1987     break;
1988   }
1989 
1990   if (Current == End)
1991     CurrentEntry = Status();
1992   return {};
1993 }
1994 
recursive_directory_iterator(FileSystem & FS_,const Twine & Path,std::error_code & EC)1995 vfs::recursive_directory_iterator::recursive_directory_iterator(FileSystem &FS_,
1996                                                            const Twine &Path,
1997                                                            std::error_code &EC)
1998     : FS(&FS_) {
1999   directory_iterator I = FS->dir_begin(Path, EC);
2000   if (I != directory_iterator()) {
2001     State = std::make_shared<IterState>();
2002     State->push(I);
2003   }
2004 }
2005 
2006 vfs::recursive_directory_iterator &
increment(std::error_code & EC)2007 recursive_directory_iterator::increment(std::error_code &EC) {
2008   assert(FS && State && !State->empty() && "incrementing past end");
2009   assert(State->top()->isStatusKnown() && "non-canonical end iterator");
2010   vfs::directory_iterator End;
2011   if (State->top()->isDirectory()) {
2012     vfs::directory_iterator I = FS->dir_begin(State->top()->getName(), EC);
2013     if (I != End) {
2014       State->push(I);
2015       return *this;
2016     }
2017   }
2018 
2019   while (!State->empty() && State->top().increment(EC) == End)
2020     State->pop();
2021 
2022   if (State->empty())
2023     State.reset(); // end iterator
2024 
2025   return *this;
2026 }
2027