1 //===- VirtualFileSystem.cpp - Virtual File System Layer ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the VirtualFileSystem interface.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Support/VirtualFileSystem.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/IntrusiveRefCntPtr.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSet.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/ADT/iterator_range.h"
27 #include "llvm/Config/llvm-config.h"
28 #include "llvm/Support/Casting.h"
29 #include "llvm/Support/Chrono.h"
30 #include "llvm/Support/Compiler.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/Errc.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/ErrorOr.h"
35 #include "llvm/Support/FileSystem.h"
36 #include "llvm/Support/MemoryBuffer.h"
37 #include "llvm/Support/Path.h"
38 #include "llvm/Support/Process.h"
39 #include "llvm/Support/SMLoc.h"
40 #include "llvm/Support/SourceMgr.h"
41 #include "llvm/Support/YAMLParser.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include <algorithm>
44 #include <atomic>
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <limits>
49 #include <map>
50 #include <memory>
51 #include <mutex>
52 #include <string>
53 #include <system_error>
54 #include <utility>
55 #include <vector>
56
57 using namespace llvm;
58 using namespace llvm::vfs;
59
60 using llvm::sys::fs::file_status;
61 using llvm::sys::fs::file_type;
62 using llvm::sys::fs::perms;
63 using llvm::sys::fs::UniqueID;
64
Status(const file_status & Status)65 Status::Status(const file_status &Status)
66 : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
67 User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
68 Type(Status.type()), Perms(Status.permissions()) {}
69
Status(StringRef Name,UniqueID UID,sys::TimePoint<> MTime,uint32_t User,uint32_t Group,uint64_t Size,file_type Type,perms Perms)70 Status::Status(StringRef Name, UniqueID UID, sys::TimePoint<> MTime,
71 uint32_t User, uint32_t Group, uint64_t Size, file_type Type,
72 perms Perms)
73 : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size),
74 Type(Type), Perms(Perms) {}
75
copyWithNewName(const Status & In,StringRef NewName)76 Status Status::copyWithNewName(const Status &In, StringRef NewName) {
77 return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
78 In.getUser(), In.getGroup(), In.getSize(), In.getType(),
79 In.getPermissions());
80 }
81
copyWithNewName(const file_status & In,StringRef NewName)82 Status Status::copyWithNewName(const file_status &In, StringRef NewName) {
83 return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
84 In.getUser(), In.getGroup(), In.getSize(), In.type(),
85 In.permissions());
86 }
87
equivalent(const Status & Other) const88 bool Status::equivalent(const Status &Other) const {
89 assert(isStatusKnown() && Other.isStatusKnown());
90 return getUniqueID() == Other.getUniqueID();
91 }
92
isDirectory() const93 bool Status::isDirectory() const { return Type == file_type::directory_file; }
94
isRegularFile() const95 bool Status::isRegularFile() const { return Type == file_type::regular_file; }
96
isOther() const97 bool Status::isOther() const {
98 return exists() && !isRegularFile() && !isDirectory() && !isSymlink();
99 }
100
isSymlink() const101 bool Status::isSymlink() const { return Type == file_type::symlink_file; }
102
isStatusKnown() const103 bool Status::isStatusKnown() const { return Type != file_type::status_error; }
104
exists() const105 bool Status::exists() const {
106 return isStatusKnown() && Type != file_type::file_not_found;
107 }
108
109 File::~File() = default;
110
111 FileSystem::~FileSystem() = default;
112
113 ErrorOr<std::unique_ptr<MemoryBuffer>>
getBufferForFile(const llvm::Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)114 FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
115 bool RequiresNullTerminator, bool IsVolatile) {
116 auto F = openFileForRead(Name);
117 if (!F)
118 return F.getError();
119
120 return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
121 }
122
makeAbsolute(SmallVectorImpl<char> & Path) const123 std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
124 if (llvm::sys::path::is_absolute(Path))
125 return {};
126
127 auto WorkingDir = getCurrentWorkingDirectory();
128 if (!WorkingDir)
129 return WorkingDir.getError();
130
131 llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
132 return {};
133 }
134
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const135 std::error_code FileSystem::getRealPath(const Twine &Path,
136 SmallVectorImpl<char> &Output) const {
137 return errc::operation_not_permitted;
138 }
139
isLocal(const Twine & Path,bool & Result)140 std::error_code FileSystem::isLocal(const Twine &Path, bool &Result) {
141 return errc::operation_not_permitted;
142 }
143
exists(const Twine & Path)144 bool FileSystem::exists(const Twine &Path) {
145 auto Status = status(Path);
146 return Status && Status->exists();
147 }
148
149 #ifndef NDEBUG
isTraversalComponent(StringRef Component)150 static bool isTraversalComponent(StringRef Component) {
151 return Component.equals("..") || Component.equals(".");
152 }
153
pathHasTraversal(StringRef Path)154 static bool pathHasTraversal(StringRef Path) {
155 using namespace llvm::sys;
156
157 for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path)))
158 if (isTraversalComponent(Comp))
159 return true;
160 return false;
161 }
162 #endif
163
164 //===-----------------------------------------------------------------------===/
165 // RealFileSystem implementation
166 //===-----------------------------------------------------------------------===/
167
168 namespace {
169
170 /// Wrapper around a raw file descriptor.
171 class RealFile : public File {
172 friend class RealFileSystem;
173
174 int FD;
175 Status S;
176 std::string RealName;
177
RealFile(int FD,StringRef NewName,StringRef NewRealPathName)178 RealFile(int FD, StringRef NewName, StringRef NewRealPathName)
179 : FD(FD), S(NewName, {}, {}, {}, {}, {},
180 llvm::sys::fs::file_type::status_error, {}),
181 RealName(NewRealPathName.str()) {
182 assert(FD >= 0 && "Invalid or inactive file descriptor");
183 }
184
185 public:
186 ~RealFile() override;
187
188 ErrorOr<Status> status() override;
189 ErrorOr<std::string> getName() override;
190 ErrorOr<std::unique_ptr<MemoryBuffer>> getBuffer(const Twine &Name,
191 int64_t FileSize,
192 bool RequiresNullTerminator,
193 bool IsVolatile) override;
194 std::error_code close() override;
195 };
196
197 } // namespace
198
~RealFile()199 RealFile::~RealFile() { close(); }
200
status()201 ErrorOr<Status> RealFile::status() {
202 assert(FD != -1 && "cannot stat closed file");
203 if (!S.isStatusKnown()) {
204 file_status RealStatus;
205 if (std::error_code EC = sys::fs::status(FD, RealStatus))
206 return EC;
207 S = Status::copyWithNewName(RealStatus, S.getName());
208 }
209 return S;
210 }
211
getName()212 ErrorOr<std::string> RealFile::getName() {
213 return RealName.empty() ? S.getName().str() : RealName;
214 }
215
216 ErrorOr<std::unique_ptr<MemoryBuffer>>
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)217 RealFile::getBuffer(const Twine &Name, int64_t FileSize,
218 bool RequiresNullTerminator, bool IsVolatile) {
219 assert(FD != -1 && "cannot get buffer for closed file");
220 return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
221 IsVolatile);
222 }
223
close()224 std::error_code RealFile::close() {
225 std::error_code EC = sys::Process::SafelyCloseFileDescriptor(FD);
226 FD = -1;
227 return EC;
228 }
229
230 namespace {
231
232 /// The file system according to your operating system.
233 class RealFileSystem : public FileSystem {
234 public:
235 ErrorOr<Status> status(const Twine &Path) override;
236 ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
237 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
238
239 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
240 std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
241 std::error_code isLocal(const Twine &Path, bool &Result) override;
242 std::error_code getRealPath(const Twine &Path,
243 SmallVectorImpl<char> &Output) const override;
244
245 private:
246 mutable std::mutex CWDMutex;
247 mutable std::string CWDCache;
248 };
249
250 } // namespace
251
status(const Twine & Path)252 ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
253 sys::fs::file_status RealStatus;
254 if (std::error_code EC = sys::fs::status(Path, RealStatus))
255 return EC;
256 return Status::copyWithNewName(RealStatus, Path.str());
257 }
258
259 ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Name)260 RealFileSystem::openFileForRead(const Twine &Name) {
261 int FD;
262 SmallString<256> RealName;
263 if (std::error_code EC =
264 sys::fs::openFileForRead(Name, FD, sys::fs::OF_None, &RealName))
265 return EC;
266 return std::unique_ptr<File>(new RealFile(FD, Name.str(), RealName.str()));
267 }
268
getCurrentWorkingDirectory() const269 llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
270 std::lock_guard<std::mutex> Lock(CWDMutex);
271 if (!CWDCache.empty())
272 return CWDCache;
273 SmallString<256> Dir;
274 if (std::error_code EC = llvm::sys::fs::current_path(Dir))
275 return EC;
276 CWDCache = Dir.str();
277 return CWDCache;
278 }
279
setCurrentWorkingDirectory(const Twine & Path)280 std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
281 // FIXME: chdir is thread hostile; on the other hand, creating the same
282 // behavior as chdir is complex: chdir resolves the path once, thus
283 // guaranteeing that all subsequent relative path operations work
284 // on the same path the original chdir resulted in. This makes a
285 // difference for example on network filesystems, where symlinks might be
286 // switched during runtime of the tool. Fixing this depends on having a
287 // file system abstraction that allows openat() style interactions.
288 if (auto EC = llvm::sys::fs::set_current_path(Path))
289 return EC;
290
291 // Invalidate cache.
292 std::lock_guard<std::mutex> Lock(CWDMutex);
293 CWDCache.clear();
294 return std::error_code();
295 }
296
isLocal(const Twine & Path,bool & Result)297 std::error_code RealFileSystem::isLocal(const Twine &Path, bool &Result) {
298 return llvm::sys::fs::is_local(Path, Result);
299 }
300
301 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const302 RealFileSystem::getRealPath(const Twine &Path,
303 SmallVectorImpl<char> &Output) const {
304 return llvm::sys::fs::real_path(Path, Output);
305 }
306
getRealFileSystem()307 IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() {
308 static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
309 return FS;
310 }
311
312 namespace {
313
314 class RealFSDirIter : public llvm::vfs::detail::DirIterImpl {
315 llvm::sys::fs::directory_iterator Iter;
316
317 public:
RealFSDirIter(const Twine & Path,std::error_code & EC)318 RealFSDirIter(const Twine &Path, std::error_code &EC) : Iter(Path, EC) {
319 if (Iter != llvm::sys::fs::directory_iterator())
320 CurrentEntry = directory_entry(Iter->path(), Iter->type());
321 }
322
increment()323 std::error_code increment() override {
324 std::error_code EC;
325 Iter.increment(EC);
326 CurrentEntry = (Iter == llvm::sys::fs::directory_iterator())
327 ? directory_entry()
328 : directory_entry(Iter->path(), Iter->type());
329 return EC;
330 }
331 };
332
333 } // namespace
334
dir_begin(const Twine & Dir,std::error_code & EC)335 directory_iterator RealFileSystem::dir_begin(const Twine &Dir,
336 std::error_code &EC) {
337 return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC));
338 }
339
340 //===-----------------------------------------------------------------------===/
341 // OverlayFileSystem implementation
342 //===-----------------------------------------------------------------------===/
343
OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS)344 OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) {
345 FSList.push_back(std::move(BaseFS));
346 }
347
pushOverlay(IntrusiveRefCntPtr<FileSystem> FS)348 void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) {
349 FSList.push_back(FS);
350 // Synchronize added file systems by duplicating the working directory from
351 // the first one in the list.
352 FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get());
353 }
354
status(const Twine & Path)355 ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
356 // FIXME: handle symlinks that cross file systems
357 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
358 ErrorOr<Status> Status = (*I)->status(Path);
359 if (Status || Status.getError() != llvm::errc::no_such_file_or_directory)
360 return Status;
361 }
362 return make_error_code(llvm::errc::no_such_file_or_directory);
363 }
364
365 ErrorOr<std::unique_ptr<File>>
openFileForRead(const llvm::Twine & Path)366 OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
367 // FIXME: handle symlinks that cross file systems
368 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
369 auto Result = (*I)->openFileForRead(Path);
370 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
371 return Result;
372 }
373 return make_error_code(llvm::errc::no_such_file_or_directory);
374 }
375
376 llvm::ErrorOr<std::string>
getCurrentWorkingDirectory() const377 OverlayFileSystem::getCurrentWorkingDirectory() const {
378 // All file systems are synchronized, just take the first working directory.
379 return FSList.front()->getCurrentWorkingDirectory();
380 }
381
382 std::error_code
setCurrentWorkingDirectory(const Twine & Path)383 OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
384 for (auto &FS : FSList)
385 if (std::error_code EC = FS->setCurrentWorkingDirectory(Path))
386 return EC;
387 return {};
388 }
389
isLocal(const Twine & Path,bool & Result)390 std::error_code OverlayFileSystem::isLocal(const Twine &Path, bool &Result) {
391 for (auto &FS : FSList)
392 if (FS->exists(Path))
393 return FS->isLocal(Path, Result);
394 return errc::no_such_file_or_directory;
395 }
396
397 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const398 OverlayFileSystem::getRealPath(const Twine &Path,
399 SmallVectorImpl<char> &Output) const {
400 for (auto &FS : FSList)
401 if (FS->exists(Path))
402 return FS->getRealPath(Path, Output);
403 return errc::no_such_file_or_directory;
404 }
405
406 llvm::vfs::detail::DirIterImpl::~DirIterImpl() = default;
407
408 namespace {
409
410 class OverlayFSDirIterImpl : public llvm::vfs::detail::DirIterImpl {
411 OverlayFileSystem &Overlays;
412 std::string Path;
413 OverlayFileSystem::iterator CurrentFS;
414 directory_iterator CurrentDirIter;
415 llvm::StringSet<> SeenNames;
416
incrementFS()417 std::error_code incrementFS() {
418 assert(CurrentFS != Overlays.overlays_end() && "incrementing past end");
419 ++CurrentFS;
420 for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) {
421 std::error_code EC;
422 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
423 if (EC && EC != errc::no_such_file_or_directory)
424 return EC;
425 if (CurrentDirIter != directory_iterator())
426 break; // found
427 }
428 return {};
429 }
430
incrementDirIter(bool IsFirstTime)431 std::error_code incrementDirIter(bool IsFirstTime) {
432 assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
433 "incrementing past end");
434 std::error_code EC;
435 if (!IsFirstTime)
436 CurrentDirIter.increment(EC);
437 if (!EC && CurrentDirIter == directory_iterator())
438 EC = incrementFS();
439 return EC;
440 }
441
incrementImpl(bool IsFirstTime)442 std::error_code incrementImpl(bool IsFirstTime) {
443 while (true) {
444 std::error_code EC = incrementDirIter(IsFirstTime);
445 if (EC || CurrentDirIter == directory_iterator()) {
446 CurrentEntry = directory_entry();
447 return EC;
448 }
449 CurrentEntry = *CurrentDirIter;
450 StringRef Name = llvm::sys::path::filename(CurrentEntry.path());
451 if (SeenNames.insert(Name).second)
452 return EC; // name not seen before
453 }
454 llvm_unreachable("returned above");
455 }
456
457 public:
OverlayFSDirIterImpl(const Twine & Path,OverlayFileSystem & FS,std::error_code & EC)458 OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS,
459 std::error_code &EC)
460 : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) {
461 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
462 EC = incrementImpl(true);
463 }
464
increment()465 std::error_code increment() override { return incrementImpl(false); }
466 };
467
468 } // namespace
469
dir_begin(const Twine & Dir,std::error_code & EC)470 directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
471 std::error_code &EC) {
472 return directory_iterator(
473 std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
474 }
475
anchor()476 void ProxyFileSystem::anchor() {}
477
478 namespace llvm {
479 namespace vfs {
480
481 namespace detail {
482
483 enum InMemoryNodeKind { IME_File, IME_Directory, IME_HardLink };
484
485 /// The in memory file system is a tree of Nodes. Every node can either be a
486 /// file , hardlink or a directory.
487 class InMemoryNode {
488 InMemoryNodeKind Kind;
489 std::string FileName;
490
491 public:
InMemoryNode(llvm::StringRef FileName,InMemoryNodeKind Kind)492 InMemoryNode(llvm::StringRef FileName, InMemoryNodeKind Kind)
493 : Kind(Kind), FileName(llvm::sys::path::filename(FileName)) {}
494 virtual ~InMemoryNode() = default;
495
496 /// Get the filename of this node (the name without the directory part).
getFileName() const497 StringRef getFileName() const { return FileName; }
getKind() const498 InMemoryNodeKind getKind() const { return Kind; }
499 virtual std::string toString(unsigned Indent) const = 0;
500 };
501
502 class InMemoryFile : public InMemoryNode {
503 Status Stat;
504 std::unique_ptr<llvm::MemoryBuffer> Buffer;
505
506 public:
InMemoryFile(Status Stat,std::unique_ptr<llvm::MemoryBuffer> Buffer)507 InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
508 : InMemoryNode(Stat.getName(), IME_File), Stat(std::move(Stat)),
509 Buffer(std::move(Buffer)) {}
510
511 /// Return the \p Status for this node. \p RequestedName should be the name
512 /// through which the caller referred to this node. It will override
513 /// \p Status::Name in the return value, to mimic the behavior of \p RealFile.
getStatus(StringRef RequestedName) const514 Status getStatus(StringRef RequestedName) const {
515 return Status::copyWithNewName(Stat, RequestedName);
516 }
getBuffer() const517 llvm::MemoryBuffer *getBuffer() const { return Buffer.get(); }
518
toString(unsigned Indent) const519 std::string toString(unsigned Indent) const override {
520 return (std::string(Indent, ' ') + Stat.getName() + "\n").str();
521 }
522
classof(const InMemoryNode * N)523 static bool classof(const InMemoryNode *N) {
524 return N->getKind() == IME_File;
525 }
526 };
527
528 namespace {
529
530 class InMemoryHardLink : public InMemoryNode {
531 const InMemoryFile &ResolvedFile;
532
533 public:
InMemoryHardLink(StringRef Path,const InMemoryFile & ResolvedFile)534 InMemoryHardLink(StringRef Path, const InMemoryFile &ResolvedFile)
535 : InMemoryNode(Path, IME_HardLink), ResolvedFile(ResolvedFile) {}
getResolvedFile() const536 const InMemoryFile &getResolvedFile() const { return ResolvedFile; }
537
toString(unsigned Indent) const538 std::string toString(unsigned Indent) const override {
539 return std::string(Indent, ' ') + "HardLink to -> " +
540 ResolvedFile.toString(0);
541 }
542
classof(const InMemoryNode * N)543 static bool classof(const InMemoryNode *N) {
544 return N->getKind() == IME_HardLink;
545 }
546 };
547
548 /// Adapt a InMemoryFile for VFS' File interface. The goal is to make
549 /// \p InMemoryFileAdaptor mimic as much as possible the behavior of
550 /// \p RealFile.
551 class InMemoryFileAdaptor : public File {
552 const InMemoryFile &Node;
553 /// The name to use when returning a Status for this file.
554 std::string RequestedName;
555
556 public:
InMemoryFileAdaptor(const InMemoryFile & Node,std::string RequestedName)557 explicit InMemoryFileAdaptor(const InMemoryFile &Node,
558 std::string RequestedName)
559 : Node(Node), RequestedName(std::move(RequestedName)) {}
560
status()561 llvm::ErrorOr<Status> status() override {
562 return Node.getStatus(RequestedName);
563 }
564
565 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)566 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
567 bool IsVolatile) override {
568 llvm::MemoryBuffer *Buf = Node.getBuffer();
569 return llvm::MemoryBuffer::getMemBuffer(
570 Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
571 }
572
close()573 std::error_code close() override { return {}; }
574 };
575 } // namespace
576
577 class InMemoryDirectory : public InMemoryNode {
578 Status Stat;
579 llvm::StringMap<std::unique_ptr<InMemoryNode>> Entries;
580
581 public:
InMemoryDirectory(Status Stat)582 InMemoryDirectory(Status Stat)
583 : InMemoryNode(Stat.getName(), IME_Directory), Stat(std::move(Stat)) {}
584
585 /// Return the \p Status for this node. \p RequestedName should be the name
586 /// through which the caller referred to this node. It will override
587 /// \p Status::Name in the return value, to mimic the behavior of \p RealFile.
getStatus(StringRef RequestedName) const588 Status getStatus(StringRef RequestedName) const {
589 return Status::copyWithNewName(Stat, RequestedName);
590 }
getChild(StringRef Name)591 InMemoryNode *getChild(StringRef Name) {
592 auto I = Entries.find(Name);
593 if (I != Entries.end())
594 return I->second.get();
595 return nullptr;
596 }
597
addChild(StringRef Name,std::unique_ptr<InMemoryNode> Child)598 InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
599 return Entries.insert(make_pair(Name, std::move(Child)))
600 .first->second.get();
601 }
602
603 using const_iterator = decltype(Entries)::const_iterator;
604
begin() const605 const_iterator begin() const { return Entries.begin(); }
end() const606 const_iterator end() const { return Entries.end(); }
607
toString(unsigned Indent) const608 std::string toString(unsigned Indent) const override {
609 std::string Result =
610 (std::string(Indent, ' ') + Stat.getName() + "\n").str();
611 for (const auto &Entry : Entries)
612 Result += Entry.second->toString(Indent + 2);
613 return Result;
614 }
615
classof(const InMemoryNode * N)616 static bool classof(const InMemoryNode *N) {
617 return N->getKind() == IME_Directory;
618 }
619 };
620
621 namespace {
getNodeStatus(const InMemoryNode * Node,StringRef RequestedName)622 Status getNodeStatus(const InMemoryNode *Node, StringRef RequestedName) {
623 if (auto Dir = dyn_cast<detail::InMemoryDirectory>(Node))
624 return Dir->getStatus(RequestedName);
625 if (auto File = dyn_cast<detail::InMemoryFile>(Node))
626 return File->getStatus(RequestedName);
627 if (auto Link = dyn_cast<detail::InMemoryHardLink>(Node))
628 return Link->getResolvedFile().getStatus(RequestedName);
629 llvm_unreachable("Unknown node type");
630 }
631 } // namespace
632 } // namespace detail
633
InMemoryFileSystem(bool UseNormalizedPaths)634 InMemoryFileSystem::InMemoryFileSystem(bool UseNormalizedPaths)
635 : Root(new detail::InMemoryDirectory(
636 Status("", getNextVirtualUniqueID(), llvm::sys::TimePoint<>(), 0, 0,
637 0, llvm::sys::fs::file_type::directory_file,
638 llvm::sys::fs::perms::all_all))),
639 UseNormalizedPaths(UseNormalizedPaths) {}
640
641 InMemoryFileSystem::~InMemoryFileSystem() = default;
642
toString() const643 std::string InMemoryFileSystem::toString() const {
644 return Root->toString(/*Indent=*/0);
645 }
646
addFile(const Twine & P,time_t ModificationTime,std::unique_ptr<llvm::MemoryBuffer> Buffer,Optional<uint32_t> User,Optional<uint32_t> Group,Optional<llvm::sys::fs::file_type> Type,Optional<llvm::sys::fs::perms> Perms,const detail::InMemoryFile * HardLinkTarget)647 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
648 std::unique_ptr<llvm::MemoryBuffer> Buffer,
649 Optional<uint32_t> User,
650 Optional<uint32_t> Group,
651 Optional<llvm::sys::fs::file_type> Type,
652 Optional<llvm::sys::fs::perms> Perms,
653 const detail::InMemoryFile *HardLinkTarget) {
654 SmallString<128> Path;
655 P.toVector(Path);
656
657 // Fix up relative paths. This just prepends the current working directory.
658 std::error_code EC = makeAbsolute(Path);
659 assert(!EC);
660 (void)EC;
661
662 if (useNormalizedPaths())
663 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
664
665 if (Path.empty())
666 return false;
667
668 detail::InMemoryDirectory *Dir = Root.get();
669 auto I = llvm::sys::path::begin(Path), E = sys::path::end(Path);
670 const auto ResolvedUser = User.getValueOr(0);
671 const auto ResolvedGroup = Group.getValueOr(0);
672 const auto ResolvedType = Type.getValueOr(sys::fs::file_type::regular_file);
673 const auto ResolvedPerms = Perms.getValueOr(sys::fs::all_all);
674 assert(!(HardLinkTarget && Buffer) && "HardLink cannot have a buffer");
675 // Any intermediate directories we create should be accessible by
676 // the owner, even if Perms says otherwise for the final path.
677 const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all;
678 while (true) {
679 StringRef Name = *I;
680 detail::InMemoryNode *Node = Dir->getChild(Name);
681 ++I;
682 if (!Node) {
683 if (I == E) {
684 // End of the path.
685 std::unique_ptr<detail::InMemoryNode> Child;
686 if (HardLinkTarget)
687 Child.reset(new detail::InMemoryHardLink(P.str(), *HardLinkTarget));
688 else {
689 // Create a new file or directory.
690 Status Stat(P.str(), getNextVirtualUniqueID(),
691 llvm::sys::toTimePoint(ModificationTime), ResolvedUser,
692 ResolvedGroup, Buffer->getBufferSize(), ResolvedType,
693 ResolvedPerms);
694 if (ResolvedType == sys::fs::file_type::directory_file) {
695 Child.reset(new detail::InMemoryDirectory(std::move(Stat)));
696 } else {
697 Child.reset(
698 new detail::InMemoryFile(std::move(Stat), std::move(Buffer)));
699 }
700 }
701 Dir->addChild(Name, std::move(Child));
702 return true;
703 }
704
705 // Create a new directory. Use the path up to here.
706 Status Stat(
707 StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
708 getNextVirtualUniqueID(), llvm::sys::toTimePoint(ModificationTime),
709 ResolvedUser, ResolvedGroup, 0, sys::fs::file_type::directory_file,
710 NewDirectoryPerms);
711 Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
712 Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
713 continue;
714 }
715
716 if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node)) {
717 Dir = NewDir;
718 } else {
719 assert((isa<detail::InMemoryFile>(Node) ||
720 isa<detail::InMemoryHardLink>(Node)) &&
721 "Must be either file, hardlink or directory!");
722
723 // Trying to insert a directory in place of a file.
724 if (I != E)
725 return false;
726
727 // Return false only if the new file is different from the existing one.
728 if (auto Link = dyn_cast<detail::InMemoryHardLink>(Node)) {
729 return Link->getResolvedFile().getBuffer()->getBuffer() ==
730 Buffer->getBuffer();
731 }
732 return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
733 Buffer->getBuffer();
734 }
735 }
736 }
737
addFile(const Twine & P,time_t ModificationTime,std::unique_ptr<llvm::MemoryBuffer> Buffer,Optional<uint32_t> User,Optional<uint32_t> Group,Optional<llvm::sys::fs::file_type> Type,Optional<llvm::sys::fs::perms> Perms)738 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
739 std::unique_ptr<llvm::MemoryBuffer> Buffer,
740 Optional<uint32_t> User,
741 Optional<uint32_t> Group,
742 Optional<llvm::sys::fs::file_type> Type,
743 Optional<llvm::sys::fs::perms> Perms) {
744 return addFile(P, ModificationTime, std::move(Buffer), User, Group, Type,
745 Perms, /*HardLinkTarget=*/nullptr);
746 }
747
addFileNoOwn(const Twine & P,time_t ModificationTime,llvm::MemoryBuffer * Buffer,Optional<uint32_t> User,Optional<uint32_t> Group,Optional<llvm::sys::fs::file_type> Type,Optional<llvm::sys::fs::perms> Perms)748 bool InMemoryFileSystem::addFileNoOwn(const Twine &P, time_t ModificationTime,
749 llvm::MemoryBuffer *Buffer,
750 Optional<uint32_t> User,
751 Optional<uint32_t> Group,
752 Optional<llvm::sys::fs::file_type> Type,
753 Optional<llvm::sys::fs::perms> Perms) {
754 return addFile(P, ModificationTime,
755 llvm::MemoryBuffer::getMemBuffer(
756 Buffer->getBuffer(), Buffer->getBufferIdentifier()),
757 std::move(User), std::move(Group), std::move(Type),
758 std::move(Perms));
759 }
760
761 static ErrorOr<const detail::InMemoryNode *>
lookupInMemoryNode(const InMemoryFileSystem & FS,detail::InMemoryDirectory * Dir,const Twine & P)762 lookupInMemoryNode(const InMemoryFileSystem &FS, detail::InMemoryDirectory *Dir,
763 const Twine &P) {
764 SmallString<128> Path;
765 P.toVector(Path);
766
767 // Fix up relative paths. This just prepends the current working directory.
768 std::error_code EC = FS.makeAbsolute(Path);
769 assert(!EC);
770 (void)EC;
771
772 if (FS.useNormalizedPaths())
773 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
774
775 if (Path.empty())
776 return Dir;
777
778 auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
779 while (true) {
780 detail::InMemoryNode *Node = Dir->getChild(*I);
781 ++I;
782 if (!Node)
783 return errc::no_such_file_or_directory;
784
785 // Return the file if it's at the end of the path.
786 if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
787 if (I == E)
788 return File;
789 return errc::no_such_file_or_directory;
790 }
791
792 // If Node is HardLink then return the resolved file.
793 if (auto File = dyn_cast<detail::InMemoryHardLink>(Node)) {
794 if (I == E)
795 return &File->getResolvedFile();
796 return errc::no_such_file_or_directory;
797 }
798 // Traverse directories.
799 Dir = cast<detail::InMemoryDirectory>(Node);
800 if (I == E)
801 return Dir;
802 }
803 }
804
addHardLink(const Twine & FromPath,const Twine & ToPath)805 bool InMemoryFileSystem::addHardLink(const Twine &FromPath,
806 const Twine &ToPath) {
807 auto FromNode = lookupInMemoryNode(*this, Root.get(), FromPath);
808 auto ToNode = lookupInMemoryNode(*this, Root.get(), ToPath);
809 // FromPath must not have been added before. ToPath must have been added
810 // before. Resolved ToPath must be a File.
811 if (!ToNode || FromNode || !isa<detail::InMemoryFile>(*ToNode))
812 return false;
813 return this->addFile(FromPath, 0, nullptr, None, None, None, None,
814 cast<detail::InMemoryFile>(*ToNode));
815 }
816
status(const Twine & Path)817 llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
818 auto Node = lookupInMemoryNode(*this, Root.get(), Path);
819 if (Node)
820 return detail::getNodeStatus(*Node, Path.str());
821 return Node.getError();
822 }
823
824 llvm::ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Path)825 InMemoryFileSystem::openFileForRead(const Twine &Path) {
826 auto Node = lookupInMemoryNode(*this, Root.get(), Path);
827 if (!Node)
828 return Node.getError();
829
830 // When we have a file provide a heap-allocated wrapper for the memory buffer
831 // to match the ownership semantics for File.
832 if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
833 return std::unique_ptr<File>(
834 new detail::InMemoryFileAdaptor(*F, Path.str()));
835
836 // FIXME: errc::not_a_file?
837 return make_error_code(llvm::errc::invalid_argument);
838 }
839
840 namespace {
841
842 /// Adaptor from InMemoryDir::iterator to directory_iterator.
843 class InMemoryDirIterator : public llvm::vfs::detail::DirIterImpl {
844 detail::InMemoryDirectory::const_iterator I;
845 detail::InMemoryDirectory::const_iterator E;
846 std::string RequestedDirName;
847
setCurrentEntry()848 void setCurrentEntry() {
849 if (I != E) {
850 SmallString<256> Path(RequestedDirName);
851 llvm::sys::path::append(Path, I->second->getFileName());
852 sys::fs::file_type Type;
853 switch (I->second->getKind()) {
854 case detail::IME_File:
855 case detail::IME_HardLink:
856 Type = sys::fs::file_type::regular_file;
857 break;
858 case detail::IME_Directory:
859 Type = sys::fs::file_type::directory_file;
860 break;
861 }
862 CurrentEntry = directory_entry(Path.str(), Type);
863 } else {
864 // When we're at the end, make CurrentEntry invalid and DirIterImpl will
865 // do the rest.
866 CurrentEntry = directory_entry();
867 }
868 }
869
870 public:
871 InMemoryDirIterator() = default;
872
InMemoryDirIterator(const detail::InMemoryDirectory & Dir,std::string RequestedDirName)873 explicit InMemoryDirIterator(const detail::InMemoryDirectory &Dir,
874 std::string RequestedDirName)
875 : I(Dir.begin()), E(Dir.end()),
876 RequestedDirName(std::move(RequestedDirName)) {
877 setCurrentEntry();
878 }
879
increment()880 std::error_code increment() override {
881 ++I;
882 setCurrentEntry();
883 return {};
884 }
885 };
886
887 } // namespace
888
dir_begin(const Twine & Dir,std::error_code & EC)889 directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
890 std::error_code &EC) {
891 auto Node = lookupInMemoryNode(*this, Root.get(), Dir);
892 if (!Node) {
893 EC = Node.getError();
894 return directory_iterator(std::make_shared<InMemoryDirIterator>());
895 }
896
897 if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
898 return directory_iterator(
899 std::make_shared<InMemoryDirIterator>(*DirNode, Dir.str()));
900
901 EC = make_error_code(llvm::errc::not_a_directory);
902 return directory_iterator(std::make_shared<InMemoryDirIterator>());
903 }
904
setCurrentWorkingDirectory(const Twine & P)905 std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
906 SmallString<128> Path;
907 P.toVector(Path);
908
909 // Fix up relative paths. This just prepends the current working directory.
910 std::error_code EC = makeAbsolute(Path);
911 assert(!EC);
912 (void)EC;
913
914 if (useNormalizedPaths())
915 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
916
917 if (!Path.empty())
918 WorkingDirectory = Path.str();
919 return {};
920 }
921
922 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const923 InMemoryFileSystem::getRealPath(const Twine &Path,
924 SmallVectorImpl<char> &Output) const {
925 auto CWD = getCurrentWorkingDirectory();
926 if (!CWD || CWD->empty())
927 return errc::operation_not_permitted;
928 Path.toVector(Output);
929 if (auto EC = makeAbsolute(Output))
930 return EC;
931 llvm::sys::path::remove_dots(Output, /*remove_dot_dot=*/true);
932 return {};
933 }
934
isLocal(const Twine & Path,bool & Result)935 std::error_code InMemoryFileSystem::isLocal(const Twine &Path, bool &Result) {
936 Result = false;
937 return {};
938 }
939
940 } // namespace vfs
941 } // namespace llvm
942
943 //===-----------------------------------------------------------------------===/
944 // RedirectingFileSystem implementation
945 //===-----------------------------------------------------------------------===/
946
947 // FIXME: reuse implementation common with OverlayFSDirIterImpl as these
948 // iterators are conceptually similar.
949 class llvm::vfs::VFSFromYamlDirIterImpl
950 : public llvm::vfs::detail::DirIterImpl {
951 std::string Dir;
952 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Current, End;
953
954 // To handle 'fallthrough' mode we need to iterate at first through
955 // RedirectingDirectoryEntry and then through ExternalFS. These operations are
956 // done sequentially, we just need to keep a track of what kind of iteration
957 // we are currently performing.
958
959 /// Flag telling if we should iterate through ExternalFS or stop at the last
960 /// RedirectingDirectoryEntry::iterator.
961 bool IterateExternalFS;
962 /// Flag telling if we have switched to iterating through ExternalFS.
963 bool IsExternalFSCurrent = false;
964 FileSystem &ExternalFS;
965 directory_iterator ExternalDirIter;
966 llvm::StringSet<> SeenNames;
967
968 /// To combine multiple iterations, different methods are responsible for
969 /// different iteration steps.
970 /// @{
971
972 /// Responsible for dispatching between RedirectingDirectoryEntry iteration
973 /// and ExternalFS iteration.
974 std::error_code incrementImpl(bool IsFirstTime);
975 /// Responsible for RedirectingDirectoryEntry iteration.
976 std::error_code incrementContent(bool IsFirstTime);
977 /// Responsible for ExternalFS iteration.
978 std::error_code incrementExternal();
979 /// @}
980
981 public:
982 VFSFromYamlDirIterImpl(
983 const Twine &Path,
984 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
985 RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
986 bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC);
987
988 std::error_code increment() override;
989 };
990
991 llvm::ErrorOr<std::string>
getCurrentWorkingDirectory() const992 RedirectingFileSystem::getCurrentWorkingDirectory() const {
993 return ExternalFS->getCurrentWorkingDirectory();
994 }
995
996 std::error_code
setCurrentWorkingDirectory(const Twine & Path)997 RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
998 return ExternalFS->setCurrentWorkingDirectory(Path);
999 }
1000
isLocal(const Twine & Path,bool & Result)1001 std::error_code RedirectingFileSystem::isLocal(const Twine &Path,
1002 bool &Result) {
1003 return ExternalFS->isLocal(Path, Result);
1004 }
1005
dir_begin(const Twine & Dir,std::error_code & EC)1006 directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir,
1007 std::error_code &EC) {
1008 ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Dir);
1009 if (!E) {
1010 EC = E.getError();
1011 if (IsFallthrough && EC == errc::no_such_file_or_directory)
1012 return ExternalFS->dir_begin(Dir, EC);
1013 return {};
1014 }
1015 ErrorOr<Status> S = status(Dir, *E);
1016 if (!S) {
1017 EC = S.getError();
1018 return {};
1019 }
1020 if (!S->isDirectory()) {
1021 EC = std::error_code(static_cast<int>(errc::not_a_directory),
1022 std::system_category());
1023 return {};
1024 }
1025
1026 auto *D = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(*E);
1027 return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(
1028 Dir, D->contents_begin(), D->contents_end(),
1029 /*IterateExternalFS=*/IsFallthrough, *ExternalFS, EC));
1030 }
1031
setExternalContentsPrefixDir(StringRef PrefixDir)1032 void RedirectingFileSystem::setExternalContentsPrefixDir(StringRef PrefixDir) {
1033 ExternalContentsPrefixDir = PrefixDir.str();
1034 }
1035
getExternalContentsPrefixDir() const1036 StringRef RedirectingFileSystem::getExternalContentsPrefixDir() const {
1037 return ExternalContentsPrefixDir;
1038 }
1039
1040 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dump() const1041 LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const {
1042 for (const auto &Root : Roots)
1043 dumpEntry(Root.get());
1044 }
1045
1046 LLVM_DUMP_METHOD void
dumpEntry(RedirectingFileSystem::Entry * E,int NumSpaces) const1047 RedirectingFileSystem::dumpEntry(RedirectingFileSystem::Entry *E,
1048 int NumSpaces) const {
1049 StringRef Name = E->getName();
1050 for (int i = 0, e = NumSpaces; i < e; ++i)
1051 dbgs() << " ";
1052 dbgs() << "'" << Name.str().c_str() << "'"
1053 << "\n";
1054
1055 if (E->getKind() == RedirectingFileSystem::EK_Directory) {
1056 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
1057 assert(DE && "Should be a directory");
1058
1059 for (std::unique_ptr<Entry> &SubEntry :
1060 llvm::make_range(DE->contents_begin(), DE->contents_end()))
1061 dumpEntry(SubEntry.get(), NumSpaces + 2);
1062 }
1063 }
1064 #endif
1065
1066 /// A helper class to hold the common YAML parsing state.
1067 class llvm::vfs::RedirectingFileSystemParser {
1068 yaml::Stream &Stream;
1069
error(yaml::Node * N,const Twine & Msg)1070 void error(yaml::Node *N, const Twine &Msg) { Stream.printError(N, Msg); }
1071
1072 // false on error
parseScalarString(yaml::Node * N,StringRef & Result,SmallVectorImpl<char> & Storage)1073 bool parseScalarString(yaml::Node *N, StringRef &Result,
1074 SmallVectorImpl<char> &Storage) {
1075 const auto *S = dyn_cast<yaml::ScalarNode>(N);
1076
1077 if (!S) {
1078 error(N, "expected string");
1079 return false;
1080 }
1081 Result = S->getValue(Storage);
1082 return true;
1083 }
1084
1085 // false on error
parseScalarBool(yaml::Node * N,bool & Result)1086 bool parseScalarBool(yaml::Node *N, bool &Result) {
1087 SmallString<5> Storage;
1088 StringRef Value;
1089 if (!parseScalarString(N, Value, Storage))
1090 return false;
1091
1092 if (Value.equals_lower("true") || Value.equals_lower("on") ||
1093 Value.equals_lower("yes") || Value == "1") {
1094 Result = true;
1095 return true;
1096 } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
1097 Value.equals_lower("no") || Value == "0") {
1098 Result = false;
1099 return true;
1100 }
1101
1102 error(N, "expected boolean value");
1103 return false;
1104 }
1105
1106 struct KeyStatus {
1107 bool Required;
1108 bool Seen = false;
1109
KeyStatusllvm::vfs::RedirectingFileSystemParser::KeyStatus1110 KeyStatus(bool Required = false) : Required(Required) {}
1111 };
1112
1113 using KeyStatusPair = std::pair<StringRef, KeyStatus>;
1114
1115 // false on error
checkDuplicateOrUnknownKey(yaml::Node * KeyNode,StringRef Key,DenseMap<StringRef,KeyStatus> & Keys)1116 bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
1117 DenseMap<StringRef, KeyStatus> &Keys) {
1118 if (!Keys.count(Key)) {
1119 error(KeyNode, "unknown key");
1120 return false;
1121 }
1122 KeyStatus &S = Keys[Key];
1123 if (S.Seen) {
1124 error(KeyNode, Twine("duplicate key '") + Key + "'");
1125 return false;
1126 }
1127 S.Seen = true;
1128 return true;
1129 }
1130
1131 // false on error
checkMissingKeys(yaml::Node * Obj,DenseMap<StringRef,KeyStatus> & Keys)1132 bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
1133 for (const auto &I : Keys) {
1134 if (I.second.Required && !I.second.Seen) {
1135 error(Obj, Twine("missing key '") + I.first + "'");
1136 return false;
1137 }
1138 }
1139 return true;
1140 }
1141
1142 RedirectingFileSystem::Entry *
lookupOrCreateEntry(RedirectingFileSystem * FS,StringRef Name,RedirectingFileSystem::Entry * ParentEntry=nullptr)1143 lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
1144 RedirectingFileSystem::Entry *ParentEntry = nullptr) {
1145 if (!ParentEntry) { // Look for a existent root
1146 for (const auto &Root : FS->Roots) {
1147 if (Name.equals(Root->getName())) {
1148 ParentEntry = Root.get();
1149 return ParentEntry;
1150 }
1151 }
1152 } else { // Advance to the next component
1153 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1154 ParentEntry);
1155 for (std::unique_ptr<RedirectingFileSystem::Entry> &Content :
1156 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1157 auto *DirContent =
1158 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1159 Content.get());
1160 if (DirContent && Name.equals(Content->getName()))
1161 return DirContent;
1162 }
1163 }
1164
1165 // ... or create a new one
1166 std::unique_ptr<RedirectingFileSystem::Entry> E =
1167 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1168 Name, Status("", getNextVirtualUniqueID(),
1169 std::chrono::system_clock::now(), 0, 0, 0,
1170 file_type::directory_file, sys::fs::all_all));
1171
1172 if (!ParentEntry) { // Add a new root to the overlay
1173 FS->Roots.push_back(std::move(E));
1174 ParentEntry = FS->Roots.back().get();
1175 return ParentEntry;
1176 }
1177
1178 auto *DE =
1179 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(ParentEntry);
1180 DE->addContent(std::move(E));
1181 return DE->getLastContent();
1182 }
1183
uniqueOverlayTree(RedirectingFileSystem * FS,RedirectingFileSystem::Entry * SrcE,RedirectingFileSystem::Entry * NewParentE=nullptr)1184 void uniqueOverlayTree(RedirectingFileSystem *FS,
1185 RedirectingFileSystem::Entry *SrcE,
1186 RedirectingFileSystem::Entry *NewParentE = nullptr) {
1187 StringRef Name = SrcE->getName();
1188 switch (SrcE->getKind()) {
1189 case RedirectingFileSystem::EK_Directory: {
1190 auto *DE =
1191 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
1192 assert(DE && "Must be a directory");
1193 // Empty directories could be present in the YAML as a way to
1194 // describe a file for a current directory after some of its subdir
1195 // is parsed. This only leads to redundant walks, ignore it.
1196 if (!Name.empty())
1197 NewParentE = lookupOrCreateEntry(FS, Name, NewParentE);
1198 for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
1199 llvm::make_range(DE->contents_begin(), DE->contents_end()))
1200 uniqueOverlayTree(FS, SubEntry.get(), NewParentE);
1201 break;
1202 }
1203 case RedirectingFileSystem::EK_File: {
1204 auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
1205 assert(FE && "Must be a file");
1206 assert(NewParentE && "Parent entry must exist");
1207 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1208 NewParentE);
1209 DE->addContent(
1210 llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
1211 Name, FE->getExternalContentsPath(), FE->getUseName()));
1212 break;
1213 }
1214 }
1215 }
1216
1217 std::unique_ptr<RedirectingFileSystem::Entry>
parseEntry(yaml::Node * N,RedirectingFileSystem * FS,bool IsRootEntry)1218 parseEntry(yaml::Node *N, RedirectingFileSystem *FS, bool IsRootEntry) {
1219 auto *M = dyn_cast<yaml::MappingNode>(N);
1220 if (!M) {
1221 error(N, "expected mapping node for file or directory entry");
1222 return nullptr;
1223 }
1224
1225 KeyStatusPair Fields[] = {
1226 KeyStatusPair("name", true),
1227 KeyStatusPair("type", true),
1228 KeyStatusPair("contents", false),
1229 KeyStatusPair("external-contents", false),
1230 KeyStatusPair("use-external-name", false),
1231 };
1232
1233 DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1234
1235 bool HasContents = false; // external or otherwise
1236 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>>
1237 EntryArrayContents;
1238 std::string ExternalContentsPath;
1239 std::string Name;
1240 yaml::Node *NameValueNode;
1241 auto UseExternalName =
1242 RedirectingFileSystem::RedirectingFileEntry::NK_NotSet;
1243 RedirectingFileSystem::EntryKind Kind;
1244
1245 for (auto &I : *M) {
1246 StringRef Key;
1247 // Reuse the buffer for key and value, since we don't look at key after
1248 // parsing value.
1249 SmallString<256> Buffer;
1250 if (!parseScalarString(I.getKey(), Key, Buffer))
1251 return nullptr;
1252
1253 if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1254 return nullptr;
1255
1256 StringRef Value;
1257 if (Key == "name") {
1258 if (!parseScalarString(I.getValue(), Value, Buffer))
1259 return nullptr;
1260
1261 NameValueNode = I.getValue();
1262 if (FS->UseCanonicalizedPaths) {
1263 SmallString<256> Path(Value);
1264 // Guarantee that old YAML files containing paths with ".." and "."
1265 // are properly canonicalized before read into the VFS.
1266 Path = sys::path::remove_leading_dotslash(Path);
1267 sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1268 Name = Path.str();
1269 } else {
1270 Name = Value;
1271 }
1272 } else if (Key == "type") {
1273 if (!parseScalarString(I.getValue(), Value, Buffer))
1274 return nullptr;
1275 if (Value == "file")
1276 Kind = RedirectingFileSystem::EK_File;
1277 else if (Value == "directory")
1278 Kind = RedirectingFileSystem::EK_Directory;
1279 else {
1280 error(I.getValue(), "unknown value for 'type'");
1281 return nullptr;
1282 }
1283 } else if (Key == "contents") {
1284 if (HasContents) {
1285 error(I.getKey(),
1286 "entry already has 'contents' or 'external-contents'");
1287 return nullptr;
1288 }
1289 HasContents = true;
1290 auto *Contents = dyn_cast<yaml::SequenceNode>(I.getValue());
1291 if (!Contents) {
1292 // FIXME: this is only for directories, what about files?
1293 error(I.getValue(), "expected array");
1294 return nullptr;
1295 }
1296
1297 for (auto &I : *Contents) {
1298 if (std::unique_ptr<RedirectingFileSystem::Entry> E =
1299 parseEntry(&I, FS, /*IsRootEntry*/ false))
1300 EntryArrayContents.push_back(std::move(E));
1301 else
1302 return nullptr;
1303 }
1304 } else if (Key == "external-contents") {
1305 if (HasContents) {
1306 error(I.getKey(),
1307 "entry already has 'contents' or 'external-contents'");
1308 return nullptr;
1309 }
1310 HasContents = true;
1311 if (!parseScalarString(I.getValue(), Value, Buffer))
1312 return nullptr;
1313
1314 SmallString<256> FullPath;
1315 if (FS->IsRelativeOverlay) {
1316 FullPath = FS->getExternalContentsPrefixDir();
1317 assert(!FullPath.empty() &&
1318 "External contents prefix directory must exist");
1319 llvm::sys::path::append(FullPath, Value);
1320 } else {
1321 FullPath = Value;
1322 }
1323
1324 if (FS->UseCanonicalizedPaths) {
1325 // Guarantee that old YAML files containing paths with ".." and "."
1326 // are properly canonicalized before read into the VFS.
1327 FullPath = sys::path::remove_leading_dotslash(FullPath);
1328 sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
1329 }
1330 ExternalContentsPath = FullPath.str();
1331 } else if (Key == "use-external-name") {
1332 bool Val;
1333 if (!parseScalarBool(I.getValue(), Val))
1334 return nullptr;
1335 UseExternalName =
1336 Val ? RedirectingFileSystem::RedirectingFileEntry::NK_External
1337 : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual;
1338 } else {
1339 llvm_unreachable("key missing from Keys");
1340 }
1341 }
1342
1343 if (Stream.failed())
1344 return nullptr;
1345
1346 // check for missing keys
1347 if (!HasContents) {
1348 error(N, "missing key 'contents' or 'external-contents'");
1349 return nullptr;
1350 }
1351 if (!checkMissingKeys(N, Keys))
1352 return nullptr;
1353
1354 // check invalid configuration
1355 if (Kind == RedirectingFileSystem::EK_Directory &&
1356 UseExternalName !=
1357 RedirectingFileSystem::RedirectingFileEntry::NK_NotSet) {
1358 error(N, "'use-external-name' is not supported for directories");
1359 return nullptr;
1360 }
1361
1362 if (IsRootEntry && !sys::path::is_absolute(Name)) {
1363 assert(NameValueNode && "Name presence should be checked earlier");
1364 error(NameValueNode,
1365 "entry with relative path at the root level is not discoverable");
1366 return nullptr;
1367 }
1368
1369 // Remove trailing slash(es), being careful not to remove the root path
1370 StringRef Trimmed(Name);
1371 size_t RootPathLen = sys::path::root_path(Trimmed).size();
1372 while (Trimmed.size() > RootPathLen &&
1373 sys::path::is_separator(Trimmed.back()))
1374 Trimmed = Trimmed.slice(0, Trimmed.size() - 1);
1375 // Get the last component
1376 StringRef LastComponent = sys::path::filename(Trimmed);
1377
1378 std::unique_ptr<RedirectingFileSystem::Entry> Result;
1379 switch (Kind) {
1380 case RedirectingFileSystem::EK_File:
1381 Result = llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
1382 LastComponent, std::move(ExternalContentsPath), UseExternalName);
1383 break;
1384 case RedirectingFileSystem::EK_Directory:
1385 Result =
1386 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1387 LastComponent, std::move(EntryArrayContents),
1388 Status("", getNextVirtualUniqueID(),
1389 std::chrono::system_clock::now(), 0, 0, 0,
1390 file_type::directory_file, sys::fs::all_all));
1391 break;
1392 }
1393
1394 StringRef Parent = sys::path::parent_path(Trimmed);
1395 if (Parent.empty())
1396 return Result;
1397
1398 // if 'name' contains multiple components, create implicit directory entries
1399 for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
1400 E = sys::path::rend(Parent);
1401 I != E; ++I) {
1402 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> Entries;
1403 Entries.push_back(std::move(Result));
1404 Result =
1405 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1406 *I, std::move(Entries),
1407 Status("", getNextVirtualUniqueID(),
1408 std::chrono::system_clock::now(), 0, 0, 0,
1409 file_type::directory_file, sys::fs::all_all));
1410 }
1411 return Result;
1412 }
1413
1414 public:
RedirectingFileSystemParser(yaml::Stream & S)1415 RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {}
1416
1417 // false on error
parse(yaml::Node * Root,RedirectingFileSystem * FS)1418 bool parse(yaml::Node *Root, RedirectingFileSystem *FS) {
1419 auto *Top = dyn_cast<yaml::MappingNode>(Root);
1420 if (!Top) {
1421 error(Root, "expected mapping node");
1422 return false;
1423 }
1424
1425 KeyStatusPair Fields[] = {
1426 KeyStatusPair("version", true),
1427 KeyStatusPair("case-sensitive", false),
1428 KeyStatusPair("use-external-names", false),
1429 KeyStatusPair("overlay-relative", false),
1430 KeyStatusPair("fallthrough", false),
1431 KeyStatusPair("roots", true),
1432 };
1433
1434 DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1435 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> RootEntries;
1436
1437 // Parse configuration and 'roots'
1438 for (auto &I : *Top) {
1439 SmallString<10> KeyBuffer;
1440 StringRef Key;
1441 if (!parseScalarString(I.getKey(), Key, KeyBuffer))
1442 return false;
1443
1444 if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1445 return false;
1446
1447 if (Key == "roots") {
1448 auto *Roots = dyn_cast<yaml::SequenceNode>(I.getValue());
1449 if (!Roots) {
1450 error(I.getValue(), "expected array");
1451 return false;
1452 }
1453
1454 for (auto &I : *Roots) {
1455 if (std::unique_ptr<RedirectingFileSystem::Entry> E =
1456 parseEntry(&I, FS, /*IsRootEntry*/ true))
1457 RootEntries.push_back(std::move(E));
1458 else
1459 return false;
1460 }
1461 } else if (Key == "version") {
1462 StringRef VersionString;
1463 SmallString<4> Storage;
1464 if (!parseScalarString(I.getValue(), VersionString, Storage))
1465 return false;
1466 int Version;
1467 if (VersionString.getAsInteger<int>(10, Version)) {
1468 error(I.getValue(), "expected integer");
1469 return false;
1470 }
1471 if (Version < 0) {
1472 error(I.getValue(), "invalid version number");
1473 return false;
1474 }
1475 if (Version != 0) {
1476 error(I.getValue(), "version mismatch, expected 0");
1477 return false;
1478 }
1479 } else if (Key == "case-sensitive") {
1480 if (!parseScalarBool(I.getValue(), FS->CaseSensitive))
1481 return false;
1482 } else if (Key == "overlay-relative") {
1483 if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay))
1484 return false;
1485 } else if (Key == "use-external-names") {
1486 if (!parseScalarBool(I.getValue(), FS->UseExternalNames))
1487 return false;
1488 } else if (Key == "fallthrough") {
1489 if (!parseScalarBool(I.getValue(), FS->IsFallthrough))
1490 return false;
1491 } else {
1492 llvm_unreachable("key missing from Keys");
1493 }
1494 }
1495
1496 if (Stream.failed())
1497 return false;
1498
1499 if (!checkMissingKeys(Top, Keys))
1500 return false;
1501
1502 // Now that we sucessefully parsed the YAML file, canonicalize the internal
1503 // representation to a proper directory tree so that we can search faster
1504 // inside the VFS.
1505 for (auto &E : RootEntries)
1506 uniqueOverlayTree(FS, E.get());
1507
1508 return true;
1509 }
1510 };
1511
1512 RedirectingFileSystem *
create(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1513 RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
1514 SourceMgr::DiagHandlerTy DiagHandler,
1515 StringRef YAMLFilePath, void *DiagContext,
1516 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1517 SourceMgr SM;
1518 yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
1519
1520 SM.setDiagHandler(DiagHandler, DiagContext);
1521 yaml::document_iterator DI = Stream.begin();
1522 yaml::Node *Root = DI->getRoot();
1523 if (DI == Stream.end() || !Root) {
1524 SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
1525 return nullptr;
1526 }
1527
1528 RedirectingFileSystemParser P(Stream);
1529
1530 std::unique_ptr<RedirectingFileSystem> FS(
1531 new RedirectingFileSystem(std::move(ExternalFS)));
1532
1533 if (!YAMLFilePath.empty()) {
1534 // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
1535 // to each 'external-contents' path.
1536 //
1537 // Example:
1538 // -ivfsoverlay dummy.cache/vfs/vfs.yaml
1539 // yields:
1540 // FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
1541 //
1542 SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
1543 std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
1544 assert(!EC && "Overlay dir final path must be absolute");
1545 (void)EC;
1546 FS->setExternalContentsPrefixDir(OverlayAbsDir);
1547 }
1548
1549 if (!P.parse(Root, FS.get()))
1550 return nullptr;
1551
1552 return FS.release();
1553 }
1554
1555 ErrorOr<RedirectingFileSystem::Entry *>
lookupPath(const Twine & Path_) const1556 RedirectingFileSystem::lookupPath(const Twine &Path_) const {
1557 SmallString<256> Path;
1558 Path_.toVector(Path);
1559
1560 // Handle relative paths
1561 if (std::error_code EC = makeAbsolute(Path))
1562 return EC;
1563
1564 // Canonicalize path by removing ".", "..", "./", etc components. This is
1565 // a VFS request, do bot bother about symlinks in the path components
1566 // but canonicalize in order to perform the correct entry search.
1567 if (UseCanonicalizedPaths) {
1568 Path = sys::path::remove_leading_dotslash(Path);
1569 sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1570 }
1571
1572 if (Path.empty())
1573 return make_error_code(llvm::errc::invalid_argument);
1574
1575 sys::path::const_iterator Start = sys::path::begin(Path);
1576 sys::path::const_iterator End = sys::path::end(Path);
1577 for (const auto &Root : Roots) {
1578 ErrorOr<RedirectingFileSystem::Entry *> Result =
1579 lookupPath(Start, End, Root.get());
1580 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1581 return Result;
1582 }
1583 return make_error_code(llvm::errc::no_such_file_or_directory);
1584 }
1585
1586 ErrorOr<RedirectingFileSystem::Entry *>
lookupPath(sys::path::const_iterator Start,sys::path::const_iterator End,RedirectingFileSystem::Entry * From) const1587 RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
1588 sys::path::const_iterator End,
1589 RedirectingFileSystem::Entry *From) const {
1590 #ifndef _WIN32
1591 assert(!isTraversalComponent(*Start) &&
1592 !isTraversalComponent(From->getName()) &&
1593 "Paths should not contain traversal components");
1594 #else
1595 // FIXME: this is here to support windows, remove it once canonicalized
1596 // paths become globally default.
1597 if (Start->equals("."))
1598 ++Start;
1599 #endif
1600
1601 StringRef FromName = From->getName();
1602
1603 // Forward the search to the next component in case this is an empty one.
1604 if (!FromName.empty()) {
1605 if (CaseSensitive ? !Start->equals(FromName)
1606 : !Start->equals_lower(FromName))
1607 // failure to match
1608 return make_error_code(llvm::errc::no_such_file_or_directory);
1609
1610 ++Start;
1611
1612 if (Start == End) {
1613 // Match!
1614 return From;
1615 }
1616 }
1617
1618 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(From);
1619 if (!DE)
1620 return make_error_code(llvm::errc::not_a_directory);
1621
1622 for (const std::unique_ptr<RedirectingFileSystem::Entry> &DirEntry :
1623 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1624 ErrorOr<RedirectingFileSystem::Entry *> Result =
1625 lookupPath(Start, End, DirEntry.get());
1626 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1627 return Result;
1628 }
1629 return make_error_code(llvm::errc::no_such_file_or_directory);
1630 }
1631
getRedirectedFileStatus(const Twine & Path,bool UseExternalNames,Status ExternalStatus)1632 static Status getRedirectedFileStatus(const Twine &Path, bool UseExternalNames,
1633 Status ExternalStatus) {
1634 Status S = ExternalStatus;
1635 if (!UseExternalNames)
1636 S = Status::copyWithNewName(S, Path.str());
1637 S.IsVFSMapped = true;
1638 return S;
1639 }
1640
status(const Twine & Path,RedirectingFileSystem::Entry * E)1641 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path,
1642 RedirectingFileSystem::Entry *E) {
1643 assert(E != nullptr);
1644 if (auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(E)) {
1645 ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
1646 assert(!S || S->getName() == F->getExternalContentsPath());
1647 if (S)
1648 return getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1649 *S);
1650 return S;
1651 } else { // directory
1652 auto *DE = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
1653 return Status::copyWithNewName(DE->getStatus(), Path.str());
1654 }
1655 }
1656
status(const Twine & Path)1657 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
1658 ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
1659 if (!Result) {
1660 if (IsFallthrough &&
1661 Result.getError() == llvm::errc::no_such_file_or_directory) {
1662 return ExternalFS->status(Path);
1663 }
1664 return Result.getError();
1665 }
1666 return status(Path, *Result);
1667 }
1668
1669 namespace {
1670
1671 /// Provide a file wrapper with an overriden status.
1672 class FileWithFixedStatus : public File {
1673 std::unique_ptr<File> InnerFile;
1674 Status S;
1675
1676 public:
FileWithFixedStatus(std::unique_ptr<File> InnerFile,Status S)1677 FileWithFixedStatus(std::unique_ptr<File> InnerFile, Status S)
1678 : InnerFile(std::move(InnerFile)), S(std::move(S)) {}
1679
status()1680 ErrorOr<Status> status() override { return S; }
1681 ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
1682
getBuffer(const Twine & Name,int64_t FileSize,bool RequiresNullTerminator,bool IsVolatile)1683 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
1684 bool IsVolatile) override {
1685 return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
1686 IsVolatile);
1687 }
1688
close()1689 std::error_code close() override { return InnerFile->close(); }
1690 };
1691
1692 } // namespace
1693
1694 ErrorOr<std::unique_ptr<File>>
openFileForRead(const Twine & Path)1695 RedirectingFileSystem::openFileForRead(const Twine &Path) {
1696 ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Path);
1697 if (!E) {
1698 if (IsFallthrough &&
1699 E.getError() == llvm::errc::no_such_file_or_directory) {
1700 return ExternalFS->openFileForRead(Path);
1701 }
1702 return E.getError();
1703 }
1704
1705 auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*E);
1706 if (!F) // FIXME: errc::not_a_file?
1707 return make_error_code(llvm::errc::invalid_argument);
1708
1709 auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
1710 if (!Result)
1711 return Result;
1712
1713 auto ExternalStatus = (*Result)->status();
1714 if (!ExternalStatus)
1715 return ExternalStatus.getError();
1716
1717 // FIXME: Update the status with the name and VFSMapped.
1718 Status S = getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1719 *ExternalStatus);
1720 return std::unique_ptr<File>(
1721 llvm::make_unique<FileWithFixedStatus>(std::move(*Result), S));
1722 }
1723
1724 std::error_code
getRealPath(const Twine & Path,SmallVectorImpl<char> & Output) const1725 RedirectingFileSystem::getRealPath(const Twine &Path,
1726 SmallVectorImpl<char> &Output) const {
1727 ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
1728 if (!Result) {
1729 if (IsFallthrough &&
1730 Result.getError() == llvm::errc::no_such_file_or_directory) {
1731 return ExternalFS->getRealPath(Path, Output);
1732 }
1733 return Result.getError();
1734 }
1735
1736 if (auto *F =
1737 dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*Result)) {
1738 return ExternalFS->getRealPath(F->getExternalContentsPath(), Output);
1739 }
1740 // Even if there is a directory entry, fall back to ExternalFS if allowed,
1741 // because directories don't have a single external contents path.
1742 return IsFallthrough ? ExternalFS->getRealPath(Path, Output)
1743 : llvm::errc::invalid_argument;
1744 }
1745
1746 IntrusiveRefCntPtr<FileSystem>
getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1747 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1748 SourceMgr::DiagHandlerTy DiagHandler,
1749 StringRef YAMLFilePath, void *DiagContext,
1750 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1751 return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
1752 YAMLFilePath, DiagContext,
1753 std::move(ExternalFS));
1754 }
1755
getVFSEntries(RedirectingFileSystem::Entry * SrcE,SmallVectorImpl<StringRef> & Path,SmallVectorImpl<YAMLVFSEntry> & Entries)1756 static void getVFSEntries(RedirectingFileSystem::Entry *SrcE,
1757 SmallVectorImpl<StringRef> &Path,
1758 SmallVectorImpl<YAMLVFSEntry> &Entries) {
1759 auto Kind = SrcE->getKind();
1760 if (Kind == RedirectingFileSystem::EK_Directory) {
1761 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
1762 assert(DE && "Must be a directory");
1763 for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
1764 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1765 Path.push_back(SubEntry->getName());
1766 getVFSEntries(SubEntry.get(), Path, Entries);
1767 Path.pop_back();
1768 }
1769 return;
1770 }
1771
1772 assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File");
1773 auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
1774 assert(FE && "Must be a file");
1775 SmallString<128> VPath;
1776 for (auto &Comp : Path)
1777 llvm::sys::path::append(VPath, Comp);
1778 Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath()));
1779 }
1780
collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,SourceMgr::DiagHandlerTy DiagHandler,StringRef YAMLFilePath,SmallVectorImpl<YAMLVFSEntry> & CollectedEntries,void * DiagContext,IntrusiveRefCntPtr<FileSystem> ExternalFS)1781 void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1782 SourceMgr::DiagHandlerTy DiagHandler,
1783 StringRef YAMLFilePath,
1784 SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
1785 void *DiagContext,
1786 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1787 RedirectingFileSystem *VFS = RedirectingFileSystem::create(
1788 std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
1789 std::move(ExternalFS));
1790 ErrorOr<RedirectingFileSystem::Entry *> RootE = VFS->lookupPath("/");
1791 if (!RootE)
1792 return;
1793 SmallVector<StringRef, 8> Components;
1794 Components.push_back("/");
1795 getVFSEntries(*RootE, Components, CollectedEntries);
1796 }
1797
getNextVirtualUniqueID()1798 UniqueID vfs::getNextVirtualUniqueID() {
1799 static std::atomic<unsigned> UID;
1800 unsigned ID = ++UID;
1801 // The following assumes that uint64_t max will never collide with a real
1802 // dev_t value from the OS.
1803 return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
1804 }
1805
addFileMapping(StringRef VirtualPath,StringRef RealPath)1806 void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
1807 assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
1808 assert(sys::path::is_absolute(RealPath) && "real path not absolute");
1809 assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
1810 Mappings.emplace_back(VirtualPath, RealPath);
1811 }
1812
1813 namespace {
1814
1815 class JSONWriter {
1816 llvm::raw_ostream &OS;
1817 SmallVector<StringRef, 16> DirStack;
1818
getDirIndent()1819 unsigned getDirIndent() { return 4 * DirStack.size(); }
getFileIndent()1820 unsigned getFileIndent() { return 4 * (DirStack.size() + 1); }
1821 bool containedIn(StringRef Parent, StringRef Path);
1822 StringRef containedPart(StringRef Parent, StringRef Path);
1823 void startDirectory(StringRef Path);
1824 void endDirectory();
1825 void writeEntry(StringRef VPath, StringRef RPath);
1826
1827 public:
JSONWriter(llvm::raw_ostream & OS)1828 JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
1829
1830 void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> UseExternalNames,
1831 Optional<bool> IsCaseSensitive, Optional<bool> IsOverlayRelative,
1832 StringRef OverlayDir);
1833 };
1834
1835 } // namespace
1836
containedIn(StringRef Parent,StringRef Path)1837 bool JSONWriter::containedIn(StringRef Parent, StringRef Path) {
1838 using namespace llvm::sys;
1839
1840 // Compare each path component.
1841 auto IParent = path::begin(Parent), EParent = path::end(Parent);
1842 for (auto IChild = path::begin(Path), EChild = path::end(Path);
1843 IParent != EParent && IChild != EChild; ++IParent, ++IChild) {
1844 if (*IParent != *IChild)
1845 return false;
1846 }
1847 // Have we exhausted the parent path?
1848 return IParent == EParent;
1849 }
1850
containedPart(StringRef Parent,StringRef Path)1851 StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) {
1852 assert(!Parent.empty());
1853 assert(containedIn(Parent, Path));
1854 return Path.slice(Parent.size() + 1, StringRef::npos);
1855 }
1856
startDirectory(StringRef Path)1857 void JSONWriter::startDirectory(StringRef Path) {
1858 StringRef Name =
1859 DirStack.empty() ? Path : containedPart(DirStack.back(), Path);
1860 DirStack.push_back(Path);
1861 unsigned Indent = getDirIndent();
1862 OS.indent(Indent) << "{\n";
1863 OS.indent(Indent + 2) << "'type': 'directory',\n";
1864 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n";
1865 OS.indent(Indent + 2) << "'contents': [\n";
1866 }
1867
endDirectory()1868 void JSONWriter::endDirectory() {
1869 unsigned Indent = getDirIndent();
1870 OS.indent(Indent + 2) << "]\n";
1871 OS.indent(Indent) << "}";
1872
1873 DirStack.pop_back();
1874 }
1875
writeEntry(StringRef VPath,StringRef RPath)1876 void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
1877 unsigned Indent = getFileIndent();
1878 OS.indent(Indent) << "{\n";
1879 OS.indent(Indent + 2) << "'type': 'file',\n";
1880 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n";
1881 OS.indent(Indent + 2) << "'external-contents': \""
1882 << llvm::yaml::escape(RPath) << "\"\n";
1883 OS.indent(Indent) << "}";
1884 }
1885
write(ArrayRef<YAMLVFSEntry> Entries,Optional<bool> UseExternalNames,Optional<bool> IsCaseSensitive,Optional<bool> IsOverlayRelative,StringRef OverlayDir)1886 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
1887 Optional<bool> UseExternalNames,
1888 Optional<bool> IsCaseSensitive,
1889 Optional<bool> IsOverlayRelative,
1890 StringRef OverlayDir) {
1891 using namespace llvm::sys;
1892
1893 OS << "{\n"
1894 " 'version': 0,\n";
1895 if (IsCaseSensitive.hasValue())
1896 OS << " 'case-sensitive': '"
1897 << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
1898 if (UseExternalNames.hasValue())
1899 OS << " 'use-external-names': '"
1900 << (UseExternalNames.getValue() ? "true" : "false") << "',\n";
1901 bool UseOverlayRelative = false;
1902 if (IsOverlayRelative.hasValue()) {
1903 UseOverlayRelative = IsOverlayRelative.getValue();
1904 OS << " 'overlay-relative': '" << (UseOverlayRelative ? "true" : "false")
1905 << "',\n";
1906 }
1907 OS << " 'roots': [\n";
1908
1909 if (!Entries.empty()) {
1910 const YAMLVFSEntry &Entry = Entries.front();
1911 startDirectory(path::parent_path(Entry.VPath));
1912
1913 StringRef RPath = Entry.RPath;
1914 if (UseOverlayRelative) {
1915 unsigned OverlayDirLen = OverlayDir.size();
1916 assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1917 "Overlay dir must be contained in RPath");
1918 RPath = RPath.slice(OverlayDirLen, RPath.size());
1919 }
1920
1921 writeEntry(path::filename(Entry.VPath), RPath);
1922
1923 for (const auto &Entry : Entries.slice(1)) {
1924 StringRef Dir = path::parent_path(Entry.VPath);
1925 if (Dir == DirStack.back())
1926 OS << ",\n";
1927 else {
1928 while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
1929 OS << "\n";
1930 endDirectory();
1931 }
1932 OS << ",\n";
1933 startDirectory(Dir);
1934 }
1935 StringRef RPath = Entry.RPath;
1936 if (UseOverlayRelative) {
1937 unsigned OverlayDirLen = OverlayDir.size();
1938 assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1939 "Overlay dir must be contained in RPath");
1940 RPath = RPath.slice(OverlayDirLen, RPath.size());
1941 }
1942 writeEntry(path::filename(Entry.VPath), RPath);
1943 }
1944
1945 while (!DirStack.empty()) {
1946 OS << "\n";
1947 endDirectory();
1948 }
1949 OS << "\n";
1950 }
1951
1952 OS << " ]\n"
1953 << "}\n";
1954 }
1955
write(llvm::raw_ostream & OS)1956 void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
1957 llvm::sort(Mappings, [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) {
1958 return LHS.VPath < RHS.VPath;
1959 });
1960
1961 JSONWriter(OS).write(Mappings, UseExternalNames, IsCaseSensitive,
1962 IsOverlayRelative, OverlayDir);
1963 }
1964
VFSFromYamlDirIterImpl(const Twine & _Path,RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,bool IterateExternalFS,FileSystem & ExternalFS,std::error_code & EC)1965 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
1966 const Twine &_Path,
1967 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
1968 RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
1969 bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC)
1970 : Dir(_Path.str()), Current(Begin), End(End),
1971 IterateExternalFS(IterateExternalFS), ExternalFS(ExternalFS) {
1972 EC = incrementImpl(/*IsFirstTime=*/true);
1973 }
1974
increment()1975 std::error_code VFSFromYamlDirIterImpl::increment() {
1976 return incrementImpl(/*IsFirstTime=*/false);
1977 }
1978
incrementExternal()1979 std::error_code VFSFromYamlDirIterImpl::incrementExternal() {
1980 assert(!(IsExternalFSCurrent && ExternalDirIter == directory_iterator()) &&
1981 "incrementing past end");
1982 std::error_code EC;
1983 if (IsExternalFSCurrent) {
1984 ExternalDirIter.increment(EC);
1985 } else if (IterateExternalFS) {
1986 ExternalDirIter = ExternalFS.dir_begin(Dir, EC);
1987 IsExternalFSCurrent = true;
1988 if (EC && EC != errc::no_such_file_or_directory)
1989 return EC;
1990 EC = {};
1991 }
1992 if (EC || ExternalDirIter == directory_iterator()) {
1993 CurrentEntry = directory_entry();
1994 } else {
1995 CurrentEntry = *ExternalDirIter;
1996 }
1997 return EC;
1998 }
1999
incrementContent(bool IsFirstTime)2000 std::error_code VFSFromYamlDirIterImpl::incrementContent(bool IsFirstTime) {
2001 assert((IsFirstTime || Current != End) && "cannot iterate past end");
2002 if (!IsFirstTime)
2003 ++Current;
2004 while (Current != End) {
2005 SmallString<128> PathStr(Dir);
2006 llvm::sys::path::append(PathStr, (*Current)->getName());
2007 sys::fs::file_type Type;
2008 switch ((*Current)->getKind()) {
2009 case RedirectingFileSystem::EK_Directory:
2010 Type = sys::fs::file_type::directory_file;
2011 break;
2012 case RedirectingFileSystem::EK_File:
2013 Type = sys::fs::file_type::regular_file;
2014 break;
2015 }
2016 CurrentEntry = directory_entry(PathStr.str(), Type);
2017 return {};
2018 }
2019 return incrementExternal();
2020 }
2021
incrementImpl(bool IsFirstTime)2022 std::error_code VFSFromYamlDirIterImpl::incrementImpl(bool IsFirstTime) {
2023 while (true) {
2024 std::error_code EC = IsExternalFSCurrent ? incrementExternal()
2025 : incrementContent(IsFirstTime);
2026 if (EC || CurrentEntry.path().empty())
2027 return EC;
2028 StringRef Name = llvm::sys::path::filename(CurrentEntry.path());
2029 if (SeenNames.insert(Name).second)
2030 return EC; // name not seen before
2031 }
2032 llvm_unreachable("returned above");
2033 }
2034
recursive_directory_iterator(FileSystem & FS_,const Twine & Path,std::error_code & EC)2035 vfs::recursive_directory_iterator::recursive_directory_iterator(
2036 FileSystem &FS_, const Twine &Path, std::error_code &EC)
2037 : FS(&FS_) {
2038 directory_iterator I = FS->dir_begin(Path, EC);
2039 if (I != directory_iterator()) {
2040 State = std::make_shared<detail::RecDirIterState>();
2041 State->Stack.push(I);
2042 }
2043 }
2044
2045 vfs::recursive_directory_iterator &
increment(std::error_code & EC)2046 recursive_directory_iterator::increment(std::error_code &EC) {
2047 assert(FS && State && !State->Stack.empty() && "incrementing past end");
2048 assert(!State->Stack.top()->path().empty() && "non-canonical end iterator");
2049 vfs::directory_iterator End;
2050
2051 if (State->HasNoPushRequest)
2052 State->HasNoPushRequest = false;
2053 else {
2054 if (State->Stack.top()->type() == sys::fs::file_type::directory_file) {
2055 vfs::directory_iterator I = FS->dir_begin(State->Stack.top()->path(), EC);
2056 if (I != End) {
2057 State->Stack.push(I);
2058 return *this;
2059 }
2060 }
2061 }
2062
2063 while (!State->Stack.empty() && State->Stack.top().increment(EC) == End)
2064 State->Stack.pop();
2065
2066 if (State->Stack.empty())
2067 State.reset(); // end iterator
2068
2069 return *this;
2070 }
2071