1 /** @file fs_main.cpp
2 * @ingroup fs
3 *
4 * @authors Copyright © 2003-2017 Jaakko Keränen <jaakko.keranen@iki.fi>
5 * @authors Copyright © 2006-2013 Daniel Swanson <danij@dengine.net>
6 * @authors Copyright © 2006 Jamie Jones <jamie_jones_au@yahoo.com.au>
7 * @authors Copyright © 1993-1996 by id Software, Inc.
8 *
9 * @par License
10 * GPL: http://www.gnu.org/licenses/gpl.html
11 *
12 * <small>This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by the
14 * Free Software Foundation; either version 2 of the License, or (at your
15 * option) any later version. This program is distributed in the hope that it
16 * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
17 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
18 * Public License for more details. You should have received a copy of the GNU
19 * General Public License along with this program; if not, write to the Free
20 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
21 * 02110-1301 USA</small>
22 */
23
24 #include "doomsday/filesys/fs_main.h"
25 #include "doomsday/filesys/fs_util.h"
26 #include "doomsday/console/exec.h"
27 #include "doomsday/console/cmd.h"
28 #include "doomsday/filesys/file.h"
29 #include "doomsday/filesys/fileid.h"
30 #include "doomsday/filesys/fileinfo.h"
31 #include "doomsday/filesys/lumpindex.h"
32 #include "doomsday/filesys/wad.h"
33 #include "doomsday/filesys/zip.h"
34
35 #include <ctime>
36 #include <QDir>
37 #include <QList>
38 #include <QtAlgorithms>
39 #include <de/App>
40 #include <de/Log>
41 #include <de/NativePath>
42 #include <de/LogBuffer>
43 #include <de/memory.h>
44 #include <de/findfile.h>
45 #include <de/FileSystem>
46
47 using namespace de;
48
49 extern uint F_GetLastModified(char const *path);
50
51 static FS1 *fileSystem;
52
53 typedef QList<FileId> FileIds;
54
55 /**
56 * Virtual (file) path => Lump name mapping.
57 *
58 * @todo We can't presently use a Map or Hash for these. Although the paths are
59 * unique, several of the existing algorithms which match using patterns
60 * assume they are sorted in a quasi load ordering.
61 */
62 typedef QPair<QString, QString> LumpMapping;
63 typedef QList<LumpMapping> LumpMappings;
64
65 /**
66 * Virtual file-directory mapping.
67 * Maps one (absolute) path in the virtual file system to another.
68 *
69 * @todo We can't presently use a Map or Hash for these. Although the paths are
70 * unique, several of the existing algorithms which match using patterns
71 * assume they are sorted in a quasi load ordering.
72 */
73 typedef QPair<QString, QString> PathMapping;
74 typedef QList<PathMapping> PathMappings;
75
76 /**
77 * @note Performance is O(n).
78 * @return @c iterator pointing to list->end() if not found.
79 */
findListFile(FS1::FileList & list,File1 & file)80 static FS1::FileList::iterator findListFile(FS1::FileList &list, File1 &file)
81 {
82 if (list.empty()) return list.end();
83 // Perform the search.
84 FS1::FileList::iterator i;
85 for (i = list.begin(); i != list.end(); ++i)
86 {
87 if (&file == &(*i)->file())
88 {
89 break; // This is the node we are looking for.
90 }
91 }
92 return i;
93 }
94
95 /**
96 * @note Performance is O(n).
97 * @return @c iterator pointing to list->end() if not found.
98 */
findListFileByPath(FS1::FileList & list,String path)99 static FS1::FileList::iterator findListFileByPath(FS1::FileList &list, String path)
100 {
101 if (list.empty()) return list.end();
102 if (path.isEmpty()) return list.end();
103
104 // Perform the search.
105 FS1::FileList::iterator i;
106 for (i = list.begin(); i != list.end(); ++i)
107 {
108 File1 &file = (*i)->file();
109 if (!file.composePath().compare(path, Qt::CaseInsensitive))
110 {
111 break; // This is the node we are looking for.
112 }
113 }
114 return i;
115 }
116
117 static bool applyPathMapping(ddstring_t *path, PathMapping const &vdm);
118
119 /**
120 * Performs a case-insensitive pattern match. The pattern can contain
121 * wildcards.
122 *
123 * @param filePath Path to match.
124 * @param pattern Pattern with * and ? as wildcards.
125 *
126 * @return @c true, if @a filePath matches the pattern.
127 */
matchFileName(String const & string,String const & pattern)128 static bool matchFileName(String const &string, String const &pattern)
129 {
130 static QChar const ASTERISK('*');
131 static QChar const QUESTION_MARK('?');
132
133 QChar const *in = string.constData(), *st = pattern.constData();
134
135 while (!in->isNull())
136 {
137 if (*st == ASTERISK)
138 {
139 st++;
140 continue;
141 }
142
143 if (*st != QUESTION_MARK && st->toLower() != in->toLower())
144 {
145 // A mismatch. Hmm. Go back to a previous '*'.
146 while (st >= pattern && *st != ASTERISK) { st--; }
147
148 if (st < pattern)
149 return false; // No match!
150 // The asterisk lets us continue.
151 }
152
153 // This character of the pattern is OK.
154 st++;
155 in++;
156 }
157
158 // Match is good if the end of the pattern was reached.
159
160 // Skip remaining asterisks.
161 while (*st == ASTERISK) { st++; }
162
163 return st->isNull();
164 }
165
DENG2_PIMPL(FS1)166 DENG2_PIMPL(FS1)
167 {
168 bool loadingForStartup; ///< @c true= Flag newly opened files as "startup".
169
170 FileList openFiles; ///< List of currently opened files.
171 FileList loadedFiles; ///< List of all loaded files present in the system.
172 uint loadedFilesCRC;
173 FileIds fileIds; ///< Database of unique identifiers for all loaded/opened files.
174
175 LumpIndex primaryIndex; ///< Primary index of all files in the system.
176 LumpIndex zipFileIndex; ///< Type-specific index for ZipFiles.
177
178 LumpMappings lumpMappings; ///< Virtual (file) path => Lump name mapping.
179 PathMappings pathMappings; ///< Virtual file-directory mapping.
180
181 Schemes schemes; ///< File subsets.
182
183 Impl(Public *i)
184 : Base(i)
185 , loadingForStartup(true)
186 , loadedFilesCRC (0)
187 , zipFileIndex (true/*paths are unique*/)
188 {}
189
190 ~Impl()
191 {
192 clearLoadedFiles();
193 clearOpenFiles();
194 clearIndexes();
195
196 fileIds.clear(); // Should be NOP, if bookkeeping is correct.
197
198 pathMappings.clear();
199 lumpMappings.clear();
200
201 clearAllSchemes();
202 }
203
204 void clearAllSchemes()
205 {
206 DENG2_FOR_EACH(Schemes, i, schemes)
207 {
208 delete *i;
209 }
210 schemes.clear();
211 }
212
213 /// @return @c true if the FileId associated with @a path was released.
214 bool releaseFileId(String path)
215 {
216 if (!path.isEmpty())
217 {
218 FileId fileId = FileId::fromPath(path);
219 FileIds::iterator place = qLowerBound(fileIds.begin(), fileIds.end(), fileId);
220 if (place != fileIds.end() && *place == fileId)
221 {
222 LOGDEV_RES_XVERBOSE_DEBUGONLY("Released FileId %s - \"%s\"", *place << fileId.path());
223 fileIds.erase(place);
224 return true;
225 }
226 }
227 return false;
228 }
229
230 void clearLoadedFiles(LumpIndex *index = 0)
231 {
232 loadedFilesCRC = 0;
233
234 // Unload in reverse load order.
235 for (int i = loadedFiles.size() - 1; i >= 0; i--)
236 {
237 File1 &file = loadedFiles[i]->file();
238 if (!index || index->catalogues(file))
239 {
240 self().deindex(file);
241 delete &file;
242 }
243 }
244 }
245
246 void clearOpenFiles()
247 {
248 while (!openFiles.isEmpty()){ delete openFiles.takeLast(); }
249 }
250
251 void clearIndexes()
252 {
253 primaryIndex.clear();
254 zipFileIndex.clear();
255 }
256
257 String findPath(de::Uri const &search)
258 {
259 // Within a subspace scheme?
260 try
261 {
262 Scheme &scheme = self().scheme(search.scheme());
263 LOG_RES_XVERBOSE("Using scheme '%s'...", scheme.name());
264
265 // Ensure the scheme's index is up to date.
266 scheme.rebuild();
267
268 // The in-scheme name is the file name sans extension.
269 String name = search.path().lastSegment().toString().fileNameWithoutExtension();
270
271 // Perform the search.
272 Scheme::FoundNodes foundNodes;
273 if (scheme.findAll(name, foundNodes))
274 {
275 // At least one node name was matched (perhaps partially).
276 DENG2_FOR_EACH_CONST(Scheme::FoundNodes, i, foundNodes)
277 {
278 PathTree::Node &node = **i;
279 if (!node.comparePath(search.path(), PathTree::NoBranch))
280 {
281 // This is the file we are looking for.
282 return node.path();
283 }
284 }
285 }
286
287 /// @todo Should return not-found here but some searches are still dependent
288 /// on falling back to a wider search. -ds
289 }
290 catch (UnknownSchemeError const &)
291 {} // Ignore this error.
292
293 // Try a wider search of the whole virtual file system.
294 QScopedPointer<File1> file(openFile(search.path(), "rb", 0, true /* allow duplicates */));
295 if (!file.isNull())
296 {
297 return file->composePath();
298 }
299
300 return ""; // Not found.
301 }
302
303 File1 *findLump(String path, String const & /*mode*/)
304 {
305 if (path.isEmpty()) return 0;
306
307 // We must have an absolute path - prepend the base path if necessary.
308 if (QDir::isRelativePath(path))
309 {
310 path = App_BasePath() / path;
311 }
312
313 // First check the Zip lump index.
314 lumpnum_t lumpNum = zipFileIndex.findLast(path);
315 if (lumpNum >= 0)
316 {
317 return &zipFileIndex[lumpNum];
318 }
319
320 // Nope. Any applicable dir/WAD redirects?
321 if (!lumpMappings.empty())
322 {
323 DENG2_FOR_EACH_CONST(LumpMappings, i, lumpMappings)
324 {
325 LumpMapping const &mapping = *i;
326 if (mapping.first.compare(path)) continue;
327
328 lumpnum_t lumpNum = self().lumpNumForName(mapping.second);
329 if (lumpNum < 0) continue;
330
331 return &self().lump(lumpNum);
332 }
333 }
334
335 return 0;
336 }
337
338 FILE *findAndOpenNativeFile(String path, String const &mymode, String &foundPath)
339 {
340 DENG2_ASSERT(!path.isEmpty());
341
342 // We must have an absolute path - prepend the CWD if necessary.
343 path = NativePath::workPath().withSeparators('/') / path;
344
345 // Translate mymode to the C-lib's fopen() mode specifiers.
346 char mode[8] = "";
347 if (mymode.contains('r')) strcat(mode, "r");
348 else if (mymode.contains('w')) strcat(mode, "w");
349 if (mymode.contains('b')) strcat(mode, "b");
350 else if (mymode.contains('t')) strcat(mode, "t");
351
352 // First try a real native file at this absolute path.
353 NativePath nativePath = NativePath(path);
354 FILE *nativeFile = fopen(nativePath.toUtf8().constData(), mode);
355 if (nativeFile)
356 {
357 foundPath = nativePath.expand().withSeparators('/');
358 return nativeFile;
359 }
360
361 // Nope. Any applicable virtual directory mappings?
362 if (!pathMappings.empty())
363 {
364 QByteArray pathUtf8 = path.toUtf8();
365 AutoStr *mapped = AutoStr_NewStd();
366 DENG2_FOR_EACH_CONST(PathMappings, i, pathMappings)
367 {
368 Str_Set(mapped, pathUtf8.constData());
369 if (!applyPathMapping(mapped, *i)) continue;
370 // The mapping was successful.
371
372 nativePath = NativePath(Str_Text(mapped));
373 nativeFile = fopen(nativePath.toUtf8().constData(), mode);
374 if (nativeFile)
375 {
376 foundPath = nativePath.expand().withSeparators('/');
377 return nativeFile;
378 }
379 }
380 }
381
382 return 0;
383 }
384
385 File1 *openFile(String path, String const& mode, size_t baseOffset,
386 bool allowDuplicate)
387 {
388 if (path.isEmpty()) return 0;
389
390 LOG_AS("FS1::openFile");
391
392 // We must have an absolute path.
393 path = App_BasePath() / path;
394
395 LOG_RES_XVERBOSE("Trying \"%s\"...", NativePath(path).pretty());
396
397 bool const reqNativeFile = mode.contains('f');
398
399 FileHandle *hndl = 0;
400 FileInfo info; // The temporary info descriptor.
401
402 // First check for lumps?
403 if (!reqNativeFile)
404 {
405 if (File1 *found = findLump(path, mode))
406 {
407 // Do not read files twice.
408 if (!allowDuplicate && !self().checkFileId(found->composeUri())) return 0;
409
410 // Get a handle to the lump we intend to open.
411 /// @todo The way this buffering works is nonsensical it should not be done here
412 /// but should instead be deferred until the content of the lump is read.
413 hndl = FileHandle::fromLump(*found);
414
415 // Prepare a temporary info descriptor.
416 info = found->info();
417 }
418 }
419
420 // Not found? - try a native file.
421 if (!hndl)
422 {
423 String foundPath;
424 if (FILE *found = findAndOpenNativeFile(path, mode, foundPath))
425 {
426 // Do not read files twice.
427 if (!allowDuplicate && !self().checkFileId(de::makeUri(foundPath)))
428 {
429 fclose(found);
430 return 0;
431 }
432
433 // Acquire a handle on the file we intend to open.
434 hndl = FileHandle::fromNativeFile(*found, baseOffset);
435
436 // Prepare the temporary info descriptor.
437 info = FileInfo(F_GetLastModified(foundPath.toUtf8().constData()));
438 }
439 }
440
441 // Nothing?
442 if (!hndl) return 0;
443
444 // Search path is used here rather than found path as the latter may have
445 // been mapped to another location. We want the file to be attributed with
446 // the path it is to be known by throughout the virtual file system.
447
448 File1 &file = self().interpret(*hndl, path, info);
449
450 if (loadingForStartup)
451 {
452 file.setStartup(true);
453 }
454
455 return &file;
456 }
457 };
458
FS1()459 FS1::FS1() : d(new Impl(this))
460 {}
461
createScheme(String name,Scheme::Flags flags)462 FS1::Scheme &FS1::createScheme(String name, Scheme::Flags flags)
463 {
464 DENG2_ASSERT(name.length() >= Scheme::min_name_length);
465
466 // Ensure this is a unique name.
467 if (knownScheme(name)) return scheme(name);
468
469 // Create a new scheme.
470 Scheme *newScheme = new Scheme(name, flags);
471 d->schemes.insert(name.toLower(), newScheme);
472 return *newScheme;
473 }
474
index(File1 & file)475 void FS1::index(File1 &file)
476 {
477 #ifdef DENG_DEBUG
478 // Ensure this hasn't yet been indexed.
479 FileList::const_iterator found = findListFile(d->loadedFiles, file);
480 if (found != d->loadedFiles.end())
481 throw Error("FS1::index", "File \"" + NativePath(file.composePath()).pretty() + "\" has already been indexed");
482 #endif
483
484 // Publish lumps to one or more indexes?
485 if (Zip *zip = maybeAs<Zip>(file))
486 {
487 if (!zip->isEmpty())
488 {
489 // Insert the lumps into their rightful places in the index.
490 for (int i = 0; i < zip->lumpCount(); ++i)
491 {
492 File1 &lump = zip->lump(i);
493
494 d->primaryIndex.catalogLump(lump);
495
496 // Zip files go into a special ZipFile index as well.
497 d->zipFileIndex.catalogLump(lump);
498 }
499 }
500 }
501 else if (Wad *wad = maybeAs<Wad>(file))
502 {
503 if (!wad->isEmpty())
504 {
505 // Insert the lumps into their rightful places in the index.
506 for (int i = 0; i < wad->lumpCount(); ++i)
507 {
508 d->primaryIndex.catalogLump(wad->lump(i));
509 }
510 }
511 }
512
513 // Add a handle to the loaded files list.
514 FileHandle *hndl = FileHandle::fromFile(file);
515 d->loadedFiles.push_back(hndl); hndl->setList(reinterpret_cast<de::FileList *>(&d->loadedFiles));
516 d->loadedFilesCRC = 0;
517 }
518
deindex(File1 & file)519 void FS1::deindex(File1 &file)
520 {
521 FileList::iterator found = findListFile(d->loadedFiles, file);
522 if (found == d->loadedFiles.end()) return; // Most peculiar..
523
524 FileHandle *fileHandle = *found;
525
526 d->releaseFileId(file.composePath());
527
528 d->zipFileIndex.pruneByFile(file);
529 d->primaryIndex.pruneByFile(file);
530
531 d->loadedFiles.erase(found);
532 d->loadedFilesCRC = 0;
533
534 delete fileHandle;
535 }
536
find(de::Uri const & search)537 File1 &FS1::find(de::Uri const &search)
538 {
539 LOG_AS("FS1::find");
540 if (!search.isEmpty())
541 {
542 try
543 {
544 String searchPath = search.resolved();
545
546 // Convert to an absolute path.
547 if (!QDir::isAbsolutePath(searchPath))
548 {
549 searchPath = App_BasePath() / searchPath;
550 }
551
552 FileList::iterator found = findListFileByPath(d->loadedFiles, searchPath);
553 if (found != d->loadedFiles.end())
554 {
555 DENG_ASSERT((*found)->hasFile());
556 return (*found)->file();
557 }
558 }
559 catch (de::Uri::ResolveError const &er)
560 {
561 // Log but otherwise ignore unresolved paths.
562 LOGDEV_RES_VERBOSE(er.asText());
563 }
564 }
565
566 /// @throw NotFoundError No files found matching the search term.
567 throw NotFoundError("FS1::find", "No files found matching '" + search.compose() + "'");
568 }
569
findPath(de::Uri const & search,int flags,ResourceClass & rclass)570 String FS1::findPath(de::Uri const &search, int flags, ResourceClass &rclass)
571 {
572 LOG_AS("FS1::findPath");
573 if (!search.isEmpty())
574 {
575 try
576 {
577 String searchPath = search.resolved();
578
579 // If an extension was specified, first look for files of the same type.
580 String ext = searchPath.fileNameExtension();
581 if (!ext.isEmpty() && ext.compare(".*"))
582 {
583 String found = d->findPath(de::Uri(search.scheme(), searchPath));
584 if (!found.isEmpty()) return found;
585
586 // If we are looking for a particular file type, get out of here.
587 if (flags & RLF_MATCH_EXTENSION) return "";
588 }
589
590 if (isNullResourceClass(rclass) || !rclass.fileTypeCount()) return "";
591
592 /*
593 * Try each expected file type name extension for this resource class.
594 */
595 String searchPathWithoutFileNameExtension = searchPath.fileNamePath() / searchPath.fileNameWithoutExtension();
596
597 DENG2_FOR_EACH_CONST(ResourceClass::FileTypes, typeIt, rclass.fileTypes())
598 {
599 DENG2_FOR_EACH_CONST(QStringList, i, (*typeIt)->knownFileNameExtensions())
600 {
601 String const &ext = *i;
602 String found = d->findPath(de::Uri(search.scheme(), searchPathWithoutFileNameExtension + ext));
603 if (!found.isEmpty()) return found;
604 }
605 };
606 }
607 catch (de::Uri::ResolveError const &er)
608 {
609 // Log but otherwise ignore unresolved paths.
610 LOGDEV_RES_VERBOSE(er.asText());
611 }
612 }
613
614 /// @throw NotFoundError No files found matching the search term.
615 throw NotFoundError("FS1::findPath", "No paths found matching '" + search.compose() + "'");
616 }
617
findPath(de::Uri const & search,int flags)618 String FS1::findPath(de::Uri const &search, int flags)
619 {
620 return findPath(search, flags, ResourceClass::classForId(RC_NULL));
621 }
622
623 #ifdef DENG_DEBUG
printFileIds(FileIds const & fileIds)624 static void printFileIds(FileIds const &fileIds)
625 {
626 uint idx = 0;
627 DENG2_FOR_EACH_CONST(FileIds, i, fileIds)
628 {
629 LOGDEV_RES_MSG(" %u - %s : \"%s\"") << idx << *i << i->path();
630 ++idx;
631 }
632 }
633 #endif
634
635 #ifdef DENG_DEBUG
printFileList(FS1::FileList & list)636 static void printFileList(FS1::FileList &list)
637 {
638 uint idx = 0;
639 DENG2_FOR_EACH_CONST(FS1::FileList, i, list)
640 {
641 FileHandle &hndl = **i;
642 File1 &file = hndl.file();
643 FileId fileId = FileId::fromPath(file.composePath());
644
645 LOGDEV_RES_VERBOSE(" %c%d: %s - \"%s\" (handle: %p)")
646 << (file.hasStartup()? '*' : ' ') << idx
647 << fileId << fileId.path() << &hndl;
648 ++idx;
649 }
650 }
651 #endif
652
unloadAllNonStartupFiles()653 int FS1::unloadAllNonStartupFiles()
654 {
655 #ifdef DENG_DEBUG
656 // List all open files with their identifiers.
657 if (LogBuffer::get().isEnabled(LogEntry::Generic | LogEntry::Verbose))
658 {
659 LOGDEV_RES_VERBOSE("Open files at reset:");
660 printFileList(d->openFiles);
661 LOGDEV_RES_VERBOSE("End\n");
662 }
663 #endif
664
665 // Perform non-startup file unloading (in reverse load order).
666 int numUnloadedFiles = 0;
667 for (int i = d->loadedFiles.size() - 1; i >= 0; i--)
668 {
669 File1 &file = d->loadedFiles[i]->file();
670 if (file.hasStartup()) continue;
671
672 deindex(file);
673 delete &file;
674 numUnloadedFiles += 1;
675 }
676
677 #ifdef DENG_DEBUG
678 // Sanity check: look for orphaned identifiers.
679 if (!d->fileIds.empty())
680 {
681 LOGDEV_RES_MSG("Orphan FileIds:");
682 printFileIds(d->fileIds);
683 }
684 #endif
685
686 return numUnloadedFiles;
687 }
688
checkFileId(de::Uri const & path)689 bool FS1::checkFileId(de::Uri const &path)
690 {
691 if (!accessFile(path)) return false;
692
693 // Calculate the identifier.
694 FileId fileId = FileId::fromPath(path.compose());
695 FileIds::iterator place = qLowerBound(d->fileIds.begin(), d->fileIds.end(), fileId);
696 if (place != d->fileIds.end() && *place == fileId) return false;
697
698 LOGDEV_RES_XVERBOSE_DEBUGONLY("checkFileId \"%s\" => %s", fileId.path() << fileId); /* path() is debug-only */
699
700 d->fileIds.insert(place, fileId);
701 return true;
702 }
703
resetFileIds()704 void FS1::resetFileIds()
705 {
706 d->fileIds.clear();
707 }
708
endStartup()709 void FS1::endStartup()
710 {
711 d->loadingForStartup = false;
712 }
713
nameIndex() const714 LumpIndex const &FS1::nameIndex() const
715 {
716 return d->primaryIndex;
717 }
718
lumpNumForName(String name)719 lumpnum_t FS1::lumpNumForName(String name)
720 {
721 LOG_AS("FS1::lumpNumForName");
722
723 if (name.isEmpty()) return -1;
724
725 // Append a .lmp extension if none is specified.
726 if (name.fileNameExtension().isEmpty())
727 {
728 name += ".lmp";
729 }
730
731 // Perform the search.
732 return d->primaryIndex.findLast(Path(name));
733 }
734
releaseFile(File1 & file)735 void FS1::releaseFile(File1 &file)
736 {
737 for (int i = d->openFiles.size() - 1; i >= 0; i--)
738 {
739 FileHandle &hndl = *(d->openFiles[i]);
740 if (&hndl.file() == &file)
741 {
742 d->openFiles.removeAt(i);
743 }
744 }
745 }
746
747 /// @return @c NULL= Not found.
findFirstWadFile(FS1::FileList & list,bool custom)748 static Wad *findFirstWadFile(FS1::FileList &list, bool custom)
749 {
750 if (list.empty()) return 0;
751 DENG2_FOR_EACH(FS1::FileList, i, list)
752 {
753 File1 &file = (*i)->file();
754 if (custom != file.hasCustom()) continue;
755
756 if (Wad *wad = maybeAs<Wad>(file))
757 {
758 return wad;
759 }
760 }
761 return 0;
762 }
763
loadedFilesCRC()764 uint FS1::loadedFilesCRC()
765 {
766 if (!d->loadedFilesCRC)
767 {
768 /**
769 * We define the CRC as that of the lump directory of the first loaded IWAD.
770 * @todo Really kludgy...
771 */
772 // CRC not calculated yet, let's do it now.
773 Wad *iwad = findFirstWadFile(d->loadedFiles, false/*not-custom*/);
774 if (!iwad) return 0;
775 d->loadedFilesCRC = iwad->calculateCRC();
776 }
777 return d->loadedFilesCRC;
778 }
779
loadedFiles() const780 FS1::FileList const &FS1::loadedFiles() const
781 {
782 return d->loadedFiles;
783 }
784
findAll(bool (* predicate)(File1 & file,void * parameters),void * parameters,FS1::FileList & found) const785 int FS1::findAll(bool (*predicate)(File1 &file, void *parameters), void *parameters,
786 FS1::FileList &found) const
787 {
788 int numFound = 0;
789 DENG2_FOR_EACH_CONST(FS1::FileList, i, d->loadedFiles)
790 {
791 // Interested in this file?
792 if (predicate && !predicate((*i)->file(), parameters)) continue; // Nope.
793
794 found.push_back(*i);
795 numFound += 1;
796 }
797 return numFound;
798 }
799
findAllPaths(Path searchPattern,int flags,FS1::PathList & found)800 int FS1::findAllPaths(Path searchPattern, int flags, FS1::PathList &found)
801 {
802 int const numFoundSoFar = found.count();
803
804 // We must have an absolute path - prepend the base path if necessary.
805 if (!QDir::isAbsolutePath(searchPattern))
806 {
807 searchPattern = App_BasePath() / searchPattern;
808 }
809
810 /*
811 * Check the Zip directory.
812 */
813 DENG2_FOR_EACH_CONST(LumpIndex::Lumps, i, d->zipFileIndex.allLumps())
814 {
815 File1 const &lump = **i;
816 PathTree::Node const &node = lump.directoryNode();
817
818 String filePath;
819 bool patternMatched;
820 if (!(flags & SearchPath::NoDescend))
821 {
822 filePath = lump.composePath();
823 patternMatched = matchFileName(filePath, searchPattern);
824 }
825 else
826 {
827 patternMatched = !node.comparePath(searchPattern, PathTree::MatchFull);
828 }
829
830 if (!patternMatched) continue;
831
832 // Not yet composed the path?
833 if (filePath.isEmpty())
834 {
835 filePath = lump.composePath();
836 }
837
838 found.push_back(PathListItem(filePath, !node.isLeaf()? A_SUBDIR : 0));
839 }
840
841 /*
842 * Check the dir/WAD direcs.
843 */
844 if (!d->lumpMappings.empty())
845 {
846 DENG2_FOR_EACH_CONST(LumpMappings, i, d->lumpMappings)
847 {
848 if (!matchFileName(i->first, searchPattern)) continue;
849
850 found.push_back(PathListItem(i->first, 0 /*only filepaths (i.e., leaves) can be mapped to lumps*/));
851 }
852
853 /// @todo Shouldn't these be sorted? -ds
854 }
855
856 /*
857 * Check native paths.
858 */
859 String searchDirectory = searchPattern.toString().fileNamePath();
860 if (!searchDirectory.isEmpty())
861 {
862 QByteArray searchDirectoryUtf8 = searchDirectory.toUtf8();
863 PathList nativeFilePaths;
864 AutoStr *wildPath = AutoStr_NewStd();
865 Str_Reserve(wildPath, searchDirectory.length() + 2 + 16); // Conservative estimate.
866
867 for (int i = -1; i < (int)d->pathMappings.count(); ++i)
868 {
869 Str_Clear(wildPath);
870 Str_Appendf(wildPath, "%s/", searchDirectoryUtf8.constData());
871
872 if (i > -1)
873 {
874 // Possible mapping?
875 if (!applyPathMapping(wildPath, d->pathMappings[i])) continue;
876 }
877 Str_AppendChar(wildPath, '*');
878
879 FindData fd;
880 if (!FindFile_FindFirst(&fd, Str_Text(wildPath)))
881 {
882 // First path found.
883 do
884 {
885 // Ignore relative directory symbolics.
886 if (Str_Compare(&fd.name, ".") && Str_Compare(&fd.name, ".."))
887 {
888 String foundPath = searchDirectory / NativePath(Str_Text(&fd.name)).withSeparators('/');
889 if (!matchFileName(foundPath, searchPattern)) continue;
890
891 nativeFilePaths.push_back(PathListItem(foundPath, fd.attrib));
892 }
893 } while (!FindFile_FindNext(&fd));
894 }
895 FindFile_Finish(&fd);
896 }
897
898 // Sort the native file paths.
899 qSort(nativeFilePaths.begin(), nativeFilePaths.end());
900
901 // Add the native file paths to the found results.
902 found.append(nativeFilePaths);
903 }
904
905 return found.count() - numFoundSoFar;
906 }
907
interpret(FileHandle & hndl,String filePath,FileInfo const & info)908 File1 &FS1::interpret(FileHandle &hndl, String filePath, FileInfo const &info)
909 {
910 DENG2_ASSERT(!filePath.isEmpty());
911
912 File1 *interpretedFile = 0;
913
914 // Firstly try the interpreter for the guessed resource types.
915 FileType const &ftypeGuess = DD_GuessFileTypeFromFileName(filePath);
916 if (NativeFileType const* fileType = dynamic_cast<NativeFileType const *>(&ftypeGuess))
917 {
918 interpretedFile = fileType->interpret(hndl, filePath, info);
919 }
920
921 // If not yet interpreted - try each recognisable format in order.
922 if (!interpretedFile)
923 {
924 FileTypes const &fileTypes = DD_FileTypes();
925 DENG2_FOR_EACH_CONST(FileTypes, i, fileTypes)
926 {
927 if (NativeFileType const *fileType = dynamic_cast<NativeFileType const *>(*i))
928 {
929 // Already tried this?
930 if (fileType == &ftypeGuess) continue;
931
932 interpretedFile = fileType->interpret(hndl, filePath, info);
933 if (interpretedFile) break;
934 }
935 }
936 }
937
938 // Still not interpreted?
939 if (!interpretedFile)
940 {
941 // Use a generic file.
942 File1 *container = (hndl.hasFile() && hndl.file().isContained())? &hndl.file().container() : 0;
943 interpretedFile = new File1(&hndl, filePath, info, container);
944 }
945
946 DENG2_ASSERT(interpretedFile);
947 return *interpretedFile;
948 }
949
openFile(String const & path,String const & mode,size_t baseOffset,bool allowDuplicate)950 FileHandle &FS1::openFile(String const &path, String const &mode, size_t baseOffset, bool allowDuplicate)
951 {
952 #ifdef DENG_DEBUG
953 for (int i = 0; i < mode.length(); ++i)
954 {
955 if (mode[i] != 'r' && mode[i] != 't' && mode[i] != 'b' && mode[i] != 'f')
956 throw Error("FS1::openFile", "Unknown argument in mode string '" + mode + "'");
957 }
958 #endif
959
960 File1 *file = d->openFile(path, mode, baseOffset, allowDuplicate);
961 if (!file) throw NotFoundError("FS1::openFile", "No files found matching '" + path + "'");
962
963 // Add a handle to the opened files list.
964 FileHandle &hndl = *FileHandle::fromFile(*file);
965 d->openFiles.push_back(&hndl); hndl.setList(reinterpret_cast<de::FileList *>(&d->openFiles));
966 return hndl;
967 }
968
openLump(File1 & lump)969 FileHandle &FS1::openLump(File1 &lump)
970 {
971 // Add a handle to the opened files list.
972 FileHandle &hndl = *FileHandle::fromLump(lump);
973 d->openFiles.push_back(&hndl); hndl.setList(reinterpret_cast<de::FileList *>(&d->openFiles));
974 return hndl;
975 }
976
accessFile(de::Uri const & search)977 bool FS1::accessFile(de::Uri const &search)
978 {
979 try
980 {
981 QScopedPointer<File1> file(d->openFile(search.resolved(), "rb", 0, true /* allow duplicates */));
982 return !file.isNull();
983 }
984 catch (de::Uri::ResolveError const &er)
985 {
986 // Log but otherwise ignore unresolved paths.
987 LOGDEV_RES_VERBOSE(er.asText());
988 }
989 return false;
990 }
991
addPathLumpMapping(String lumpName,String destination)992 void FS1::addPathLumpMapping(String lumpName, String destination)
993 {
994 if (lumpName.isEmpty() || destination.isEmpty()) return;
995
996 // We require an absolute path - prepend the CWD if necessary.
997 if (QDir::isRelativePath(destination))
998 {
999 String workPath = DENG2_APP->currentWorkPath().withSeparators('/');
1000 destination = workPath / destination;
1001 }
1002
1003 // Have already mapped this path?
1004 LumpMappings::iterator found = d->lumpMappings.begin();
1005 for (; found != d->lumpMappings.end(); ++found)
1006 {
1007 LumpMapping const &ldm = *found;
1008 if (!ldm.first.compare(destination, Qt::CaseInsensitive))
1009 break;
1010 }
1011
1012 LumpMapping *ldm;
1013 if (found == d->lumpMappings.end())
1014 {
1015 // No. Acquire another mapping.
1016 d->lumpMappings.push_back(LumpMapping(destination, lumpName));
1017 ldm = &d->lumpMappings.back();
1018 }
1019 else
1020 {
1021 // Remap to another lump.
1022 ldm = &*found;
1023 ldm->second = lumpName;
1024 }
1025
1026 LOG_RES_MSG("Path \"%s\" now mapped to lump \"%s\"") << NativePath(ldm->first).pretty() << ldm->second;
1027 }
1028
clearPathLumpMappings()1029 void FS1::clearPathLumpMappings()
1030 {
1031 d->lumpMappings.clear();
1032 }
1033
1034 /// @return @c true iff the mapping matched the path.
applyPathMapping(ddstring_t * path,PathMapping const & pm)1035 static bool applyPathMapping(ddstring_t *path, PathMapping const &pm)
1036 {
1037 if (!path) return false;
1038 QByteArray destUtf8 = pm.first.toUtf8();
1039 AutoStr *dest = AutoStr_FromTextStd(destUtf8.constData());
1040 if (qstrnicmp(Str_Text(path), Str_Text(dest), Str_Length(dest))) return false;
1041
1042 // Replace the beginning with the source path.
1043 QByteArray sourceUtf8 = pm.second.toUtf8();
1044 AutoStr *temp = AutoStr_FromTextStd(sourceUtf8.constData());
1045 Str_PartAppend(temp, Str_Text(path), pm.first.length(), Str_Length(path) - pm.first.length());
1046 Str_Copy(path, temp);
1047 return true;
1048 }
1049
addPathMapping(String source,String destination)1050 void FS1::addPathMapping(String source, String destination)
1051 {
1052 if (source.isEmpty() || destination.isEmpty()) return;
1053
1054 // Have already mapped this source path?
1055 PathMappings::iterator found = d->pathMappings.begin();
1056 for (; found != d->pathMappings.end(); ++found)
1057 {
1058 PathMapping const &pm = *found;
1059 if (!pm.second.compare(source, Qt::CaseInsensitive))
1060 break;
1061 }
1062
1063 PathMapping* pm;
1064 if (found == d->pathMappings.end())
1065 {
1066 // No. Acquire another mapping.
1067 d->pathMappings.push_back(PathMapping(destination, source));
1068 pm = &d->pathMappings.back();
1069 }
1070 else
1071 {
1072 // Remap to another destination.
1073 pm = &*found;
1074 pm->first = destination;
1075 }
1076
1077 LOG_RES_MSG("Path \"%s\" now mapped to \"%s\"")
1078 << NativePath(pm->second).pretty() << NativePath(pm->first).pretty();
1079 }
1080
clearPathMappings()1081 void FS1::clearPathMappings()
1082 {
1083 d->pathMappings.clear();
1084 }
1085
printDirectory(Path path)1086 void FS1::printDirectory(Path path)
1087 {
1088 LOG_RES_MSG(_E(b) "Directory: %s") << NativePath(path).pretty();
1089
1090 // We are interested in *everything*.
1091 path = path / "*";
1092
1093 PathList found;
1094 if (findAllPaths(path, 0, found))
1095 {
1096 qSort(found.begin(), found.end());
1097
1098 DENG2_FOR_EACH_CONST(PathList, i, found)
1099 {
1100 LOG_RES_MSG(" %s") << NativePath(i->path).pretty();
1101 }
1102 }
1103 }
1104
knownScheme(String name)1105 bool FS1::knownScheme(String name)
1106 {
1107 if (!name.isEmpty())
1108 {
1109 Schemes::iterator found = d->schemes.find(name.toLower());
1110 if (found != d->schemes.end()) return true;
1111 }
1112 return false;
1113 }
1114
scheme(String name)1115 FS1::Scheme &FS1::scheme(String name)
1116 {
1117 if (!name.isEmpty())
1118 {
1119 Schemes::iterator found = d->schemes.find(name.toLower());
1120 if (found != d->schemes.end()) return **found;
1121 }
1122 /// @throw UnknownSchemeError An unknown scheme was referenced.
1123 throw UnknownSchemeError("FS1::scheme", "No scheme found matching '" + name + "'");
1124 }
1125
allSchemes()1126 FS1::Schemes const &FS1::allSchemes()
1127 {
1128 return d->schemes;
1129 }
1130
1131 /// Print contents of directories as Doomsday sees them.
D_CMD(Dir)1132 D_CMD(Dir)
1133 {
1134 DENG2_UNUSED(src);
1135 if (argc > 1)
1136 {
1137 for (int i = 1; i < argc; ++i)
1138 {
1139 String path = NativePath(argv[i]).expand().withSeparators('/');
1140 App_FileSystem().printDirectory(path);
1141 }
1142 }
1143 else
1144 {
1145 App_FileSystem().printDirectory(String("/"));
1146 }
1147 return true;
1148 }
1149
1150 /// Dump a copy of a virtual file to the runtime directory.
D_CMD(DumpLump)1151 D_CMD(DumpLump)
1152 {
1153 DENG2_UNUSED2(src, argc);
1154
1155 if (fileSystem)
1156 {
1157 lumpnum_t lumpNum = App_FileSystem().lumpNumForName(argv[1]);
1158 if (lumpNum >= 0)
1159 {
1160 return F_DumpFile(App_FileSystem().lump(lumpNum), 0);
1161 }
1162 LOG_RES_ERROR("No such lump");
1163 return false;
1164 }
1165 return false;
1166 }
1167
1168 /// List virtual files inside containers.
D_CMD(ListLumps)1169 D_CMD(ListLumps)
1170 {
1171 DENG2_UNUSED3(src, argc, argv);
1172
1173 if (!fileSystem) return false;
1174
1175 LumpIndex const &lumpIndex = App_FileSystem().nameIndex();
1176 int const numRecords = lumpIndex.size();
1177 int const numIndexDigits = de::max(3, M_NumDigits(numRecords));
1178
1179 LOG_RES_MSG("LumpIndex %p (%i records):") << &lumpIndex << numRecords;
1180
1181 int idx = 0;
1182 DENG2_FOR_EACH_CONST(LumpIndex::Lumps, i, lumpIndex.allLumps())
1183 {
1184 File1 const &lump = **i;
1185 String containerPath = NativePath(lump.container().composePath()).pretty();
1186 String lumpPath = NativePath(lump.composePath()).pretty();
1187
1188 LOG_RES_MSG(String("%1 - \"%2:%3\" (size: %4 bytes%5)")
1189 .arg(idx++, numIndexDigits, 10, QChar('0'))
1190 .arg(containerPath)
1191 .arg(lumpPath)
1192 .arg(lump.info().size)
1193 .arg(lump.info().isCompressed()? " compressed" : ""));
1194 }
1195 LOG_RES_MSG("---End of lumps---");
1196
1197 return true;
1198 }
1199
1200 /// List presently loaded files in original load order.
D_CMD(ListFiles)1201 D_CMD(ListFiles)
1202 {
1203 DENG2_UNUSED3(src, argc, argv);
1204
1205 LOG_RES_MSG(_E(b) "Loaded Files " _E(l) "(in load order)" _E(w) ":");
1206
1207 int totalFiles = 0;
1208 int totalPackages = 0;
1209 if (fileSystem)
1210 {
1211 FS1::FileList const &allLoadedFiles = App_FileSystem().loadedFiles();
1212 DENG2_FOR_EACH_CONST(FS1::FileList, i, allLoadedFiles)
1213 {
1214 File1 &file = (*i)->file();
1215 uint crc = 0;
1216
1217 int fileCount = 1;
1218 if (de::Zip *zip = maybeAs<de::Zip>(file))
1219 {
1220 fileCount = zip->lumpCount();
1221 }
1222 else if (de::Wad *wad = maybeAs<de::Wad>(file))
1223 {
1224 fileCount = wad->lumpCount();
1225 crc = (!file.hasCustom()? wad->calculateCRC() : 0);
1226 }
1227
1228 LOG_RES_MSG(" %s " _E(2)_E(>) "(%i %s%s)%s")
1229 << NativePath(file.composePath()).pretty()
1230 << fileCount << (fileCount != 1 ? "files" : "file")
1231 << (file.hasStartup()? ", startup" : "")
1232 << (crc? QString(" [%1]").arg(crc, 0, 16) : "");
1233
1234 totalFiles += fileCount;
1235 ++totalPackages;
1236 }
1237 }
1238
1239 LOG_RES_MSG(_E(b)"Total: " _E(.) "%i files in %i packages")
1240 << totalFiles << totalPackages;
1241
1242 if (auto *svFiles = FS::get().tryLocate<Folder const>("/sys/server/public"))
1243 {
1244 LOG_RES_MSG("Server files:\n" _E(m) "%s") << svFiles->contentsAsText();
1245 }
1246
1247 return true;
1248 }
1249
consoleRegister()1250 void FS1::consoleRegister()
1251 {
1252 C_CMD("dir", "", Dir);
1253 C_CMD("ls", "", Dir); // Alias
1254 C_CMD("dir", "s*", Dir);
1255 C_CMD("ls", "s*", Dir); // Alias
1256
1257 C_CMD("dump", "s", DumpLump);
1258 C_CMD("listfiles", "", ListFiles);
1259 C_CMD("listlumps", "", ListLumps);
1260 }
1261
App_FileSystem()1262 FS1 &App_FileSystem()
1263 {
1264 if (!fileSystem) throw Error("App_FileSystem", "File system not yet initialized");
1265 return *fileSystem;
1266 }
1267
App_BasePath()1268 String App_BasePath()
1269 {
1270 return App::app().nativeBasePath().withSeparators('/');
1271 }
1272
F_Init()1273 void F_Init()
1274 {
1275 DENG2_ASSERT(!fileSystem);
1276 fileSystem = new FS1();
1277 }
1278
F_Shutdown()1279 void F_Shutdown()
1280 {
1281 if (!fileSystem) return;
1282 delete fileSystem; fileSystem = 0;
1283 }
1284
F_LumpIndex()1285 void const *F_LumpIndex()
1286 {
1287 return &App_FileSystem().nameIndex();
1288 }
1289