1 // Copyright (C) 2014-2018 Manuel Schneider
2 
3 #include <QDebug>
4 #include <QDir>
5 #include <QDirIterator>
6 #include <QFutureWatcher>
7 #include <QMessageBox>
8 #include <QObject>
9 #include <QPointer>
10 #include <QSettings>
11 #include <QStandardPaths>
12 #include <QtConcurrent>
13 #include <QThreadPool>
14 #include <QTimer>
15 #include <memory>
16 #include <functional>
17 #include <vector>
18 #include <set>
19 #include "configwidget.h"
20 #include "file.h"
21 #include "standardfile.h"
22 #include "extension.h"
23 #include "indextreenode.h"
24 #include "albert/util/offlineindex.h"
25 #include "albert/util/standarditem.h"
26 #include "albert/util/standardactions.h"
27 #include "xdg/iconlookup.h"
28 using namespace Core;
29 using namespace std;
30 
31 
32 namespace {
33 
34 const char* CFG_PATHS           = "paths";
35 const char* CFG_FILTERS         = "filters";
36 const QStringList DEF_FILTERS   = { "inode/directory", "application/*" };
37 const char* CFG_FUZZY           = "fuzzy";
38 const bool  DEF_FUZZY           = false;
39 const char* CFG_INDEX_HIDDEN    = "indexhidden";
40 const bool  DEF_INDEX_HIDDEN    = false;
41 const char* CFG_FOLLOW_SYMLINKS = "follow_symlinks";
42 const bool  DEF_FOLLOW_SYMLINKS = false;
43 const char* CFG_SCAN_INTERVAL   = "scan_interval";
44 const uint  DEF_SCAN_INTERVAL   = 15;
45 
46 
47 
48 class OfflineIndexBuilderVisitor : public Files::Visitor {
49     Core::OfflineIndex &offlineIndex;
50 public:
OfflineIndexBuilderVisitor(Core::OfflineIndex & offlineIndex)51     OfflineIndexBuilderVisitor(Core::OfflineIndex &offlineIndex)
52         : offlineIndex(offlineIndex) { }
53 
visit(Files::IndexTreeNode * node)54     void visit(Files::IndexTreeNode *node) override {
55         for ( const shared_ptr<Files::File> &item : node->items() )
56             offlineIndex.add(item);
57     }
58 };
59 
60 
61 class CounterVisitor : public Files::Visitor {
62 public:
63     uint itemCount = 0;
64     uint dirCount = 0;
visit(Files::IndexTreeNode * node)65     void visit(Files::IndexTreeNode *node) override {
66         ++dirCount;
67         itemCount += node->items().size();
68     }
69 };
70 
71 }
72 
73 
74 
75 /** ***************************************************************************/
76 /** ***************************************************************************/
77 /** ***************************************************************************/
78 /** ***************************************************************************/
79 class Files::Private
80 {
81 public:
Private(Extension * q)82     Private(Extension *q) : q(q), abort(false), rerun(false) {}
83 
84     Extension *q;
85 
86     QPointer<ConfigWidget> widget;
87 
88     QStringList indexRootDirs;
89     IndexSettings indexSettings;
90     vector<shared_ptr<IndexTreeNode>> indexTrees;
91     unique_ptr<QFutureWatcher<Core::OfflineIndex*>> futureWatcher;
92     Core::OfflineIndex offlineIndex;
93     QTimer indexIntervalTimer;
94     bool abort;
95     bool rerun;
96 
97 
98     void finishIndexing();
99     void startIndexing();
100     Core::OfflineIndex *indexFiles();
101 };
102 
103 
104 
105 /** ***************************************************************************/
startIndexing()106 void Files::Private::startIndexing() {
107 
108     // Abort and rerun
109     if ( futureWatcher ) {
110         emit q->statusInfo("Waiting for indexer to shut down ...");
111         abort = true;
112         rerun = true;
113         return;
114     }
115 
116     // Run finishIndexing when the indexing thread finished
117     futureWatcher.reset(new QFutureWatcher<Core::OfflineIndex*>);
118     QObject::connect(futureWatcher.get(), &QFutureWatcher<Core::OfflineIndex*>::finished,
119                      [this](){ this->finishIndexing(); });
120 
121     // Restart the timer (Index update may have been started manually)
122     if (indexIntervalTimer.interval() != 0)
123         indexIntervalTimer.start();
124 
125     // Run the indexer thread
126     qInfo() << "Start indexing files.";
127     futureWatcher->setFuture(QtConcurrent::run(this, &Private::indexFiles));
128 
129     // Notification
130     emit q->statusInfo("Indexing files ...");
131 }
132 
133 
134 
135 /** ***************************************************************************/
finishIndexing()136 void Files::Private::finishIndexing() {
137 
138     // In case of abortion the returned data is invalid
139     if ( !abort ) {
140         OfflineIndex *retval = futureWatcher->future().result();
141         if (retval) {
142             offlineIndex = std::move(*retval);
143             delete retval;
144         }
145 
146         // Notification
147         CounterVisitor counterVisitor;
148         for (const auto & tree : indexTrees )
149             tree->accept(counterVisitor);
150         qInfo() << qPrintable(QString("Indexed %1 files in %2 directories.")
151                               .arg(counterVisitor.itemCount).arg(counterVisitor.dirCount));
152         emit q->statusInfo(QString("Indexed %1 files in %2 directories.")
153                            .arg(counterVisitor.itemCount).arg(counterVisitor.dirCount));
154     }
155 
156     futureWatcher.reset();
157     abort = false;
158 
159     if ( rerun ) {
160         rerun = false;
161         startIndexing();
162     }
163 }
164 
165 
166 
167 /** ***************************************************************************/
indexFiles()168 OfflineIndex* Files::Private::indexFiles() {
169 
170     // Remove the subtrees not wanted anymore
171     auto it = indexTrees.begin();
172     while ( it != indexTrees.end() ) {
173         if ( indexRootDirs.contains((*it)->path()) )
174             ++it;
175         else {
176             (*it)->removeDownlinks();
177             it = indexTrees.erase(it);
178         }
179     }
180 
181     // Start the indexing
182     for ( const QString &rootDir : indexRootDirs ) {
183 
184         emit q->statusInfo(QString("Indexing %1…").arg(rootDir));
185 
186         // If this root dir does not exist create it
187         auto it = find_if(indexTrees.begin(), indexTrees.end(),
188                           [&rootDir](const shared_ptr<IndexTreeNode>& tree){ return tree->path() == rootDir; });
189         if ( it == indexTrees.end() ) {
190             indexTrees.push_back(make_shared<IndexTreeNode>(rootDir));
191             indexTrees.back()->update(abort, indexSettings);
192         }
193         else
194             (*it)->update(abort, indexSettings);
195 
196 
197         if ( abort )
198             return nullptr;
199     }
200 
201     // Serialize data
202     qDebug() << "Serializing files…";
203     emit q->statusInfo("Serializing index data…");
204     QFile file(q->cacheLocation().filePath("fileindex.json"));
205     if (file.open(QIODevice::WriteOnly)) {
206         QJsonArray array;
207         for (auto& tree : this->indexTrees)
208             array.push_back(tree->serialize());
209         QJsonDocument doc(array);
210         file.write(doc.toJson(QJsonDocument::Compact));
211         file.close();
212     }
213     else
214         qWarning() << "Couldn't write to file:" << file.fileName();
215 
216 
217     // Build offline index
218     qDebug() << "Building inverted file index…";
219     emit q->statusInfo("Building inverted index…");
220     Core::OfflineIndex *offline = new Core::OfflineIndex(indexSettings.fuzzy());
221     OfflineIndexBuilderVisitor visitor(*offline);
222     for (auto& tree : this->indexTrees)
223         tree->accept(visitor);
224     return offline;
225 }
226 
227 
228 /** ***************************************************************************/
229 /** ***************************************************************************/
230 /** ***************************************************************************/
231 /** ***************************************************************************/
Extension()232 Files::Extension::Extension()
233     : Core::Extension("org.albert.extension.files"),
234       Core::QueryHandler(Core::Plugin::id()),
235       d(new Private(this)) {
236 
237     registerQueryHandler(this);
238 
239     // Load settings
240     d->indexSettings.setFilters(settings().value(CFG_FILTERS, DEF_FILTERS).toStringList());
241     d->indexSettings.setIndexHidden(settings().value(CFG_INDEX_HIDDEN, DEF_INDEX_HIDDEN).toBool());
242     d->indexSettings.setFollowSymlinks(settings().value(CFG_FOLLOW_SYMLINKS, DEF_FOLLOW_SYMLINKS).toBool());
243     d->indexSettings.setFuzzy(settings().value(CFG_FUZZY, DEF_FUZZY).toBool());
244     d->indexSettings.setForceUpdate(false);
245     d->offlineIndex.setFuzzy(d->indexSettings.fuzzy());
246     d->indexIntervalTimer.setInterval(settings().value(CFG_SCAN_INTERVAL, DEF_SCAN_INTERVAL).toInt()*60000); // Will be started in the initial index update
247     d->indexRootDirs = settings().value(CFG_PATHS, QDir::homePath()).toStringList();
248 
249     // Index timer
250     connect(&d->indexIntervalTimer, &QTimer::timeout, this, &Extension::updateIndex);
251 
252     // If the root dirs change write it to the settings
253     connect(this, &Extension::pathsChanged, [this](const QStringList& dirs){
254         settings().setValue(CFG_PATHS, dirs);
255     });
256 
257     // Deserialize data
258     qDebug() << "Loading file index from cache.";
259     QFile file(cacheLocation().filePath("fileindex.json"));
260     if ( file.exists() ) {
261         if (file.open(QIODevice::ReadOnly)) {
262             QJsonDocument loadDoc{QJsonDocument::fromJson(file.readAll())};
263             for ( const QJsonValueRef value : loadDoc.array()){
264                 d->indexTrees.push_back(make_shared<IndexTreeNode>());  // Invalid node
265                 d->indexTrees.back()->deserialize(value.toObject());
266             }
267             file.close();
268 
269             // Build offline index
270             qDebug() << "Building inverted file index.";
271             OfflineIndexBuilderVisitor visitor(d->offlineIndex);
272             for (auto& tree : d->indexTrees)
273                 tree->accept(visitor);
274         }
275         else
276             qWarning() << "Could not read from file: " << file.fileName();
277     }
278 
279     // Trigger an initial update
280     updateIndex();
281 }
282 
283 
284 
285 /** ***************************************************************************/
~Extension()286 Files::Extension::~Extension() {
287 
288     // The indexer thread has sideeffects wait for termination
289     d->abort = true;
290     d->rerun = false;
291     if ( d->futureWatcher ){
292         disconnect(d->futureWatcher.get(), 0, 0, 0);
293         d->futureWatcher->waitForFinished();
294     }
295 }
296 
297 
298 
299 /** ***************************************************************************/
widget(QWidget * parent)300 QWidget *Files::Extension::widget(QWidget *parent) {
301     if (d->widget.isNull())
302         d->widget = new ConfigWidget(this, parent);
303     return d->widget;
304 }
305 
306 
307 
308 /** ***************************************************************************/
handleQuery(Core::Query * query) const309 void Files::Extension::handleQuery(Core::Query * query) const {
310 
311     if ( query->trigger()=="/" || query->trigger()=="~" ) {
312 
313         QFileInfo queryFileInfo(query->rawString());
314 
315         // Substitute tilde
316         if ( query->rawString()[0] == '~' )
317             queryFileInfo.setFile(QDir::homePath()+query->string());
318 
319         // Get all matching files
320         QFileInfo pathInfo(queryFileInfo.path());
321         if ( pathInfo.exists() && pathInfo.isDir() ) {
322 
323             QMimeDatabase mimeDatabase;
324             QDir dir(pathInfo.filePath());
325             QString commonPrefix;
326             QString queryFileName = queryFileInfo.fileName();
327             vector<shared_ptr<StandardItem>> items;
328 
329             for (const QFileInfo& fileInfo : dir.entryInfoList(QDir::AllEntries|QDir::Hidden|QDir::NoDotAndDotDot,
330                                                                QDir::DirsFirst|QDir::Name|QDir::IgnoreCase) ) {
331                 QString fileName = fileInfo.fileName();
332 
333                 if ( fileName.startsWith(queryFileName) ) {
334 
335                     if (fileInfo.isDir())
336                         fileName.append(QDir::separator());
337 
338                     if (commonPrefix.isNull())
339                         commonPrefix = fileName;
340                     else {
341                         auto pair = mismatch(commonPrefix.begin() , commonPrefix.end(), fileName.begin(), fileName.end());
342                         commonPrefix.resize(distance(commonPrefix.begin(), pair.first));
343                     }
344 
345                     QMimeType mimetype = mimeDatabase.mimeTypeForFile(fileInfo.filePath());
346                     QString icon = XDG::IconLookup::iconPath({mimetype.iconName(), mimetype.genericIconName(), "unknown"});
347                     if (icon.isEmpty())
348                         icon = (mimetype.iconName() == "inode-directory") ? ":directory" : ":unknown";
349 
350                     auto item = make_shared<StandardItem>(fileInfo.filePath(),
351                                                           icon,
352                                                           fileName,
353                                                           fileInfo.filePath());
354                     item->setActions(File::buildFileActions(fileInfo.filePath()));
355                     items.push_back(move(item));
356                 }
357             }
358             for (auto &item : items) {
359                 item->setCompletion(dir.filePath(commonPrefix));
360                 query->addMatch(std::move(item));
361             }
362         }
363     }
364     else
365     {
366         if ( query->string().isEmpty() )
367             return;
368 
369         if ( QString("albert scan files").startsWith(query->string()) ) {
370 
371             auto item = make_shared<StandardItem>("files.action.index");
372             item->setText("albert scan files");
373             item->setSubtext("Update the file index");
374             item->setIconPath(":app_icon");
375             // Const cast is fine since the action will not be called here
376             item->addAction(make_shared<FuncAction>("Update the file index",
377                                                     [this](){ const_cast<Extension*>(this)->updateIndex();}));
378 
379             query->addMatch(move(item));
380         }
381 
382         // Search for matches
383         const vector<shared_ptr<IndexableItem>> &indexables = d->offlineIndex.search(query->string());
384 
385         // Add results to query
386         vector<pair<shared_ptr<Core::Item>,uint>> results;
387         for (const shared_ptr<Core::IndexableItem> &item : indexables)
388             // TODO `Search` has to determine the relevance. Set to 0 for now
389             results.emplace_back(static_pointer_cast<File>(item), 0);
390 
391         query->addMatches(make_move_iterator(results.begin()),
392                           make_move_iterator(results.end()));
393     }
394 }
395 
396 
397 
398 /** ***************************************************************************/
paths() const399 const QStringList &Files::Extension::paths() const {
400     return d->indexRootDirs;
401 }
402 
403 
404 
405 /** ***************************************************************************/
setPaths(const QStringList & paths)406 void Files::Extension::setPaths(const QStringList &paths) {
407 
408     if (d->indexRootDirs == paths)
409         return;
410 
411     d->indexRootDirs.clear();
412 
413     // Check sanity and add path
414     for ( const QString& path : paths ) {
415 
416         QFileInfo fileInfo(path);
417         QString absPath = fileInfo.absoluteFilePath();
418 
419         if (d->indexRootDirs.contains(absPath)) {
420             qWarning() << QString("Duplicate paths: %1.").arg(path);
421             continue;
422         }
423 
424         if (!fileInfo.exists()) {
425             qWarning() << QString("Path does not exist: %1.").arg(path);
426             continue;
427         }
428 
429         if(!fileInfo.isDir()) {
430             qWarning() << QString("Path is not a directory: %1.").arg(path);
431             continue;
432         }
433 
434         d->indexRootDirs << absPath;
435     }
436 
437     sort(d->indexRootDirs.begin(), d->indexRootDirs.end());
438 
439     emit pathsChanged(d->indexRootDirs);
440 
441     // Store to settings
442     settings().setValue(CFG_PATHS, d->indexRootDirs);
443 
444 }
445 
446 
447 
448 /** ***************************************************************************/
restorePaths()449 void Files::Extension::restorePaths() {
450     // Add standard path
451     d->indexRootDirs.clear();
452     d->indexRootDirs << QStandardPaths::writableLocation(QStandardPaths::HomeLocation);
453     emit pathsChanged(d->indexRootDirs);
454 }
455 
456 
457 
458 /** ***************************************************************************/
updateIndex()459 void Files::Extension::updateIndex() {
460     d->startIndexing();
461 }
462 
463 
464 
465 /** ***************************************************************************/
indexHidden() const466 bool Files::Extension::indexHidden() const {
467     return d->indexSettings.indexHidden();
468 }
469 
470 
471 
472 /** ***************************************************************************/
setIndexHidden(bool b)473 void Files::Extension::setIndexHidden(bool b)  {
474     settings().setValue(CFG_INDEX_HIDDEN, b);
475     d->indexSettings.setIndexHidden(b);
476 }
477 
478 
479 
480 /** ***************************************************************************/
followSymlinks() const481 bool Files::Extension::followSymlinks() const {
482     return d->indexSettings.followSymlinks();
483 }
484 
485 
486 
487 /** ***************************************************************************/
setFollowSymlinks(bool b)488 void Files::Extension::setFollowSymlinks(bool b)  {
489     settings().setValue(CFG_FOLLOW_SYMLINKS, b);
490     d->indexSettings.setFollowSymlinks(b);
491 }
492 
493 
494 
495 /** ***************************************************************************/
scanInterval() const496 unsigned int Files::Extension::scanInterval() const {
497     return static_cast<uint>(d->indexIntervalTimer.interval()/60000);
498 }
499 
500 
501 
502 /** ***************************************************************************/
setScanInterval(uint minutes)503 void Files::Extension::setScanInterval(uint minutes) {
504     settings().setValue(CFG_SCAN_INTERVAL, minutes);
505     (minutes == 0) ? d->indexIntervalTimer.stop()
506                    : d->indexIntervalTimer.start(static_cast<int>(minutes*60000));
507 }
508 
509 
510 
511 /** ***************************************************************************/
fuzzy() const512 bool Files::Extension::fuzzy() const {
513     return d->offlineIndex.fuzzy();
514 }
515 
516 
517 
518 /** ***************************************************************************/
setFuzzy(bool b)519 void Files::Extension::setFuzzy(bool b) {
520     settings().setValue(CFG_FUZZY, b);
521     d->offlineIndex.setFuzzy(b);
522 }
523 
524 
525 
526 /** ***************************************************************************/
filters() const527 QStringList Files::Extension::filters() const {
528     QStringList retval;
529     for ( auto const & regex : d->indexSettings.filters() )
530         retval.push_back(regex.pattern());
531     return retval;
532 }
533 
534 
535 
536 /** ***************************************************************************/
setFilters(const QStringList & filters)537 void Files::Extension::setFilters(const QStringList &filters) {
538     settings().setValue(CFG_FILTERS, filters);
539     d->indexSettings.setFilters(filters);
540 }
541