1 // Copyright (C) 2014-2018 Manuel Schneider
2
3 #include <QDebug>
4 #include <QDir>
5 #include <QDirIterator>
6 #include <QFutureWatcher>
7 #include <QMessageBox>
8 #include <QObject>
9 #include <QPointer>
10 #include <QSettings>
11 #include <QStandardPaths>
12 #include <QtConcurrent>
13 #include <QThreadPool>
14 #include <QTimer>
15 #include <memory>
16 #include <functional>
17 #include <vector>
18 #include <set>
19 #include "configwidget.h"
20 #include "file.h"
21 #include "standardfile.h"
22 #include "extension.h"
23 #include "indextreenode.h"
24 #include "albert/util/offlineindex.h"
25 #include "albert/util/standarditem.h"
26 #include "albert/util/standardactions.h"
27 #include "xdg/iconlookup.h"
28 using namespace Core;
29 using namespace std;
30
31
32 namespace {
33
34 const char* CFG_PATHS = "paths";
35 const char* CFG_FILTERS = "filters";
36 const QStringList DEF_FILTERS = { "inode/directory", "application/*" };
37 const char* CFG_FUZZY = "fuzzy";
38 const bool DEF_FUZZY = false;
39 const char* CFG_INDEX_HIDDEN = "indexhidden";
40 const bool DEF_INDEX_HIDDEN = false;
41 const char* CFG_FOLLOW_SYMLINKS = "follow_symlinks";
42 const bool DEF_FOLLOW_SYMLINKS = false;
43 const char* CFG_SCAN_INTERVAL = "scan_interval";
44 const uint DEF_SCAN_INTERVAL = 15;
45
46
47
48 class OfflineIndexBuilderVisitor : public Files::Visitor {
49 Core::OfflineIndex &offlineIndex;
50 public:
OfflineIndexBuilderVisitor(Core::OfflineIndex & offlineIndex)51 OfflineIndexBuilderVisitor(Core::OfflineIndex &offlineIndex)
52 : offlineIndex(offlineIndex) { }
53
visit(Files::IndexTreeNode * node)54 void visit(Files::IndexTreeNode *node) override {
55 for ( const shared_ptr<Files::File> &item : node->items() )
56 offlineIndex.add(item);
57 }
58 };
59
60
61 class CounterVisitor : public Files::Visitor {
62 public:
63 uint itemCount = 0;
64 uint dirCount = 0;
visit(Files::IndexTreeNode * node)65 void visit(Files::IndexTreeNode *node) override {
66 ++dirCount;
67 itemCount += node->items().size();
68 }
69 };
70
71 }
72
73
74
75 /** ***************************************************************************/
76 /** ***************************************************************************/
77 /** ***************************************************************************/
78 /** ***************************************************************************/
79 class Files::Private
80 {
81 public:
Private(Extension * q)82 Private(Extension *q) : q(q), abort(false), rerun(false) {}
83
84 Extension *q;
85
86 QPointer<ConfigWidget> widget;
87
88 QStringList indexRootDirs;
89 IndexSettings indexSettings;
90 vector<shared_ptr<IndexTreeNode>> indexTrees;
91 unique_ptr<QFutureWatcher<Core::OfflineIndex*>> futureWatcher;
92 Core::OfflineIndex offlineIndex;
93 QTimer indexIntervalTimer;
94 bool abort;
95 bool rerun;
96
97
98 void finishIndexing();
99 void startIndexing();
100 Core::OfflineIndex *indexFiles();
101 };
102
103
104
105 /** ***************************************************************************/
startIndexing()106 void Files::Private::startIndexing() {
107
108 // Abort and rerun
109 if ( futureWatcher ) {
110 emit q->statusInfo("Waiting for indexer to shut down ...");
111 abort = true;
112 rerun = true;
113 return;
114 }
115
116 // Run finishIndexing when the indexing thread finished
117 futureWatcher.reset(new QFutureWatcher<Core::OfflineIndex*>);
118 QObject::connect(futureWatcher.get(), &QFutureWatcher<Core::OfflineIndex*>::finished,
119 [this](){ this->finishIndexing(); });
120
121 // Restart the timer (Index update may have been started manually)
122 if (indexIntervalTimer.interval() != 0)
123 indexIntervalTimer.start();
124
125 // Run the indexer thread
126 qInfo() << "Start indexing files.";
127 futureWatcher->setFuture(QtConcurrent::run(this, &Private::indexFiles));
128
129 // Notification
130 emit q->statusInfo("Indexing files ...");
131 }
132
133
134
135 /** ***************************************************************************/
finishIndexing()136 void Files::Private::finishIndexing() {
137
138 // In case of abortion the returned data is invalid
139 if ( !abort ) {
140 OfflineIndex *retval = futureWatcher->future().result();
141 if (retval) {
142 offlineIndex = std::move(*retval);
143 delete retval;
144 }
145
146 // Notification
147 CounterVisitor counterVisitor;
148 for (const auto & tree : indexTrees )
149 tree->accept(counterVisitor);
150 qInfo() << qPrintable(QString("Indexed %1 files in %2 directories.")
151 .arg(counterVisitor.itemCount).arg(counterVisitor.dirCount));
152 emit q->statusInfo(QString("Indexed %1 files in %2 directories.")
153 .arg(counterVisitor.itemCount).arg(counterVisitor.dirCount));
154 }
155
156 futureWatcher.reset();
157 abort = false;
158
159 if ( rerun ) {
160 rerun = false;
161 startIndexing();
162 }
163 }
164
165
166
167 /** ***************************************************************************/
indexFiles()168 OfflineIndex* Files::Private::indexFiles() {
169
170 // Remove the subtrees not wanted anymore
171 auto it = indexTrees.begin();
172 while ( it != indexTrees.end() ) {
173 if ( indexRootDirs.contains((*it)->path()) )
174 ++it;
175 else {
176 (*it)->removeDownlinks();
177 it = indexTrees.erase(it);
178 }
179 }
180
181 // Start the indexing
182 for ( const QString &rootDir : indexRootDirs ) {
183
184 emit q->statusInfo(QString("Indexing %1…").arg(rootDir));
185
186 // If this root dir does not exist create it
187 auto it = find_if(indexTrees.begin(), indexTrees.end(),
188 [&rootDir](const shared_ptr<IndexTreeNode>& tree){ return tree->path() == rootDir; });
189 if ( it == indexTrees.end() ) {
190 indexTrees.push_back(make_shared<IndexTreeNode>(rootDir));
191 indexTrees.back()->update(abort, indexSettings);
192 }
193 else
194 (*it)->update(abort, indexSettings);
195
196
197 if ( abort )
198 return nullptr;
199 }
200
201 // Serialize data
202 qDebug() << "Serializing files…";
203 emit q->statusInfo("Serializing index data…");
204 QFile file(q->cacheLocation().filePath("fileindex.json"));
205 if (file.open(QIODevice::WriteOnly)) {
206 QJsonArray array;
207 for (auto& tree : this->indexTrees)
208 array.push_back(tree->serialize());
209 QJsonDocument doc(array);
210 file.write(doc.toJson(QJsonDocument::Compact));
211 file.close();
212 }
213 else
214 qWarning() << "Couldn't write to file:" << file.fileName();
215
216
217 // Build offline index
218 qDebug() << "Building inverted file index…";
219 emit q->statusInfo("Building inverted index…");
220 Core::OfflineIndex *offline = new Core::OfflineIndex(indexSettings.fuzzy());
221 OfflineIndexBuilderVisitor visitor(*offline);
222 for (auto& tree : this->indexTrees)
223 tree->accept(visitor);
224 return offline;
225 }
226
227
228 /** ***************************************************************************/
229 /** ***************************************************************************/
230 /** ***************************************************************************/
231 /** ***************************************************************************/
Extension()232 Files::Extension::Extension()
233 : Core::Extension("org.albert.extension.files"),
234 Core::QueryHandler(Core::Plugin::id()),
235 d(new Private(this)) {
236
237 registerQueryHandler(this);
238
239 // Load settings
240 d->indexSettings.setFilters(settings().value(CFG_FILTERS, DEF_FILTERS).toStringList());
241 d->indexSettings.setIndexHidden(settings().value(CFG_INDEX_HIDDEN, DEF_INDEX_HIDDEN).toBool());
242 d->indexSettings.setFollowSymlinks(settings().value(CFG_FOLLOW_SYMLINKS, DEF_FOLLOW_SYMLINKS).toBool());
243 d->indexSettings.setFuzzy(settings().value(CFG_FUZZY, DEF_FUZZY).toBool());
244 d->indexSettings.setForceUpdate(false);
245 d->offlineIndex.setFuzzy(d->indexSettings.fuzzy());
246 d->indexIntervalTimer.setInterval(settings().value(CFG_SCAN_INTERVAL, DEF_SCAN_INTERVAL).toInt()*60000); // Will be started in the initial index update
247 d->indexRootDirs = settings().value(CFG_PATHS, QDir::homePath()).toStringList();
248
249 // Index timer
250 connect(&d->indexIntervalTimer, &QTimer::timeout, this, &Extension::updateIndex);
251
252 // If the root dirs change write it to the settings
253 connect(this, &Extension::pathsChanged, [this](const QStringList& dirs){
254 settings().setValue(CFG_PATHS, dirs);
255 });
256
257 // Deserialize data
258 qDebug() << "Loading file index from cache.";
259 QFile file(cacheLocation().filePath("fileindex.json"));
260 if ( file.exists() ) {
261 if (file.open(QIODevice::ReadOnly)) {
262 QJsonDocument loadDoc{QJsonDocument::fromJson(file.readAll())};
263 for ( const QJsonValueRef value : loadDoc.array()){
264 d->indexTrees.push_back(make_shared<IndexTreeNode>()); // Invalid node
265 d->indexTrees.back()->deserialize(value.toObject());
266 }
267 file.close();
268
269 // Build offline index
270 qDebug() << "Building inverted file index.";
271 OfflineIndexBuilderVisitor visitor(d->offlineIndex);
272 for (auto& tree : d->indexTrees)
273 tree->accept(visitor);
274 }
275 else
276 qWarning() << "Could not read from file: " << file.fileName();
277 }
278
279 // Trigger an initial update
280 updateIndex();
281 }
282
283
284
285 /** ***************************************************************************/
~Extension()286 Files::Extension::~Extension() {
287
288 // The indexer thread has sideeffects wait for termination
289 d->abort = true;
290 d->rerun = false;
291 if ( d->futureWatcher ){
292 disconnect(d->futureWatcher.get(), 0, 0, 0);
293 d->futureWatcher->waitForFinished();
294 }
295 }
296
297
298
299 /** ***************************************************************************/
widget(QWidget * parent)300 QWidget *Files::Extension::widget(QWidget *parent) {
301 if (d->widget.isNull())
302 d->widget = new ConfigWidget(this, parent);
303 return d->widget;
304 }
305
306
307
308 /** ***************************************************************************/
handleQuery(Core::Query * query) const309 void Files::Extension::handleQuery(Core::Query * query) const {
310
311 if ( query->trigger()=="/" || query->trigger()=="~" ) {
312
313 QFileInfo queryFileInfo(query->rawString());
314
315 // Substitute tilde
316 if ( query->rawString()[0] == '~' )
317 queryFileInfo.setFile(QDir::homePath()+query->string());
318
319 // Get all matching files
320 QFileInfo pathInfo(queryFileInfo.path());
321 if ( pathInfo.exists() && pathInfo.isDir() ) {
322
323 QMimeDatabase mimeDatabase;
324 QDir dir(pathInfo.filePath());
325 QString commonPrefix;
326 QString queryFileName = queryFileInfo.fileName();
327 vector<shared_ptr<StandardItem>> items;
328
329 for (const QFileInfo& fileInfo : dir.entryInfoList(QDir::AllEntries|QDir::Hidden|QDir::NoDotAndDotDot,
330 QDir::DirsFirst|QDir::Name|QDir::IgnoreCase) ) {
331 QString fileName = fileInfo.fileName();
332
333 if ( fileName.startsWith(queryFileName) ) {
334
335 if (fileInfo.isDir())
336 fileName.append(QDir::separator());
337
338 if (commonPrefix.isNull())
339 commonPrefix = fileName;
340 else {
341 auto pair = mismatch(commonPrefix.begin() , commonPrefix.end(), fileName.begin(), fileName.end());
342 commonPrefix.resize(distance(commonPrefix.begin(), pair.first));
343 }
344
345 QMimeType mimetype = mimeDatabase.mimeTypeForFile(fileInfo.filePath());
346 QString icon = XDG::IconLookup::iconPath({mimetype.iconName(), mimetype.genericIconName(), "unknown"});
347 if (icon.isEmpty())
348 icon = (mimetype.iconName() == "inode-directory") ? ":directory" : ":unknown";
349
350 auto item = make_shared<StandardItem>(fileInfo.filePath(),
351 icon,
352 fileName,
353 fileInfo.filePath());
354 item->setActions(File::buildFileActions(fileInfo.filePath()));
355 items.push_back(move(item));
356 }
357 }
358 for (auto &item : items) {
359 item->setCompletion(dir.filePath(commonPrefix));
360 query->addMatch(std::move(item));
361 }
362 }
363 }
364 else
365 {
366 if ( query->string().isEmpty() )
367 return;
368
369 if ( QString("albert scan files").startsWith(query->string()) ) {
370
371 auto item = make_shared<StandardItem>("files.action.index");
372 item->setText("albert scan files");
373 item->setSubtext("Update the file index");
374 item->setIconPath(":app_icon");
375 // Const cast is fine since the action will not be called here
376 item->addAction(make_shared<FuncAction>("Update the file index",
377 [this](){ const_cast<Extension*>(this)->updateIndex();}));
378
379 query->addMatch(move(item));
380 }
381
382 // Search for matches
383 const vector<shared_ptr<IndexableItem>> &indexables = d->offlineIndex.search(query->string());
384
385 // Add results to query
386 vector<pair<shared_ptr<Core::Item>,uint>> results;
387 for (const shared_ptr<Core::IndexableItem> &item : indexables)
388 // TODO `Search` has to determine the relevance. Set to 0 for now
389 results.emplace_back(static_pointer_cast<File>(item), 0);
390
391 query->addMatches(make_move_iterator(results.begin()),
392 make_move_iterator(results.end()));
393 }
394 }
395
396
397
398 /** ***************************************************************************/
paths() const399 const QStringList &Files::Extension::paths() const {
400 return d->indexRootDirs;
401 }
402
403
404
405 /** ***************************************************************************/
setPaths(const QStringList & paths)406 void Files::Extension::setPaths(const QStringList &paths) {
407
408 if (d->indexRootDirs == paths)
409 return;
410
411 d->indexRootDirs.clear();
412
413 // Check sanity and add path
414 for ( const QString& path : paths ) {
415
416 QFileInfo fileInfo(path);
417 QString absPath = fileInfo.absoluteFilePath();
418
419 if (d->indexRootDirs.contains(absPath)) {
420 qWarning() << QString("Duplicate paths: %1.").arg(path);
421 continue;
422 }
423
424 if (!fileInfo.exists()) {
425 qWarning() << QString("Path does not exist: %1.").arg(path);
426 continue;
427 }
428
429 if(!fileInfo.isDir()) {
430 qWarning() << QString("Path is not a directory: %1.").arg(path);
431 continue;
432 }
433
434 d->indexRootDirs << absPath;
435 }
436
437 sort(d->indexRootDirs.begin(), d->indexRootDirs.end());
438
439 emit pathsChanged(d->indexRootDirs);
440
441 // Store to settings
442 settings().setValue(CFG_PATHS, d->indexRootDirs);
443
444 }
445
446
447
448 /** ***************************************************************************/
restorePaths()449 void Files::Extension::restorePaths() {
450 // Add standard path
451 d->indexRootDirs.clear();
452 d->indexRootDirs << QStandardPaths::writableLocation(QStandardPaths::HomeLocation);
453 emit pathsChanged(d->indexRootDirs);
454 }
455
456
457
458 /** ***************************************************************************/
updateIndex()459 void Files::Extension::updateIndex() {
460 d->startIndexing();
461 }
462
463
464
465 /** ***************************************************************************/
indexHidden() const466 bool Files::Extension::indexHidden() const {
467 return d->indexSettings.indexHidden();
468 }
469
470
471
472 /** ***************************************************************************/
setIndexHidden(bool b)473 void Files::Extension::setIndexHidden(bool b) {
474 settings().setValue(CFG_INDEX_HIDDEN, b);
475 d->indexSettings.setIndexHidden(b);
476 }
477
478
479
480 /** ***************************************************************************/
followSymlinks() const481 bool Files::Extension::followSymlinks() const {
482 return d->indexSettings.followSymlinks();
483 }
484
485
486
487 /** ***************************************************************************/
setFollowSymlinks(bool b)488 void Files::Extension::setFollowSymlinks(bool b) {
489 settings().setValue(CFG_FOLLOW_SYMLINKS, b);
490 d->indexSettings.setFollowSymlinks(b);
491 }
492
493
494
495 /** ***************************************************************************/
scanInterval() const496 unsigned int Files::Extension::scanInterval() const {
497 return static_cast<uint>(d->indexIntervalTimer.interval()/60000);
498 }
499
500
501
502 /** ***************************************************************************/
setScanInterval(uint minutes)503 void Files::Extension::setScanInterval(uint minutes) {
504 settings().setValue(CFG_SCAN_INTERVAL, minutes);
505 (minutes == 0) ? d->indexIntervalTimer.stop()
506 : d->indexIntervalTimer.start(static_cast<int>(minutes*60000));
507 }
508
509
510
511 /** ***************************************************************************/
fuzzy() const512 bool Files::Extension::fuzzy() const {
513 return d->offlineIndex.fuzzy();
514 }
515
516
517
518 /** ***************************************************************************/
setFuzzy(bool b)519 void Files::Extension::setFuzzy(bool b) {
520 settings().setValue(CFG_FUZZY, b);
521 d->offlineIndex.setFuzzy(b);
522 }
523
524
525
526 /** ***************************************************************************/
filters() const527 QStringList Files::Extension::filters() const {
528 QStringList retval;
529 for ( auto const & regex : d->indexSettings.filters() )
530 retval.push_back(regex.pattern());
531 return retval;
532 }
533
534
535
536 /** ***************************************************************************/
setFilters(const QStringList & filters)537 void Files::Extension::setFilters(const QStringList &filters) {
538 settings().setValue(CFG_FILTERS, filters);
539 d->indexSettings.setFilters(filters);
540 }
541