1 /*
2 * Copyright (C) by Duncan Mac-Vicar P. <duncan@kde.org>
3 * Copyright (C) by Klaas Freitag <freitag@owncloud.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 */
15
16 #include "syncengine.h"
17 #include "account.h"
18 #include "owncloudpropagator.h"
19 #include "common/syncjournaldb.h"
20 #include "common/syncjournalfilerecord.h"
21 #include "discoveryphase.h"
22 #include "creds/abstractcredentials.h"
23 #include "common/syncfilestatus.h"
24 #include "csync_exclude.h"
25 #include "filesystem.h"
26 #include "propagateremotedelete.h"
27 #include "propagatedownload.h"
28 #include "common/asserts.h"
29 #include "discovery.h"
30 #include "common/vfs.h"
31
32 #ifdef Q_OS_WIN
33 #include <windows.h>
34 #else
35 #include <unistd.h>
36 #endif
37
38 #include <climits>
39 #include <assert.h>
40 #include <chrono>
41
42 #include <QCoreApplication>
43 #include <QSslSocket>
44 #include <QDir>
45 #include <QLoggingCategory>
46 #include <QMutexLocker>
47 #include <QThread>
48 #include <QStringList>
49 #include <QTextStream>
50 #include <QTime>
51 #include <QUrl>
52 #include <QSslCertificate>
53 #include <QProcess>
54 #include <QElapsedTimer>
55 #include <qtextcodec.h>
56
57 namespace OCC {
58
59 Q_LOGGING_CATEGORY(lcEngine, "sync.engine", QtInfoMsg)
60
61 bool SyncEngine::s_anySyncRunning = false;
62
63 /** When the client touches a file, block change notifications for this duration (ms)
64 *
65 * On Linux and Windows the file watcher can't distinguish a change that originates
66 * from the client (like a download during a sync operation) and an external change.
67 * To work around that, all files the client touches are recorded and file change
68 * notifications for these are blocked for some time. This value controls for how
69 * long.
70 *
71 * Reasons this delay can't be very small:
72 * - it takes time for the change notification to arrive and to be processed by the client
73 * - some time could pass between the client recording that a file will be touched
74 * and its filesystem operation finishing, triggering the notification
75 */
76 static const std::chrono::milliseconds s_touchedFilesMaxAgeMs(3 * 1000);
77
78 // doc in header
79 std::chrono::milliseconds SyncEngine::minimumFileAgeForUpload(2000);
80
SyncEngine(AccountPtr account,const QString & localPath,const QString & remotePath,OCC::SyncJournalDb * journal)81 SyncEngine::SyncEngine(AccountPtr account, const QString &localPath,
82 const QString &remotePath, OCC::SyncJournalDb *journal)
83 : _account(account)
84 , _needsUpdate(false)
85 , _syncRunning(false)
86 , _localPath(localPath)
87 , _remotePath(remotePath)
88 , _journal(journal)
89 , _progressInfo(new ProgressInfo)
90 , _hasNoneFiles(false)
91 , _hasRemoveFile(false)
92 , _uploadLimit(0)
93 , _downloadLimit(0)
94 , _anotherSyncNeeded(NoFollowUpSync)
95 {
96 qRegisterMetaType<SyncFileItem>("SyncFileItem");
97 qRegisterMetaType<SyncFileItemPtr>("SyncFileItemPtr");
98 qRegisterMetaType<SyncFileItem::Status>("SyncFileItem::Status");
99 qRegisterMetaType<SyncFileStatus>("SyncFileStatus");
100 qRegisterMetaType<SyncFileItemVector>("SyncFileItemVector");
101 qRegisterMetaType<SyncFileItem::Direction>("SyncFileItem::Direction");
102
103 // Everything in the SyncEngine expects a trailing slash for the localPath.
104 OC_ASSERT(localPath.endsWith(QLatin1Char('/')));
105
106 _excludedFiles.reset(new ExcludedFiles);
107
108 _syncFileStatusTracker.reset(new SyncFileStatusTracker(this));
109
110 _clearTouchedFilesTimer.setSingleShot(true);
111 _clearTouchedFilesTimer.setInterval(30 * 1000);
112 connect(&_clearTouchedFilesTimer, &QTimer::timeout, this, &SyncEngine::slotClearTouchedFiles);
113 }
114
~SyncEngine()115 SyncEngine::~SyncEngine()
116 {
117 abort();
118 _excludedFiles.reset();
119 }
120
121 /**
122 * Check if the item is in the blacklist.
123 * If it should not be sync'ed because of the blacklist, update the item with the error instruction
124 * and proper error message, and return true.
125 * If the item is not in the blacklist, or the blacklist is stale, return false.
126 */
checkErrorBlacklisting(SyncFileItem & item)127 bool SyncEngine::checkErrorBlacklisting(SyncFileItem &item)
128 {
129 if (!_journal) {
130 qCCritical(lcEngine) << "Journal is undefined!";
131 return false;
132 }
133
134 SyncJournalErrorBlacklistRecord entry = _journal->errorBlacklistEntry(item._file);
135 item._hasBlacklistEntry = false;
136
137 if (!entry.isValid()) {
138 return false;
139 }
140
141 item._hasBlacklistEntry = true;
142
143 // If duration has expired, it's not blacklisted anymore
144 time_t now = Utility::qDateTimeToTime_t(QDateTime::currentDateTimeUtc());
145 if (now >= entry._lastTryTime + entry._ignoreDuration) {
146 qCInfo(lcEngine) << "blacklist entry for " << item._file << " has expired!";
147 return false;
148 }
149
150 // If the file has changed locally or on the server, the blacklist
151 // entry no longer applies
152 if (item._direction == SyncFileItem::Up) { // check the modtime
153 if (item._modtime == 0 || entry._lastTryModtime == 0) {
154 return false;
155 } else if (item._modtime != entry._lastTryModtime) {
156 qCInfo(lcEngine) << item._file << " is blacklisted, but has changed mtime!";
157 return false;
158 } else if (item._renameTarget != entry._renameTarget) {
159 qCInfo(lcEngine) << item._file << " is blacklisted, but rename target changed from" << entry._renameTarget;
160 return false;
161 }
162 } else if (item._direction == SyncFileItem::Down) {
163 // download, check the etag.
164 if (item._etag.isEmpty() || entry._lastTryEtag.isEmpty()) {
165 qCInfo(lcEngine) << item._file << "one ETag is empty, no blacklisting";
166 return false;
167 } else if (item._etag != entry._lastTryEtag) {
168 qCInfo(lcEngine) << item._file << " is blacklisted, but has changed etag!";
169 return false;
170 }
171 }
172
173 int waitSeconds = entry._lastTryTime + entry._ignoreDuration - now;
174 qCInfo(lcEngine) << "Item is on blacklist: " << entry._file
175 << "retries:" << entry._retryCount
176 << "for another" << waitSeconds << "s";
177
178 // We need to indicate that we skip this file due to blacklisting
179 // for reporting and for making sure we don't update the blacklist
180 // entry yet.
181 // Classification is this _instruction and _status
182 item._instruction = CSYNC_INSTRUCTION_IGNORE;
183 item._status = SyncFileItem::BlacklistedError;
184
185 auto waitSecondsStr = Utility::durationToDescriptiveString1(1000 * waitSeconds);
186 item._errorString = tr("%1 (skipped due to earlier error, trying again in %2)").arg(entry._errorString, waitSecondsStr);
187
188 if (entry._errorCategory == SyncJournalErrorBlacklistRecord::InsufficientRemoteStorage) {
189 slotInsufficientRemoteStorage();
190 }
191
192 return true;
193 }
194
isFileTransferInstruction(SyncInstructions instruction)195 static bool isFileTransferInstruction(SyncInstructions instruction)
196 {
197 return instruction == CSYNC_INSTRUCTION_CONFLICT
198 || instruction == CSYNC_INSTRUCTION_NEW
199 || instruction == CSYNC_INSTRUCTION_SYNC
200 || instruction == CSYNC_INSTRUCTION_TYPE_CHANGE;
201 }
202
deleteStaleDownloadInfos(const SyncFileItemVector & syncItems)203 void SyncEngine::deleteStaleDownloadInfos(const SyncFileItemVector &syncItems)
204 {
205 // Find all downloadinfo paths that we want to preserve.
206 QSet<QString> download_file_paths;
207 foreach (const SyncFileItemPtr &it, syncItems) {
208 if (it->_direction == SyncFileItem::Down
209 && it->_type == ItemTypeFile
210 && isFileTransferInstruction(it->_instruction)) {
211 download_file_paths.insert(it->_file);
212 }
213 }
214
215 // Delete from journal and from filesystem.
216 const QVector<SyncJournalDb::DownloadInfo> deleted_infos =
217 _journal->getAndDeleteStaleDownloadInfos(download_file_paths);
218 foreach (const SyncJournalDb::DownloadInfo &deleted_info, deleted_infos) {
219 const QString tmppath = _propagator->fullLocalPath(deleted_info._tmpfile);
220 qCInfo(lcEngine) << "Deleting stale temporary file: " << tmppath;
221 FileSystem::remove(tmppath);
222 }
223 }
224
deleteStaleUploadInfos(const SyncFileItemVector & syncItems)225 void SyncEngine::deleteStaleUploadInfos(const SyncFileItemVector &syncItems)
226 {
227 // Find all blacklisted paths that we want to preserve.
228 QSet<QString> upload_file_paths;
229 foreach (const SyncFileItemPtr &it, syncItems) {
230 if (it->_direction == SyncFileItem::Up
231 && it->_type == ItemTypeFile
232 && isFileTransferInstruction(it->_instruction)) {
233 upload_file_paths.insert(it->_file);
234 }
235 }
236
237 // Delete from journal.
238 auto ids = _journal->deleteStaleUploadInfos(upload_file_paths);
239
240 // Delete the stales chunk on the server.
241 if (account()->capabilities().chunkingNg()) {
242 foreach (uint transferId, ids) {
243 if (!transferId)
244 continue; // Was not a chunked upload
245 QUrl url = Utility::concatUrlPath(account()->url(), QLatin1String("remote.php/dav/uploads/") + account()->davUser() + QLatin1Char('/') + QString::number(transferId));
246 (new DeleteJob(account(), url, this))->start();
247 }
248 }
249 }
250
deleteStaleErrorBlacklistEntries(const SyncFileItemVector & syncItems)251 void SyncEngine::deleteStaleErrorBlacklistEntries(const SyncFileItemVector &syncItems)
252 {
253 // Find all blacklisted paths that we want to preserve.
254 QSet<QString> blacklist_file_paths;
255 foreach (const SyncFileItemPtr &it, syncItems) {
256 if (it->_hasBlacklistEntry)
257 blacklist_file_paths.insert(it->_file);
258 }
259
260 // Delete from journal.
261 _journal->deleteStaleErrorBlacklistEntries(blacklist_file_paths);
262 }
263
conflictRecordMaintenance()264 void SyncEngine::conflictRecordMaintenance()
265 {
266 // Remove stale conflict entries from the database
267 // by checking which files still exist and removing the
268 // missing ones.
269 auto conflictRecordPaths = _journal->conflictRecordPaths();
270 for (const auto &path : conflictRecordPaths) {
271 auto fsPath = _propagator->fullLocalPath(QString::fromUtf8(path));
272 if (!QFileInfo(fsPath).exists()) {
273 _journal->deleteConflictRecord(path);
274 }
275 }
276
277 // Did the sync see any conflict files that don't yet have records?
278 // If so, add them now.
279 //
280 // This happens when the conflicts table is new or when conflict files
281 // are downlaoded but the server doesn't send conflict headers.
282 for (const auto &path : _seenConflictFiles) {
283 OC_ASSERT(Utility::isConflictFile(path));
284
285 auto bapath = path.toUtf8();
286 if (!conflictRecordPaths.contains(bapath)) {
287 ConflictRecord record;
288 record.path = bapath;
289 auto basePath = Utility::conflictFileBaseNameFromPattern(bapath);
290 record.initialBasePath = basePath;
291
292 // Determine fileid of target file
293 SyncJournalFileRecord baseRecord;
294 if (_journal->getFileRecord(basePath, &baseRecord) && baseRecord.isValid()) {
295 record.baseFileId = baseRecord._fileId;
296 }
297
298 _journal->setConflictRecord(record);
299 }
300 }
301 }
302
303
slotItemDiscovered(const OCC::SyncFileItemPtr & item)304 void OCC::SyncEngine::slotItemDiscovered(const OCC::SyncFileItemPtr &item)
305 {
306 if (Utility::isConflictFile(item->_file))
307 _seenConflictFiles.insert(item->_file);
308 if (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA && !item->isDirectory()) {
309 // For directories, metadata-only updates will be done after all their files are propagated.
310
311 // Update the database now already: New remote fileid or Etag or RemotePerm
312 // Or for files that were detected as "resolved conflict".
313 // Or a local inode/mtime change
314
315 // In case of "resolved conflict": there should have been a conflict because they
316 // both were new, or both had their local mtime or remote etag modified, but the
317 // size and mtime is the same on the server. This typically happens when the
318 // database is removed. Nothing will be done for those files, but we still need
319 // to update the database.
320
321 // This metadata update *could* be a propagation job of its own, but since it's
322 // quick to do and we don't want to create a potentially large number of
323 // mini-jobs later on, we just update metadata right now.
324
325 if (item->_direction == SyncFileItem::Down) {
326 QString filePath = _localPath + item->_file;
327
328 // If the 'W' remote permission changed, update the local filesystem
329 SyncJournalFileRecord prev;
330 if (_journal->getFileRecord(item->_file, &prev)
331 && prev.isValid()
332 && prev._remotePerm.hasPermission(RemotePermissions::CanWrite) != item->_remotePerm.hasPermission(RemotePermissions::CanWrite)) {
333 const bool isReadOnly = !item->_remotePerm.isNull() && !item->_remotePerm.hasPermission(RemotePermissions::CanWrite);
334 FileSystem::setFileReadOnlyWeak(filePath, isReadOnly);
335 }
336 auto rec = item->toSyncJournalFileRecordWithInode(filePath);
337 if (rec._checksumHeader.isEmpty())
338 rec._checksumHeader = prev._checksumHeader;
339 rec._serverHasIgnoredFiles |= prev._serverHasIgnoredFiles;
340
341 // Ensure it's a placeholder file on disk
342 if (item->_type == ItemTypeFile) {
343 const auto result = _syncOptions._vfs->convertToPlaceholder(filePath, *item);
344 if (!result) {
345 item->_instruction = CSYNC_INSTRUCTION_ERROR;
346 item->_errorString = tr("Could not update file : %1").arg(result.error());
347 return;
348 }
349 }
350
351 // Update on-disk virtual file metadata
352 if (item->_type == ItemTypeVirtualFile) {
353 auto r = _syncOptions._vfs->updateMetadata(filePath, item->_modtime, item->_size, item->_fileId);
354 if (!r) {
355 item->_instruction = CSYNC_INSTRUCTION_ERROR;
356 item->_errorString = tr("Could not update virtual file metadata: %1").arg(r.error());
357 return;
358 }
359 }
360
361 // Updating the db happens on success
362 _journal->setFileRecord(rec);
363
364 // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example
365 emit itemCompleted(item);
366 } else {
367 // Update only outdated data from the disk.
368 _journal->updateLocalMetadata(item->_file, item->_modtime, item->_size, item->_inode);
369 }
370 _hasNoneFiles = true;
371 return;
372 } else if (item->_instruction == CSYNC_INSTRUCTION_NONE) {
373 _hasNoneFiles = true;
374 if (_account->capabilities().uploadConflictFiles() && Utility::isConflictFile(item->_file)) {
375 // For uploaded conflict files, files with no action performed on them should
376 // be displayed: but we mustn't overwrite the instruction if something happens
377 // to the file!
378 item->_errorString = tr("Unresolved conflict.");
379 item->_instruction = CSYNC_INSTRUCTION_IGNORE;
380 item->_status = SyncFileItem::Conflict;
381 }
382 return;
383 } else if (item->_instruction == CSYNC_INSTRUCTION_REMOVE && !item->_isSelectiveSync) {
384 _hasRemoveFile = true;
385 } else if (item->_instruction == CSYNC_INSTRUCTION_RENAME) {
386 _hasNoneFiles = true; // If a file (or every file) has been renamed, it means not al files where deleted
387 } else if (item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE
388 || item->_instruction == CSYNC_INSTRUCTION_SYNC) {
389 if (item->_direction == SyncFileItem::Up) {
390 // An upload of an existing file means that the file was left unchanged on the server
391 // This counts as a NONE for detecting if all the files on the server were changed
392 _hasNoneFiles = true;
393 }
394 }
395
396 // check for blacklisting of this item.
397 // if the item is on blacklist, the instruction was set to ERROR
398 checkErrorBlacklisting(*item);
399 _needsUpdate = true;
400
401 // Insert sorted
402 auto it = std::lower_bound( _syncItems.begin(), _syncItems.end(), item ); // the _syncItems is sorted
403 _syncItems.insert( it, item );
404
405 slotNewItem(item);
406
407 if (item->isDirectory()) {
408 slotFolderDiscovered(item->_etag.isEmpty(), item->_file);
409 }
410 }
411
startSync()412 void SyncEngine::startSync()
413 {
414 if (_journal->exists()) {
415 QVector<SyncJournalDb::PollInfo> pollInfos = _journal->getPollInfos();
416 if (!pollInfos.isEmpty()) {
417 qCInfo(lcEngine) << "Finish Poll jobs before starting a sync";
418 CleanupPollsJob *job = new CleanupPollsJob(pollInfos, _account,
419 _journal, _localPath, _syncOptions._vfs, this);
420 connect(job, &CleanupPollsJob::finished, this, &SyncEngine::startSync);
421 connect(job, &CleanupPollsJob::aborted, this, &SyncEngine::slotCleanPollsJobAborted);
422 job->start();
423 return;
424 }
425 }
426
427 if (_syncRunning) {
428 OC_ASSERT(false);
429 return;
430 }
431
432 s_anySyncRunning = true;
433 _syncRunning = true;
434 _anotherSyncNeeded = NoFollowUpSync;
435 _clearTouchedFilesTimer.stop();
436
437 _hasNoneFiles = false;
438 _hasRemoveFile = false;
439 _seenConflictFiles.clear();
440
441 _progressInfo->reset();
442
443 if (!QDir(_localPath).exists()) {
444 _anotherSyncNeeded = DelayedFollowUp;
445 // No _tr, it should only occur in non-mirall
446 syncError(QStringLiteral("Unable to find local sync folder."));
447 finalize(false);
448 return;
449 }
450
451 // Check free size on disk first.
452 const qint64 minFree = criticalFreeSpaceLimit();
453 const qint64 freeBytes = Utility::freeDiskSpace(_localPath);
454 if (freeBytes >= 0) {
455 if (freeBytes < minFree) {
456 qCWarning(lcEngine()) << "Too little space available at" << _localPath << ". Have"
457 << freeBytes << "bytes and require at least" << minFree << "bytes";
458 _anotherSyncNeeded = DelayedFollowUp;
459 syncError(tr("Only %1 are available, need at least %2 to start",
460 "Placeholders are postfixed with file sizes using Utility::octetsToString()")
461 .arg(
462 Utility::octetsToString(freeBytes),
463 Utility::octetsToString(minFree)));
464 finalize(false);
465 return;
466 } else {
467 qCInfo(lcEngine) << "There are" << freeBytes << "bytes available at" << _localPath;
468 }
469 } else {
470 qCWarning(lcEngine) << "Could not determine free space available at" << _localPath;
471 }
472
473 _syncItems.clear();
474 _needsUpdate = false;
475
476 if (!_journal->exists()) {
477 qCInfo(lcEngine) << "New sync (no sync journal exists)";
478 } else {
479 qCInfo(lcEngine) << "Sync with existing sync journal";
480 }
481
482 qCInfo(lcEngine) << "Using Qt " << qVersion() << " SSL library " << QSslSocket::sslLibraryVersionString() << " on " << Utility::platformName();
483
484 // This creates the DB if it does not exist yet.
485 if (!_journal->open()) {
486 qCWarning(lcEngine) << "No way to create a sync journal!";
487 syncError(tr("Unable to open or create the local sync database. Make sure you have write access in the sync folder."));
488 finalize(false);
489 return;
490 // database creation error!
491 }
492
493 // Functionality like selective sync might have set up etag storage
494 // filtering via schedulePathForRemoteDiscovery(). This *is* the next sync, so
495 // undo the filter to allow this sync to retrieve and store the correct etags.
496 _journal->clearEtagStorageFilter();
497
498 _excludedFiles->setExcludeConflictFiles(!_account->capabilities().uploadConflictFiles());
499
500 _lastLocalDiscoveryStyle = _localDiscoveryStyle;
501
502 if (_syncOptions._vfs->mode() == Vfs::WithSuffix && _syncOptions._vfs->fileSuffix().isEmpty()) {
503 syncError(tr("Using virtual files with suffix, but suffix is not set"));
504 finalize(false);
505 return;
506 }
507
508 bool ok;
509 auto selectiveSyncBlackList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, &ok);
510 if (ok) {
511 bool usingSelectiveSync = (!selectiveSyncBlackList.isEmpty());
512 qCInfo(lcEngine) << (usingSelectiveSync ? "Using Selective Sync" : "NOT Using Selective Sync");
513 } else {
514 qCWarning(lcEngine) << "Could not retrieve selective sync list from DB";
515 syncError(tr("Unable to read the blacklist from the local database"));
516 finalize(false);
517 return;
518 }
519
520 _stopWatch.start();
521 _progressInfo->_status = ProgressInfo::Starting;
522 emit transmissionProgress(*_progressInfo);
523
524 qCInfo(lcEngine) << "#### Discovery start ####################################################";
525 qCInfo(lcEngine) << "Server" << account()->serverVersion()
526 << (account()->isHttp2Supported() ? "Using HTTP/2" : "");
527 _progressInfo->_status = ProgressInfo::Discovery;
528 emit transmissionProgress(*_progressInfo);
529
530 _discoveryPhase.reset(new DiscoveryPhase);
531 _discoveryPhase->_account = _account;
532 _discoveryPhase->_excludes = _excludedFiles.data();
533 _discoveryPhase->_statedb = _journal;
534 _discoveryPhase->_localDir = _localPath;
535 if (!_discoveryPhase->_localDir.endsWith(QLatin1Char('/')))
536 _discoveryPhase->_localDir+=QLatin1Char('/');
537 _discoveryPhase->_remoteFolder = _remotePath;
538 if (!_discoveryPhase->_remoteFolder.endsWith(QLatin1Char('/')))
539 _discoveryPhase->_remoteFolder+=QLatin1Char('/');
540 _discoveryPhase->_syncOptions = _syncOptions;
541 _discoveryPhase->_shouldDiscoverLocaly = [this](const QString &s) { return shouldDiscoverLocally(s); };
542 _discoveryPhase->setSelectiveSyncBlackList(selectiveSyncBlackList);
543 _discoveryPhase->setSelectiveSyncWhiteList(_journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok));
544 if (!ok) {
545 qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
546 syncError(tr("Unable to read from the sync journal."));
547 finalize(false);
548 return;
549 }
550
551 // Check for invalid character in old server version
552 QString invalidFilenamePattern = _account->capabilities().invalidFilenameRegex();
553 if (invalidFilenamePattern.isNull()
554 && _account->serverVersionInt() < Account::makeServerVersion(8, 1, 0)) {
555 // Server versions older than 8.1 don't support some characters in filenames.
556 // If the capability is not set, default to a pattern that avoids uploading
557 // files with names that contain these.
558 // It's important to respect the capability also for older servers -- the
559 // version check doesn't make sense for custom servers.
560 invalidFilenamePattern = QLatin1String("[\\\\:?*\"<>|]");
561 }
562 if (!invalidFilenamePattern.isEmpty())
563 _discoveryPhase->_invalidFilenameRx = QRegExp(invalidFilenamePattern);
564 _discoveryPhase->_serverBlacklistedFiles = _account->capabilities().blacklistedFiles();
565 _discoveryPhase->_ignoreHiddenFiles = ignoreHiddenFiles();
566
567 connect(_discoveryPhase.data(), &DiscoveryPhase::itemDiscovered, this, &SyncEngine::slotItemDiscovered);
568 connect(_discoveryPhase.data(), &DiscoveryPhase::newBigFolder, this, &SyncEngine::newBigFolder);
569 connect(_discoveryPhase.data(), &DiscoveryPhase::fatalError, this, [this](const QString &errorString) {
570 syncError(errorString);
571 finalize(false);
572 });
573 connect(_discoveryPhase.data(), &DiscoveryPhase::finished, this, &SyncEngine::slotDiscoveryFinished);
574 connect(_discoveryPhase.data(), &DiscoveryPhase::silentlyExcluded,
575 _syncFileStatusTracker.data(), &SyncFileStatusTracker::slotAddSilentlyExcluded);
576
577 auto discoveryJob = new ProcessDirectoryJob(
578 _discoveryPhase.data(), PinState::AlwaysLocal, _discoveryPhase.data());
579 _discoveryPhase->startJob(discoveryJob);
580 connect(discoveryJob, &ProcessDirectoryJob::etag, this, &SyncEngine::slotRootEtagReceived);
581 }
582
slotFolderDiscovered(bool local,const QString & folder)583 void SyncEngine::slotFolderDiscovered(bool local, const QString &folder)
584 {
585 // Don't wanna overload the UI
586 if (!_lastUpdateProgressCallbackCall.isValid()) {
587 _lastUpdateProgressCallbackCall.start(); // first call
588 } else if (_lastUpdateProgressCallbackCall.elapsed() < 200) {
589 return;
590 } else {
591 _lastUpdateProgressCallbackCall.start();
592 }
593
594 if (local) {
595 _progressInfo->_currentDiscoveredLocalFolder = folder;
596 _progressInfo->_currentDiscoveredRemoteFolder.clear();
597 } else {
598 _progressInfo->_currentDiscoveredRemoteFolder = folder;
599 _progressInfo->_currentDiscoveredLocalFolder.clear();
600 }
601 emit transmissionProgress(*_progressInfo);
602 }
603
slotRootEtagReceived(const QString & e)604 void SyncEngine::slotRootEtagReceived(const QString &e)
605 {
606 if (_remoteRootEtag.isEmpty()) {
607 qCDebug(lcEngine) << "Root etag:" << e;
608 _remoteRootEtag = e;
609 emit rootEtag(_remoteRootEtag);
610 }
611 }
612
slotNewItem(const SyncFileItemPtr & item)613 void SyncEngine::slotNewItem(const SyncFileItemPtr &item)
614 {
615 _progressInfo->adjustTotalsForFile(*item);
616 }
617
slotDiscoveryFinished()618 void SyncEngine::slotDiscoveryFinished()
619 {
620 if (!_discoveryPhase) {
621 // There was an error that was already taken care of
622 return;
623 }
624
625 qCInfo(lcEngine) << "#### Discovery end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Discovery Finished")) << "ms";
626
627 // Sanity check
628 if (!_journal->open()) {
629 qCWarning(lcEngine) << "Bailing out, DB failure";
630 syncError(tr("Cannot open the sync journal"));
631 finalize(false);
632 return;
633 } else {
634 // Commits a possibly existing (should not though) transaction and starts a new one for the propagate phase
635 _journal->commitIfNeededAndStartNewTransaction(QStringLiteral("Post discovery"));
636 }
637
638 _progressInfo->_currentDiscoveredRemoteFolder.clear();
639 _progressInfo->_currentDiscoveredLocalFolder.clear();
640 _progressInfo->_status = ProgressInfo::Reconcile;
641 emit transmissionProgress(*_progressInfo);
642
643 // qCInfo(lcEngine) << "Permissions of the root folder: " << _csync_ctx->remote.root_perms.toString();
644 auto finish = [this]{
645
646
647 auto databaseFingerprint = _journal->dataFingerprint();
648 // If databaseFingerprint is empty, this means that there was no information in the database
649 // (for example, upgrading from a previous version, or first sync, or server not supporting fingerprint)
650 if (!databaseFingerprint.isEmpty() && _discoveryPhase
651 && _discoveryPhase->_dataFingerprint != databaseFingerprint) {
652 qCInfo(lcEngine) << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryPhase->_dataFingerprint;
653 restoreOldFiles(_syncItems);
654 }
655
656 if (_discoveryPhase->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) {
657 _anotherSyncNeeded = ImmediateFollowUp;
658 }
659
660 Q_ASSERT(std::is_sorted(_syncItems.begin(), _syncItems.end()));
661
662 const auto regex = syncOptions().fileRegex();
663 if (regex.isValid()) {
664 QSet<QStringRef> names;
665 for (auto &i : _syncItems) {
666 if (regex.match(i->_file).hasMatch()) {
667 int index = -1;
668 QStringRef ref;
669 do {
670 ref = i->_file.midRef(0, index);
671 names.insert(ref);
672 index = ref.lastIndexOf(QLatin1Char('/'));
673 } while (index > 0);
674 }
675 }
676 _syncItems.erase(std::remove_if(_syncItems.begin(), _syncItems.end(), [&names](auto i) {
677 return !names.contains(QStringRef { &i->_file });
678 }),
679 _syncItems.end());
680 }
681
682 qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate) #################################################### " << _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate)")) << "ms";
683
684 _localDiscoveryPaths.clear();
685
686 // To announce the beginning of the sync
687 emit aboutToPropagate(_syncItems);
688
689 qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate OK) #################################################### "<< _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate OK)")) << "ms";
690
691 // it's important to do this before ProgressInfo::start(), to announce start of new sync
692 _progressInfo->_status = ProgressInfo::Propagation;
693 emit transmissionProgress(*_progressInfo);
694 _progressInfo->startEstimateUpdates();
695
696 // post update phase script: allow to tweak stuff by a custom script in debug mode.
697 if (!qEnvironmentVariableIsEmpty("OWNCLOUD_POST_UPDATE_SCRIPT")) {
698 #ifndef NDEBUG
699 const QString script = qEnvironmentVariable("OWNCLOUD_POST_UPDATE_SCRIPT");
700
701 qCDebug(lcEngine) << "Post Update Script: " << script;
702 QProcess::execute(script);
703 #else
704 qCWarning(lcEngine) << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG";
705 #endif
706 }
707
708 // do a database commit
709 _journal->commit(QStringLiteral("post treewalk"));
710
711 _propagator = QSharedPointer<OwncloudPropagator>(
712 new OwncloudPropagator(_account, _localPath, _remotePath, _journal));
713 _propagator->setSyncOptions(_syncOptions);
714 connect(_propagator.data(), &OwncloudPropagator::itemCompleted,
715 this, &SyncEngine::slotItemCompleted);
716 connect(_propagator.data(), &OwncloudPropagator::progress,
717 this, &SyncEngine::slotProgress);
718 connect(_propagator.data(), &OwncloudPropagator::updateFileTotal,
719 this, &SyncEngine::updateFileTotal);
720 connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotPropagationFinished, Qt::QueuedConnection);
721 connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile);
722 connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile);
723 connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage);
724 connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage);
725 connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem);
726
727 // apply the network limits to the propagator
728 setNetworkLimits(_uploadLimit, _downloadLimit);
729
730 deleteStaleDownloadInfos(_syncItems);
731 deleteStaleUploadInfos(_syncItems);
732 deleteStaleErrorBlacklistEntries(_syncItems);
733 _journal->commit(QStringLiteral("post stale entry removal"));
734
735 // Emit the started signal only after the propagator has been set up.
736 if (_needsUpdate)
737 emit(started());
738
739 _propagator->start(std::move(_syncItems));
740
741 qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Post-Reconcile Finished")) << "ms";
742 };
743
744 if (!_hasNoneFiles && _hasRemoveFile) {
745 qCInfo(lcEngine) << "All the files are going to be changed, asking the user";
746 int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client
747 foreach (const auto &it, _syncItems) {
748 if (it->_instruction == CSYNC_INSTRUCTION_REMOVE) {
749 side += it->_direction == SyncFileItem::Down ? 1 : -1;
750 }
751 }
752
753 QPointer<QObject> guard = new QObject();
754 QPointer<QObject> self = this;
755 auto callback = [this, self, finish, guard](bool cancel) -> void {
756 // use a guard to ensure its only called once...
757 // qpointer to self to ensure we still exist
758 if (!guard || !self) {
759 return;
760 }
761 guard->deleteLater();
762 if (cancel) {
763 qCInfo(lcEngine) << "User aborted sync";
764 finalize(false);
765 return;
766 } else {
767 finish();
768 }
769 };
770 emit aboutToRemoveAllFiles(side >= 0 ? SyncFileItem::Down : SyncFileItem::Up, callback);
771 return;
772 }
773 finish();
774 }
775
slotCleanPollsJobAborted(const QString & error)776 void SyncEngine::slotCleanPollsJobAborted(const QString &error)
777 {
778 syncError(error);
779 finalize(false);
780 }
781
setNetworkLimits(int upload,int download)782 void SyncEngine::setNetworkLimits(int upload, int download)
783 {
784 _uploadLimit = upload;
785 _downloadLimit = download;
786
787 if (!_propagator)
788 return;
789
790 _propagator->_uploadLimit = upload;
791 _propagator->_downloadLimit = download;
792
793 if (upload != 0 || download != 0) {
794 qCInfo(lcEngine) << "Network Limits (down/up) " << upload << download;
795 }
796 }
797
slotItemCompleted(const SyncFileItemPtr & item)798 void SyncEngine::slotItemCompleted(const SyncFileItemPtr &item)
799 {
800 _progressInfo->setProgressComplete(*item);
801
802 emit transmissionProgress(*_progressInfo);
803 emit itemCompleted(item);
804 }
805
slotPropagationFinished(bool success)806 void SyncEngine::slotPropagationFinished(bool success)
807 {
808 if (_propagator->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) {
809 _anotherSyncNeeded = ImmediateFollowUp;
810 }
811
812 if (success && _discoveryPhase) {
813 _journal->setDataFingerprint(_discoveryPhase->_dataFingerprint);
814 }
815
816 conflictRecordMaintenance();
817
818 _journal->deleteStaleFlagsEntries();
819 _journal->commit(QStringLiteral("All Finished."), false);
820
821 // Send final progress information even if no
822 // files needed propagation, but clear the lastCompletedItem
823 // so we don't count this twice (like Recent Files)
824 _progressInfo->_lastCompletedItem = SyncFileItem();
825 _progressInfo->_status = ProgressInfo::Done;
826 emit transmissionProgress(*_progressInfo);
827
828 finalize(success);
829 }
830
finalize(bool success)831 void SyncEngine::finalize(bool success)
832 {
833 qCInfo(lcEngine) << "Sync run took " << _stopWatch.addLapTime(QStringLiteral("Sync Finished")) << "ms";
834 _stopWatch.stop();
835
836 if (_discoveryPhase) {
837 _discoveryPhase.take()->deleteLater();
838 }
839 s_anySyncRunning = false;
840 _syncRunning = false;
841 emit finished(success);
842
843 // Delete the propagator only after emitting the signal.
844 _propagator.clear();
845 _seenConflictFiles.clear();
846 _uniqueErrors.clear();
847 _localDiscoveryPaths.clear();
848 _localDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly;
849
850 _clearTouchedFilesTimer.start();
851 }
852
slotProgress(const SyncFileItem & item,qint64 current)853 void SyncEngine::slotProgress(const SyncFileItem &item, qint64 current)
854 {
855 _progressInfo->setProgressItem(item, current);
856 emit transmissionProgress(*_progressInfo);
857 }
858
updateFileTotal(const SyncFileItem & item,qint64 newSize)859 void SyncEngine::updateFileTotal(const SyncFileItem &item, qint64 newSize)
860 {
861 _progressInfo->updateTotalsForFile(item, newSize);
862 emit transmissionProgress(*_progressInfo);
863 }
restoreOldFiles(SyncFileItemVector & syncItems)864 void SyncEngine::restoreOldFiles(SyncFileItemVector &syncItems)
865 {
866 /* When the server is trying to send us lots of file in the past, this means that a backup
867 was restored in the server. In that case, we should not simply overwrite the newer file
868 on the file system with the older file from the backup on the server. Instead, we will
869 upload the client file. But we still downloaded the old file in a conflict file just in case
870 */
871
872 for (auto it = syncItems.begin(); it != syncItems.end(); ++it) {
873 if ((*it)->_direction != SyncFileItem::Down)
874 continue;
875
876 switch ((*it)->_instruction) {
877 case CSYNC_INSTRUCTION_SYNC:
878 qCWarning(lcEngine) << "restoreOldFiles: RESTORING" << (*it)->_file;
879 (*it)->_instruction = CSYNC_INSTRUCTION_CONFLICT;
880 break;
881 case CSYNC_INSTRUCTION_REMOVE:
882 qCWarning(lcEngine) << "restoreOldFiles: RESTORING" << (*it)->_file;
883 (*it)->_instruction = CSYNC_INSTRUCTION_NEW;
884 (*it)->_direction = SyncFileItem::Up;
885 break;
886 case CSYNC_INSTRUCTION_RENAME:
887 case CSYNC_INSTRUCTION_NEW:
888 // Ideally we should try to revert the rename or remove, but this would be dangerous
889 // without re-doing the reconcile phase. So just let it happen.
890 break;
891 default:
892 break;
893 }
894 }
895 }
896
slotAddTouchedFile(const QString & fn)897 void SyncEngine::slotAddTouchedFile(const QString &fn)
898 {
899 QElapsedTimer now;
900 now.start();
901 QString file = QDir::cleanPath(fn);
902
903 // Iterate from the oldest and remove anything older than 15 seconds.
904 while (true) {
905 auto first = _touchedFiles.begin();
906 if (first == _touchedFiles.end())
907 break;
908 // Compare to our new QElapsedTimer instead of using elapsed().
909 // This avoids querying the current time from the OS for every loop.
910 auto elapsed = std::chrono::milliseconds(now.msecsSinceReference() - first.key().msecsSinceReference());
911 if (elapsed <= s_touchedFilesMaxAgeMs) {
912 // We found the first path younger than the maximum age, keep the rest.
913 break;
914 }
915
916 _touchedFiles.erase(first);
917 }
918
919 // This should be the largest QElapsedTimer yet, use constEnd() as hint.
920 _touchedFiles.insert(_touchedFiles.constEnd(), now, file);
921 }
922
slotClearTouchedFiles()923 void SyncEngine::slotClearTouchedFiles()
924 {
925 _touchedFiles.clear();
926 }
927
wasFileTouched(const QString & fn) const928 bool SyncEngine::wasFileTouched(const QString &fn) const
929 {
930 // Start from the end (most recent) and look for our path. Check the time just in case.
931 auto begin = _touchedFiles.constBegin();
932 for (auto it = _touchedFiles.constEnd(); it != begin; --it) {
933 if ((it-1).value() == fn)
934 return std::chrono::milliseconds((it-1).key().elapsed()) <= s_touchedFilesMaxAgeMs;
935 }
936 return false;
937 }
938
account() const939 AccountPtr SyncEngine::account() const
940 {
941 return _account;
942 }
943
setLocalDiscoveryOptions(LocalDiscoveryStyle style,std::set<QString> paths)944 void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QString> paths)
945 {
946 _localDiscoveryStyle = style;
947 _localDiscoveryPaths = std::move(paths);
948
949 // Normalize to make sure that no path is a contained in another.
950 // Note: for simplicity, this code consider anything less than '/' as a path separator, so for
951 // example, this will remove "foo.bar" if "foo" is in the list. This will mean we might have
952 // some false positive, but that's Ok.
953 // This invariant is used in SyncEngine::shouldDiscoverLocally
954 QString prev;
955 auto it = _localDiscoveryPaths.begin();
956 while(it != _localDiscoveryPaths.end()) {
957 if (!prev.isNull() && it->startsWith(prev) && (prev.endsWith(QLatin1Char('/')) || *it == prev || it->at(prev.size()) <= QLatin1Char('/'))) {
958 it = _localDiscoveryPaths.erase(it);
959 } else {
960 prev = *it;
961 ++it;
962 }
963 }
964 }
965
shouldDiscoverLocally(const QString & path) const966 bool SyncEngine::shouldDiscoverLocally(const QString &path) const
967 {
968 if (_localDiscoveryStyle == LocalDiscoveryStyle::FilesystemOnly)
969 return true;
970
971 // The intention is that if "A/X" is in _localDiscoveryPaths:
972 // - parent folders like "/", "A" will be discovered (to make sure the discovery reaches the
973 // point where something new happened)
974 // - the folder itself "A/X" will be discovered
975 // - subfolders like "A/X/Y" will be discovered (so data inside a new or renamed folder will be
976 // discovered in full)
977 // Check out TestLocalDiscovery::testLocalDiscoveryDecision()
978
979 auto it = _localDiscoveryPaths.lower_bound(path);
980 if (it == _localDiscoveryPaths.end() || !it->startsWith(path)) {
981 // Maybe a subfolder of something in the list?
982 if (it != _localDiscoveryPaths.begin() && path.startsWith(*(--it))) {
983 return it->endsWith(QLatin1Char('/')) || (path.size() > it->size() && path.at(it->size()) <= QLatin1Char('/'));
984 }
985 return false;
986 }
987
988 // maybe an exact match or an empty path?
989 if (it->size() == path.size() || path.isEmpty())
990 return true;
991
992 // Maybe a parent folder of something in the list?
993 // check for a prefix + / match
994 forever {
995 if (it->size() > path.size() && it->at(path.size()) == QLatin1Char('/'))
996 return true;
997 ++it;
998 if (it == _localDiscoveryPaths.end() || !it->startsWith(path))
999 return false;
1000 }
1001 return false;
1002 }
1003
wipeVirtualFiles(const QString & localPath,SyncJournalDb & journal,Vfs & vfs)1004 void SyncEngine::wipeVirtualFiles(const QString &localPath, SyncJournalDb &journal, Vfs &vfs)
1005 {
1006 qCInfo(lcEngine) << "Wiping virtual files inside" << localPath;
1007 journal.getFilesBelowPath(QByteArray(), [&](const SyncJournalFileRecord &rec) {
1008 if (rec._type != ItemTypeVirtualFile && rec._type != ItemTypeVirtualFileDownload)
1009 return;
1010
1011 qCDebug(lcEngine) << "Removing db record for" << rec._path;
1012 journal.deleteFileRecord(QString::fromUtf8(rec._path));
1013
1014 // If the local file is a dehydrated placeholder, wipe it too.
1015 // Otherwise leave it to allow the next sync to have a new-new conflict.
1016 QString localFile = localPath + QString::fromUtf8(rec._path);
1017 if (QFile::exists(localFile) && vfs.isDehydratedPlaceholder(localFile)) {
1018 qCDebug(lcEngine) << "Removing local dehydrated placeholder" << rec._path;
1019 QFile::remove(localFile);
1020 }
1021 });
1022
1023 journal.forceRemoteDiscoveryNextSync();
1024
1025 // Postcondition: No ItemTypeVirtualFile / ItemTypeVirtualFileDownload left in the db.
1026 // But hydrated placeholders may still be around.
1027 }
1028
abort()1029 void SyncEngine::abort()
1030 {
1031 if (_propagator)
1032 qCInfo(lcEngine) << "Aborting sync";
1033
1034 if (_propagator) {
1035 // If we're already in the propagation phase, aborting that is sufficient
1036 _propagator->abort();
1037 } else if (_discoveryPhase) {
1038 // Delete the discovery and all child jobs after ensuring
1039 // it can't finish and start the propagator
1040 disconnect(_discoveryPhase.data(), nullptr, this, nullptr);
1041 _discoveryPhase.take()->deleteLater();
1042
1043 syncError(tr("Aborted"));
1044 finalize(false);
1045 }
1046 }
1047
slotSummaryError(const QString & message)1048 void SyncEngine::slotSummaryError(const QString &message)
1049 {
1050 if (_uniqueErrors.contains(message))
1051 return;
1052
1053 _uniqueErrors.insert(message);
1054 emit syncError(message, ErrorCategory::Normal);
1055 }
1056
slotInsufficientLocalStorage()1057 void SyncEngine::slotInsufficientLocalStorage()
1058 {
1059 slotSummaryError(
1060 tr("Disk space is low: Downloads that would reduce free space "
1061 "below %1 were skipped.")
1062 .arg(Utility::octetsToString(freeSpaceLimit())));
1063 }
1064
slotInsufficientRemoteStorage()1065 void SyncEngine::slotInsufficientRemoteStorage()
1066 {
1067 auto msg = tr("There is insufficient space available on the server for some uploads.");
1068 if (_uniqueErrors.contains(msg))
1069 return;
1070
1071 _uniqueErrors.insert(msg);
1072 emit syncError(msg, ErrorCategory::InsufficientRemoteStorage);
1073 }
1074
1075 } // namespace OCC
1076