1 /*
2  *    This software is in the public domain, furnished "as is", without technical
3  *    support, and with no warranty, express or implied, as to its usefulness for
4  *    any purpose.
5  *
6  */
7 
8 #include <QtTest>
9 #include "syncenginetestutils.h"
10 #include <syncengine.h>
11 
12 using namespace OCC;
13 
14 /* Upload a 1/3 of a file of given size.
15  * fakeFolder needs to be synchronized */
partialUpload(FakeFolder & fakeFolder,const QString & name,qint64 size)16 static void partialUpload(FakeFolder &fakeFolder, const QString &name, qint64 size)
17 {
18     QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
19     QCOMPARE(fakeFolder.uploadState().children.count(), 0); // The state should be clean
20 
21     fakeFolder.localModifier().insert(name, size);
22     // Abort when the upload is at 1/3
23     qint64 sizeWhenAbort = -1;
24     auto con = QObject::connect(&fakeFolder.syncEngine(),  &SyncEngine::transmissionProgress,
25                                     [&](const ProgressInfo &progress) {
26                 if (progress.completedSize() > (progress.totalSize() /3 )) {
27                     sizeWhenAbort = progress.completedSize();
28                     fakeFolder.syncEngine().abort();
29                 }
30     });
31 
32     QVERIFY(!fakeFolder.syncOnce()); // there should have been an error
33     QObject::disconnect(con);
34     QVERIFY(sizeWhenAbort > 0);
35     QVERIFY(sizeWhenAbort < size);
36 
37     QCOMPARE(fakeFolder.uploadState().children.count(), 1); // the transfer was done with chunking
38     auto upStateChildren = fakeFolder.uploadState().children.first().children;
39     QCOMPARE(sizeWhenAbort, std::accumulate(upStateChildren.cbegin(), upStateChildren.cend(), 0,
40                                             [](int s, const FileInfo &i) { return s + i.size; }));
41 }
42 
43 // Reduce max chunk size a bit so we get more chunks
setChunkSize(SyncEngine & engine,qint64 size)44 static void setChunkSize(SyncEngine &engine, qint64 size)
45 {
46     SyncOptions options;
47     options._maxChunkSize = size;
48     options._initialChunkSize = size;
49     options._minChunkSize = size;
50     engine.setSyncOptions(options);
51 }
52 
53 class TestChunkingNG : public QObject
54 {
55     Q_OBJECT
56 
57 private slots:
58 
testFileUpload()59     void testFileUpload() {
60         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
61         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
62         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
63         const int size = 10 * 1000 * 1000; // 10 MB
64 
65         fakeFolder.localModifier().insert("A/a0", size);
66         QVERIFY(fakeFolder.syncOnce());
67         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
68         QCOMPARE(fakeFolder.uploadState().children.count(), 1); // the transfer was done with chunking
69         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
70 
71         // Check that another upload of the same file also work.
72         fakeFolder.localModifier().appendByte("A/a0");
73         QVERIFY(fakeFolder.syncOnce());
74         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
75         QCOMPARE(fakeFolder.uploadState().children.count(), 2); // the transfer was done with chunking
76     }
77 
78     // Test resuming when there's a confusing chunk added
testResume1()79     void testResume1() {
80         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
81         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
82         const int size = 10 * 1000 * 1000; // 10 MB
83         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
84 
85         partialUpload(fakeFolder, "A/a0", size);
86         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
87         auto chunkingId = fakeFolder.uploadState().children.first().name;
88         const auto &chunkMap = fakeFolder.uploadState().children.first().children;
89         qint64 uploadedSize = std::accumulate(chunkMap.begin(), chunkMap.end(), 0LL, [](qint64 s, const FileInfo &f) { return s + f.size; });
90         QVERIFY(uploadedSize > 2 * 1000 * 1000); // at least 2 MB
91 
92         // Add a fake chunk to make sure it gets deleted
93         fakeFolder.uploadState().children.first().insert("10000", size);
94 
95         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
96             if (op == QNetworkAccessManager::PutOperation) {
97                 // Test that we properly resuming and are not sending past data again.
98                 Q_ASSERT(request.rawHeader("OC-Chunk-Offset").toLongLong() >= uploadedSize);
99             } else if (op == QNetworkAccessManager::DeleteOperation) {
100                 Q_ASSERT(request.url().path().endsWith("/10000"));
101             }
102             return nullptr;
103         });
104 
105         QVERIFY(fakeFolder.syncOnce());
106 
107         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
108         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
109         // The same chunk id was re-used
110         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
111         QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
112     }
113 
114     // Test resuming when one of the uploaded chunks got removed
testResume2()115     void testResume2() {
116         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
117         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
118         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
119         const int size = 30 * 1000 * 1000; // 30 MB
120         partialUpload(fakeFolder, "A/a0", size);
121         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
122         auto chunkingId = fakeFolder.uploadState().children.first().name;
123         const auto &chunkMap = fakeFolder.uploadState().children.first().children;
124         qint64 uploadedSize = std::accumulate(chunkMap.begin(), chunkMap.end(), 0LL, [](qint64 s, const FileInfo &f) { return s + f.size; });
125         QVERIFY(uploadedSize > 2 * 1000 * 1000); // at least 50 MB
126         QVERIFY(chunkMap.size() >= 3); // at least three chunks
127 
128         QStringList chunksToDelete;
129 
130         // Remove the second chunk, so all further chunks will be deleted and resent
131         auto firstChunk = chunkMap.first();
132         auto secondChunk = *(chunkMap.begin() + 1);
133         for (const auto& name : chunkMap.keys().mid(2)) {
134             chunksToDelete.append(name);
135         }
136         fakeFolder.uploadState().children.first().remove(secondChunk.name);
137 
138         QStringList deletedPaths;
139         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
140             if (op == QNetworkAccessManager::PutOperation) {
141                 // Test that we properly resuming, not resending the first chunk
142                 Q_ASSERT(request.rawHeader("OC-Chunk-Offset").toLongLong() >= firstChunk.size);
143             } else if (op == QNetworkAccessManager::DeleteOperation) {
144                 deletedPaths.append(request.url().path());
145             }
146             return nullptr;
147         });
148 
149         QVERIFY(fakeFolder.syncOnce());
150 
151         for (const auto& toDelete : chunksToDelete) {
152             bool wasDeleted = false;
153             for (const auto& deleted : deletedPaths) {
154                 if (deleted.mid(deleted.lastIndexOf('/') + 1) == toDelete) {
155                     wasDeleted = true;
156                     break;
157                 }
158             }
159             QVERIFY(wasDeleted);
160         }
161 
162         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
163         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
164         // The same chunk id was re-used
165         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
166         QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
167     }
168 
169     // Test resuming when all chunks are already present
testResume3()170     void testResume3() {
171         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
172         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
173         const int size = 30 * 1000 * 1000; // 30 MB
174         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
175 
176         partialUpload(fakeFolder, "A/a0", size);
177         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
178         auto chunkingId = fakeFolder.uploadState().children.first().name;
179         const auto &chunkMap = fakeFolder.uploadState().children.first().children;
180         qint64 uploadedSize = std::accumulate(chunkMap.begin(), chunkMap.end(), 0LL, [](qint64 s, const FileInfo &f) { return s + f.size; });
181         QVERIFY(uploadedSize > 5 * 1000 * 1000); // at least 5 MB
182 
183         // Add a chunk that makes the file completely uploaded
184         fakeFolder.uploadState().children.first().insert(
185             QString::number(chunkMap.size()).rightJustified(16, '0'), size - uploadedSize);
186 
187         bool sawPut = false;
188         bool sawDelete = false;
189         bool sawMove = false;
190         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
191             if (op == QNetworkAccessManager::PutOperation) {
192                 sawPut = true;
193             } else if (op == QNetworkAccessManager::DeleteOperation) {
194                 sawDelete = true;
195             } else if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
196                 sawMove = true;
197             }
198             return nullptr;
199         });
200 
201         QVERIFY(fakeFolder.syncOnce());
202         QVERIFY(sawMove);
203         QVERIFY(!sawPut);
204         QVERIFY(!sawDelete);
205 
206         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
207         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
208         // The same chunk id was re-used
209         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
210         QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
211     }
212 
213     // Test resuming (or rather not resuming!) for the error case of the sum of
214     // chunk sizes being larger than the file size
testResume4()215     void testResume4() {
216         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
217         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
218         const int size = 30 * 1000 * 1000; // 30 MB
219         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
220 
221         partialUpload(fakeFolder, "A/a0", size);
222         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
223         auto chunkingId = fakeFolder.uploadState().children.first().name;
224         const auto &chunkMap = fakeFolder.uploadState().children.first().children;
225         qint64 uploadedSize = std::accumulate(chunkMap.begin(), chunkMap.end(), 0LL, [](qint64 s, const FileInfo &f) { return s + f.size; });
226         QVERIFY(uploadedSize > 5 * 1000 * 1000); // at least 5 MB
227 
228         // Add a chunk that makes the file more than completely uploaded
229         fakeFolder.uploadState().children.first().insert(
230             QString::number(chunkMap.size()).rightJustified(16, '0'), size - uploadedSize + 100);
231 
232         QVERIFY(fakeFolder.syncOnce());
233 
234         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
235         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
236         // Used a new transfer id but wiped the old one
237         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
238         QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
239     }
240 
241     // Check what happens when we abort during the final MOVE and the
242     // the final MOVE takes longer than the abort-delay
testLateAbortHard()243     void testLateAbortHard()
244     {
245         FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
246         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" } } }, { "checksums", QVariantMap{ { "supportedTypes", QStringList() << "SHA1" } } } });
247         const int size = 15 * 1000 * 1000; // 15 MB
248         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
249 
250         // Make the MOVE never reply, but trigger a client-abort and apply the change remotely
251         QObject parent;
252         QByteArray moveChecksumHeader;
253         int nGET = 0;
254         int responseDelay = 100000; // bigger than abort-wait timeout
255         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
256             if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
257                 QTimer::singleShot(50, &parent, [&]() { fakeFolder.syncEngine().abort(); });
258                 moveChecksumHeader = request.rawHeader("OC-Checksum");
259                 return new DelayedReply<FakeChunkMoveReply>(responseDelay, fakeFolder.uploadState(), fakeFolder.remoteModifier(), op, request, &parent);
260             } else if (op == QNetworkAccessManager::GetOperation) {
261                 nGET++;
262             }
263             return nullptr;
264         });
265 
266 
267         // Test 1: NEW file aborted
268         fakeFolder.localModifier().insert("A/a0", size);
269         QVERIFY(!fakeFolder.syncOnce()); // error: abort!
270 
271         // Now the next sync gets a NEW/NEW conflict and since there's no checksum
272         // it just becomes a UPDATE_METADATA
273         auto checkEtagUpdated = [&](SyncFileItemVector &items) {
274             QCOMPARE(items.size(), 1);
275             QCOMPARE(items[0]->_file, QLatin1String("A"));
276             SyncJournalFileRecord record;
277             QVERIFY(fakeFolder.syncJournal().getFileRecord(QByteArray("A/a0"), &record));
278             QCOMPARE(record._etag, fakeFolder.remoteModifier().find("A/a0")->etag);
279         };
280         auto connection = connect(&fakeFolder.syncEngine(), &SyncEngine::aboutToPropagate, checkEtagUpdated);
281         QVERIFY(fakeFolder.syncOnce());
282         disconnect(connection);
283         QCOMPARE(nGET, 0);
284         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
285 
286 
287         // Test 2: modified file upload aborted
288         fakeFolder.localModifier().appendByte("A/a0");
289         QVERIFY(!fakeFolder.syncOnce()); // error: abort!
290 
291         // An EVAL/EVAL conflict is also UPDATE_METADATA when there's no checksums
292         connection = connect(&fakeFolder.syncEngine(), &SyncEngine::aboutToPropagate, checkEtagUpdated);
293         QVERIFY(fakeFolder.syncOnce());
294         disconnect(connection);
295         QCOMPARE(nGET, 0);
296         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
297 
298 
299         // Test 3: modified file upload aborted, with good checksums
300         fakeFolder.localModifier().appendByte("A/a0");
301         QVERIFY(!fakeFolder.syncOnce()); // error: abort!
302 
303         // Set the remote checksum -- the test setup doesn't do it automatically
304         QVERIFY(!moveChecksumHeader.isEmpty());
305         fakeFolder.remoteModifier().find("A/a0")->checksums = moveChecksumHeader;
306 
307         QVERIFY(fakeFolder.syncOnce());
308         disconnect(connection);
309         QCOMPARE(nGET, 0); // no new download, just a metadata update!
310         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
311 
312 
313         // Test 4: New file, that gets deleted locally before the next sync
314         fakeFolder.localModifier().insert("A/a3", size);
315         QVERIFY(!fakeFolder.syncOnce()); // error: abort!
316         fakeFolder.localModifier().remove("A/a3");
317 
318         // bug: in this case we must expect a re-download of A/A3
319         QVERIFY(fakeFolder.syncOnce());
320         QCOMPARE(nGET, 1);
321         QVERIFY(fakeFolder.currentLocalState().find("A/a3"));
322         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
323     }
324 
325     // Check what happens when we abort during the final MOVE and the
326     // the final MOVE is short enough for the abort-delay to help
testLateAbortRecoverable()327     void testLateAbortRecoverable()
328     {
329         FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
330         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" } } }, { "checksums", QVariantMap{ { "supportedTypes", QStringList() << "SHA1" } } } });
331         const int size = 15 * 1000 * 1000; // 15 MB
332         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
333 
334         // Make the MOVE never reply, but trigger a client-abort and apply the change remotely
335         QObject parent;
336         int responseDelay = 200; // smaller than abort-wait timeout
337         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
338             if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
339                 QTimer::singleShot(50, &parent, [&]() { fakeFolder.syncEngine().abort(); });
340                 return new DelayedReply<FakeChunkMoveReply>(responseDelay, fakeFolder.uploadState(), fakeFolder.remoteModifier(), op, request, &parent);
341             }
342             return nullptr;
343         });
344 
345         // Test 1: NEW file aborted
346         fakeFolder.localModifier().insert("A/a0", size);
347         QVERIFY(fakeFolder.syncOnce());
348         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
349 
350         // Test 2: modified file upload aborted
351         fakeFolder.localModifier().appendByte("A/a0");
352         QVERIFY(fakeFolder.syncOnce());
353         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
354     }
355 
356     // We modify the file locally after it has been partially uploaded
testRemoveStale1()357     void testRemoveStale1() {
358 
359         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
360         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
361         const int size = 10 * 1000 * 1000; // 10 MB
362         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
363 
364         partialUpload(fakeFolder, "A/a0", size);
365         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
366         auto chunkingId = fakeFolder.uploadState().children.first().name;
367 
368 
369         fakeFolder.localModifier().setContents("A/a0", 'B');
370         fakeFolder.localModifier().appendByte("A/a0");
371 
372         QVERIFY(fakeFolder.syncOnce());
373 
374         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
375         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size + 1);
376         // A different chunk id was used, and the previous one is removed
377         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
378         QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
379     }
380 
381     // We remove the file locally after it has been partially uploaded
testRemoveStale2()382     void testRemoveStale2() {
383 
384         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
385         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
386         const int size = 10 * 1000 * 1000; // 10 MB
387         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
388 
389         partialUpload(fakeFolder, "A/a0", size);
390         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
391 
392         fakeFolder.localModifier().remove("A/a0");
393 
394         QVERIFY(fakeFolder.syncOnce());
395         QCOMPARE(fakeFolder.uploadState().children.count(), 0);
396     }
397 
398 
testCreateConflictWhileSyncing()399     void testCreateConflictWhileSyncing() {
400         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
401         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
402         const int size = 10 * 1000 * 1000; // 10 MB
403         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
404 
405         // Put a file on the server and download it.
406         fakeFolder.remoteModifier().insert("A/a0", size);
407         QVERIFY(fakeFolder.syncOnce());
408         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
409 
410         // Modify the file localy and start the upload
411         fakeFolder.localModifier().setContents("A/a0", 'B');
412         fakeFolder.localModifier().appendByte("A/a0");
413 
414         // But in the middle of the sync, modify the file on the server
415         QMetaObject::Connection con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
416                                     [&](const ProgressInfo &progress) {
417                 if (progress.completedSize() > (progress.totalSize() / 2 )) {
418                     fakeFolder.remoteModifier().setContents("A/a0", 'C');
419                     QObject::disconnect(con);
420                 }
421         });
422 
423         QVERIFY(!fakeFolder.syncOnce());
424         // There was a precondition failed error, this means wen need to sync again
425         QCOMPARE(fakeFolder.syncEngine().isAnotherSyncNeeded(), ImmediateFollowUp);
426 
427         QCOMPARE(fakeFolder.uploadState().children.count(), 1); // We did not clean the chunks at this point
428 
429         // Now we will download the server file and create a conflict
430         QVERIFY(fakeFolder.syncOnce());
431         auto localState = fakeFolder.currentLocalState();
432 
433         // A0 is the one from the server
434         QCOMPARE(localState.find("A/a0")->size, size);
435         QCOMPARE(localState.find("A/a0")->contentChar, 'C');
436 
437         // There is a conflict file with our version
438         auto &stateAChildren = localState.find("A")->children;
439         auto it = std::find_if(stateAChildren.cbegin(), stateAChildren.cend(), [&](const FileInfo &fi) {
440             return fi.name.startsWith("a0 (conflicted copy");
441         });
442         QVERIFY(it != stateAChildren.cend());
443         QCOMPARE(it->contentChar, 'B');
444         QCOMPARE(it->size, size+1);
445 
446         // Remove the conflict file so the comparison works!
447         fakeFolder.localModifier().remove("A/" + it->name);
448 
449         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
450 
451         QCOMPARE(fakeFolder.uploadState().children.count(), 0); // The last sync cleaned the chunks
452     }
453 
testModifyLocalFileWhileUploading()454     void testModifyLocalFileWhileUploading() {
455 
456         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
457         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
458         const int size = 10 * 1000 * 1000; // 10 MB
459         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
460 
461         fakeFolder.localModifier().insert("A/a0", size);
462 
463         // middle of the sync, modify the file
464         QMetaObject::Connection con = QObject::connect(&fakeFolder.syncEngine(), &SyncEngine::transmissionProgress,
465                                     [&](const ProgressInfo &progress) {
466                 if (progress.completedSize() > (progress.totalSize() / 2 )) {
467                     fakeFolder.localModifier().setContents("A/a0", 'B');
468                     fakeFolder.localModifier().appendByte("A/a0");
469                     QObject::disconnect(con);
470                 }
471         });
472 
473         QVERIFY(!fakeFolder.syncOnce());
474 
475         // There should be a followup sync
476         QCOMPARE(fakeFolder.syncEngine().isAnotherSyncNeeded(), ImmediateFollowUp);
477 
478         QCOMPARE(fakeFolder.uploadState().children.count(), 1); // We did not clean the chunks at this point
479         auto chunkingId = fakeFolder.uploadState().children.first().name;
480 
481         // Now we make a new sync which should upload the file for good.
482         QVERIFY(fakeFolder.syncOnce());
483 
484         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
485         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size+1);
486 
487         // A different chunk id was used, and the previous one is removed
488         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
489         QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
490     }
491 
492 
testResumeServerDeletedChunks()493     void testResumeServerDeletedChunks() {
494 
495         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
496         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
497         const int size = 30 * 1000 * 1000; // 30 MB
498         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
499         partialUpload(fakeFolder, "A/a0", size);
500         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
501         auto chunkingId = fakeFolder.uploadState().children.first().name;
502 
503         // Delete the chunks on the server
504         fakeFolder.uploadState().children.clear();
505         QVERIFY(fakeFolder.syncOnce());
506 
507         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
508         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
509 
510         // A different chunk id was used
511         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
512         QVERIFY(fakeFolder.uploadState().children.first().name != chunkingId);
513     }
514 
515     // Check what happens when the connection is dropped on the PUT (non-chunking) or MOVE (chunking)
516     // for on the issue #5106
connectionDroppedBeforeEtagRecieved_data()517     void connectionDroppedBeforeEtagRecieved_data()
518     {
519         QTest::addColumn<bool>("chunking");
520         QTest::newRow("big file") << true;
521         QTest::newRow("small file") << false;
522     }
connectionDroppedBeforeEtagRecieved()523     void connectionDroppedBeforeEtagRecieved()
524     {
525         QFETCH(bool, chunking);
526         FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
527         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" } } }, { "checksums", QVariantMap{ { "supportedTypes", QStringList() << "SHA1" } } } });
528         const int size = chunking ? 1 * 1000 * 1000 : 300;
529         setChunkSize(fakeFolder.syncEngine(), 300 * 1000);
530 
531         // Make the MOVE never reply, but trigger a client-abort and apply the change remotely
532         QByteArray checksumHeader;
533         int nGET = 0;
534         QScopedValueRollback<int> setHttpTimeout(AbstractNetworkJob::httpTimeout, 1);
535         int responseDelay = AbstractNetworkJob::httpTimeout * 1000 * 1000; // much bigger than http timeout (so a timeout will occur)
536         // This will perform the operation on the server, but the reply will not come to the client
537         fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *outgoingData) -> QNetworkReply * {
538             if (!chunking) {
539                 Q_ASSERT(!request.url().path().contains("/uploads/")
540                     && "Should not touch uploads endpoint when not chunking");
541             }
542             if (!chunking && op == QNetworkAccessManager::PutOperation) {
543                 checksumHeader = request.rawHeader("OC-Checksum");
544                 return new DelayedReply<FakePutReply>(responseDelay, fakeFolder.remoteModifier(), op, request, outgoingData->readAll(), &fakeFolder.syncEngine());
545             } else if (chunking && request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
546                 checksumHeader = request.rawHeader("OC-Checksum");
547                 return new DelayedReply<FakeChunkMoveReply>(responseDelay, fakeFolder.uploadState(), fakeFolder.remoteModifier(), op, request, &fakeFolder.syncEngine());
548             } else if (op == QNetworkAccessManager::GetOperation) {
549                 nGET++;
550             }
551             return nullptr;
552         });
553 
554         // Test 1: a NEW file
555         fakeFolder.localModifier().insert("A/a0", size);
556         QVERIFY(!fakeFolder.syncOnce()); // timeout!
557         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); // but the upload succeeded
558         QVERIFY(!checksumHeader.isEmpty());
559         fakeFolder.remoteModifier().find("A/a0")->checksums = checksumHeader; // The test system don't do that automatically
560         // Should be resolved properly
561         QVERIFY(fakeFolder.syncOnce());
562         QCOMPARE(nGET, 0);
563         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
564 
565         // Test 2: Modify the file further
566         fakeFolder.localModifier().appendByte("A/a0");
567         QVERIFY(!fakeFolder.syncOnce()); // timeout!
568         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); // but the upload succeeded
569         fakeFolder.remoteModifier().find("A/a0")->checksums = checksumHeader;
570         // modify again, should not cause conflict
571         fakeFolder.localModifier().appendByte("A/a0");
572         QVERIFY(!fakeFolder.syncOnce()); // now it's trying to upload the modified file
573         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
574         fakeFolder.remoteModifier().find("A/a0")->checksums = checksumHeader;
575         QVERIFY(fakeFolder.syncOnce());
576         QCOMPARE(nGET, 0);
577     }
578 
testPercentEncoding()579     void testPercentEncoding() {
580         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
581         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
582         const int size = 5 * 1000 * 1000;
583         setChunkSize(fakeFolder.syncEngine(), 1 * 1000 * 1000);
584 
585         fakeFolder.localModifier().insert("A/file % \u20ac", size);
586         QVERIFY(fakeFolder.syncOnce());
587         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
588 
589         // Only the second upload contains an "If" header
590         fakeFolder.localModifier().appendByte("A/file % \u20ac");
591         QVERIFY(fakeFolder.syncOnce());
592         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
593     }
594 
595     // Test uploading large files (2.5GiB)
testVeryBigFiles()596     void testVeryBigFiles() {
597         FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
598         fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ {"chunking", "1.0"} } } });
599         const qint64 size = 2.5 * 1024 * 1024 * 1024; // 2.5 GiB
600 
601         // Partial upload of big files
602         partialUpload(fakeFolder, "A/a0", size);
603         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
604         auto chunkingId = fakeFolder.uploadState().children.first().name;
605 
606         // Now resume
607         QVERIFY(fakeFolder.syncOnce());
608         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
609         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size);
610 
611         // The same chunk id was re-used
612         QCOMPARE(fakeFolder.uploadState().children.count(), 1);
613         QCOMPARE(fakeFolder.uploadState().children.first().name, chunkingId);
614 
615 
616         // Upload another file again, this time without interruption
617         fakeFolder.localModifier().appendByte("A/a0");
618         QVERIFY(fakeFolder.syncOnce());
619         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
620         QCOMPARE(fakeFolder.currentRemoteState().find("A/a0")->size, size + 1);
621     }
622 
623 
624 };
625 
626 QTEST_GUILESS_MAIN(TestChunkingNG)
627 #include "testchunkingng.moc"
628