1 /* This file is part of the KDE project
2
3 Copyright (C) 2008 Lukas Appelhans <l.appelhans@gmx.de>
4 Copyright (C) 2009 Matthias Fuchs <mat69@gmx.net>
5
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public
8 License as published by the Free Software Foundation; either
9 version 2 of the License, or (at your option) any later version.
10 */
11 #include "datasourcefactory.h"
12 #include "bitset.h"
13 #include "settings.h"
14
15 #include "core/filedeleter.h"
16 #include "core/kget.h"
17 #include "core/signature.h"
18 #include "core/verifier.h"
19
20 #include <cmath>
21
22 #include <QDir>
23 #include <QTimer>
24 #include <QVarLengthArray>
25 #include <QDomText>
26
27 #include <KIO/FileJob>
28 #include <KLocalizedString>
29 #include <KMessageBox>
30 #include <KMountPoint>
31
32 #include "kget_debug.h"
33
34
35 #include <qplatformdefs.h>
36
37 const int SPEEDTIMER = 1000;//1 second...
38
DataSourceFactory(QObject * parent,const QUrl & dest,KIO::filesize_t size,KIO::fileoffset_t segSize)39 DataSourceFactory::DataSourceFactory(QObject *parent, const QUrl &dest, KIO::filesize_t size, KIO::fileoffset_t segSize)
40 : QObject(parent),
41 m_capabilities(),
42 m_dest(dest),
43 m_size(size),
44 m_downloadedSize(0),
45 m_segSize(segSize),
46 m_speed(0),
47 m_percent(0),
48 m_tempOffset(0),
49 m_startedChunks(nullptr),
50 m_finishedChunks(nullptr),
51 m_putJob(nullptr),
52 m_doDownload(true),
53 m_open(false),
54 m_blocked(false),
55 m_startTried(false),
56 m_findFilesizeTried(false),
57 m_assignTried(false),
58 m_movingFile(false),
59 m_finished(false),
60 m_downloadInitialized(false),
61 m_sizeInitiallyDefined(m_size),
62 m_sizeFoundOnFinish(false),
63 m_maxMirrorsUsed(3),
64 m_speedTimer(nullptr),
65 m_status(Job::Stopped),
66 m_statusBeforeMove(m_status),
67 m_verifier(nullptr),
68 m_signature(nullptr)
69 {
70 qCDebug(KGET_DEBUG) << "Initialize DataSourceFactory: Dest: " + m_dest.toLocalFile() + "Size: " + QString::number(m_size) + "SegSize: " + QString::number(m_segSize);
71
72 m_prevDownloadedSizes.append(0);
73 }
74
~DataSourceFactory()75 DataSourceFactory::~DataSourceFactory()
76 {
77 killPutJob();
78 delete m_startedChunks;
79 delete m_finishedChunks;
80 }
81
init()82 void DataSourceFactory::init()
83 {
84
85 if (!m_doDownload)
86 {
87 return;
88 }
89
90 if (!m_speedTimer)
91 {
92 m_speedTimer = new QTimer(this);
93 m_speedTimer->setInterval(SPEEDTIMER);
94 connect(m_speedTimer, &QTimer::timeout, this, &DataSourceFactory::speedChanged);
95 }
96
97 if (m_segSize && m_size)
98 {
99 const int hasRemainder = (m_size % m_segSize == 0) ? 0 : 1;
100 const int bitSetSize = (m_size / m_segSize) + hasRemainder;//round up if needed
101 if (!m_startedChunks && bitSetSize)
102 {
103 m_startedChunks = new BitSet(bitSetSize);
104 }
105 if (!m_finishedChunks && bitSetSize)
106 {
107 m_finishedChunks = new BitSet(bitSetSize);
108 }
109 }
110 }
111
deinit()112 void DataSourceFactory::deinit()
113 {
114 if (m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
115 FileDeleter::deleteFile(m_dest);
116 }
117 }
118
findFileSize()119 void DataSourceFactory::findFileSize()
120 {
121 qCDebug(KGET_DEBUG) << "Find the filesize" << this;
122 if (!m_size && !m_dest.isEmpty() && !m_sources.isEmpty()) {
123 foreach (TransferDataSource *source, m_sources) {
124 if (source->capabilities() & Transfer::Cap_FindFilesize) {
125 connect(source, &TransferDataSource::foundFileSize, this, &DataSourceFactory::slotFoundFileSize);
126 connect(source, &TransferDataSource::finishedDownload, this, &DataSourceFactory::slotFinishedDownload);
127
128 m_speedTimer->start();
129 source->findFileSize(m_segSize);
130 changeStatus(Job::Running);
131 slotUpdateCapabilities();
132 return;
133 }
134 }
135 }
136 }
137
slotFoundFileSize(TransferDataSource * source,KIO::filesize_t fileSize,const QPair<int,int> & segmentRange)138 void DataSourceFactory::slotFoundFileSize(TransferDataSource *source, KIO::filesize_t fileSize, const QPair<int, int> &segmentRange)
139 {
140 m_size = fileSize;
141 qCDebug(KGET_DEBUG) << source << "found size" << m_size << "and is assigned segments" << segmentRange << this;
142 Q_EMIT dataSourceFactoryChange(Transfer::Tc_TotalSize);
143
144 init();
145
146 if ((segmentRange.first != -1) && (segmentRange.second != -1)) {
147 m_startedChunks->setRange(segmentRange.first, segmentRange.second, true);
148 }
149
150 if (m_startTried) {
151 start();
152 }
153 }
154
slotFinishedDownload(TransferDataSource * source,KIO::filesize_t size)155 void DataSourceFactory::slotFinishedDownload(TransferDataSource *source, KIO::filesize_t size)
156 {
157 Q_UNUSED(source)
158 Q_UNUSED(size)
159
160 m_speedTimer->stop();
161 m_finished = true;
162 }
163
checkLocalFile()164 bool DataSourceFactory::checkLocalFile()
165 {
166 QString dest_orig = m_dest.toLocalFile();
167 QString _dest_part(dest_orig);
168
169 QT_STATBUF buff_part;
170 bool bPartExists = (QT_STAT( _dest_part.toUtf8().constData(), &buff_part ) != -1);
171 if(!bPartExists)
172 {
173 QString _dest = dest_orig;
174 int fd = -1;
175 mode_t initialMode = 0666;
176
177 fd = QT_OPEN(_dest.toUtf8().constData(), O_CREAT | O_TRUNC | O_WRONLY, initialMode);
178 if ( fd < 0 )
179 {
180 qCDebug(KGET_DEBUG) << " error";
181 return false;
182 }
183 else
184 {
185 close(fd);
186 }
187 }//TODO: Port to use Qt functions maybe
188
189 qCDebug(KGET_DEBUG) << "success";
190 return true;
191 }
192
start()193 void DataSourceFactory::start()
194 {
195 qCDebug(KGET_DEBUG) << "Start DataSourceFactory";
196 if (m_movingFile || (m_status == Job::Finished))
197 {
198 return;
199 }
200 if (!m_doDownload)
201 {
202 m_startTried = true;
203 return;
204 }
205
206 //the file already exists, even though DataSourceFactory has not been initialized remove it
207 //to avoid problems like over methods not finished removing it because of a redownload
208 if (!m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
209 qCDebug(KGET_DEBUG) << "Removing existing file.";
210 m_startTried = true;
211 FileDeleter::deleteFile(m_dest, this, SLOT(slotRemovedFile()));
212 return;
213 }
214
215 m_downloadInitialized = true;
216
217 //create all dirs needed
218 QDir dir;
219 dir.mkpath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
220 if (checkLocalFile()) {
221 if (!m_putJob) {
222 m_putJob = KIO::open(m_dest, QIODevice::WriteOnly | QIODevice::ReadOnly);
223 connect(m_putJob, &KIO::FileJob::open, this, &DataSourceFactory::slotOpen);
224 connect(m_putJob, &QObject::destroyed, this, &DataSourceFactory::slotPutJobDestroyed);
225 m_startTried = true;
226 return;
227 }
228 } else {
229 //could not create file, maybe device not mounted so abort
230 m_startTried = true;
231 changeStatus(Job::Aborted);
232 return;
233 }
234
235 init();
236
237 if (!m_open) {
238 m_startTried = true;
239 return;
240 }
241
242 if (!m_size) {
243 if (!m_findFilesizeTried && m_sources.count()) {
244 m_findFilesizeTried = true;
245 findFileSize();
246 }
247 m_startTried = true;
248 return;
249 }
250
251 if (assignNeeded()) {
252 if (m_sources.count()) {
253 qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
254 //simply assign a TransferDataSource
255 assignSegments(*m_sources.begin());
256 } else if (m_unusedUrls.count()) {
257 qCDebug(KGET_DEBUG) << "Assigning an unused mirror";
258 //takes the first unused mirror
259 addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
260 }
261 }
262
263 if (m_assignTried) {
264 m_assignTried = false;
265
266 foreach(TransferDataSource *source, m_sources) {
267 assignSegments(source);
268 }
269 }
270
271 if (m_open) {
272 //check if the filesystem supports a file of m_size
273 const static KIO::filesize_t maxFatSize = 4294967295;
274 if (m_size > maxFatSize) {
275 KMountPoint::Ptr mountPoint = KMountPoint::currentMountPoints().findByPath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
276 if (mountPoint) {
277 if (mountPoint->mountType() == "vfat") {//TODO check what is reported on Windows for vfat
278 stop();
279 KMessageBox::error(nullptr, i18n("Filesize is larger than maximum file size supported by VFAT."), i18n("Error"));
280 return;
281 }
282 }
283 }
284
285 QFile::resize(m_dest.toLocalFile(), m_size);//TODO should we keep that?
286 m_speedTimer->start();
287
288 foreach (TransferDataSource *source, m_sources) {
289 source->start();
290 }
291
292 m_startTried = false;
293 changeStatus(Job::Running);
294 }
295 slotUpdateCapabilities();
296 }
297
slotRemovedFile()298 void DataSourceFactory::slotRemovedFile()
299 {
300 qCDebug(KGET_DEBUG) << "File has been removed" << this;
301 if (m_startTried) {
302 m_startTried = false;
303 start();
304 }
305 }
306
slotOpen(KIO::Job * job)307 void DataSourceFactory::slotOpen(KIO::Job *job)
308 {
309 Q_UNUSED(job)
310 qCDebug(KGET_DEBUG) << "File opened" << this;
311
312 if (!m_speedTimer)
313 {
314 init();
315 }
316
317 connect(m_putJob, &KIO::FileJob::position, this, &DataSourceFactory::slotOffset);
318 connect(m_putJob, &KIO::FileJob::written, this, &DataSourceFactory::slotDataWritten);
319 m_open = true;
320
321 if (m_startTried)
322 {
323 start();
324 }
325 }
326
stop()327 void DataSourceFactory::stop()
328 {
329 qCDebug(KGET_DEBUG) << "Stopping" << this;
330 if (m_movingFile || (m_status == Job::Finished))
331 {
332 return;
333 }
334
335 if (m_speedTimer)
336 {
337 m_speedTimer->stop();
338 }
339
340 foreach (TransferDataSource *source, m_sources) {
341 source->stop();
342 }
343 m_startTried = false;
344 m_findFilesizeTried = false;
345 changeStatus(Job::Stopped);
346
347 slotUpdateCapabilities();
348 }
349
setDoDownload(bool doDownload)350 void DataSourceFactory::setDoDownload(bool doDownload)
351 {
352 if (m_doDownload == doDownload) {
353 return;
354 }
355
356 m_doDownload = doDownload;
357 if (m_doDownload)
358 {
359 if (m_startTried)
360 {
361 start();
362 }
363 }
364 else
365 {
366 if (m_status == Job::Running)
367 {
368 stop();
369 }
370 }
371 }
372
addMirror(const QUrl & url,int numParallelConnections)373 void DataSourceFactory::addMirror(const QUrl &url, int numParallelConnections)
374 {
375 addMirror(url, true, numParallelConnections, false);
376 }
377
addMirror(const QUrl & url,bool used,int numParallelConnections)378 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections)
379 {
380 addMirror(url, used, numParallelConnections, true);
381 }
382
addMirror(const QUrl & url,bool used,int numParallelConnections,bool usedDefined)383 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections, bool usedDefined)
384 {
385 if (!url.isValid() || url.isEmpty())
386 {
387 qCDebug(KGET_DEBUG) << "Url is not usable: " << url.toString();
388 return;
389 }
390 if (numParallelConnections <= 0)
391 {
392 numParallelConnections = 1;
393 }
394 if (!usedDefined)
395 {
396 used = true;
397 }
398
399 if (used)
400 {
401 //there is already a TransferDataSource with that url, reuse it and modify numParallelSegments
402 if (m_sources.contains(url))
403 {
404 TransferDataSource *source = m_sources[url];
405 source->setParallelSegments(numParallelConnections);
406 if (source->changeNeeded() > 0) {
407 assignSegments(source);
408 } else {
409 for (int i = source->changeNeeded(); i < 0; ++i)
410 {
411 const QPair<int, int> removed = source->removeConnection();
412 qCDebug(KGET_DEBUG) << "Remove connection with segments" << removed;
413 const int start = removed.first;
414 const int end = removed.second;
415 if ((start != -1) && (end != -1)) {
416 m_startedChunks->setRange(start, end, false);
417 }
418 }
419 }
420 }
421 else
422 {
423 if (m_sources.count() < m_maxMirrorsUsed)
424 {
425 TransferDataSource *source = KGet::createTransferDataSource(url, QDomElement(), this);
426 if (source)
427 {
428 qCDebug(KGET_DEBUG) << "Successfully created a TransferDataSource for " << url.toString() << this;
429
430 //url might have been an unused Mirror, so remove it in any case
431 const int index = m_unusedUrls.indexOf(url);
432 if (index > -1 )
433 {
434 m_unusedUrls.removeAt(index);
435 m_unusedConnections.removeAt(index);
436 }
437
438 m_sources[url] = source;
439 m_sources[url]->setParallelSegments(numParallelConnections);
440 if (m_sizeInitiallyDefined) {
441 source->setSupposedSize(m_size);
442 }
443
444 connect(source, &TransferDataSource::capabilitiesChanged, this, &DataSourceFactory::slotUpdateCapabilities);
445 connect(source, SIGNAL(brokenSegments(TransferDataSource*,QPair<int,int>)), this, SLOT(brokenSegments(TransferDataSource*,QPair<int,int>)));
446 connect(source, &TransferDataSource::broken, this, &DataSourceFactory::broken);
447 connect(source, &TransferDataSource::finishedSegment, this, &DataSourceFactory::finishedSegment);
448 connect(source, SIGNAL(data(KIO::fileoffset_t,QByteArray,bool&)), this, SLOT(slotWriteData(KIO::fileoffset_t,QByteArray,bool&)));
449 connect(source, SIGNAL(freeSegments(TransferDataSource*,QPair<int,int>)), this, SLOT(slotFreeSegments(TransferDataSource*,QPair<int,int>)));
450 connect(source, &TransferDataSource::log, this, &DataSourceFactory::log);
451 connect(source, &TransferDataSource::urlChanged, this, &DataSourceFactory::slotUrlChanged);
452
453 slotUpdateCapabilities();
454
455 assignSegments(source);
456
457 //the job is already running, so also start the TransferDataSource
458 if (!m_assignTried && !m_startTried && m_putJob && m_open && (m_status == Job::Running))
459 {
460 source->start();
461 }
462 }
463 else
464 {
465 qCDebug(KGET_DEBUG) << "A TransferDataSource could not be created for " << url.toString();
466 }
467 }
468 else if (usedDefined)
469 {
470 //increase the number of allowed mirrors as the user wants to use this one!
471 ++m_maxMirrorsUsed;
472 addMirror(url, used, numParallelConnections, usedDefined);
473 return;
474 }
475 else
476 {
477 m_unusedUrls.append(url);
478 m_unusedConnections.append(numParallelConnections);
479 }
480 }
481 }
482 else
483 {
484 if (m_sources.contains(url))
485 {
486 removeMirror(url);
487 }
488 else
489 {
490 m_unusedUrls.append(url);
491 m_unusedConnections.append(numParallelConnections);
492 }
493 }
494 }
495
slotUrlChanged(const QUrl & old,const QUrl & newUrl)496 void DataSourceFactory::slotUrlChanged(const QUrl &old, const QUrl &newUrl)
497 {
498 TransferDataSource * ds = m_sources.take(old);
499 m_sources[newUrl] = ds;
500 Q_EMIT dataSourceFactoryChange(Transfer::Tc_Source | Transfer::Tc_FileName);
501 }
502
removeMirror(const QUrl & url)503 void DataSourceFactory::removeMirror(const QUrl &url)
504 {
505 qCDebug(KGET_DEBUG) << "Removing mirror: " << url;
506 if (m_sources.contains(url))
507 {
508 TransferDataSource *source = m_sources[url];
509 source->stop();
510 const QList<QPair<int, int> > assigned = source->assignedSegments();
511 m_sources.remove(url);
512 m_unusedUrls.append(url);
513 m_unusedConnections.append(source->parallelSegments());
514 delete source;
515
516 for (int i = 0; i < assigned.count(); ++i)
517 {
518 const int start = assigned[i].first;
519 const int end = assigned[i].second;
520 if ((start != -1) && (end != -1)) {
521 m_startedChunks->setRange(start, end, false);
522 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
523 }
524 }
525 }
526
527 if ((m_status == Job::Running) && assignNeeded()) {
528 //here we only handle the case when there are existing TransferDataSources,
529 //the other case is triggered when stopping and then starting again
530 if (m_sources.count()) {
531 qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
532 //simply assign a TransferDataSource
533 assignSegments(*m_sources.begin());
534 }
535 }
536 }
537
setMirrors(const QHash<QUrl,QPair<bool,int>> & mirrors)538 void DataSourceFactory::setMirrors(const QHash<QUrl, QPair<bool, int> > &mirrors)
539 {
540 //first remove the not set DataSources
541 QList<QUrl> oldUrls = m_sources.keys();
542 QList<QUrl> newUrls = mirrors.keys();
543
544 foreach (const QUrl &url, oldUrls)
545 {
546 if (!newUrls.contains(url))
547 {
548 removeMirror(url);
549 }
550 }
551
552 //remove all unused Mirrors and simply readd them below
553 m_unusedUrls.clear();
554 m_unusedConnections.clear();
555
556 //second modify the existing DataSources and add the new ones
557 QHash<QUrl, QPair<bool, int> >::const_iterator it;
558 QHash<QUrl, QPair<bool, int> >::const_iterator itEnd = mirrors.constEnd();
559 for (it = mirrors.constBegin(); it != itEnd; ++it)
560 {
561 addMirror(it.key(), it.value().first, it.value().second, true);
562 }
563 }
564
mirrors() const565 QHash<QUrl, QPair<bool, int> > DataSourceFactory::mirrors() const
566 {
567 QHash<QUrl, QPair<bool, int> > mirrors;
568
569 QHash<QUrl, TransferDataSource*>::const_iterator it;
570 QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
571 for (it = m_sources.constBegin(); it != itEnd; ++it) {
572 mirrors[it.key()] = QPair<bool, int>(true, (*it)->parallelSegments());
573 }
574
575 for (int i = 0; i < m_unusedUrls.count(); ++i) {
576 mirrors[m_unusedUrls[i]] = QPair<bool, int>(false, m_unusedConnections[i]);
577 }
578
579 return mirrors;
580 }
581
assignNeeded() const582 bool DataSourceFactory::assignNeeded() const
583 {
584 bool assignNeeded = true;
585 QHash<QUrl, TransferDataSource*>::const_iterator it;
586 QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
587 for (it = m_sources.constBegin(); it != itEnd; ++it) {
588 if ((*it)->currentSegments()) {
589 //at least one TransferDataSource is still running, so no assign needed
590 assignNeeded = false;
591 break;
592 }
593 }
594 return assignNeeded;
595 }
596
brokenSegments(TransferDataSource * source,const QPair<int,int> & segmentRange)597 void DataSourceFactory::brokenSegments(TransferDataSource *source, const QPair<int, int> &segmentRange)
598 {
599 qCDebug(KGET_DEBUG) << "Segments" << segmentRange << "broken," << source;
600 if (!source || !m_startedChunks || !m_finishedChunks || (segmentRange.first < 0) || (segmentRange.second < 0) || (static_cast<quint32>(segmentRange.second) > m_finishedChunks->getNumBits()))
601 {
602 return;
603 }
604
605 const int start = segmentRange.first;
606 const int end = segmentRange.second;
607 if ((start != -1) && (end != -1)) {
608 m_startedChunks->setRange(start, end, false);
609 }
610
611 removeMirror(source->sourceUrl());
612 }
613
614
broken(TransferDataSource * source,TransferDataSource::Error error)615 void DataSourceFactory::broken(TransferDataSource *source, TransferDataSource::Error error)
616 {
617 qCDebug(KGET_DEBUG) << source << "is broken with error" << error;
618 const QString url = source->sourceUrl().toString();
619
620 removeMirror(source->sourceUrl());
621
622 if (error == TransferDataSource::WrongDownloadSize)
623 {
624 KMessageBox::error(nullptr, i18nc("A mirror is removed when the file has the wrong download size", "%1 removed as it did report a wrong file size.", url), i18n("Error"));
625 }
626 }
627
slotFreeSegments(TransferDataSource * source,QPair<int,int> segmentRange)628 void DataSourceFactory::slotFreeSegments(TransferDataSource *source, QPair<int, int> segmentRange)
629 {
630 qCDebug(KGET_DEBUG) << "Segments freed:" << source << segmentRange;
631
632 const int start = segmentRange.first;
633 const int end = segmentRange.second;
634 if ((start != -1) && (end != -1)) {
635 m_startedChunks->setRange(start, end, false);
636 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
637 }
638
639 assignSegments(source);
640 }
641
finishedSegment(TransferDataSource * source,int segmentNumber,bool connectionFinished)642 void DataSourceFactory::finishedSegment(TransferDataSource *source, int segmentNumber, bool connectionFinished)
643 {
644 if (!source || (segmentNumber < 0) || (static_cast<quint32>(segmentNumber) > m_finishedChunks->getNumBits()))
645 {
646 qCDebug(KGET_DEBUG) << "Incorrect data";
647 return;
648 }
649
650 m_finishedChunks->set(segmentNumber, true);
651
652 if (!connectionFinished)
653 {
654 qCDebug(KGET_DEBUG) << "Some segments still not finished";
655 return;
656 }
657
658 m_finished = m_finishedChunks->allOn();
659 if (m_finished)
660 {
661 qDebug() << "All segments have been downloaded.";
662 return;
663 }
664
665 assignSegments(source);
666 }
667
assignSegments(TransferDataSource * source)668 void DataSourceFactory::assignSegments(TransferDataSource *source)
669 {
670 if (!m_startedChunks || !m_finishedChunks)
671 {
672 qCDebug(KGET_DEBUG) << "Assign tried";
673 m_assignTried = true;
674 return;
675 }
676 if (m_finishedChunks->allOn())
677 {
678 qCDebug(KGET_DEBUG) << "All segments are finished already.";
679 return;
680 }
681
682 //no more segments needed for that TransferDataSource
683 if (source->changeNeeded() <= 0) {
684 qCDebug(KGET_DEBUG) << "No change needed for" << source;
685 return;
686 }
687
688 //find the segments that should be assigned to the transferDataSource
689 int newStart = -1;
690 int newEnd = -1;
691
692 //a split needed
693 if (m_startedChunks->allOn()) {
694 int unfinished = 0;
695 TransferDataSource *target = nullptr;
696 foreach (TransferDataSource *source, m_sources) {
697 int temp = source->countUnfinishedSegments();
698 if (temp > unfinished) {
699 unfinished = temp;
700 target = source;
701 }
702 }
703 if (!unfinished || !target) {
704 return;
705 }
706
707 const QPair<int, int> splitResult = target->split();
708 newStart = splitResult.first;
709 newEnd = splitResult.second;
710 } else {
711 m_startedChunks->getContinuousRange(&newStart, &newEnd, false);
712 }
713
714 if ((newStart == -1) || (newEnd == -1))
715 {
716 qCDebug(KGET_DEBUG) << "No segment can be assigned.";
717 return;
718 }
719
720 const KIO::fileoffset_t rest = m_size % m_segSize;
721
722 //the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
723 const KIO::fileoffset_t lastSegSize = ((static_cast<uint>(newEnd + 1) == m_startedChunks->getNumBits() && rest) ? rest : m_segSize);
724
725 qCDebug(KGET_DEBUG) << "Segments assigned:" << newStart << "-" << newEnd << "segment-size:" << m_segSize << "rest:" << rest;
726 m_startedChunks->setRange(newStart, newEnd, true);
727 source->addSegments(qMakePair(m_segSize, lastSegSize), qMakePair(newStart, newEnd));
728
729 //there should still be segments added to this transfer
730 if (source->changeNeeded() > 0) {
731 assignSegments(source);
732 }
733 }
734
735 //TODO implement checks if the correct offsets etc. are used + error recovering e.g. when something else
736 //touches the file
slotWriteData(KIO::fileoffset_t offset,const QByteArray & data,bool & worked)737 void DataSourceFactory::slotWriteData(KIO::fileoffset_t offset, const QByteArray &data, bool &worked)
738 {
739 worked = !m_blocked && !m_movingFile && m_open;
740 if (m_blocked || m_movingFile || !m_open)
741 {
742 return;
743 }
744
745 m_blocked = true;
746 m_tempOffset = offset;
747 m_tempData = data;
748 m_putJob->seek(offset);
749 }
750
slotOffset(KIO::Job * job,KIO::filesize_t offset)751 void DataSourceFactory::slotOffset(KIO::Job *job , KIO::filesize_t offset)
752 {
753 Q_UNUSED(job)
754 Q_UNUSED(offset)
755
756 m_putJob->write(m_tempData);
757 }
758
slotDataWritten(KIO::Job * job,KIO::filesize_t written)759 void DataSourceFactory::slotDataWritten(KIO::Job *job, KIO::filesize_t written)
760 {
761 Q_UNUSED(job)
762
763 auto tempSize = static_cast<KIO::filesize_t>(m_tempData.size());
764 //the complete data has been written
765 if (written == tempSize)//TODO if not same cache it temporarily!
766 {
767 m_downloadedSize += written;
768 Q_EMIT dataSourceFactoryChange(Transfer::Tc_DownloadedSize);
769 // m_tempCache.clear();
770 }
771
772 if (m_finished)
773 {
774 m_speedTimer->stop();
775 killPutJob();
776 changeStatus(Job::Finished);
777 }
778 m_tempData.clear();
779 m_blocked = false;
780 }
781
slotPercent(KJob * job,ulong p)782 void DataSourceFactory::slotPercent(KJob* job, ulong p)
783 {
784 Q_UNUSED(job)
785 m_percent = p;
786 Q_EMIT dataSourceFactoryChange(Transfer::Tc_Percent);
787 }
788
speedChanged()789 void DataSourceFactory::speedChanged()
790 {
791 m_speed = (m_downloadedSize - m_prevDownloadedSizes.first()) / (SPEEDTIMER * m_prevDownloadedSizes.size() / 1000);//downloaded in 1 second
792
793 m_prevDownloadedSizes.append(m_downloadedSize);
794 if(m_prevDownloadedSizes.size() > 10)
795 m_prevDownloadedSizes.removeFirst();
796
797 ulong percent = (m_size ? (m_downloadedSize * 100 / m_size) : 0);
798 const bool percentChanged = (percent != m_percent);
799 m_percent = percent;
800
801 Transfer::ChangesFlags change = (percentChanged ? (Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent) : Transfer::Tc_DownloadSpeed);
802 Q_EMIT dataSourceFactoryChange(change);
803 }
804
killPutJob()805 void DataSourceFactory::killPutJob()
806 {
807 if (m_putJob)
808 {
809 qCDebug(KGET_DEBUG) << "Closing the file";
810 m_open = false;
811 m_putJob->close();
812 m_putJob = nullptr;
813 }
814 }
815
slotPutJobDestroyed(QObject * job)816 void DataSourceFactory::slotPutJobDestroyed(QObject *job)
817 {
818 Q_UNUSED(job)
819
820 m_putJob = nullptr;
821 }
822
setNewDestination(const QUrl & newDestination)823 bool DataSourceFactory::setNewDestination(const QUrl &newDestination)
824 {
825 m_newDest = newDestination;
826 if (m_newDest.isValid() && (m_newDest != m_dest))
827 {
828 //No files created yet, simply change the urls
829 if (!m_downloadInitialized) {
830 m_dest = m_newDest;
831 if (m_verifier) {
832 verifier()->setDestination(m_dest);
833 }
834 if (m_signature) {
835 signature()->setDestination(m_dest);
836 }
837
838 return true;
839 } else if (QFile::exists(m_dest.toString())) {
840 //create all dirs needed
841 QDir dir;
842 dir.mkpath(m_newDest.adjusted(QUrl::RemoveFilename).toString());
843
844 m_statusBeforeMove = m_status;
845 stop();
846 changeStatus(Job::Moving);
847 m_movingFile = true;
848
849 //still a write in progress
850 if (m_blocked)
851 {
852 QTimer::singleShot(1000, this, &DataSourceFactory::startMove);
853 }
854 else
855 {
856 startMove();
857 }
858 return true;
859 }
860 }
861 return false;
862 }
863
startMove()864 void DataSourceFactory::startMove()
865 {
866 killPutJob();
867
868 KIO::Job *move = KIO::file_move(m_dest, m_newDest, -1, KIO::HideProgressInfo);
869 connect(move, &KJob::result, this, &DataSourceFactory::newDestResult);
870 connect(move, SIGNAL(percent(KJob*,ulong)), this, SLOT(slotPercent(KJob*,ulong)));
871
872 m_dest = m_newDest;
873 verifier()->setDestination(m_dest);
874 signature()->setDestination(m_dest);
875 }
876
newDestResult(KJob * job)877 void DataSourceFactory::newDestResult(KJob *job)
878 {
879 Q_UNUSED(job)//TODO handle errors etc.?
880
881 m_movingFile = false;
882 changeStatus(m_statusBeforeMove);
883 if (m_status == Job::Running)
884 {
885 start();
886 }
887 }
888
repair()889 void DataSourceFactory::repair()
890 {
891 if (verifier()->status() != Verifier::NotVerified)
892 {
893 return;
894 }
895
896 m_finished = false;
897
898 connect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>,KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>,KIO::filesize_t)));
899
900 verifier()->brokenPieces();
901 }
902
slotRepair(const QList<KIO::fileoffset_t> & offsets,KIO::filesize_t length)903 void DataSourceFactory::slotRepair(const QList<KIO::fileoffset_t> &offsets, KIO::filesize_t length)
904 {
905 disconnect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>,KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>,KIO::filesize_t)));
906
907 if (!m_startedChunks || !m_finishedChunks)
908 {
909 qCDebug(KGET_DEBUG) << "Redownload everything";
910 m_downloadedSize = 0;
911 }
912 else
913 {
914 if (offsets.isEmpty()) {
915 m_startedChunks->clear();
916 m_finishedChunks->clear();
917 }
918 qCDebug(KGET_DEBUG) << "Redownload broken pieces";
919 for (int i = 0; i < offsets.count(); ++i) {
920 const int start = offsets[i] / m_segSize;
921 const int end = std::ceil(length / static_cast<double>(m_segSize)) - 1 + start;
922 m_startedChunks->setRange(start, end, false);
923 m_finishedChunks->setRange(start, end, false);
924 }
925
926 m_downloadedSize = m_segSize * m_finishedChunks->numOnBits();
927 }
928 m_prevDownloadedSizes.clear();
929 m_prevDownloadedSizes.append(m_downloadedSize);
930
931 //remove all current mirrors and readd the first unused mirror
932 const QList<QUrl> mirrors = m_sources.keys();//FIXME only remove the mirrors of the broken segments?! --> for that m_assignedChunks would need to be saved was well
933 foreach (const QUrl &mirror, mirrors)
934 {
935 removeMirror(mirror);
936 }
937 addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
938
939 m_speed = 0;
940
941 Transfer::ChangesFlags change = (Transfer::Tc_DownloadSpeed | Transfer::Tc_DownloadedSize);
942 if (m_size) {
943 change |= Transfer::Tc_Percent;
944 m_percent = (m_downloadedSize * 100 / m_size);
945 }
946 Q_EMIT dataSourceFactoryChange(change);
947 m_status = Job::Stopped;
948
949 start();
950 }
951
load(const QDomElement * element)952 void DataSourceFactory::load(const QDomElement *element)
953 {
954 if (!element)
955 {
956 return;
957 }
958
959 QDomElement e = element->firstChildElement("factory");
960 if (e.isNull())
961 {
962 e = element->firstChildElement("factories").firstChildElement("factory");
963 }
964
965 //only try to load values if they haven't been set
966 if (m_dest.isEmpty())
967 {
968 m_dest = QUrl(e.attribute("dest"));
969 }
970
971 verifier()->load(e);
972 signature()->load(e);
973
974 Transfer::ChangesFlags change = Transfer::Tc_None;
975
976 if (!m_size) {
977 m_size = e.attribute("size").toULongLong();
978 change |= Transfer::Tc_TotalSize;
979 }
980 KIO::fileoffset_t tempSegSize = e.attribute("segmentSize").toLongLong();
981 if (tempSegSize)
982 {
983 m_segSize = tempSegSize;
984 }
985 if (!m_downloadedSize) {
986 m_downloadedSize = e.attribute("processedSize").toULongLong();
987 change |= Transfer::Tc_DownloadedSize;
988 if (m_size) {
989 m_percent = (m_downloadedSize * 100 / m_size);
990 change |= Transfer::Tc_Percent;
991 }
992 }
993 if (e.hasAttribute("doDownload"))
994 {
995 m_doDownload = QVariant(e.attribute("doDownload")).toBool();
996 }
997 if (e.hasAttribute("downloadInitialized")) {
998 m_downloadInitialized = QVariant(e.attribute("downloadInitialized")).toBool();
999 }
1000 if (e.hasAttribute("maxMirrorsUsed"))
1001 {
1002 bool worked;
1003 m_maxMirrorsUsed = e.attribute("maxMirrorsUsed").toInt(&worked);
1004 m_maxMirrorsUsed = (worked ? m_maxMirrorsUsed : 3);
1005 }
1006 m_sizeInitiallyDefined = QVariant(e.attribute("sizeInitiallyDefined", "false")).toBool();
1007 m_sizeFoundOnFinish = QVariant(e.attribute("sizeFoundOnFinish", "false")).toBool();
1008
1009 //load the finishedChunks
1010 const QDomElement chunks = e.firstChildElement("chunks");
1011 const QDomNodeList chunkList = chunks.elementsByTagName("chunk");
1012
1013 const quint32 numBits = chunks.attribute("numBits").toInt();
1014 const int numBytes = chunks.attribute("numBytes").toInt();
1015 QVarLengthArray<quint8> data(numBytes);
1016
1017 if (numBytes && (numBytes == chunkList.length()))
1018 {
1019 for (int i = 0; i < numBytes; ++i)
1020 {
1021 const quint8 value = chunkList.at(i).toElement().text().toInt();
1022 data[i] = value;
1023 }
1024
1025 if (!m_finishedChunks)
1026 {
1027 m_finishedChunks = new BitSet(data.data(), numBits);
1028 qCDebug(KGET_DEBUG) << m_finishedChunks->numOnBits() << " bits on of " << numBits << " bits.";
1029 }
1030
1031 //set the finished chunks to started
1032 if (!m_startedChunks)
1033 {
1034 m_startedChunks = new BitSet(data.data(), numBits);
1035 }
1036
1037 }
1038 m_prevDownloadedSizes.clear();
1039 m_prevDownloadedSizes.append(m_downloadedSize);
1040
1041 init();
1042
1043 //add the used urls
1044 const QDomNodeList urls = e.firstChildElement("urls").elementsByTagName("url");
1045 for (int i = 0; i < urls.count(); ++i)
1046 {
1047 const QDomElement urlElement = urls.at(i).toElement();
1048 const QUrl url = QUrl(urlElement.text());
1049 const int connections = urlElement.attribute("numParallelSegments").toInt();
1050 addMirror(url, true, connections, true);
1051 }
1052
1053 //add the unused urls
1054 const QDomNodeList unusedUrls = e.firstChildElement("unusedUrls").elementsByTagName("url");
1055 for (int i = 0; i < unusedUrls.count(); ++i)
1056 {
1057 const QDomElement urlElement = unusedUrls.at(i).toElement();
1058 const QUrl url = QUrl(urlElement.text());
1059 const int connections = urlElement.attribute("numParallelSegments").toInt();
1060 addMirror(url, false, connections, true);
1061 }
1062
1063 //m_status = static_cast<Job::Status>(e.attribute("status").toInt());
1064
1065 if (change != Transfer::Tc_None) {
1066 Q_EMIT dataSourceFactoryChange(change);
1067 }
1068 }
1069
changeStatus(Job::Status status)1070 void DataSourceFactory::changeStatus(Job::Status status)
1071 {
1072 Transfer::ChangesFlags change = Transfer::Tc_Status;
1073 m_status = status;
1074
1075 switch (m_status)
1076 {
1077 case Job::Aborted:
1078 case Job::Moving:
1079 case Job::Stopped:
1080 m_speed = 0;
1081 change |= Transfer::Tc_DownloadSpeed;
1082 break;
1083 case Job::Running:
1084 break;
1085 case Job::Finished:
1086 m_speed = 0;
1087 m_percent = 100;
1088
1089 if (m_size) {
1090 m_downloadedSize = m_size;
1091 change |= Transfer::Tc_DownloadedSize;
1092 } else if (m_downloadedSize) {
1093 m_sizeFoundOnFinish = true;
1094 m_size = m_downloadedSize;
1095 change |= Transfer::Tc_TotalSize;
1096 }
1097
1098 change |= Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent;
1099
1100 if (Settings::checksumAutomaticVerification() && verifier()->isVerifyable()) {
1101 verifier()->verify();
1102 }
1103 if (Settings::signatureAutomaticVerification() && signature()->isVerifyable()) {
1104 signature()->verify();
1105 }
1106
1107 slotUpdateCapabilities();
1108 break;
1109 default:
1110 //TODO handle Delayed
1111 break;
1112 }
1113
1114 Q_EMIT dataSourceFactoryChange(change);
1115 }
1116
save(const QDomElement & element)1117 void DataSourceFactory::save(const QDomElement &element)
1118 {
1119 if (element.isNull())
1120 {
1121 return;
1122 }
1123
1124 QDomElement e = element;
1125 QDomDocument doc(e.ownerDocument());
1126
1127
1128 QDomElement factories = e.firstChildElement("factories");
1129 if (factories.isNull())
1130 {
1131 factories = doc.createElement("factories");
1132 }
1133
1134 QDomElement factory = doc.createElement("factory");
1135 factory.setAttribute("dest", m_dest.url());
1136
1137 if (!m_finishedChunks || m_sizeFoundOnFinish) {
1138 factory.setAttribute("processedSize", m_downloadedSize);
1139 }
1140 factory.setAttribute("size", m_size);
1141 factory.setAttribute("segmentSize", m_segSize);
1142 factory.setAttribute("status", m_status);
1143 factory.setAttribute("doDownload", m_doDownload);
1144 factory.setAttribute("downloadInitialized", m_downloadInitialized);
1145 factory.setAttribute("maxMirrorsUsed", m_maxMirrorsUsed);
1146 factory.setAttribute("sizeInitiallyDefined", m_sizeInitiallyDefined);
1147 factory.setAttribute("sizeFoundOnFinish", m_sizeFoundOnFinish);
1148
1149 verifier()->save(factory);
1150 signature()->save(factory);
1151
1152 //set the finished chunks, but only if chunks were actually used
1153 if (m_finishedChunks && !m_sizeFoundOnFinish) {
1154 const KIO::fileoffset_t rest = m_size % m_segSize;
1155 //the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
1156 const KIO::fileoffset_t lastSegSize = (rest ? rest : m_segSize);
1157
1158 //not m_downloadedSize is stored, but the bytes that do not have to be redownloaded
1159 const bool lastOn = m_finishedChunks->get(m_finishedChunks->getNumBits() - 1);
1160 factory.setAttribute("processedSize", m_segSize * (m_finishedChunks->numOnBits() - lastOn) + lastOn * lastSegSize);
1161
1162 QDomElement chunks = doc.createElement("chunks");
1163 chunks.setAttribute("numBits", m_finishedChunks->getNumBits());
1164 chunks.setAttribute("numBytes", m_finishedChunks->getNumBytes());
1165
1166 const quint8 *data = m_finishedChunks->getData();
1167 for (quint32 i = 0; i < m_finishedChunks->getNumBytes(); ++i)
1168 {
1169 QDomElement chunk = doc.createElement("chunk");
1170 QDomText num = doc.createTextNode(QString::number(data[i]));
1171 chunk.setAttribute("number", i);
1172 chunk.appendChild(num);
1173 chunks.appendChild(chunk);
1174 }
1175 factory.appendChild(chunks);
1176 }
1177
1178 //set the used urls
1179 QDomElement urls = doc.createElement("urls");
1180 QHash<QUrl, TransferDataSource*>::const_iterator it;
1181 QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
1182 for (it = m_sources.constBegin(); it != itEnd; ++it) {
1183 QDomElement url = doc.createElement("url");
1184 const QDomText text = doc.createTextNode(it.key().url());
1185 url.appendChild(text);
1186 url.setAttribute("numParallelSegments", (*it)->parallelSegments());
1187 urls.appendChild(url);
1188 factory.appendChild(urls);
1189 }
1190
1191 //set the unused urls
1192 urls = doc.createElement("unusedUrls");
1193 for (int i = 0; i < m_unusedUrls.count(); ++i) {
1194 QDomElement url = doc.createElement("url");
1195 const QDomText text = doc.createTextNode(m_unusedUrls.at(i).url());
1196 url.appendChild(text);
1197 url.setAttribute("numParallelSegments", m_unusedConnections.at(i));
1198 urls.appendChild(url);
1199 factory.appendChild(urls);
1200 }
1201
1202 factories.appendChild(factory);
1203 e.appendChild(factories);
1204 }
1205
1206
isValid() const1207 bool DataSourceFactory::isValid() const
1208 {
1209 //FIXME
1210 return true;
1211 bool valid = (m_size && m_segSize && m_dest.isValid() && !m_sources.isEmpty());
1212
1213 //if the download is finished the it is valid no matter what is set or not
1214 if (m_status == Job::Finished)
1215 {
1216 valid = true;
1217 }
1218 qDebug() << m_size << m_segSize << m_dest.isValid() << !m_sources.isEmpty();
1219 return valid;
1220 }
1221
verifier()1222 Verifier *DataSourceFactory::verifier()
1223 {
1224 if (!m_verifier) {
1225 m_verifier = new Verifier(m_dest, this);
1226 }
1227
1228 return m_verifier;
1229 }
1230
signature()1231 Signature *DataSourceFactory::signature()
1232 {
1233 if (!m_signature) {
1234 m_signature = new Signature(m_dest, this);
1235 }
1236
1237 return m_signature;
1238 }
1239
slotUpdateCapabilities()1240 void DataSourceFactory::slotUpdateCapabilities()
1241 {
1242 const Transfer::Capabilities oldCaps = capabilities();
1243 Transfer::Capabilities newCaps = {};
1244
1245 if ((status() == Job::Finished) || (status() == Job::Stopped)) {
1246 newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1247 } else {
1248 foreach (TransferDataSource *source, m_sources) {
1249 if (!source->assignedSegments().isEmpty()) {
1250 if (newCaps) {
1251 newCaps &= source->capabilities();
1252 } else {
1253 newCaps = source->capabilities();
1254 }
1255 }
1256 }
1257 }
1258
1259 if (newCaps & Transfer::Cap_Resuming) {
1260 newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1261 }
1262
1263 newCaps |= Transfer::Cap_MultipleMirrors;
1264
1265 if (oldCaps != newCaps) {
1266 m_capabilities = newCaps;
1267 Q_EMIT capabilitiesChanged();
1268 }
1269 }
1270
1271
1272