1 /* This file is part of the KDE project
2 
3    Copyright (C) 2008 Lukas Appelhans <l.appelhans@gmx.de>
4    Copyright (C) 2009 Matthias Fuchs <mat69@gmx.net>
5 
6    This program is free software; you can redistribute it and/or
7    modify it under the terms of the GNU General Public
8    License as published by the Free Software Foundation; either
9    version 2 of the License, or (at your option) any later version.
10 */
11 #include "datasourcefactory.h"
12 #include "bitset.h"
13 #include "settings.h"
14 
15 #include "core/filedeleter.h"
16 #include "core/kget.h"
17 #include "core/signature.h"
18 #include "core/verifier.h"
19 
20 #include <cmath>
21 
22 #include <QDir>
23 #include <QTimer>
24 #include <QVarLengthArray>
25 #include <QDomText>
26 
27 #include <KIO/FileJob>
28 #include <KLocalizedString>
29 #include <KMessageBox>
30 #include <KMountPoint>
31 
32 #include "kget_debug.h"
33 
34 
35 #include <qplatformdefs.h>
36 
37 const int SPEEDTIMER = 1000;//1 second...
38 
DataSourceFactory(QObject * parent,const QUrl & dest,KIO::filesize_t size,KIO::fileoffset_t segSize)39 DataSourceFactory::DataSourceFactory(QObject *parent, const QUrl &dest, KIO::filesize_t size, KIO::fileoffset_t segSize)
40   : QObject(parent),
41     m_capabilities(),
42     m_dest(dest),
43     m_size(size),
44     m_downloadedSize(0),
45     m_segSize(segSize),
46     m_speed(0),
47     m_percent(0),
48     m_tempOffset(0),
49     m_startedChunks(nullptr),
50     m_finishedChunks(nullptr),
51     m_putJob(nullptr),
52     m_doDownload(true),
53     m_open(false),
54     m_blocked(false),
55     m_startTried(false),
56     m_findFilesizeTried(false),
57     m_assignTried(false),
58     m_movingFile(false),
59     m_finished(false),
60     m_downloadInitialized(false),
61     m_sizeInitiallyDefined(m_size),
62     m_sizeFoundOnFinish(false),
63     m_maxMirrorsUsed(3),
64     m_speedTimer(nullptr),
65     m_status(Job::Stopped),
66     m_statusBeforeMove(m_status),
67     m_verifier(nullptr),
68     m_signature(nullptr)
69 {
70     qCDebug(KGET_DEBUG) << "Initialize DataSourceFactory: Dest: " + m_dest.toLocalFile() + "Size: " + QString::number(m_size) + "SegSize: " + QString::number(m_segSize);
71 
72     m_prevDownloadedSizes.append(0);
73 }
74 
~DataSourceFactory()75 DataSourceFactory::~DataSourceFactory()
76 {
77     killPutJob();
78     delete m_startedChunks;
79     delete m_finishedChunks;
80 }
81 
init()82 void DataSourceFactory::init()
83 {
84 
85     if (!m_doDownload)
86     {
87         return;
88     }
89 
90     if (!m_speedTimer)
91     {
92         m_speedTimer = new QTimer(this);
93         m_speedTimer->setInterval(SPEEDTIMER);
94         connect(m_speedTimer, &QTimer::timeout, this, &DataSourceFactory::speedChanged);
95     }
96 
97     if (m_segSize && m_size)
98     {
99         const int hasRemainder = (m_size % m_segSize == 0) ? 0 : 1;
100         const int bitSetSize = (m_size / m_segSize) + hasRemainder;//round up if needed
101         if (!m_startedChunks && bitSetSize)
102         {
103             m_startedChunks = new BitSet(bitSetSize);
104         }
105         if (!m_finishedChunks && bitSetSize)
106         {
107             m_finishedChunks = new BitSet(bitSetSize);
108         }
109     }
110 }
111 
deinit()112 void DataSourceFactory::deinit()
113 {
114     if (m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
115         FileDeleter::deleteFile(m_dest);
116     }
117 }
118 
findFileSize()119 void DataSourceFactory::findFileSize()
120 {
121     qCDebug(KGET_DEBUG) << "Find the filesize" << this;
122     if (!m_size && !m_dest.isEmpty() && !m_sources.isEmpty()) {
123         foreach (TransferDataSource *source, m_sources) {
124             if (source->capabilities() & Transfer::Cap_FindFilesize) {
125                 connect(source, &TransferDataSource::foundFileSize, this, &DataSourceFactory::slotFoundFileSize);
126                 connect(source, &TransferDataSource::finishedDownload, this, &DataSourceFactory::slotFinishedDownload);
127 
128                 m_speedTimer->start();
129                 source->findFileSize(m_segSize);
130                 changeStatus(Job::Running);
131                 slotUpdateCapabilities();
132                 return;
133             }
134         }
135     }
136 }
137 
slotFoundFileSize(TransferDataSource * source,KIO::filesize_t fileSize,const QPair<int,int> & segmentRange)138 void DataSourceFactory::slotFoundFileSize(TransferDataSource *source, KIO::filesize_t fileSize, const QPair<int, int> &segmentRange)
139 {
140     m_size = fileSize;
141     qCDebug(KGET_DEBUG) << source << "found size" << m_size << "and is assigned segments" << segmentRange << this;
142     Q_EMIT dataSourceFactoryChange(Transfer::Tc_TotalSize);
143 
144     init();
145 
146     if ((segmentRange.first != -1) && (segmentRange.second != -1)) {
147         m_startedChunks->setRange(segmentRange.first, segmentRange.second, true);
148     }
149 
150     if (m_startTried) {
151         start();
152     }
153 }
154 
slotFinishedDownload(TransferDataSource * source,KIO::filesize_t size)155 void DataSourceFactory::slotFinishedDownload(TransferDataSource *source, KIO::filesize_t size)
156 {
157     Q_UNUSED(source)
158     Q_UNUSED(size)
159 
160     m_speedTimer->stop();
161     m_finished = true;
162 }
163 
checkLocalFile()164 bool DataSourceFactory::checkLocalFile()
165 {
166     QString dest_orig = m_dest.toLocalFile();
167     QString _dest_part(dest_orig);
168 
169     QT_STATBUF buff_part;
170     bool bPartExists = (QT_STAT( _dest_part.toUtf8().constData(), &buff_part ) != -1);
171     if(!bPartExists)
172     {
173         QString _dest = dest_orig;
174         int fd = -1;
175         mode_t initialMode = 0666;
176 
177         fd = QT_OPEN(_dest.toUtf8().constData(), O_CREAT | O_TRUNC | O_WRONLY, initialMode);
178         if ( fd < 0 )
179         {
180             qCDebug(KGET_DEBUG) << " error";
181             return false;
182         }
183         else
184         {
185             close(fd);
186         }
187     }//TODO: Port to use Qt functions maybe
188 
189     qCDebug(KGET_DEBUG) << "success";
190     return true;
191 }
192 
start()193 void DataSourceFactory::start()
194 {
195     qCDebug(KGET_DEBUG) << "Start DataSourceFactory";
196     if (m_movingFile || (m_status == Job::Finished))
197     {
198         return;
199     }
200     if (!m_doDownload)
201     {
202         m_startTried = true;
203         return;
204     }
205 
206     //the file already exists, even though DataSourceFactory has not been initialized remove it
207     //to avoid problems like over methods not finished removing it because of a redownload
208     if (!m_downloadInitialized && QFile::exists(m_dest.toLocalFile())) {
209         qCDebug(KGET_DEBUG) << "Removing existing file.";
210         m_startTried = true;
211         FileDeleter::deleteFile(m_dest, this, SLOT(slotRemovedFile()));
212         return;
213     }
214 
215     m_downloadInitialized = true;
216 
217     //create all dirs needed
218     QDir dir;
219     dir.mkpath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
220     if (checkLocalFile()) {
221         if (!m_putJob) {
222             m_putJob = KIO::open(m_dest, QIODevice::WriteOnly | QIODevice::ReadOnly);
223             connect(m_putJob, &KIO::FileJob::open, this, &DataSourceFactory::slotOpen);
224             connect(m_putJob, &QObject::destroyed, this, &DataSourceFactory::slotPutJobDestroyed);
225             m_startTried = true;
226             return;
227         }
228     } else {
229         //could not create file, maybe device not mounted so abort
230         m_startTried = true;
231         changeStatus(Job::Aborted);
232         return;
233     }
234 
235     init();
236 
237     if (!m_open) {
238         m_startTried = true;
239         return;
240     }
241 
242     if (!m_size) {
243         if (!m_findFilesizeTried && m_sources.count()) {
244             m_findFilesizeTried = true;
245             findFileSize();
246         }
247         m_startTried = true;
248         return;
249     }
250 
251     if (assignNeeded()) {
252         if (m_sources.count()) {
253             qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
254             //simply assign a TransferDataSource
255             assignSegments(*m_sources.begin());
256         } else if (m_unusedUrls.count()) {
257             qCDebug(KGET_DEBUG) << "Assigning an unused mirror";
258             //takes the first unused mirror
259             addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
260         }
261     }
262 
263     if (m_assignTried) {
264         m_assignTried = false;
265 
266         foreach(TransferDataSource *source, m_sources) {
267             assignSegments(source);
268         }
269     }
270 
271     if (m_open) {
272         //check if the filesystem supports a file of m_size
273         const static KIO::filesize_t maxFatSize = 4294967295;
274         if (m_size > maxFatSize) {
275             KMountPoint::Ptr mountPoint = KMountPoint::currentMountPoints().findByPath(m_dest.adjusted(QUrl::RemoveFilename).toLocalFile());
276             if (mountPoint) {
277                 if (mountPoint->mountType() == "vfat") {//TODO check what is reported on Windows for vfat
278                     stop();
279                     KMessageBox::error(nullptr, i18n("Filesize is larger than maximum file size supported by VFAT."), i18n("Error"));
280                     return;
281                 }
282             }
283         }
284 
285         QFile::resize(m_dest.toLocalFile(), m_size);//TODO should we keep that?
286         m_speedTimer->start();
287 
288         foreach (TransferDataSource *source, m_sources) {
289             source->start();
290         }
291 
292         m_startTried = false;
293         changeStatus(Job::Running);
294     }
295     slotUpdateCapabilities();
296 }
297 
slotRemovedFile()298 void DataSourceFactory::slotRemovedFile()
299 {
300     qCDebug(KGET_DEBUG) << "File has been removed" << this;
301     if (m_startTried) {
302         m_startTried = false;
303         start();
304     }
305 }
306 
slotOpen(KIO::Job * job)307 void DataSourceFactory::slotOpen(KIO::Job *job)
308 {
309     Q_UNUSED(job)
310     qCDebug(KGET_DEBUG) << "File opened" << this;
311 
312     if (!m_speedTimer)
313     {
314         init();
315     }
316 
317     connect(m_putJob, &KIO::FileJob::position, this, &DataSourceFactory::slotOffset);
318     connect(m_putJob, &KIO::FileJob::written, this, &DataSourceFactory::slotDataWritten);
319     m_open = true;
320 
321     if (m_startTried)
322     {
323         start();
324     }
325 }
326 
stop()327 void DataSourceFactory::stop()
328 {
329     qCDebug(KGET_DEBUG) << "Stopping" << this;
330     if (m_movingFile || (m_status == Job::Finished))
331     {
332         return;
333     }
334 
335     if (m_speedTimer)
336     {
337         m_speedTimer->stop();
338     }
339 
340     foreach (TransferDataSource *source, m_sources) {
341         source->stop();
342     }
343     m_startTried = false;
344     m_findFilesizeTried = false;
345     changeStatus(Job::Stopped);
346 
347     slotUpdateCapabilities();
348 }
349 
setDoDownload(bool doDownload)350 void DataSourceFactory::setDoDownload(bool doDownload)
351 {
352     if (m_doDownload == doDownload) {
353         return;
354     }
355 
356     m_doDownload = doDownload;
357     if (m_doDownload)
358     {
359         if (m_startTried)
360         {
361             start();
362         }
363     }
364     else
365     {
366         if (m_status == Job::Running)
367         {
368             stop();
369         }
370     }
371 }
372 
addMirror(const QUrl & url,int numParallelConnections)373 void DataSourceFactory::addMirror(const QUrl &url, int numParallelConnections)
374 {
375     addMirror(url, true, numParallelConnections, false);
376 }
377 
addMirror(const QUrl & url,bool used,int numParallelConnections)378 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections)
379 {
380     addMirror(url, used, numParallelConnections, true);
381 }
382 
addMirror(const QUrl & url,bool used,int numParallelConnections,bool usedDefined)383 void DataSourceFactory::addMirror(const QUrl &url, bool used, int numParallelConnections, bool usedDefined)
384 {
385     if (!url.isValid() || url.isEmpty())
386     {
387         qCDebug(KGET_DEBUG) << "Url is not usable: " << url.toString();
388         return;
389     }
390     if (numParallelConnections <= 0)
391     {
392         numParallelConnections = 1;
393     }
394     if (!usedDefined)
395     {
396         used = true;
397     }
398 
399     if (used)
400     {
401         //there is already a TransferDataSource with that url, reuse it and modify numParallelSegments
402         if (m_sources.contains(url))
403         {
404             TransferDataSource *source = m_sources[url];
405             source->setParallelSegments(numParallelConnections);
406             if (source->changeNeeded() > 0) {
407                 assignSegments(source);
408             } else {
409                 for (int i = source->changeNeeded(); i < 0; ++i)
410                 {
411                     const QPair<int, int> removed = source->removeConnection();
412                     qCDebug(KGET_DEBUG) << "Remove connection with segments" << removed;
413                     const int start = removed.first;
414                     const int end = removed.second;
415                     if ((start != -1) && (end != -1)) {
416                         m_startedChunks->setRange(start, end, false);
417                     }
418                 }
419             }
420         }
421         else
422         {
423             if (m_sources.count() < m_maxMirrorsUsed)
424             {
425                 TransferDataSource *source = KGet::createTransferDataSource(url, QDomElement(), this);
426                 if (source)
427                 {
428                     qCDebug(KGET_DEBUG) << "Successfully created a TransferDataSource for " << url.toString() << this;
429 
430                     //url might have been an unused Mirror, so remove it in any case
431                     const int index = m_unusedUrls.indexOf(url);
432                     if (index > -1 )
433                     {
434                         m_unusedUrls.removeAt(index);
435                         m_unusedConnections.removeAt(index);
436                     }
437 
438                     m_sources[url] = source;
439                     m_sources[url]->setParallelSegments(numParallelConnections);
440                     if (m_sizeInitiallyDefined) {
441                         source->setSupposedSize(m_size);
442                     }
443 
444                     connect(source, &TransferDataSource::capabilitiesChanged, this, &DataSourceFactory::slotUpdateCapabilities);
445                     connect(source, SIGNAL(brokenSegments(TransferDataSource*,QPair<int,int>)), this, SLOT(brokenSegments(TransferDataSource*,QPair<int,int>)));
446                     connect(source, &TransferDataSource::broken, this, &DataSourceFactory::broken);
447                     connect(source, &TransferDataSource::finishedSegment, this, &DataSourceFactory::finishedSegment);
448                     connect(source, SIGNAL(data(KIO::fileoffset_t,QByteArray,bool&)), this, SLOT(slotWriteData(KIO::fileoffset_t,QByteArray,bool&)));
449                     connect(source, SIGNAL(freeSegments(TransferDataSource*,QPair<int,int>)), this, SLOT(slotFreeSegments(TransferDataSource*,QPair<int,int>)));
450                     connect(source, &TransferDataSource::log, this, &DataSourceFactory::log);
451                     connect(source, &TransferDataSource::urlChanged, this, &DataSourceFactory::slotUrlChanged);
452 
453                     slotUpdateCapabilities();
454 
455                     assignSegments(source);
456 
457                     //the job is already running, so also start the TransferDataSource
458                     if (!m_assignTried && !m_startTried && m_putJob && m_open && (m_status == Job::Running))
459                     {
460                         source->start();
461                     }
462                 }
463                 else
464                 {
465                     qCDebug(KGET_DEBUG) << "A TransferDataSource could not be created for " << url.toString();
466                 }
467             }
468             else if (usedDefined)
469             {
470                 //increase the number of allowed mirrors as the user wants to use this one!
471                 ++m_maxMirrorsUsed;
472                 addMirror(url, used, numParallelConnections, usedDefined);
473                 return;
474             }
475             else
476             {
477                 m_unusedUrls.append(url);
478                 m_unusedConnections.append(numParallelConnections);
479             }
480         }
481     }
482     else
483     {
484         if (m_sources.contains(url))
485         {
486             removeMirror(url);
487         }
488         else
489         {
490             m_unusedUrls.append(url);
491             m_unusedConnections.append(numParallelConnections);
492         }
493     }
494 }
495 
slotUrlChanged(const QUrl & old,const QUrl & newUrl)496 void DataSourceFactory::slotUrlChanged(const QUrl &old, const QUrl &newUrl)
497 {
498     TransferDataSource * ds = m_sources.take(old);
499     m_sources[newUrl] = ds;
500     Q_EMIT dataSourceFactoryChange(Transfer::Tc_Source | Transfer::Tc_FileName);
501 }
502 
removeMirror(const QUrl & url)503 void DataSourceFactory::removeMirror(const QUrl &url)
504 {
505     qCDebug(KGET_DEBUG) << "Removing mirror: " << url;
506     if (m_sources.contains(url))
507     {
508         TransferDataSource *source = m_sources[url];
509         source->stop();
510         const QList<QPair<int, int> > assigned = source->assignedSegments();
511         m_sources.remove(url);
512         m_unusedUrls.append(url);
513         m_unusedConnections.append(source->parallelSegments());
514         delete source;
515 
516         for (int i = 0; i < assigned.count(); ++i)
517         {
518             const int start = assigned[i].first;
519             const int end = assigned[i].second;
520             if ((start != -1) && (end != -1)) {
521                 m_startedChunks->setRange(start, end, false);
522                 qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
523             }
524         }
525     }
526 
527     if ((m_status == Job::Running) && assignNeeded()) {
528         //here we only handle the case when there are existing TransferDataSources,
529         //the other case is triggered when stopping and then starting again
530         if (m_sources.count()) {
531             qCDebug(KGET_DEBUG) << "Assigning a TransferDataSource.";
532             //simply assign a TransferDataSource
533             assignSegments(*m_sources.begin());
534         }
535     }
536 }
537 
setMirrors(const QHash<QUrl,QPair<bool,int>> & mirrors)538 void DataSourceFactory::setMirrors(const QHash<QUrl, QPair<bool, int> > &mirrors)
539 {
540     //first remove the not set DataSources
541     QList<QUrl> oldUrls = m_sources.keys();
542     QList<QUrl> newUrls = mirrors.keys();
543 
544     foreach (const QUrl &url, oldUrls)
545     {
546         if (!newUrls.contains(url))
547         {
548             removeMirror(url);
549         }
550     }
551 
552     //remove all unused Mirrors and simply readd them below
553     m_unusedUrls.clear();
554     m_unusedConnections.clear();
555 
556     //second modify the existing DataSources and add the new ones
557     QHash<QUrl, QPair<bool, int> >::const_iterator it;
558     QHash<QUrl, QPair<bool, int> >::const_iterator itEnd = mirrors.constEnd();
559     for (it = mirrors.constBegin(); it != itEnd; ++it)
560     {
561         addMirror(it.key(), it.value().first, it.value().second, true);
562     }
563 }
564 
mirrors() const565 QHash<QUrl, QPair<bool, int> > DataSourceFactory::mirrors() const
566 {
567     QHash<QUrl, QPair<bool, int> > mirrors;
568 
569     QHash<QUrl, TransferDataSource*>::const_iterator it;
570     QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
571     for (it = m_sources.constBegin(); it != itEnd; ++it) {
572         mirrors[it.key()] = QPair<bool, int>(true, (*it)->parallelSegments());
573     }
574 
575     for (int i = 0; i < m_unusedUrls.count(); ++i) {
576         mirrors[m_unusedUrls[i]] = QPair<bool, int>(false, m_unusedConnections[i]);
577     }
578 
579     return mirrors;
580 }
581 
assignNeeded() const582 bool DataSourceFactory::assignNeeded() const
583 {
584     bool assignNeeded = true;
585     QHash<QUrl, TransferDataSource*>::const_iterator it;
586     QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
587     for (it = m_sources.constBegin(); it != itEnd; ++it) {
588         if ((*it)->currentSegments()) {
589             //at least one TransferDataSource is still running, so no assign needed
590             assignNeeded = false;
591             break;
592         }
593     }
594     return assignNeeded;
595 }
596 
brokenSegments(TransferDataSource * source,const QPair<int,int> & segmentRange)597 void DataSourceFactory::brokenSegments(TransferDataSource *source, const QPair<int, int> &segmentRange)
598 {
599     qCDebug(KGET_DEBUG) << "Segments" << segmentRange << "broken," << source;
600     if (!source || !m_startedChunks || !m_finishedChunks || (segmentRange.first < 0) || (segmentRange.second < 0) || (static_cast<quint32>(segmentRange.second) > m_finishedChunks->getNumBits()))
601     {
602         return;
603     }
604 
605     const int start = segmentRange.first;
606     const int end = segmentRange.second;
607     if ((start != -1) && (end != -1)) {
608         m_startedChunks->setRange(start, end, false);
609     }
610 
611     removeMirror(source->sourceUrl());
612 }
613 
614 
broken(TransferDataSource * source,TransferDataSource::Error error)615 void DataSourceFactory::broken(TransferDataSource *source, TransferDataSource::Error error)
616 {
617     qCDebug(KGET_DEBUG) << source << "is broken with error" << error;
618     const QString url = source->sourceUrl().toString();
619 
620     removeMirror(source->sourceUrl());
621 
622     if (error == TransferDataSource::WrongDownloadSize)
623     {
624         KMessageBox::error(nullptr, i18nc("A mirror is removed when the file has the wrong download size", "%1 removed as it did report a wrong file size.", url), i18n("Error"));
625     }
626 }
627 
slotFreeSegments(TransferDataSource * source,QPair<int,int> segmentRange)628 void DataSourceFactory::slotFreeSegments(TransferDataSource *source, QPair<int, int> segmentRange)
629 {
630     qCDebug(KGET_DEBUG) << "Segments freed:" << source << segmentRange;
631 
632     const int start = segmentRange.first;
633     const int end = segmentRange.second;
634     if ((start != -1) && (end != -1)) {
635         m_startedChunks->setRange(start, end, false);
636         qCDebug(KGET_DEBUG) << "Segmentrange" << start << '-' << end << "not assigned anymore.";
637     }
638 
639     assignSegments(source);
640 }
641 
finishedSegment(TransferDataSource * source,int segmentNumber,bool connectionFinished)642 void DataSourceFactory::finishedSegment(TransferDataSource *source, int segmentNumber, bool connectionFinished)
643 {
644     if (!source || (segmentNumber < 0) || (static_cast<quint32>(segmentNumber) > m_finishedChunks->getNumBits()))
645     {
646         qCDebug(KGET_DEBUG) << "Incorrect data";
647         return;
648     }
649 
650     m_finishedChunks->set(segmentNumber, true);
651 
652     if (!connectionFinished)
653     {
654         qCDebug(KGET_DEBUG) << "Some segments still not finished";
655         return;
656     }
657 
658     m_finished = m_finishedChunks->allOn();
659     if (m_finished)
660     {
661         qDebug() << "All segments have been downloaded.";
662         return;
663     }
664 
665     assignSegments(source);
666 }
667 
assignSegments(TransferDataSource * source)668 void DataSourceFactory::assignSegments(TransferDataSource *source)
669 {
670     if (!m_startedChunks || !m_finishedChunks)
671     {
672         qCDebug(KGET_DEBUG) << "Assign tried";
673         m_assignTried = true;
674         return;
675     }
676     if (m_finishedChunks->allOn())
677     {
678         qCDebug(KGET_DEBUG) << "All segments are finished already.";
679         return;
680     }
681 
682     //no more segments needed for that TransferDataSource
683     if (source->changeNeeded() <= 0) {
684         qCDebug(KGET_DEBUG) << "No change needed for" << source;
685         return;
686     }
687 
688     //find the segments that should be assigned to the transferDataSource
689     int newStart = -1;
690     int newEnd = -1;
691 
692     //a split needed
693     if (m_startedChunks->allOn()) {
694         int unfinished = 0;
695         TransferDataSource *target = nullptr;
696         foreach (TransferDataSource *source, m_sources) {
697             int temp = source->countUnfinishedSegments();
698             if (temp > unfinished) {
699                 unfinished = temp;
700                 target = source;
701             }
702         }
703         if (!unfinished || !target) {
704             return;
705         }
706 
707         const QPair<int, int> splitResult = target->split();
708         newStart = splitResult.first;
709         newEnd = splitResult.second;
710     } else {
711         m_startedChunks->getContinuousRange(&newStart, &newEnd, false);
712     }
713 
714     if ((newStart == -1) || (newEnd == -1))
715     {
716         qCDebug(KGET_DEBUG) << "No segment can be assigned.";
717         return;
718     }
719 
720     const KIO::fileoffset_t rest = m_size % m_segSize;
721 
722     //the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
723     const KIO::fileoffset_t lastSegSize = ((static_cast<uint>(newEnd + 1) == m_startedChunks->getNumBits() && rest) ? rest : m_segSize);
724 
725     qCDebug(KGET_DEBUG) << "Segments assigned:" << newStart << "-" << newEnd << "segment-size:" << m_segSize << "rest:" << rest;
726     m_startedChunks->setRange(newStart, newEnd, true);
727     source->addSegments(qMakePair(m_segSize, lastSegSize), qMakePair(newStart, newEnd));
728 
729     //there should still be segments added to this transfer
730     if (source->changeNeeded() > 0) {
731         assignSegments(source);
732     }
733 }
734 
735 //TODO implement checks if the correct offsets etc. are used + error recovering e.g. when something else
736 //touches the file
slotWriteData(KIO::fileoffset_t offset,const QByteArray & data,bool & worked)737 void DataSourceFactory::slotWriteData(KIO::fileoffset_t offset, const QByteArray &data, bool &worked)
738 {
739     worked = !m_blocked && !m_movingFile && m_open;
740     if (m_blocked || m_movingFile || !m_open)
741     {
742         return;
743     }
744 
745     m_blocked = true;
746     m_tempOffset = offset;
747     m_tempData = data;
748     m_putJob->seek(offset);
749 }
750 
slotOffset(KIO::Job * job,KIO::filesize_t offset)751 void DataSourceFactory::slotOffset(KIO::Job *job , KIO::filesize_t offset)
752 {
753     Q_UNUSED(job)
754     Q_UNUSED(offset)
755 
756     m_putJob->write(m_tempData);
757 }
758 
slotDataWritten(KIO::Job * job,KIO::filesize_t written)759 void DataSourceFactory::slotDataWritten(KIO::Job *job, KIO::filesize_t written)
760 {
761     Q_UNUSED(job)
762 
763     auto tempSize = static_cast<KIO::filesize_t>(m_tempData.size());
764     //the complete data has been written
765     if (written == tempSize)//TODO if not same cache it temporarily!
766     {
767         m_downloadedSize += written;
768         Q_EMIT dataSourceFactoryChange(Transfer::Tc_DownloadedSize);
769 //             m_tempCache.clear();
770     }
771 
772     if (m_finished)
773     {
774         m_speedTimer->stop();
775         killPutJob();
776         changeStatus(Job::Finished);
777     }
778     m_tempData.clear();
779     m_blocked = false;
780 }
781 
slotPercent(KJob * job,ulong p)782 void DataSourceFactory::slotPercent(KJob* job, ulong p)
783 {
784     Q_UNUSED(job)
785     m_percent = p;
786     Q_EMIT dataSourceFactoryChange(Transfer::Tc_Percent);
787 }
788 
speedChanged()789 void DataSourceFactory::speedChanged()
790 {
791     m_speed = (m_downloadedSize - m_prevDownloadedSizes.first()) / (SPEEDTIMER * m_prevDownloadedSizes.size() / 1000);//downloaded in 1 second
792 
793     m_prevDownloadedSizes.append(m_downloadedSize);
794     if(m_prevDownloadedSizes.size() > 10)
795         m_prevDownloadedSizes.removeFirst();
796 
797     ulong percent = (m_size ? (m_downloadedSize * 100 / m_size) : 0);
798     const bool percentChanged = (percent != m_percent);
799     m_percent = percent;
800 
801     Transfer::ChangesFlags change = (percentChanged ? (Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent) : Transfer::Tc_DownloadSpeed);
802     Q_EMIT dataSourceFactoryChange(change);
803 }
804 
killPutJob()805 void DataSourceFactory::killPutJob()
806 {
807     if (m_putJob)
808     {
809         qCDebug(KGET_DEBUG) << "Closing the file";
810         m_open = false;
811         m_putJob->close();
812         m_putJob = nullptr;
813     }
814 }
815 
slotPutJobDestroyed(QObject * job)816 void DataSourceFactory::slotPutJobDestroyed(QObject *job)
817 {
818     Q_UNUSED(job)
819 
820     m_putJob = nullptr;
821 }
822 
setNewDestination(const QUrl & newDestination)823 bool DataSourceFactory::setNewDestination(const QUrl &newDestination)
824 {
825     m_newDest = newDestination;
826     if (m_newDest.isValid() && (m_newDest != m_dest))
827     {
828         //No files created yet, simply change the urls
829         if (!m_downloadInitialized) {
830             m_dest = m_newDest;
831             if (m_verifier) {
832                 verifier()->setDestination(m_dest);
833             }
834             if (m_signature) {
835                 signature()->setDestination(m_dest);
836             }
837 
838             return true;
839         } else if (QFile::exists(m_dest.toString())) {
840             //create all dirs needed
841             QDir dir;
842             dir.mkpath(m_newDest.adjusted(QUrl::RemoveFilename).toString());
843 
844             m_statusBeforeMove = m_status;
845             stop();
846             changeStatus(Job::Moving);
847             m_movingFile = true;
848 
849             //still a write in progress
850             if (m_blocked)
851             {
852                 QTimer::singleShot(1000, this, &DataSourceFactory::startMove);
853             }
854             else
855             {
856                 startMove();
857             }
858             return true;
859         }
860     }
861     return false;
862 }
863 
startMove()864 void DataSourceFactory::startMove()
865 {
866     killPutJob();
867 
868     KIO::Job *move = KIO::file_move(m_dest, m_newDest, -1, KIO::HideProgressInfo);
869     connect(move, &KJob::result, this, &DataSourceFactory::newDestResult);
870     connect(move, SIGNAL(percent(KJob*,ulong)), this, SLOT(slotPercent(KJob*,ulong)));
871 
872     m_dest = m_newDest;
873     verifier()->setDestination(m_dest);
874     signature()->setDestination(m_dest);
875 }
876 
newDestResult(KJob * job)877 void DataSourceFactory::newDestResult(KJob *job)
878 {
879     Q_UNUSED(job)//TODO handle errors etc.?
880 
881     m_movingFile = false;
882     changeStatus(m_statusBeforeMove);
883     if (m_status == Job::Running)
884     {
885         start();
886     }
887 }
888 
repair()889 void DataSourceFactory::repair()
890 {
891     if (verifier()->status() != Verifier::NotVerified)
892     {
893         return;
894     }
895 
896     m_finished = false;
897 
898     connect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>,KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>,KIO::filesize_t)));
899 
900     verifier()->brokenPieces();
901 }
902 
slotRepair(const QList<KIO::fileoffset_t> & offsets,KIO::filesize_t length)903 void DataSourceFactory::slotRepair(const QList<KIO::fileoffset_t> &offsets, KIO::filesize_t length)
904 {
905     disconnect(verifier(), SIGNAL(brokenPieces(QList<KIO::fileoffset_t>,KIO::filesize_t)), this, SLOT(slotRepair(QList<KIO::fileoffset_t>,KIO::filesize_t)));
906 
907     if (!m_startedChunks || !m_finishedChunks)
908     {
909         qCDebug(KGET_DEBUG) << "Redownload everything";
910         m_downloadedSize = 0;
911     }
912     else
913     {
914         if (offsets.isEmpty()) {
915             m_startedChunks->clear();
916             m_finishedChunks->clear();
917         }
918         qCDebug(KGET_DEBUG) << "Redownload broken pieces";
919         for (int i = 0; i < offsets.count(); ++i) {
920             const int start = offsets[i] / m_segSize;
921             const int end = std::ceil(length / static_cast<double>(m_segSize)) - 1 + start;
922             m_startedChunks->setRange(start, end, false);
923             m_finishedChunks->setRange(start, end, false);
924         }
925 
926         m_downloadedSize = m_segSize * m_finishedChunks->numOnBits();
927     }
928     m_prevDownloadedSizes.clear();
929     m_prevDownloadedSizes.append(m_downloadedSize);
930 
931     //remove all current mirrors and readd the first unused mirror
932     const QList<QUrl> mirrors = m_sources.keys();//FIXME only remove the mirrors of the broken segments?! --> for that m_assignedChunks would need to be saved was well
933     foreach (const QUrl &mirror, mirrors)
934     {
935         removeMirror(mirror);
936     }
937     addMirror(m_unusedUrls.takeFirst(), true, m_unusedConnections.takeFirst());
938 
939     m_speed = 0;
940 
941     Transfer::ChangesFlags change = (Transfer::Tc_DownloadSpeed | Transfer::Tc_DownloadedSize);
942     if (m_size) {
943         change |= Transfer::Tc_Percent;
944         m_percent = (m_downloadedSize * 100 / m_size);
945     }
946     Q_EMIT dataSourceFactoryChange(change);
947     m_status = Job::Stopped;
948 
949     start();
950 }
951 
load(const QDomElement * element)952 void DataSourceFactory::load(const QDomElement *element)
953 {
954     if (!element)
955     {
956         return;
957     }
958 
959     QDomElement e = element->firstChildElement("factory");
960     if (e.isNull())
961     {
962         e = element->firstChildElement("factories").firstChildElement("factory");
963     }
964 
965     //only try to load values if they haven't been set
966     if (m_dest.isEmpty())
967     {
968         m_dest = QUrl(e.attribute("dest"));
969     }
970 
971     verifier()->load(e);
972     signature()->load(e);
973 
974     Transfer::ChangesFlags change = Transfer::Tc_None;
975 
976     if (!m_size) {
977         m_size = e.attribute("size").toULongLong();
978         change |= Transfer::Tc_TotalSize;
979     }
980     KIO::fileoffset_t tempSegSize = e.attribute("segmentSize").toLongLong();
981     if (tempSegSize)
982     {
983         m_segSize = tempSegSize;
984     }
985     if (!m_downloadedSize) {
986         m_downloadedSize = e.attribute("processedSize").toULongLong();
987         change |= Transfer::Tc_DownloadedSize;
988         if (m_size) {
989             m_percent = (m_downloadedSize * 100 / m_size);
990             change |= Transfer::Tc_Percent;
991         }
992     }
993     if (e.hasAttribute("doDownload"))
994     {
995         m_doDownload = QVariant(e.attribute("doDownload")).toBool();
996     }
997     if (e.hasAttribute("downloadInitialized")) {
998         m_downloadInitialized = QVariant(e.attribute("downloadInitialized")).toBool();
999     }
1000     if (e.hasAttribute("maxMirrorsUsed"))
1001     {
1002         bool worked;
1003         m_maxMirrorsUsed = e.attribute("maxMirrorsUsed").toInt(&worked);
1004         m_maxMirrorsUsed = (worked ? m_maxMirrorsUsed : 3);
1005     }
1006     m_sizeInitiallyDefined = QVariant(e.attribute("sizeInitiallyDefined", "false")).toBool();
1007     m_sizeFoundOnFinish = QVariant(e.attribute("sizeFoundOnFinish", "false")).toBool();
1008 
1009     //load the finishedChunks
1010     const QDomElement chunks = e.firstChildElement("chunks");
1011     const QDomNodeList chunkList = chunks.elementsByTagName("chunk");
1012 
1013     const quint32 numBits = chunks.attribute("numBits").toInt();
1014     const int numBytes = chunks.attribute("numBytes").toInt();
1015     QVarLengthArray<quint8> data(numBytes);
1016 
1017     if (numBytes && (numBytes == chunkList.length()))
1018     {
1019         for (int i = 0; i < numBytes; ++i)
1020         {
1021             const quint8 value = chunkList.at(i).toElement().text().toInt();
1022             data[i] = value;
1023         }
1024 
1025         if (!m_finishedChunks)
1026         {
1027             m_finishedChunks = new BitSet(data.data(), numBits);
1028             qCDebug(KGET_DEBUG) << m_finishedChunks->numOnBits() << " bits on of " << numBits << " bits.";
1029         }
1030 
1031         //set the finished chunks to started
1032         if (!m_startedChunks)
1033         {
1034             m_startedChunks = new BitSet(data.data(), numBits);
1035         }
1036 
1037     }
1038     m_prevDownloadedSizes.clear();
1039     m_prevDownloadedSizes.append(m_downloadedSize);
1040 
1041     init();
1042 
1043     //add the used urls
1044     const QDomNodeList urls = e.firstChildElement("urls").elementsByTagName("url");
1045     for (int i = 0; i < urls.count(); ++i)
1046     {
1047         const QDomElement urlElement = urls.at(i).toElement();
1048         const QUrl url = QUrl(urlElement.text());
1049         const int connections = urlElement.attribute("numParallelSegments").toInt();
1050         addMirror(url, true, connections, true);
1051     }
1052 
1053     //add the unused urls
1054     const QDomNodeList unusedUrls = e.firstChildElement("unusedUrls").elementsByTagName("url");
1055     for (int i = 0; i < unusedUrls.count(); ++i)
1056     {
1057         const QDomElement urlElement = unusedUrls.at(i).toElement();
1058         const QUrl url = QUrl(urlElement.text());
1059         const int connections = urlElement.attribute("numParallelSegments").toInt();
1060         addMirror(url, false, connections, true);
1061     }
1062 
1063     //m_status = static_cast<Job::Status>(e.attribute("status").toInt());
1064 
1065     if (change != Transfer::Tc_None) {
1066         Q_EMIT dataSourceFactoryChange(change);
1067     }
1068 }
1069 
changeStatus(Job::Status status)1070 void DataSourceFactory::changeStatus(Job::Status status)
1071 {
1072     Transfer::ChangesFlags change = Transfer::Tc_Status;
1073     m_status = status;
1074 
1075     switch (m_status)
1076     {
1077         case Job::Aborted:
1078         case Job::Moving:
1079         case Job::Stopped:
1080             m_speed = 0;
1081             change |= Transfer::Tc_DownloadSpeed;
1082             break;
1083         case Job::Running:
1084             break;
1085         case Job::Finished:
1086             m_speed = 0;
1087             m_percent = 100;
1088 
1089             if (m_size) {
1090                 m_downloadedSize = m_size;
1091                 change |= Transfer::Tc_DownloadedSize;
1092             } else if (m_downloadedSize) {
1093                 m_sizeFoundOnFinish = true;
1094                 m_size = m_downloadedSize;
1095                 change |= Transfer::Tc_TotalSize;
1096             }
1097 
1098             change |= Transfer::Tc_DownloadSpeed | Transfer::Tc_Percent;
1099 
1100             if (Settings::checksumAutomaticVerification() && verifier()->isVerifyable()) {
1101                 verifier()->verify();
1102             }
1103             if (Settings::signatureAutomaticVerification() && signature()->isVerifyable()) {
1104                 signature()->verify();
1105             }
1106 
1107             slotUpdateCapabilities();
1108             break;
1109         default:
1110             //TODO handle Delayed
1111             break;
1112     }
1113 
1114     Q_EMIT dataSourceFactoryChange(change);
1115 }
1116 
save(const QDomElement & element)1117 void DataSourceFactory::save(const QDomElement &element)
1118 {
1119     if (element.isNull())
1120     {
1121         return;
1122     }
1123 
1124     QDomElement e = element;
1125     QDomDocument doc(e.ownerDocument());
1126 
1127 
1128     QDomElement factories = e.firstChildElement("factories");
1129     if (factories.isNull())
1130     {
1131         factories = doc.createElement("factories");
1132     }
1133 
1134     QDomElement factory = doc.createElement("factory");
1135     factory.setAttribute("dest", m_dest.url());
1136 
1137     if (!m_finishedChunks || m_sizeFoundOnFinish) {
1138         factory.setAttribute("processedSize", m_downloadedSize);
1139     }
1140     factory.setAttribute("size", m_size);
1141     factory.setAttribute("segmentSize", m_segSize);
1142     factory.setAttribute("status", m_status);
1143     factory.setAttribute("doDownload", m_doDownload);
1144     factory.setAttribute("downloadInitialized", m_downloadInitialized);
1145     factory.setAttribute("maxMirrorsUsed", m_maxMirrorsUsed);
1146     factory.setAttribute("sizeInitiallyDefined", m_sizeInitiallyDefined);
1147     factory.setAttribute("sizeFoundOnFinish", m_sizeFoundOnFinish);
1148 
1149     verifier()->save(factory);
1150     signature()->save(factory);
1151 
1152     //set the finished chunks, but only if chunks were actually used
1153     if (m_finishedChunks && !m_sizeFoundOnFinish) {
1154         const KIO::fileoffset_t rest = m_size % m_segSize;
1155         //the lastSegsize is rest, but only if there is a rest and it is the last segment of the download
1156         const KIO::fileoffset_t lastSegSize = (rest ? rest : m_segSize);
1157 
1158         //not m_downloadedSize is stored, but the bytes that do not have to be redownloaded
1159         const bool lastOn = m_finishedChunks->get(m_finishedChunks->getNumBits() - 1);
1160         factory.setAttribute("processedSize", m_segSize * (m_finishedChunks->numOnBits() - lastOn) + lastOn * lastSegSize);
1161 
1162         QDomElement chunks = doc.createElement("chunks");
1163         chunks.setAttribute("numBits", m_finishedChunks->getNumBits());
1164         chunks.setAttribute("numBytes", m_finishedChunks->getNumBytes());
1165 
1166         const quint8 *data = m_finishedChunks->getData();
1167         for (quint32 i = 0; i < m_finishedChunks->getNumBytes(); ++i)
1168         {
1169             QDomElement chunk = doc.createElement("chunk");
1170             QDomText num = doc.createTextNode(QString::number(data[i]));
1171             chunk.setAttribute("number", i);
1172             chunk.appendChild(num);
1173             chunks.appendChild(chunk);
1174         }
1175         factory.appendChild(chunks);
1176     }
1177 
1178     //set the used urls
1179     QDomElement urls = doc.createElement("urls");
1180     QHash<QUrl, TransferDataSource*>::const_iterator it;
1181     QHash<QUrl, TransferDataSource*>::const_iterator itEnd = m_sources.constEnd();
1182     for (it = m_sources.constBegin(); it != itEnd; ++it) {
1183         QDomElement url = doc.createElement("url");
1184         const QDomText text = doc.createTextNode(it.key().url());
1185         url.appendChild(text);
1186         url.setAttribute("numParallelSegments", (*it)->parallelSegments());
1187         urls.appendChild(url);
1188         factory.appendChild(urls);
1189     }
1190 
1191     //set the unused urls
1192     urls = doc.createElement("unusedUrls");
1193     for (int i = 0; i < m_unusedUrls.count(); ++i) {
1194         QDomElement url = doc.createElement("url");
1195         const QDomText text = doc.createTextNode(m_unusedUrls.at(i).url());
1196         url.appendChild(text);
1197         url.setAttribute("numParallelSegments", m_unusedConnections.at(i));
1198         urls.appendChild(url);
1199         factory.appendChild(urls);
1200     }
1201 
1202     factories.appendChild(factory);
1203     e.appendChild(factories);
1204 }
1205 
1206 
isValid() const1207 bool DataSourceFactory::isValid() const
1208 {
1209     //FIXME
1210     return true;
1211     bool valid = (m_size && m_segSize && m_dest.isValid() && !m_sources.isEmpty());
1212 
1213     //if the download is finished the it is valid no matter what is set or not
1214     if (m_status == Job::Finished)
1215     {
1216         valid = true;
1217     }
1218     qDebug() << m_size << m_segSize << m_dest.isValid() << !m_sources.isEmpty();
1219     return valid;
1220 }
1221 
verifier()1222 Verifier *DataSourceFactory::verifier()
1223 {
1224     if (!m_verifier) {
1225         m_verifier = new Verifier(m_dest, this);
1226     }
1227 
1228     return m_verifier;
1229 }
1230 
signature()1231 Signature *DataSourceFactory::signature()
1232 {
1233     if (!m_signature) {
1234         m_signature = new Signature(m_dest, this);
1235     }
1236 
1237     return m_signature;
1238 }
1239 
slotUpdateCapabilities()1240 void DataSourceFactory::slotUpdateCapabilities()
1241 {
1242     const Transfer::Capabilities oldCaps = capabilities();
1243     Transfer::Capabilities newCaps = {};
1244 
1245     if ((status() == Job::Finished) || (status() == Job::Stopped)) {
1246         newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1247     } else {
1248         foreach (TransferDataSource *source, m_sources) {
1249             if (!source->assignedSegments().isEmpty()) {
1250                 if (newCaps) {
1251                     newCaps &= source->capabilities();
1252                 } else {
1253                     newCaps = source->capabilities();
1254                 }
1255             }
1256         }
1257     }
1258 
1259     if (newCaps & Transfer::Cap_Resuming) {
1260         newCaps |= Transfer::Cap_Moving | Transfer::Cap_Renaming;
1261     }
1262 
1263     newCaps |= Transfer::Cap_MultipleMirrors;
1264 
1265     if (oldCaps != newCaps) {
1266         m_capabilities = newCaps;
1267         Q_EMIT capabilitiesChanged();
1268     }
1269 }
1270 
1271 
1272