1 /* <!-- copyright */
2 /*
3  * aria2 - The high speed download utility
4  *
5  * Copyright (C) 2006 Tatsuhiro Tsujikawa
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  *
21  * In addition, as a special exception, the copyright holders give
22  * permission to link the code of portions of this program with the
23  * OpenSSL library under certain conditions as described in each
24  * individual source file, and distribute linked combinations
25  * including the two.
26  * You must obey the GNU General Public License in all respects
27  * for all of the code used other than OpenSSL.  If you modify
28  * file(s) with this exception, you may extend this exception to your
29  * version of the file(s), but you are not obligated to do so.  If you
30  * do not wish to do so, delete this exception statement from your
31  * version.  If you delete this exception statement from all source
32  * files in the program, then also delete it here.
33  */
34 /* copyright --> */
35 #include "RequestGroup.h"
36 
37 #include <cassert>
38 #include <algorithm>
39 
40 #include "PostDownloadHandler.h"
41 #include "DownloadEngine.h"
42 #include "SegmentMan.h"
43 #include "NullProgressInfoFile.h"
44 #include "Dependency.h"
45 #include "prefs.h"
46 #include "CreateRequestCommand.h"
47 #include "File.h"
48 #include "message.h"
49 #include "util.h"
50 #include "LogFactory.h"
51 #include "Logger.h"
52 #include "DiskAdaptor.h"
53 #include "DiskWriterFactory.h"
54 #include "RecoverableException.h"
55 #include "StreamCheckIntegrityEntry.h"
56 #include "CheckIntegrityCommand.h"
57 #include "UnknownLengthPieceStorage.h"
58 #include "DownloadContext.h"
59 #include "DlAbortEx.h"
60 #include "DownloadFailureException.h"
61 #include "RequestGroupMan.h"
62 #include "DefaultBtProgressInfoFile.h"
63 #include "DefaultPieceStorage.h"
64 #include "download_handlers.h"
65 #include "MemoryBufferPreDownloadHandler.h"
66 #include "DownloadHandlerConstants.h"
67 #include "Option.h"
68 #include "FileEntry.h"
69 #include "Request.h"
70 #include "FileAllocationIterator.h"
71 #include "fmt.h"
72 #include "A2STR.h"
73 #include "URISelector.h"
74 #include "InorderURISelector.h"
75 #include "PieceSelector.h"
76 #include "a2functional.h"
77 #include "SocketCore.h"
78 #include "SimpleRandomizer.h"
79 #include "Segment.h"
80 #include "SocketRecvBuffer.h"
81 #include "RequestGroupCriteria.h"
82 #include "CheckIntegrityCommand.h"
83 #include "ChecksumCheckIntegrityEntry.h"
84 #ifdef ENABLE_BITTORRENT
85 #  include "bittorrent_helper.h"
86 #  include "BtRegistry.h"
87 #  include "BtCheckIntegrityEntry.h"
88 #  include "DefaultPeerStorage.h"
89 #  include "DefaultBtAnnounce.h"
90 #  include "BtRuntime.h"
91 #  include "BtSetup.h"
92 #  include "BtPostDownloadHandler.h"
93 #  include "DHTSetup.h"
94 #  include "DHTRegistry.h"
95 #  include "DHTNode.h"
96 #  include "DHTRoutingTable.h"
97 #  include "DHTTaskQueue.h"
98 #  include "DHTTaskFactory.h"
99 #  include "DHTTokenTracker.h"
100 #  include "DHTMessageDispatcher.h"
101 #  include "DHTMessageReceiver.h"
102 #  include "DHTMessageFactory.h"
103 #  include "DHTMessageCallback.h"
104 #  include "BtMessageFactory.h"
105 #  include "BtRequestFactory.h"
106 #  include "BtMessageDispatcher.h"
107 #  include "BtMessageReceiver.h"
108 #  include "PeerConnection.h"
109 #  include "ExtensionMessageFactory.h"
110 #  include "DHTPeerAnnounceStorage.h"
111 #  include "DHTEntryPointNameResolveCommand.h"
112 #  include "LongestSequencePieceSelector.h"
113 #  include "PriorityPieceSelector.h"
114 #  include "bittorrent_helper.h"
115 #endif // ENABLE_BITTORRENT
116 #ifdef ENABLE_METALINK
117 #  include "MetalinkPostDownloadHandler.h"
118 #endif // ENABLE_METALINK
119 
120 namespace aria2 {
121 
RequestGroup(const std::shared_ptr<GroupId> & gid,const std::shared_ptr<Option> & option)122 RequestGroup::RequestGroup(const std::shared_ptr<GroupId>& gid,
123                            const std::shared_ptr<Option>& option)
124     : belongsToGID_(0),
125       gid_(gid),
126       option_(option),
127       progressInfoFile_(std::make_shared<NullProgressInfoFile>()),
128       uriSelector_(make_unique<InorderURISelector>()),
129       requestGroupMan_(nullptr),
130 #ifdef ENABLE_BITTORRENT
131       btRuntime_(nullptr),
132       peerStorage_(nullptr),
133 #endif // ENABLE_BITTORRENT
134       followingGID_(0),
135       lastModifiedTime_(Time::null()),
136       timeout_(option->getAsInt(PREF_TIMEOUT)),
137       state_(STATE_WAITING),
138       numConcurrentCommand_(option->getAsInt(PREF_SPLIT)),
139       numStreamConnection_(0),
140       numStreamCommand_(0),
141       numCommand_(0),
142       fileNotFoundCount_(0),
143       maxDownloadSpeedLimit_(option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)),
144       maxUploadSpeedLimit_(option->getAsInt(PREF_MAX_UPLOAD_LIMIT)),
145       resumeFailureCount_(0),
146       haltReason_(RequestGroup::NONE),
147       lastErrorCode_(error_code::UNDEFINED),
148       saveControlFile_(true),
149       preLocalFileCheckEnabled_(true),
150       haltRequested_(false),
151       forceHaltRequested_(false),
152       pauseRequested_(false),
153       restartRequested_(false),
154       inMemoryDownload_(false),
155       seedOnly_(false)
156 {
157   fileAllocationEnabled_ = option_->get(PREF_FILE_ALLOCATION) != V_NONE;
158   if (!option_->getAsBool(PREF_DRY_RUN)) {
159     initializePreDownloadHandler();
160     initializePostDownloadHandler();
161   }
162 }
163 
164 RequestGroup::~RequestGroup() = default;
165 
isCheckIntegrityReady()166 bool RequestGroup::isCheckIntegrityReady()
167 {
168   return option_->getAsBool(PREF_CHECK_INTEGRITY) &&
169          ((downloadContext_->isChecksumVerificationAvailable() &&
170            downloadFinishedByFileLength()) ||
171           downloadContext_->isPieceHashVerificationAvailable());
172 }
173 
downloadFinished() const174 bool RequestGroup::downloadFinished() const
175 {
176   if (!pieceStorage_) {
177     return false;
178   }
179   return pieceStorage_->downloadFinished();
180 }
181 
allDownloadFinished() const182 bool RequestGroup::allDownloadFinished() const
183 {
184   if (!pieceStorage_) {
185     return false;
186   }
187   return pieceStorage_->allDownloadFinished();
188 }
189 
downloadResult() const190 std::pair<error_code::Value, std::string> RequestGroup::downloadResult() const
191 {
192   if (downloadFinished() && !downloadContext_->isChecksumVerificationNeeded()) {
193     return std::make_pair(error_code::FINISHED, "");
194   }
195 
196   if (haltReason_ == RequestGroup::USER_REQUEST) {
197     return std::make_pair(error_code::REMOVED, "");
198   }
199 
200   if (lastErrorCode_ == error_code::UNDEFINED) {
201     if (haltReason_ == RequestGroup::SHUTDOWN_SIGNAL) {
202       return std::make_pair(error_code::IN_PROGRESS, "");
203     }
204     return std::make_pair(error_code::UNKNOWN_ERROR, "");
205   }
206 
207   return std::make_pair(lastErrorCode_, lastErrorMessage_);
208 }
209 
closeFile()210 void RequestGroup::closeFile()
211 {
212   if (pieceStorage_) {
213     pieceStorage_->flushWrDiskCacheEntry(true);
214     pieceStorage_->getDiskAdaptor()->flushOSBuffers();
215     pieceStorage_->getDiskAdaptor()->closeFile();
216   }
217 }
218 
219 // TODO The function name is not intuitive at all.. it does not convey
220 // that this function open file.
createCheckIntegrityEntry()221 std::unique_ptr<CheckIntegrityEntry> RequestGroup::createCheckIntegrityEntry()
222 {
223   auto infoFile = std::make_shared<DefaultBtProgressInfoFile>(
224       downloadContext_, pieceStorage_, option_.get());
225 
226   if (option_->getAsBool(PREF_CHECK_INTEGRITY) &&
227       downloadContext_->isPieceHashVerificationAvailable()) {
228     // When checking piece hash, we don't care file is downloaded and
229     // infoFile exists.
230     loadAndOpenFile(infoFile);
231     return make_unique<StreamCheckIntegrityEntry>(this);
232   }
233 
234   if (isPreLocalFileCheckEnabled() &&
235       (infoFile->exists() || (File(getFirstFilePath()).exists() &&
236                               option_->getAsBool(PREF_CONTINUE)))) {
237     // If infoFile exists or -c option is given, we need to check
238     // download has been completed (which is determined after
239     // loadAndOpenFile()). If so, use ChecksumCheckIntegrityEntry when
240     // verification is enabled, because CreateRequestCommand does not
241     // issue checksum verification and download fails without it.
242     loadAndOpenFile(infoFile);
243     if (downloadFinished()) {
244       if (downloadContext_->isChecksumVerificationNeeded()) {
245         A2_LOG_INFO(MSG_HASH_CHECK_NOT_DONE);
246         auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
247         tempEntry->setRedownload(true);
248         return std::move(tempEntry);
249       }
250       downloadContext_->setChecksumVerified(true);
251       A2_LOG_NOTICE(fmt(MSG_DOWNLOAD_ALREADY_COMPLETED, gid_->toHex().c_str(),
252                         downloadContext_->getBasePath().c_str()));
253       return nullptr;
254     }
255     return make_unique<StreamCheckIntegrityEntry>(this);
256   }
257 
258   if (downloadFinishedByFileLength() &&
259       downloadContext_->isChecksumVerificationAvailable()) {
260     pieceStorage_->markAllPiecesDone();
261     loadAndOpenFile(infoFile);
262     auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
263     tempEntry->setRedownload(true);
264     return std::move(tempEntry);
265   }
266 
267   loadAndOpenFile(infoFile);
268   return make_unique<StreamCheckIntegrityEntry>(this);
269 }
270 
createInitialCommand(std::vector<std::unique_ptr<Command>> & commands,DownloadEngine * e)271 void RequestGroup::createInitialCommand(
272     std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
273 {
274   // Start session timer here.  When file size becomes known, it will
275   // be reset again in *FileAllocationEntry, because hash check and
276   // file allocation takes a time.  For downloads in which file size
277   // is unknown, session timer will not be reset.
278   downloadContext_->resetDownloadStartTime();
279 #ifdef ENABLE_BITTORRENT
280   if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
281     auto torrentAttrs = bittorrent::getTorrentAttrs(downloadContext_);
282     bool metadataGetMode = torrentAttrs->metadata.empty();
283     if (option_->getAsBool(PREF_DRY_RUN)) {
284       throw DOWNLOAD_FAILURE_EXCEPTION(
285           "Cancel BitTorrent download in dry-run context.");
286     }
287     auto& btRegistry = e->getBtRegistry();
288     if (btRegistry->getDownloadContext(torrentAttrs->infoHash)) {
289       // TODO If metadataGetMode == false and each FileEntry has
290       // URI, then go without BT.
291       throw DOWNLOAD_FAILURE_EXCEPTION2(
292           fmt("InfoHash %s is already registered.",
293               bittorrent::getInfoHashString(downloadContext_).c_str()),
294           error_code::DUPLICATE_INFO_HASH);
295     }
296     if (metadataGetMode) {
297       // Use UnknownLengthPieceStorage.
298       initPieceStorage();
299     }
300     else if (e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
301       throw DOWNLOAD_FAILURE_EXCEPTION2(
302           fmt(EX_DUPLICATE_FILE_DOWNLOAD,
303               downloadContext_->getBasePath().c_str()),
304           error_code::DUPLICATE_DOWNLOAD);
305     }
306     else {
307       initPieceStorage();
308       if (downloadContext_->getFileEntries().size() > 1) {
309         pieceStorage_->setupFileFilter();
310       }
311     }
312 
313     std::shared_ptr<DefaultBtProgressInfoFile> progressInfoFile;
314     if (!metadataGetMode) {
315       progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
316           downloadContext_, pieceStorage_, option_.get());
317     }
318 
319     auto btRuntime = std::make_shared<BtRuntime>();
320     btRuntime->setMaxPeers(option_->getAsInt(PREF_BT_MAX_PEERS));
321     btRuntime_ = btRuntime.get();
322     if (progressInfoFile) {
323       progressInfoFile->setBtRuntime(btRuntime);
324     }
325 
326     auto peerStorage = std::make_shared<DefaultPeerStorage>();
327     peerStorage->setBtRuntime(btRuntime);
328     peerStorage->setPieceStorage(pieceStorage_);
329     peerStorage_ = peerStorage.get();
330     if (progressInfoFile) {
331       progressInfoFile->setPeerStorage(peerStorage);
332     }
333 
334     auto btAnnounce = std::make_shared<DefaultBtAnnounce>(
335         downloadContext_.get(), option_.get());
336     btAnnounce->setBtRuntime(btRuntime);
337     btAnnounce->setPieceStorage(pieceStorage_);
338     btAnnounce->setPeerStorage(peerStorage);
339     btAnnounce->setUserDefinedInterval(
340         std::chrono::seconds(option_->getAsInt(PREF_BT_TRACKER_INTERVAL)));
341     btAnnounce->shuffleAnnounce();
342 
343     assert(!btRegistry->get(gid_->getNumericId()));
344     btRegistry->put(
345         gid_->getNumericId(),
346         make_unique<BtObject>(
347             downloadContext_, pieceStorage_, peerStorage, btAnnounce, btRuntime,
348             (progressInfoFile ? progressInfoFile : progressInfoFile_)));
349 
350     if (option_->getAsBool(PREF_ENABLE_DHT) ||
351         (!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
352          option_->getAsBool(PREF_ENABLE_DHT6))) {
353 
354       if (option_->getAsBool(PREF_ENABLE_DHT)) {
355         std::vector<std::unique_ptr<Command>> c, rc;
356         std::tie(c, rc) = DHTSetup().setup(e, AF_INET);
357 
358         e->addCommand(std::move(c));
359         for (auto& a : rc) {
360           e->addRoutineCommand(std::move(a));
361         }
362       }
363 
364       if (!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
365           option_->getAsBool(PREF_ENABLE_DHT6)) {
366         std::vector<std::unique_ptr<Command>> c, rc;
367         std::tie(c, rc) = DHTSetup().setup(e, AF_INET6);
368 
369         e->addCommand(std::move(c));
370         for (auto& a : rc) {
371           e->addRoutineCommand(std::move(a));
372         }
373       }
374       const auto& nodes = torrentAttrs->nodes;
375       if (!torrentAttrs->privateTorrent && !nodes.empty()) {
376         if (DHTRegistry::isInitialized()) {
377           auto command = make_unique<DHTEntryPointNameResolveCommand>(
378               e->newCUID(), e, AF_INET, nodes);
379           const auto& data = DHTRegistry::getData();
380           command->setTaskQueue(data.taskQueue.get());
381           command->setTaskFactory(data.taskFactory.get());
382           command->setRoutingTable(data.routingTable.get());
383           command->setLocalNode(data.localNode);
384           e->addCommand(std::move(command));
385         }
386 
387         if (DHTRegistry::isInitialized6()) {
388           auto command = make_unique<DHTEntryPointNameResolveCommand>(
389               e->newCUID(), e, AF_INET6, nodes);
390           const auto& data = DHTRegistry::getData6();
391           command->setTaskQueue(data.taskQueue.get());
392           command->setTaskFactory(data.taskFactory.get());
393           command->setRoutingTable(data.routingTable.get());
394           command->setLocalNode(data.localNode);
395           e->addCommand(std::move(command));
396         }
397       }
398     }
399     else if (metadataGetMode) {
400       A2_LOG_NOTICE(_("For BitTorrent Magnet URI, enabling DHT is strongly"
401                       " recommended. See --enable-dht option."));
402     }
403 
404     if (metadataGetMode) {
405       BtCheckIntegrityEntry{this}.onDownloadIncomplete(commands, e);
406       return;
407     }
408 
409     removeDefunctControlFile(progressInfoFile);
410     {
411       int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
412       if (actualFileSize == downloadContext_->getTotalLength()) {
413         // First, make DiskAdaptor read-only mode to allow the
414         // program to seed file in read-only media.
415         pieceStorage_->getDiskAdaptor()->enableReadOnly();
416       }
417       else {
418         // Open file in writable mode to allow the program
419         // truncate the file to downloadContext_->getTotalLength()
420         A2_LOG_DEBUG(fmt("File size not match. File is opened in writable"
421                          " mode. Expected:%" PRId64 " Actual:%" PRId64 "",
422                          downloadContext_->getTotalLength(), actualFileSize));
423       }
424     }
425     // Call Load, Save and file allocation command here
426     if (progressInfoFile->exists()) {
427       // load .aria2 file if it exists.
428       progressInfoFile->load();
429       pieceStorage_->getDiskAdaptor()->openFile();
430     }
431     else if (pieceStorage_->getDiskAdaptor()->fileExists()) {
432       if (!option_->getAsBool(PREF_CHECK_INTEGRITY) &&
433           !option_->getAsBool(PREF_ALLOW_OVERWRITE) &&
434           !option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
435         // TODO we need this->haltRequested = true?
436         throw DOWNLOAD_FAILURE_EXCEPTION2(
437             fmt(MSG_FILE_ALREADY_EXISTS,
438                 downloadContext_->getBasePath().c_str()),
439             error_code::FILE_ALREADY_EXISTS);
440       }
441       pieceStorage_->getDiskAdaptor()->openFile();
442       if (option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
443         pieceStorage_->markAllPiecesDone();
444       }
445     }
446     else {
447       pieceStorage_->getDiskAdaptor()->openFile();
448     }
449     progressInfoFile_ = progressInfoFile;
450 
451     auto entry = make_unique<BtCheckIntegrityEntry>(this);
452     // --bt-seed-unverified=true is given and download has completed, skip
453     // validation for piece hashes.
454     if (option_->getAsBool(PREF_BT_SEED_UNVERIFIED) &&
455         pieceStorage_->downloadFinished()) {
456       entry->onDownloadFinished(commands, e);
457     }
458     else {
459       processCheckIntegrityEntry(commands, std::move(entry), e);
460     }
461     return;
462   }
463 #endif // ENABLE_BITTORRENT
464 
465   if (downloadContext_->getFileEntries().size() == 1) {
466     // TODO I assume here when totallength is set to DownloadContext and it is
467     // not 0, then filepath is also set DownloadContext correctly....
468     if (option_->getAsBool(PREF_DRY_RUN) ||
469         downloadContext_->getTotalLength() == 0) {
470       createNextCommand(commands, e, 1);
471       return;
472     }
473     auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
474         downloadContext_, nullptr, option_.get());
475     adjustFilename(progressInfoFile);
476     initPieceStorage();
477     auto checkEntry = createCheckIntegrityEntry();
478     if (checkEntry) {
479       processCheckIntegrityEntry(commands, std::move(checkEntry), e);
480     }
481     return;
482   }
483 
484   // TODO --dry-run is not supported for multifile download for now.
485   if (option_->getAsBool(PREF_DRY_RUN)) {
486     throw DOWNLOAD_FAILURE_EXCEPTION(
487         "--dry-run in multi-file download is not supported yet.");
488   }
489   // TODO file size is known in this context?
490 
491   // In this context, multiple FileEntry objects are in
492   // DownloadContext.
493   if (e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
494     throw DOWNLOAD_FAILURE_EXCEPTION2(
495         fmt(EX_DUPLICATE_FILE_DOWNLOAD,
496             downloadContext_->getBasePath().c_str()),
497         error_code::DUPLICATE_DOWNLOAD);
498   }
499   initPieceStorage();
500   if (downloadContext_->getFileEntries().size() > 1) {
501     pieceStorage_->setupFileFilter();
502   }
503   auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
504       downloadContext_, pieceStorage_, option_.get());
505   removeDefunctControlFile(progressInfoFile);
506   // Call Load, Save and file allocation command here
507   if (progressInfoFile->exists()) {
508     // load .aria2 file if it exists.
509     progressInfoFile->load();
510     pieceStorage_->getDiskAdaptor()->openFile();
511   }
512   else if (pieceStorage_->getDiskAdaptor()->fileExists()) {
513     if (!isCheckIntegrityReady() && !option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
514       // TODO we need this->haltRequested = true?
515       throw DOWNLOAD_FAILURE_EXCEPTION2(
516           fmt(MSG_FILE_ALREADY_EXISTS, downloadContext_->getBasePath().c_str()),
517           error_code::FILE_ALREADY_EXISTS);
518     }
519     pieceStorage_->getDiskAdaptor()->openFile();
520   }
521   else {
522     pieceStorage_->getDiskAdaptor()->openFile();
523   }
524   progressInfoFile_ = progressInfoFile;
525   processCheckIntegrityEntry(commands,
526                              make_unique<StreamCheckIntegrityEntry>(this), e);
527 }
528 
processCheckIntegrityEntry(std::vector<std::unique_ptr<Command>> & commands,std::unique_ptr<CheckIntegrityEntry> entry,DownloadEngine * e)529 void RequestGroup::processCheckIntegrityEntry(
530     std::vector<std::unique_ptr<Command>>& commands,
531     std::unique_ptr<CheckIntegrityEntry> entry, DownloadEngine* e)
532 {
533   int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
534   if (actualFileSize > downloadContext_->getTotalLength()) {
535     entry->cutTrailingGarbage();
536   }
537   if ((option_->getAsBool(PREF_CHECK_INTEGRITY) ||
538        downloadContext_->isChecksumVerificationNeeded()) &&
539       entry->isValidationReady()) {
540     entry->initValidator();
541     // Don't save control file(.aria2 file) when user presses
542     // control-c key while aria2 is checking hashes. If control file
543     // doesn't exist when aria2 launched, the completed length in
544     // saved control file will be 0 byte and this confuses user.
545     // enableSaveControlFile() will be called after hash checking is
546     // done. See CheckIntegrityCommand.
547     disableSaveControlFile();
548     e->getCheckIntegrityMan()->pushEntry(std::move(entry));
549     return;
550   }
551 
552   entry->onDownloadIncomplete(commands, e);
553 }
554 
initPieceStorage()555 void RequestGroup::initPieceStorage()
556 {
557   std::shared_ptr<PieceStorage> tempPieceStorage;
558   if (downloadContext_->knowsTotalLength() &&
559       // Following conditions are needed for chunked encoding with
560       // content-length = 0. Google's dl server used this before.
561       (downloadContext_->getTotalLength() > 0
562 #ifdef ENABLE_BITTORRENT
563        || downloadContext_->hasAttribute(CTX_ATTR_BT)
564 #endif // ENABLE_BITTORRENT
565            )) {
566 #ifdef ENABLE_BITTORRENT
567     auto ps =
568         std::make_shared<DefaultPieceStorage>(downloadContext_, option_.get());
569     if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
570       if (isUriSuppliedForRequsetFileEntry(
571               downloadContext_->getFileEntries().begin(),
572               downloadContext_->getFileEntries().end())) {
573         // Use LongestSequencePieceSelector when HTTP/FTP/BitTorrent
574         // integrated downloads.
575         A2_LOG_DEBUG("Using LongestSequencePieceSelector");
576         ps->setPieceSelector(make_unique<LongestSequencePieceSelector>());
577       }
578       if (option_->defined(PREF_BT_PRIORITIZE_PIECE)) {
579         std::vector<size_t> result;
580         util::parsePrioritizePieceRange(result,
581                                         option_->get(PREF_BT_PRIORITIZE_PIECE),
582                                         downloadContext_->getFileEntries(),
583                                         downloadContext_->getPieceLength());
584         if (!result.empty()) {
585           std::shuffle(std::begin(result), std::end(result),
586                        *SimpleRandomizer::getInstance());
587           auto priSelector =
588               make_unique<PriorityPieceSelector>(ps->popPieceSelector());
589           priSelector->setPriorityPiece(std::begin(result), std::end(result));
590           ps->setPieceSelector(std::move(priSelector));
591         }
592       }
593     }
594 #else  // !ENABLE_BITTORRENT
595     auto ps =
596         std::make_shared<DefaultPieceStorage>(downloadContext_, option_.get());
597 #endif // !ENABLE_BITTORRENT
598     if (requestGroupMan_) {
599       ps->setWrDiskCache(requestGroupMan_->getWrDiskCache());
600     }
601     if (diskWriterFactory_) {
602       ps->setDiskWriterFactory(diskWriterFactory_);
603     }
604     tempPieceStorage = ps;
605   }
606   else {
607     auto ps = std::make_shared<UnknownLengthPieceStorage>(downloadContext_);
608     if (diskWriterFactory_) {
609       ps->setDiskWriterFactory(diskWriterFactory_);
610     }
611     tempPieceStorage = ps;
612   }
613   tempPieceStorage->initStorage();
614   if (requestGroupMan_) {
615     tempPieceStorage->getDiskAdaptor()->setOpenedFileCounter(
616         requestGroupMan_->getOpenedFileCounter());
617   }
618   segmentMan_ =
619       std::make_shared<SegmentMan>(downloadContext_, tempPieceStorage);
620   pieceStorage_ = tempPieceStorage;
621 
622 #ifdef __MINGW32__
623   // Windows build: --file-allocation=falloc uses SetFileValidData
624   // which requires SE_MANAGE_VOLUME_NAME privilege.  SetFileValidData
625   // has security implications (see
626   // https://msdn.microsoft.com/en-us/library/windows/desktop/aa365544%28v=vs.85%29.aspx).
627   static auto gainPrivilegeAttempted = false;
628 
629   if (!gainPrivilegeAttempted &&
630       pieceStorage_->getDiskAdaptor()->getFileAllocationMethod() ==
631           DiskAdaptor::FILE_ALLOC_FALLOC &&
632       isFileAllocationEnabled()) {
633     if (!util::gainPrivilege(SE_MANAGE_VOLUME_NAME)) {
634       A2_LOG_WARN("--file-allocation=falloc will not work properly.");
635     }
636     else {
637       A2_LOG_DEBUG("SE_MANAGE_VOLUME_NAME privilege acquired");
638 
639       A2_LOG_WARN(
640           "--file-allocation=falloc will use SetFileValidData() API, and "
641           "aria2 uses uninitialized disk space which may contain "
642           "confidential data as the download file space. If it is "
643           "undesirable, --file-allocation=prealloc is slower, but safer "
644           "option.");
645     }
646 
647     gainPrivilegeAttempted = true;
648   }
649 #endif // __MINGW32__
650 }
651 
dropPieceStorage()652 void RequestGroup::dropPieceStorage()
653 {
654   segmentMan_.reset();
655   pieceStorage_.reset();
656 }
657 
downloadFinishedByFileLength()658 bool RequestGroup::downloadFinishedByFileLength()
659 {
660   // assuming that a control file doesn't exist.
661   if (!isPreLocalFileCheckEnabled() ||
662       option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
663     return false;
664   }
665   if (!downloadContext_->knowsTotalLength()) {
666     return false;
667   }
668   File outfile(getFirstFilePath());
669   if (outfile.exists() &&
670       downloadContext_->getTotalLength() == outfile.size()) {
671     return true;
672   }
673   return false;
674 }
675 
adjustFilename(const std::shared_ptr<BtProgressInfoFile> & infoFile)676 void RequestGroup::adjustFilename(
677     const std::shared_ptr<BtProgressInfoFile>& infoFile)
678 {
679   if (!isPreLocalFileCheckEnabled()) {
680     // OK, no need to care about filename.
681     return;
682   }
683   // TODO need this?
684   if (requestGroupMan_) {
685     if (requestGroupMan_->isSameFileBeingDownloaded(this)) {
686       // The file name must be renamed
687       tryAutoFileRenaming();
688       A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
689       return;
690     }
691   }
692   if (!option_->getAsBool(PREF_DRY_RUN) &&
693       option_->getAsBool(PREF_REMOVE_CONTROL_FILE) && infoFile->exists()) {
694     infoFile->removeFile();
695     A2_LOG_NOTICE(fmt(_("Removed control file for %s because it is requested by"
696                         " user."),
697                       infoFile->getFilename().c_str()));
698   }
699 
700   if (infoFile->exists()) {
701     // Use current filename
702     return;
703   }
704 
705   File outfile(getFirstFilePath());
706   if (outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
707       outfile.size() <= downloadContext_->getTotalLength()) {
708     // File exists but user decided to resume it.
709   }
710   else if (outfile.exists() && isCheckIntegrityReady()) {
711     // check-integrity existing file
712   }
713   else {
714     shouldCancelDownloadForSafety();
715   }
716 }
717 
removeDefunctControlFile(const std::shared_ptr<BtProgressInfoFile> & progressInfoFile)718 void RequestGroup::removeDefunctControlFile(
719     const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
720 {
721   // Remove the control file if download file doesn't exist
722   if (progressInfoFile->exists() &&
723       !pieceStorage_->getDiskAdaptor()->fileExists()) {
724     progressInfoFile->removeFile();
725     A2_LOG_NOTICE(fmt(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
726                       progressInfoFile->getFilename().c_str(),
727                       downloadContext_->getBasePath().c_str()));
728   }
729 }
730 
loadAndOpenFile(const std::shared_ptr<BtProgressInfoFile> & progressInfoFile)731 void RequestGroup::loadAndOpenFile(
732     const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
733 {
734   try {
735     if (!isPreLocalFileCheckEnabled()) {
736       pieceStorage_->getDiskAdaptor()->initAndOpenFile();
737       return;
738     }
739     removeDefunctControlFile(progressInfoFile);
740     if (progressInfoFile->exists()) {
741       progressInfoFile->load();
742       pieceStorage_->getDiskAdaptor()->openExistingFile();
743     }
744     else {
745       File outfile(getFirstFilePath());
746       if (outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
747           outfile.size() <= getTotalLength()) {
748         pieceStorage_->getDiskAdaptor()->openExistingFile();
749         pieceStorage_->markPiecesDone(outfile.size());
750       }
751       else if (outfile.exists() && isCheckIntegrityReady()) {
752         pieceStorage_->getDiskAdaptor()->openExistingFile();
753       }
754       else {
755         pieceStorage_->getDiskAdaptor()->initAndOpenFile();
756       }
757     }
758     setProgressInfoFile(progressInfoFile);
759   }
760   catch (RecoverableException& e) {
761     throw DOWNLOAD_FAILURE_EXCEPTION2(EX_DOWNLOAD_ABORTED, e);
762   }
763 }
764 
765 // assuming that a control file does not exist
shouldCancelDownloadForSafety()766 void RequestGroup::shouldCancelDownloadForSafety()
767 {
768   if (option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
769     return;
770   }
771   File outfile(getFirstFilePath());
772   if (!outfile.exists()) {
773     return;
774   }
775 
776   tryAutoFileRenaming();
777   A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
778 }
779 
tryAutoFileRenaming()780 void RequestGroup::tryAutoFileRenaming()
781 {
782   if (!option_->getAsBool(PREF_AUTO_FILE_RENAMING)) {
783     throw DOWNLOAD_FAILURE_EXCEPTION2(
784         fmt(MSG_FILE_ALREADY_EXISTS, getFirstFilePath().c_str()),
785         error_code::FILE_ALREADY_EXISTS);
786   }
787 
788   std::string filepath = getFirstFilePath();
789   if (filepath.empty()) {
790     throw DOWNLOAD_FAILURE_EXCEPTION2(
791         fmt("File renaming failed: %s", getFirstFilePath().c_str()),
792         error_code::FILE_RENAMING_FAILED);
793   }
794   auto fn = filepath;
795   std::string ext;
796   const auto idx = fn.find_last_of(".");
797   const auto slash = fn.find_last_of("\\/");
798   // Do extract the extension, as in "file.ext" = "file" and ".ext",
799   // but do not consider ".file" to be a file name without extension instead
800   // of a blank file name and an extension of ".file"
801   if (idx != std::string::npos &&
802       // fn has no path component and starts with a dot, but has no extension
803       // otherwise
804       idx != 0 &&
805       // has a file path component if we found a slash.
806       // if slash == idx - 1 this means a form of "*/.*", so the file name
807       // starts with a dot, has no extension otherwise, and therefore do not
808       // extract an extension either
809       (slash == std::string::npos || slash < idx - 1)) {
810     ext = fn.substr(idx);
811     fn = fn.substr(0, idx);
812   }
813   for (int i = 1; i < 10000; ++i) {
814     auto newfilename = fmt("%s.%d%s", fn.c_str(), i, ext.c_str());
815     File newfile(newfilename);
816     File ctrlfile(newfile.getPath() + DefaultBtProgressInfoFile::getSuffix());
817     if (!newfile.exists() || (newfile.exists() && ctrlfile.exists())) {
818       downloadContext_->getFirstFileEntry()->setPath(newfile.getPath());
819       return;
820     }
821   }
822   throw DOWNLOAD_FAILURE_EXCEPTION2(
823       fmt("File renaming failed: %s", getFirstFilePath().c_str()),
824       error_code::FILE_RENAMING_FAILED);
825 }
826 
createNextCommandWithAdj(std::vector<std::unique_ptr<Command>> & commands,DownloadEngine * e,int numAdj)827 void RequestGroup::createNextCommandWithAdj(
828     std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
829     int numAdj)
830 {
831   int numCommand;
832   if (getTotalLength() == 0) {
833     numCommand = 1 + numAdj;
834   }
835   else {
836     numCommand = std::min(downloadContext_->getNumPieces(),
837                           static_cast<size_t>(numConcurrentCommand_));
838     numCommand += numAdj;
839   }
840 
841   if (numCommand > 0) {
842     createNextCommand(commands, e, numCommand);
843   }
844 }
845 
createNextCommand(std::vector<std::unique_ptr<Command>> & commands,DownloadEngine * e)846 void RequestGroup::createNextCommand(
847     std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
848 {
849   int numCommand;
850   if (getTotalLength() == 0) {
851     if (numStreamCommand_ > 0) {
852       numCommand = 0;
853     }
854     else {
855       numCommand = 1;
856     }
857   }
858   else if (numStreamCommand_ >= numConcurrentCommand_) {
859     numCommand = 0;
860   }
861   else {
862     numCommand = std::min(
863         downloadContext_->getNumPieces(),
864         static_cast<size_t>(numConcurrentCommand_ - numStreamCommand_));
865   }
866 
867   if (numCommand > 0) {
868     createNextCommand(commands, e, numCommand);
869   }
870 }
871 
createNextCommand(std::vector<std::unique_ptr<Command>> & commands,DownloadEngine * e,int numCommand)872 void RequestGroup::createNextCommand(
873     std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
874     int numCommand)
875 {
876   for (; numCommand > 0; --numCommand) {
877     commands.push_back(
878         make_unique<CreateRequestCommand>(e->newCUID(), this, e));
879   }
880   if (!commands.empty()) {
881     e->setNoWait(true);
882   }
883 }
884 
getFirstFilePath() const885 std::string RequestGroup::getFirstFilePath() const
886 {
887   assert(downloadContext_);
888   if (inMemoryDownload()) {
889     return "[MEMORY]" +
890            File(downloadContext_->getFirstFileEntry()->getPath()).getBasename();
891   }
892   return downloadContext_->getFirstFileEntry()->getPath();
893 }
894 
getTotalLength() const895 int64_t RequestGroup::getTotalLength() const
896 {
897   if (!pieceStorage_) {
898     return 0;
899   }
900 
901   if (pieceStorage_->isSelectiveDownloadingMode()) {
902     return pieceStorage_->getFilteredTotalLength();
903   }
904 
905   return pieceStorage_->getTotalLength();
906 }
907 
getCompletedLength() const908 int64_t RequestGroup::getCompletedLength() const
909 {
910   if (!pieceStorage_) {
911     return 0;
912   }
913 
914   if (pieceStorage_->isSelectiveDownloadingMode()) {
915     return pieceStorage_->getFilteredCompletedLength();
916   }
917 
918   return pieceStorage_->getCompletedLength();
919 }
920 
validateFilename(const std::string & expectedFilename,const std::string & actualFilename) const921 void RequestGroup::validateFilename(const std::string& expectedFilename,
922                                     const std::string& actualFilename) const
923 {
924   if (expectedFilename.empty()) {
925     return;
926   }
927 
928   if (expectedFilename != actualFilename) {
929     throw DL_ABORT_EX(fmt(EX_FILENAME_MISMATCH, expectedFilename.c_str(),
930                           actualFilename.c_str()));
931   }
932 }
933 
validateTotalLength(int64_t expectedTotalLength,int64_t actualTotalLength) const934 void RequestGroup::validateTotalLength(int64_t expectedTotalLength,
935                                        int64_t actualTotalLength) const
936 {
937   if (expectedTotalLength <= 0) {
938     return;
939   }
940 
941   if (expectedTotalLength != actualTotalLength) {
942     throw DL_ABORT_EX(
943         fmt(EX_SIZE_MISMATCH, expectedTotalLength, actualTotalLength));
944   }
945 }
946 
validateFilename(const std::string & actualFilename) const947 void RequestGroup::validateFilename(const std::string& actualFilename) const
948 {
949   validateFilename(downloadContext_->getFileEntries().front()->getBasename(),
950                    actualFilename);
951 }
952 
validateTotalLength(int64_t actualTotalLength) const953 void RequestGroup::validateTotalLength(int64_t actualTotalLength) const
954 {
955   validateTotalLength(getTotalLength(), actualTotalLength);
956 }
957 
increaseStreamCommand()958 void RequestGroup::increaseStreamCommand() { ++numStreamCommand_; }
959 
decreaseStreamCommand()960 void RequestGroup::decreaseStreamCommand() { --numStreamCommand_; }
961 
increaseStreamConnection()962 void RequestGroup::increaseStreamConnection() { ++numStreamConnection_; }
963 
decreaseStreamConnection()964 void RequestGroup::decreaseStreamConnection() { --numStreamConnection_; }
965 
getNumConnection() const966 int RequestGroup::getNumConnection() const
967 {
968   int numConnection = numStreamConnection_;
969 #ifdef ENABLE_BITTORRENT
970   if (btRuntime_) {
971     numConnection += btRuntime_->getConnections();
972   }
973 #endif // ENABLE_BITTORRENT
974   return numConnection;
975 }
976 
increaseNumCommand()977 void RequestGroup::increaseNumCommand() { ++numCommand_; }
978 
decreaseNumCommand()979 void RequestGroup::decreaseNumCommand()
980 {
981   --numCommand_;
982   if (!numCommand_ && requestGroupMan_) {
983     A2_LOG_DEBUG(fmt("GID#%s - Request queue check", gid_->toHex().c_str()));
984     requestGroupMan_->requestQueueCheck();
985   }
986 }
987 
calculateStat() const988 TransferStat RequestGroup::calculateStat() const
989 {
990   TransferStat stat = downloadContext_->getNetStat().toTransferStat();
991 #ifdef ENABLE_BITTORRENT
992   if (btRuntime_) {
993     stat.allTimeUploadLength =
994         btRuntime_->getUploadLengthAtStartup() + stat.sessionUploadLength;
995   }
996 #endif // ENABLE_BITTORRENT
997   return stat;
998 }
999 
setHaltRequested(bool f,HaltReason haltReason)1000 void RequestGroup::setHaltRequested(bool f, HaltReason haltReason)
1001 {
1002   haltRequested_ = f;
1003   if (haltRequested_) {
1004     pauseRequested_ = false;
1005     haltReason_ = haltReason;
1006   }
1007 #ifdef ENABLE_BITTORRENT
1008   if (btRuntime_) {
1009     btRuntime_->setHalt(f);
1010   }
1011 #endif // ENABLE_BITTORRENT
1012 }
1013 
setForceHaltRequested(bool f,HaltReason haltReason)1014 void RequestGroup::setForceHaltRequested(bool f, HaltReason haltReason)
1015 {
1016   setHaltRequested(f, haltReason);
1017   forceHaltRequested_ = f;
1018 }
1019 
setPauseRequested(bool f)1020 void RequestGroup::setPauseRequested(bool f) { pauseRequested_ = f; }
1021 
setRestartRequested(bool f)1022 void RequestGroup::setRestartRequested(bool f) { restartRequested_ = f; }
1023 
releaseRuntimeResource(DownloadEngine * e)1024 void RequestGroup::releaseRuntimeResource(DownloadEngine* e)
1025 {
1026 #ifdef ENABLE_BITTORRENT
1027   e->getBtRegistry()->remove(gid_->getNumericId());
1028   btRuntime_ = nullptr;
1029   peerStorage_ = nullptr;
1030 #endif // ENABLE_BITTORRENT
1031   if (pieceStorage_) {
1032     pieceStorage_->removeAdvertisedPiece(Timer::zero());
1033   }
1034   // Don't reset segmentMan_ and pieceStorage_ here to provide
1035   // progress information via RPC
1036   progressInfoFile_ = std::make_shared<NullProgressInfoFile>();
1037   downloadContext_->releaseRuntimeResource();
1038   // Reset seedOnly_, so that we can handle pause/unpause-ing seeding
1039   // torrent with --bt-detach-seed-only.
1040   seedOnly_ = false;
1041 }
1042 
preDownloadProcessing()1043 void RequestGroup::preDownloadProcessing()
1044 {
1045   A2_LOG_DEBUG(fmt("Finding PreDownloadHandler for path %s.",
1046                    getFirstFilePath().c_str()));
1047   try {
1048     for (const auto& pdh : preDownloadHandlers_) {
1049       if (pdh->canHandle(this)) {
1050         pdh->execute(this);
1051         return;
1052       }
1053     }
1054   }
1055   catch (RecoverableException& ex) {
1056     A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
1057     return;
1058   }
1059 
1060   A2_LOG_DEBUG("No PreDownloadHandler found.");
1061   return;
1062 }
1063 
postDownloadProcessing(std::vector<std::shared_ptr<RequestGroup>> & groups)1064 void RequestGroup::postDownloadProcessing(
1065     std::vector<std::shared_ptr<RequestGroup>>& groups)
1066 {
1067   A2_LOG_DEBUG(fmt("Finding PostDownloadHandler for path %s.",
1068                    getFirstFilePath().c_str()));
1069   try {
1070     for (const auto& pdh : postDownloadHandlers_) {
1071       if (pdh->canHandle(this)) {
1072         pdh->getNextRequestGroups(groups, this);
1073         return;
1074       }
1075     }
1076   }
1077   catch (RecoverableException& ex) {
1078     A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
1079   }
1080 
1081   A2_LOG_DEBUG("No PostDownloadHandler found.");
1082 }
1083 
initializePreDownloadHandler()1084 void RequestGroup::initializePreDownloadHandler()
1085 {
1086 #ifdef ENABLE_BITTORRENT
1087   if (option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
1088     preDownloadHandlers_.push_back(
1089         download_handlers::getBtPreDownloadHandler());
1090   }
1091 #endif // ENABLE_BITTORRENT
1092 #ifdef ENABLE_METALINK
1093   if (option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
1094     preDownloadHandlers_.push_back(
1095         download_handlers::getMetalinkPreDownloadHandler());
1096   }
1097 #endif // ENABLE_METALINK
1098 }
1099 
initializePostDownloadHandler()1100 void RequestGroup::initializePostDownloadHandler()
1101 {
1102 #ifdef ENABLE_BITTORRENT
1103   if (option_->getAsBool(PREF_FOLLOW_TORRENT) ||
1104       option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
1105     postDownloadHandlers_.push_back(
1106         download_handlers::getBtPostDownloadHandler());
1107   }
1108 #endif // ENABLE_BITTORRENT
1109 #ifdef ENABLE_METALINK
1110   if (option_->getAsBool(PREF_FOLLOW_METALINK) ||
1111       option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
1112     postDownloadHandlers_.push_back(
1113         download_handlers::getMetalinkPostDownloadHandler());
1114   }
1115 #endif // ENABLE_METALINK
1116 }
1117 
isDependencyResolved()1118 bool RequestGroup::isDependencyResolved()
1119 {
1120   if (!dependency_) {
1121     return true;
1122   }
1123   return dependency_->resolve();
1124 }
1125 
dependsOn(const std::shared_ptr<Dependency> & dep)1126 void RequestGroup::dependsOn(const std::shared_ptr<Dependency>& dep)
1127 {
1128   dependency_ = dep;
1129 }
1130 
setDiskWriterFactory(const std::shared_ptr<DiskWriterFactory> & diskWriterFactory)1131 void RequestGroup::setDiskWriterFactory(
1132     const std::shared_ptr<DiskWriterFactory>& diskWriterFactory)
1133 {
1134   diskWriterFactory_ = diskWriterFactory;
1135 }
1136 
addPostDownloadHandler(const PostDownloadHandler * handler)1137 void RequestGroup::addPostDownloadHandler(const PostDownloadHandler* handler)
1138 {
1139   postDownloadHandlers_.push_back(handler);
1140 }
1141 
addPreDownloadHandler(const PreDownloadHandler * handler)1142 void RequestGroup::addPreDownloadHandler(const PreDownloadHandler* handler)
1143 {
1144   preDownloadHandlers_.push_back(handler);
1145 }
1146 
clearPostDownloadHandler()1147 void RequestGroup::clearPostDownloadHandler() { postDownloadHandlers_.clear(); }
1148 
clearPreDownloadHandler()1149 void RequestGroup::clearPreDownloadHandler() { preDownloadHandlers_.clear(); }
1150 
setPieceStorage(const std::shared_ptr<PieceStorage> & pieceStorage)1151 void RequestGroup::setPieceStorage(
1152     const std::shared_ptr<PieceStorage>& pieceStorage)
1153 {
1154   pieceStorage_ = pieceStorage;
1155 }
1156 
setProgressInfoFile(const std::shared_ptr<BtProgressInfoFile> & progressInfoFile)1157 void RequestGroup::setProgressInfoFile(
1158     const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
1159 {
1160   progressInfoFile_ = progressInfoFile;
1161 }
1162 
needsFileAllocation() const1163 bool RequestGroup::needsFileAllocation() const
1164 {
1165   return isFileAllocationEnabled() &&
1166          option_->getAsLLInt(PREF_NO_FILE_ALLOCATION_LIMIT) <=
1167              getTotalLength() &&
1168          !pieceStorage_->getDiskAdaptor()->fileAllocationIterator()->finished();
1169 }
1170 
createDownloadResult() const1171 std::shared_ptr<DownloadResult> RequestGroup::createDownloadResult() const
1172 {
1173   A2_LOG_DEBUG(fmt("GID#%s - Creating DownloadResult.", gid_->toHex().c_str()));
1174   TransferStat st = calculateStat();
1175   auto res = std::make_shared<DownloadResult>();
1176   res->gid = gid_;
1177   res->attrs = downloadContext_->getAttributes();
1178   res->fileEntries = downloadContext_->getFileEntries();
1179   res->inMemoryDownload = inMemoryDownload_;
1180   res->sessionDownloadLength = st.sessionDownloadLength;
1181   res->sessionTime = std::chrono::duration_cast<std::chrono::milliseconds>(
1182       downloadContext_->calculateSessionTime());
1183 
1184   auto result = downloadResult();
1185   res->result = result.first;
1186   res->resultMessage = result.second;
1187   res->followedBy = followedByGIDs_;
1188   res->following = followingGID_;
1189   res->belongsTo = belongsToGID_;
1190   res->option = option_;
1191   res->metadataInfo = metadataInfo_;
1192   res->totalLength = getTotalLength();
1193   res->completedLength = getCompletedLength();
1194   res->uploadLength = st.allTimeUploadLength;
1195   if (pieceStorage_ && pieceStorage_->getBitfieldLength() > 0) {
1196     res->bitfield.assign(pieceStorage_->getBitfield(),
1197                          pieceStorage_->getBitfield() +
1198                              pieceStorage_->getBitfieldLength());
1199   }
1200 #ifdef ENABLE_BITTORRENT
1201   if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
1202     const unsigned char* p = bittorrent::getInfoHash(downloadContext_);
1203     res->infoHash.assign(p, p + INFO_HASH_LENGTH);
1204   }
1205 #endif // ENABLE_BITTORRENT
1206   res->pieceLength = downloadContext_->getPieceLength();
1207   res->numPieces = downloadContext_->getNumPieces();
1208   res->dir = option_->get(PREF_DIR);
1209   return res;
1210 }
1211 
reportDownloadFinished()1212 void RequestGroup::reportDownloadFinished()
1213 {
1214   A2_LOG_NOTICE(fmt(MSG_FILE_DOWNLOAD_COMPLETED,
1215                     inMemoryDownload()
1216                         ? getFirstFilePath().c_str()
1217                         : downloadContext_->getBasePath().c_str()));
1218   uriSelector_->resetCounters();
1219 #ifdef ENABLE_BITTORRENT
1220   if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
1221     TransferStat stat = calculateStat();
1222     int64_t completedLength = getCompletedLength();
1223     double shareRatio = completedLength == 0
1224                             ? 0.0
1225                             : 1.0 * stat.allTimeUploadLength / completedLength;
1226     auto attrs = bittorrent::getTorrentAttrs(downloadContext_);
1227     if (!attrs->metadata.empty()) {
1228       A2_LOG_NOTICE(fmt(MSG_SHARE_RATIO_REPORT, shareRatio,
1229                         util::abbrevSize(stat.allTimeUploadLength).c_str(),
1230                         util::abbrevSize(completedLength).c_str()));
1231     }
1232   }
1233 #endif // ENABLE_BITTORRENT
1234 }
1235 
setURISelector(std::unique_ptr<URISelector> uriSelector)1236 void RequestGroup::setURISelector(std::unique_ptr<URISelector> uriSelector)
1237 {
1238   uriSelector_ = std::move(uriSelector);
1239 }
1240 
applyLastModifiedTimeToLocalFiles()1241 void RequestGroup::applyLastModifiedTimeToLocalFiles()
1242 {
1243   if (!pieceStorage_ || !lastModifiedTime_.good()) {
1244     return;
1245   }
1246   A2_LOG_INFO(fmt("Applying Last-Modified time: %s",
1247                   lastModifiedTime_.toHTTPDate().c_str()));
1248   size_t n = pieceStorage_->getDiskAdaptor()->utime(Time(), lastModifiedTime_);
1249   A2_LOG_INFO(fmt("Last-Modified attrs of %lu files were updated.",
1250                   static_cast<unsigned long>(n)));
1251 }
1252 
updateLastModifiedTime(const Time & time)1253 void RequestGroup::updateLastModifiedTime(const Time& time)
1254 {
1255   if (time.good() && lastModifiedTime_ < time) {
1256     lastModifiedTime_ = time;
1257   }
1258 }
1259 
increaseAndValidateFileNotFoundCount()1260 void RequestGroup::increaseAndValidateFileNotFoundCount()
1261 {
1262   ++fileNotFoundCount_;
1263   const int maxCount = option_->getAsInt(PREF_MAX_FILE_NOT_FOUND);
1264   if (maxCount > 0 && fileNotFoundCount_ >= maxCount &&
1265       downloadContext_->getNetStat().getSessionDownloadLength() == 0) {
1266     throw DOWNLOAD_FAILURE_EXCEPTION2(
1267         fmt("Reached max-file-not-found count=%d", maxCount),
1268         error_code::MAX_FILE_NOT_FOUND);
1269   }
1270 }
1271 
markInMemoryDownload()1272 void RequestGroup::markInMemoryDownload() { inMemoryDownload_ = true; }
1273 
setTimeout(std::chrono::seconds timeout)1274 void RequestGroup::setTimeout(std::chrono::seconds timeout)
1275 {
1276   timeout_ = std::move(timeout);
1277 }
1278 
doesDownloadSpeedExceed()1279 bool RequestGroup::doesDownloadSpeedExceed()
1280 {
1281   int spd = downloadContext_->getNetStat().calculateDownloadSpeed();
1282   return maxDownloadSpeedLimit_ > 0 && maxDownloadSpeedLimit_ < spd;
1283 }
1284 
doesUploadSpeedExceed()1285 bool RequestGroup::doesUploadSpeedExceed()
1286 {
1287   int spd = downloadContext_->getNetStat().calculateUploadSpeed();
1288   return maxUploadSpeedLimit_ > 0 && maxUploadSpeedLimit_ < spd;
1289 }
1290 
saveControlFile() const1291 void RequestGroup::saveControlFile() const
1292 {
1293   if (saveControlFile_) {
1294     if (pieceStorage_) {
1295       pieceStorage_->flushWrDiskCacheEntry(false);
1296       pieceStorage_->getDiskAdaptor()->flushOSBuffers();
1297     }
1298     progressInfoFile_->save();
1299   }
1300 }
1301 
removeControlFile() const1302 void RequestGroup::removeControlFile() const
1303 {
1304   progressInfoFile_->removeFile();
1305 }
1306 
setDownloadContext(const std::shared_ptr<DownloadContext> & downloadContext)1307 void RequestGroup::setDownloadContext(
1308     const std::shared_ptr<DownloadContext>& downloadContext)
1309 {
1310   downloadContext_ = downloadContext;
1311   if (downloadContext_) {
1312     downloadContext_->setOwnerRequestGroup(this);
1313   }
1314 }
1315 
p2pInvolved() const1316 bool RequestGroup::p2pInvolved() const
1317 {
1318 #ifdef ENABLE_BITTORRENT
1319   return downloadContext_->hasAttribute(CTX_ATTR_BT);
1320 #else  // !ENABLE_BITTORRENT
1321   return false;
1322 #endif // !ENABLE_BITTORRENT
1323 }
1324 
enableSeedOnly()1325 void RequestGroup::enableSeedOnly()
1326 {
1327   if (seedOnly_ || !option_->getAsBool(PREF_BT_DETACH_SEED_ONLY)) {
1328     return;
1329   }
1330 
1331   if (requestGroupMan_) {
1332     seedOnly_ = true;
1333 
1334     requestGroupMan_->decreaseNumActive();
1335     requestGroupMan_->requestQueueCheck();
1336   }
1337 }
1338 
isSeeder() const1339 bool RequestGroup::isSeeder() const
1340 {
1341 #ifdef ENABLE_BITTORRENT
1342   return downloadContext_->hasAttribute(CTX_ATTR_BT) &&
1343          !bittorrent::getTorrentAttrs(downloadContext_)->metadata.empty() &&
1344          downloadFinished();
1345 #else  // !ENABLE_BITTORRENT
1346   return false;
1347 #endif // !ENABLE_BITTORRENT
1348 }
1349 
setPendingOption(std::shared_ptr<Option> option)1350 void RequestGroup::setPendingOption(std::shared_ptr<Option> option)
1351 {
1352   pendingOption_ = std::move(option);
1353 }
1354 
1355 } // namespace aria2
1356