1 /*
2 * Copyright (C) 2001-2012 Jacek Sieka, arnetheduck on gmail point com
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 */
18
19 #include "stdinc.h"
20
21 #include "QueueManager.h"
22 #include "format.h"
23
24 #include "ClientManager.h"
25 #include "ConnectionManager.h"
26 #include "DirectoryListing.h"
27 #include "Download.h"
28 #include "DownloadManager.h"
29 #include "HashManager.h"
30 #include "LogManager.h"
31 #include "SearchManager.h"
32 #include "ShareManager.h"
33 #include "SimpleXML.h"
34 #include "StringTokenizer.h"
35 #include "Transfer.h"
36 #include "UserConnection.h"
37 #include "version.h"
38 #include "SearchResult.h"
39 #include "MerkleCheckOutputStream.h"
40 #include "SFVReader.h"
41 #include "FilteredFile.h"
42 #include "FinishedItem.h"
43 #include "FinishedManager.h"
44 #include "ZUtils.h"
45
46 #ifdef WITH_DHT
47 #include "dht/IndexManager.h"
48 #endif
49 #include <climits>
50
51 #if !defined(_WIN32) && !defined(PATH_MAX) // Extra PATH_MAX check for Mac OS X
52 #include <sys/syslimits.h>
53 #endif
54
55 #ifdef ff
56 #undef ff
57 #endif
58
59 namespace dcpp {
60
add(const string & aTarget,int64_t aSize,int aFlags,QueueItem::Priority p,const string & aTempTarget,time_t aAdded,const TTHValue & root)61 QueueItem* QueueManager::FileQueue::add(const string& aTarget, int64_t aSize,
62 int aFlags, QueueItem::Priority p, const string& aTempTarget,
63 time_t aAdded, const TTHValue& root)
64 {
65 if(p == QueueItem::DEFAULT) {
66 p = QueueItem::NORMAL;
67 if(aSize <= SETTING(PRIO_HIGHEST_SIZE)*1024) {
68 p = QueueItem::HIGHEST;
69 } else if(aSize <= SETTING(PRIO_HIGH_SIZE)*1024) {
70 p = QueueItem::HIGH;
71 } else if(aSize <= SETTING(PRIO_NORMAL_SIZE)*1024) {
72 p = QueueItem::NORMAL;
73 } else if(aSize <= SETTING(PRIO_LOW_SIZE)*1024) {
74 p = QueueItem::LOW;
75 } else if(SETTING(PRIO_LOWEST)) {
76 p = QueueItem::LOWEST;
77 }
78 }
79
80 QueueItem* qi = new QueueItem(aTarget, aSize, p, aFlags, aAdded, root);
81
82 if(qi->isSet(QueueItem::FLAG_USER_LIST)) {
83 qi->setPriority(QueueItem::HIGHEST);
84 }
85
86 qi->setTempTarget(aTempTarget);
87
88 dcassert(find(aTarget) == NULL);
89 add(qi);
90 return qi;
91 }
92
add(QueueItem * qi)93 void QueueManager::FileQueue::add(QueueItem* qi) {
94 if(lastInsert == queue.end())
95 lastInsert = queue.insert(make_pair(const_cast<string*>(&qi->getTarget()), qi)).first;
96 else
97 lastInsert = queue.insert(lastInsert, make_pair(const_cast<string*>(&qi->getTarget()), qi));
98 }
99
remove(QueueItem * qi)100 void QueueManager::FileQueue::remove(QueueItem* qi) {
101 if(lastInsert != queue.end() && Util::stricmp(*lastInsert->first, qi->getTarget()) == 0)
102 ++lastInsert;
103 queue.erase(const_cast<string*>(&qi->getTarget()));
104 delete qi;
105 }
106
find(const string & target)107 QueueItem* QueueManager::FileQueue::find(const string& target) {
108 QueueItem::StringIter i = queue.find(const_cast<string*>(&target));
109 return (i == queue.end()) ? NULL : i->second;
110 }
111
find(QueueItem::List & sl,int64_t aSize,const string & suffix)112 void QueueManager::FileQueue::find(QueueItem::List& sl, int64_t aSize, const string& suffix) {
113 for(QueueItem::StringIter i = queue.begin(); i != queue.end(); ++i) {
114 if(i->second->getSize() == aSize) {
115 const string& t = i->second->getTarget();
116 if(suffix.empty() || (suffix.length() < t.length() &&
117 Util::stricmp(suffix.c_str(), t.c_str() + (t.length() - suffix.length())) == 0) )
118 sl.push_back(i->second);
119 }
120 }
121 }
122
find(QueueItem::List & ql,const TTHValue & tth)123 void QueueManager::FileQueue::find(QueueItem::List& ql, const TTHValue& tth) {
124 for(QueueItem::StringIter i = queue.begin(); i != queue.end(); ++i) {
125 QueueItem* qi = i->second;
126 if(qi->getTTH() == tth) {
127 ql.push_back(qi);
128 }
129 }
130 }
131
exists(const TTHValue & tth) const132 bool QueueManager::FileQueue::exists(const TTHValue& tth) const {
133 for(QueueItem::StringMap::const_iterator i = queue.begin(); i != queue.end(); ++i)
134 if(i->second->getTTH() == tth)
135 return true;
136 return false;
137 }
138
findCandidate(QueueItem * cand,QueueItem::StringIter start,QueueItem::StringIter end,const StringList & recent)139 static QueueItem* findCandidate(QueueItem* cand, QueueItem::StringIter start, QueueItem::StringIter end, const StringList& recent) {
140 for(QueueItem::StringIter i = start; i != end; ++i) {
141 QueueItem* q = i->second;
142
143 // We prefer to search for things that are not running...
144 if((cand != NULL) && q->isRunning())
145 continue;
146 // No finished files
147 if(q->isFinished())
148 continue;
149 // No user lists
150 if(q->isSet(QueueItem::FLAG_USER_LIST))
151 continue;
152 // No paused downloads
153 if(q->getPriority() == QueueItem::PAUSED)
154 continue;
155 // No files that already have more than AUTO_SEARCH_LIMIT online sources
156 if(q->countOnlineUsers() >= SETTING(AUTO_SEARCH_LIMIT))
157 continue;
158 // Did we search for it recently?
159 if(find(recent.begin(), recent.end(), q->getTarget()) != recent.end())
160 continue;
161
162 cand = q;
163
164 if(cand->isWaiting())
165 break;
166 }
167 return cand;
168 }
169
findAutoSearch(StringList & recent)170 QueueItem* QueueManager::FileQueue::findAutoSearch(StringList& recent) {
171 // We pick a start position at random, hoping that we will find something to search for...
172 QueueItem::StringMap::size_type start = (QueueItem::StringMap::size_type)Util::rand((uint32_t)queue.size());
173
174 QueueItem::StringIter i = queue.begin();
175 advance(i, start);
176
177 QueueItem* cand = findCandidate(NULL, i, queue.end(), recent);
178 if(cand == NULL || cand->isRunning()) {
179 cand = findCandidate(cand, queue.begin(), i, recent);
180 }
181 return cand;
182 }
183
move(QueueItem * qi,const string & aTarget)184 void QueueManager::FileQueue::move(QueueItem* qi, const string& aTarget) {
185 if(lastInsert != queue.end() && Util::stricmp(*lastInsert->first, qi->getTarget()) == 0)
186 lastInsert = queue.end();
187 queue.erase(const_cast<string*>(&qi->getTarget()));
188 qi->setTarget(aTarget);
189 add(qi);
190 }
191
getQueueInfo(const UserPtr & aUser,string & aTarget,int64_t & aSize,int & aFlags)192 bool QueueManager::getQueueInfo(const UserPtr& aUser, string& aTarget, int64_t& aSize, int& aFlags) noexcept {
193 Lock l(cs);
194 QueueItem* qi = userQueue.getNext(aUser);
195 if(qi == NULL)
196 return false;
197
198 aTarget = qi->getTarget();
199 aSize = qi->getSize();
200 aFlags = qi->getFlags();
201
202 return true;
203 }
204
add(QueueItem * qi)205 void QueueManager::UserQueue::add(QueueItem* qi) {
206 for(QueueItem::SourceConstIter i = qi->getSources().begin(); i != qi->getSources().end(); ++i) {
207 add(qi, i->getUser());
208 }
209 }
210
add(QueueItem * qi,const UserPtr & aUser)211 void QueueManager::UserQueue::add(QueueItem* qi, const UserPtr& aUser) {
212 QueueItem::List& l = userQueue[qi->getPriority()][aUser];
213
214 if(qi->getDownloadedBytes() > 0) {
215 l.push_front(qi);
216 } else {
217 l.push_back(qi);
218 }
219 }
220
getNext(const UserPtr & aUser,QueueItem::Priority minPrio,int64_t wantedSize,int64_t lastSpeed,bool allowRemove)221 QueueItem* QueueManager::UserQueue::getNext(const UserPtr& aUser, QueueItem::Priority minPrio, int64_t wantedSize,int64_t lastSpeed,bool allowRemove) {
222 int p = QueueItem::LAST - 1;
223 string lastError = Util::emptyString;
224
225 do {
226 QueueItem::UserListIter i = userQueue[p].find(aUser);
227 if(i != userQueue[p].end()) {
228 dcassert(!i->second.empty());
229 for(QueueItem::Iter j = i->second.begin(); j != i->second.end(); ++j) {
230 QueueItem* qi = *j;
231 QueueItem::SourceConstIter source = qi->getSource(aUser);
232 if(source->isSet(QueueItem::Source::FLAG_PARTIAL)) {
233 // check partial source
234 int64_t blockSize = HashManager::getInstance()->getBlockSize(qi->getTTH());
235 if(blockSize == 0)
236 blockSize = qi->getSize();
237
238 Segment segment = qi->getNextSegment(blockSize, wantedSize, lastSpeed, source->getPartialSource());
239 if(allowRemove && segment.getStart() != -1 && segment.getSize() == 0) {
240 // no other partial chunk from this user, remove him from queue
241 remove(qi, aUser);
242 qi->removeSource(aUser, QueueItem::Source::FLAG_NO_NEED_PARTS);
243 lastError = _("No needed part");
244 p++;
245 break;
246 }
247 }
248 if(qi->isWaiting()) {
249 return qi;
250 }
251
252 // No segmented downloading when getting the tree
253 if(qi->getDownloads()[0]->getType() == Transfer::TYPE_TREE) {
254 continue;
255 }
256 if(!qi->isSet(QueueItem::FLAG_USER_LIST)) {
257 int64_t blockSize = HashManager::getInstance()->getBlockSize(qi->getTTH());
258 if(blockSize == 0)
259 blockSize = qi->getSize();
260 if(qi->getNextSegment(blockSize, wantedSize,lastSpeed, source->getPartialSource()).getSize() == 0) {
261 dcdebug("No segment for %s in %s, block " I64_FMT "\n",
262 aUser->getCID().toBase32().c_str(), qi->getTarget().c_str(),
263 static_cast<long long int>(blockSize));
264 continue;
265 }
266 }
267 return qi;
268 }
269 }
270 p--;
271 } while(p >= minPrio);
272
273 return NULL;
274 }
275
addDownload(QueueItem * qi,Download * d)276 void QueueManager::UserQueue::addDownload(QueueItem* qi, Download* d) {
277 qi->getDownloads().push_back(d);
278
279 // Only one download per user...
280 dcassert(running.find(d->getUser()) == running.end());
281 running[d->getUser()] = qi;
282 }
283
removeDownload(QueueItem * qi,const UserPtr & user)284 void QueueManager::UserQueue::removeDownload(QueueItem* qi, const UserPtr& user) {
285 running.erase(user);
286
287 for(DownloadList::iterator i = qi->getDownloads().begin(); i != qi->getDownloads().end(); ++i) {
288 if((*i)->getUser() == user) {
289 qi->getDownloads().erase(i);
290 break;
291 }
292 }
293 }
294
setPriority(QueueItem * qi,QueueItem::Priority p)295 void QueueManager::UserQueue::setPriority(QueueItem* qi, QueueItem::Priority p) {
296 remove(qi, false);
297 qi->setPriority(p);
298 add(qi);
299 }
300
getQueued(const UserPtr & aUser) const301 int64_t QueueManager::UserQueue::getQueued(const UserPtr& aUser) const {
302 int64_t total = 0;
303 for(size_t i = QueueItem::LOWEST; i < QueueItem::LAST; ++i) {
304 const QueueItem::UserListMap& ulm = userQueue[i];
305 QueueItem::UserListMap::const_iterator iulm = ulm.find(aUser);
306 if(iulm == ulm.end()) {
307 continue;
308 }
309
310 for(QueueItem::List::const_iterator j = iulm->second.begin(); j != iulm->second.end(); ++j) {
311 const QueueItem::Ptr qi = *j;
312 if(qi->getSize() != -1) {
313 total += qi->getSize() - qi->getDownloadedBytes();
314 }
315 }
316 }
317 return total;
318 }
319
getRunning(const UserPtr & aUser)320 QueueItem* QueueManager::UserQueue::getRunning(const UserPtr& aUser) {
321 QueueItem::UserIter i = running.find(aUser);
322 return (i == running.end()) ? 0 : i->second;
323 }
324
remove(QueueItem * qi,bool removeRunning)325 void QueueManager::UserQueue::remove(QueueItem* qi, bool removeRunning) {
326 for(QueueItem::SourceConstIter i = qi->getSources().begin(); i != qi->getSources().end(); ++i) {
327 remove(qi, i->getUser(), removeRunning);
328 }
329 }
330
remove(QueueItem * qi,const UserPtr & aUser,bool removeRunning)331 void QueueManager::UserQueue::remove(QueueItem* qi, const UserPtr& aUser, bool removeRunning) {
332 if(removeRunning && qi == getRunning(aUser)) {
333 removeDownload(qi, aUser);
334 }
335
336 dcassert(qi->isSource(aUser));
337 QueueItem::UserListMap& ulm = userQueue[qi->getPriority()];
338 QueueItem::UserListIter j = ulm.find(aUser);
339 dcassert(j != ulm.end());
340 QueueItem::List& l = j->second;
341 QueueItem::Iter i = find(l.begin(), l.end(), qi);
342 dcassert(i != l.end());
343 l.erase(i);
344
345 if(l.empty()) {
346 ulm.erase(j);
347 }
348 }
349
moveFile(const string & source,const string & target)350 void QueueManager::FileMover::moveFile(const string& source, const string& target) {
351 Lock l(cs);
352 files.push_back(make_pair(source, target));
353 if(!active) {
354 active = true;
355 start();
356 }
357 }
358
run()359 int QueueManager::FileMover::run() {
360 setThreadName("FileMover");
361 for(;;) {
362 FilePair next;
363 {
364 Lock l(cs);
365 if(files.empty()) {
366 active = false;
367 return 0;
368 }
369 next = files.back();
370 files.pop_back();
371 }
372 moveFile_(next.first, next.second);
373 }
374 }
375
add(const string & file)376 void QueueManager::Rechecker::add(const string& file) {
377 Lock l(cs);
378 files.push_back(file);
379 if(!active) {
380 active = true;
381 start();
382 }
383 }
384
run()385 int QueueManager::Rechecker::run() {
386 setThreadName("Rechecker");
387 while(true) {
388 string file;
389 {
390 Lock l(cs);
391 StringIter i = files.begin();
392 if(i == files.end()) {
393 active = false;
394 return 0;
395 }
396 file = *i;
397 files.erase(i);
398 }
399
400 QueueItem* q;
401 int64_t tempSize;
402 TTHValue tth;
403
404 {
405 Lock l(qm->cs);
406
407 q = qm->fileQueue.find(file);
408 if(!q || q->isSet(QueueItem::FLAG_USER_LIST))
409 continue;
410
411 qm->fire(QueueManagerListener::RecheckStarted(), q->getTarget());
412 dcdebug("Rechecking %s\n", file.c_str());
413
414 tempSize = File::getSize(q->getTempTarget());
415
416 if(tempSize == -1) {
417 qm->fire(QueueManagerListener::RecheckNoFile(), q->getTarget());
418 continue;
419 }
420
421 if(tempSize < 64*1024) {
422 qm->fire(QueueManagerListener::RecheckFileTooSmall(), q->getTarget());
423 continue;
424 }
425
426 if(tempSize != q->getSize()) {
427 File(q->getTempTarget(), File::WRITE, File::OPEN).setSize(q->getSize());
428 }
429
430 if(q->isRunning()) {
431 qm->fire(QueueManagerListener::RecheckDownloadsRunning(), q->getTarget());
432 continue;
433 }
434
435 tth = q->getTTH();
436 }
437
438 TigerTree tt;
439 bool gotTree = HashManager::getInstance()->getTree(tth, tt);
440
441 string tempTarget;
442
443 {
444 Lock l(qm->cs);
445
446 // get q again in case it has been (re)moved
447 q = qm->fileQueue.find(file);
448 if(!q)
449 continue;
450
451 if(!gotTree) {
452 qm->fire(QueueManagerListener::RecheckNoTree(), q->getTarget());
453 continue;
454 }
455
456 //Clear segments
457 q->resetDownloaded();
458
459 tempTarget = q->getTempTarget();
460 }
461
462 //Merklecheck
463 int64_t startPos=0;
464 DummyOutputStream dummy;
465 int64_t blockSize = tt.getBlockSize();
466 bool hasBadBlocks = false;
467
468 vector<uint8_t> buf((size_t)min((int64_t)1024*1024, blockSize));
469
470 typedef pair<int64_t, int64_t> SizePair;
471 typedef vector<SizePair> Sizes;
472 Sizes sizes;
473
474 {
475 File inFile(tempTarget, File::READ, File::OPEN);
476
477 while(startPos < tempSize) {
478 try {
479 MerkleCheckOutputStream<TigerTree, false> check(tt, &dummy, startPos);
480
481 inFile.setPos(startPos);
482 int64_t bytesLeft = min((tempSize - startPos),blockSize); //Take care of the last incomplete block
483 int64_t segmentSize = bytesLeft;
484 while(bytesLeft > 0) {
485 size_t n = (size_t)min((int64_t)buf.size(), bytesLeft);
486 size_t nr = inFile.read(&buf[0], n);
487 check.write(&buf[0], nr);
488 bytesLeft -= nr;
489 if(bytesLeft > 0 && nr == 0) {
490 // Huh??
491 throw Exception();
492 }
493 }
494 check.flush();
495
496 sizes.push_back(make_pair(startPos, segmentSize));
497 } catch(const Exception&) {
498 hasBadBlocks = true;
499 dcdebug("Found bad block at " I64_FMT "\n", static_cast<long long int>(startPos));
500 }
501 startPos += blockSize;
502 }
503 }
504
505 Lock l(qm->cs);
506
507 // get q again in case it has been (re)moved
508 q = qm->fileQueue.find(file);
509 if(!q)
510 continue;
511
512 //If no bad blocks then the file probably got stuck in the temp folder for some reason
513 if(!hasBadBlocks) {
514 qm->moveStuckFile(q);
515 continue;
516 }
517
518 for(Sizes::const_iterator i = sizes.begin(); i != sizes.end(); ++i)
519 q->addSegment(Segment(i->first, i->second));
520
521 qm->rechecked(q);
522 }
523 return 0;
524 }
525
QueueManager()526 QueueManager::QueueManager() :
527 lastSave(0),
528 queueFile(Util::getPath(Util::PATH_USER_CONFIG) + "Queue.xml"),
529 rechecker(this),
530 dirty(true),
531 nextSearch(0)
532 {
533 TimerManager::getInstance()->addListener(this);
534 SearchManager::getInstance()->addListener(this);
535 ClientManager::getInstance()->addListener(this);
536
537 File::ensureDirectory(Util::getListPath());
538 }
539
~QueueManager()540 QueueManager::~QueueManager() {
541 SearchManager::getInstance()->removeListener(this);
542 TimerManager::getInstance()->removeListener(this);
543 ClientManager::getInstance()->removeListener(this);
544
545 if(!BOOLSETTING(KEEP_LISTS)) {
546 string path = Util::getListPath();
547
548 std::sort(protectedFileLists.begin(), protectedFileLists.end());
549
550 StringList filelists = File::findFiles(path, "*.xml.bz2");
551 std::sort(filelists.begin(), filelists.end());
552 std::for_each(filelists.begin(), std::set_difference(filelists.begin(), filelists.end(),
553 protectedFileLists.begin(), protectedFileLists.end(), filelists.begin()), &File::deleteFile);
554
555 filelists = File::findFiles(path, "*.DcLst");
556 std::sort(filelists.begin(), filelists.end());
557 std::for_each(filelists.begin(), std::set_difference(filelists.begin(), filelists.end(),
558 protectedFileLists.begin(), protectedFileLists.end(), filelists.begin()), &File::deleteFile);
559 }
560 }
561
getTTH(const string & name,TTHValue & tth)562 bool QueueManager::getTTH(const string& name, TTHValue& tth) noexcept {
563 Lock l(cs);
564 QueueItem* qi = fileQueue.find(name);
565 if(qi) {
566 tth = qi->getTTH();
567 return true;
568 }
569 return false;
570 }
571
572 struct PartsInfoReqParam{
573 PartsInfo parts;
574 string tth;
575 string myNick;
576 string hubIpPort;
577 string ip;
578 uint16_t udpPort;
579 };
580
on(TimerManagerListener::Minute,uint64_t aTick)581 void QueueManager::on(TimerManagerListener::Minute, uint64_t aTick) noexcept {
582 string fn;
583 string searchString;
584 vector<const PartsInfoReqParam*> params;
585 TTHValue* tthPub = NULL;
586 {
587 Lock l(cs);
588 //find max 10 pfs sources to exchange parts
589 //the source basis interval is 5 minutes
590 PFSSourceList sl;
591 fileQueue.findPFSSources(sl);
592
593 for(PFSSourceList::const_iterator i = sl.begin(); i != sl.end(); ++i){
594 QueueItem::PartialSource::Ptr source = (*i->first).getPartialSource();
595 const QueueItem* qi = i->second;
596
597 PartsInfoReqParam* param = new PartsInfoReqParam;
598
599 int64_t blockSize = HashManager::getInstance()->getBlockSize(qi->getTTH());
600 if(blockSize == 0)
601 blockSize = qi->getSize();
602 qi->getPartialInfo(param->parts, blockSize);
603
604 param->tth = qi->getTTH().toBase32();
605 param->ip = source->getIp();
606 param->udpPort = source->getUdpPort();
607 param->myNick = source->getMyNick();
608 param->hubIpPort = source->getHubIpPort();
609
610 params.push_back(param);
611
612 source->setPendingQueryCount(source->getPendingQueryCount() + 1);
613 source->setNextQueryTime(aTick + 300000); // 5 minutes
614 }
615
616 #ifdef WITH_DHT
617 if(BOOLSETTING(USE_DHT) && SETTING(INCOMING_CONNECTIONS) != SettingsManager::INCOMING_FIREWALL_PASSIVE)
618 tthPub = fileQueue.findPFSPubTTH();
619 #endif
620
621 if(BOOLSETTING(AUTO_SEARCH) && (aTick >= nextSearch) && (fileQueue.getSize() > 0)) {
622 // We keep 30 recent searches to avoid duplicate searches
623 while((recent.size() >= fileQueue.getSize()) || (recent.size() > 30)) {
624 recent.erase(recent.begin());
625 }
626
627 QueueItem* qi = fileQueue.findAutoSearch(recent);
628 if(qi) {
629 searchString = qi->getTTH().toBase32();
630 recent.push_back(qi->getTarget());
631 nextSearch = aTick + (SETTING(AUTO_SEARCH_TIME) * 60000);
632 if (BOOLSETTING(REPORT_ALTERNATES))
633 LogManager::getInstance()->message(str(F_("Searching TTH alternates for: %1%")%Util::getFileName(qi->getTargetFileName())));
634 }
635 }
636 }
637 // Request parts info from partial file sharing sources
638 for(vector<const PartsInfoReqParam*>::const_iterator i = params.begin(); i != params.end(); ++i){
639 const PartsInfoReqParam* param = *i;
640 dcassert(param->udpPort > 0);
641
642 try {
643 AdcCommand cmd = SearchManager::getInstance()->toPSR(true, param->myNick, param->hubIpPort, param->tth, param->parts);
644 Socket s;
645 s.writeTo(param->ip, param->udpPort, cmd.toString(ClientManager::getInstance()->getMyCID()));
646 } catch(...) {
647 dcdebug("Partial search caught error\n");
648 }
649
650 delete param;
651 }
652
653 // DHT PFS announce
654 if(tthPub)
655 {
656 #ifdef WITH_DHT
657 dht::IndexManager::getInstance()->publishPartialFile(*tthPub);
658 #endif
659 delete tthPub;
660 }
661
662 if(!searchString.empty()) {
663 SearchManager::getInstance()->search(searchString, 0, SearchManager::TYPE_TTH, SearchManager::SIZE_DONTCARE, "auto");
664 }
665 }
666
addList(const HintedUser & aUser,int aFlags,const string & aInitialDir)667 void QueueManager::addList(const HintedUser& aUser, int aFlags, const string& aInitialDir /* = Util::emptyString */) {
668 add(aInitialDir, -1, TTHValue(), aUser, QueueItem::FLAG_USER_LIST | aFlags);
669 }
670
getListPath(const HintedUser & user)671 string QueueManager::getListPath(const HintedUser& user) {
672 StringList nicks = ClientManager::getInstance()->getNicks(user);
673 string nick = nicks.empty() ? Util::emptyString : Util::cleanPathChars(nicks[0]) + ".";
674 return checkTarget(Util::getListPath() + nick + user.user->getCID().toBase32(), /*checkExistence*/ false);
675 }
676 //NOTE: freedcpp
add(const string & aTarget,int64_t aSize,const TTHValue & root)677 void QueueManager::add(const string& aTarget, int64_t aSize, const TTHValue& root)
678 {
679 // Check if we're not downloading something already in our share
680 if (BOOLSETTING(DONT_DL_ALREADY_SHARED))
681 {
682 if (ShareManager::getInstance()->isTTHShared(root))
683 {
684 throw QueueException(_("A file with the same hash already exists in your share"));
685 }
686 }
687 // Check that target contains at least one directory...we don't want headless files...
688 // Check that the file doesn't already exist...
689 const string target = checkTarget(aTarget, aSize);
690
691 // Check if it's a zero-byte file, if so, create and return...
692 if (aSize == 0)
693 {
694 if (!BOOLSETTING(SKIP_ZERO_BYTE))
695 {
696 File::ensureDirectory(target);
697 File f(target, File::WRITE, File::CREATE);
698 }
699 return;
700 }
701
702 Lock l(cs);
703
704 // This will be pretty slow on large queues...
705 if (BOOLSETTING(DONT_DL_ALREADY_QUEUED))
706 {
707 QueueItem::List ql;
708 fileQueue.find(ql, root);
709 if (!ql.empty())
710 throw QueueException(_("This file is already queued"));
711 }
712
713 QueueItem* q = fileQueue.find(target);
714
715 if(q == NULL)
716 {
717 q = fileQueue.add(target, aSize, 0, QueueItem::DEFAULT/*QueueItem::Priority*/, Util::emptyString/*aTempTarget*/,
718 GET_TIME()/*time_t aAdded*/, root/*TTHValue& root*/);
719 fire(QueueManagerListener::Added(), q);
720 } else {
721 if(q->getSize() != aSize)
722 {
723 throw QueueException(_("A file with a different size already exists in the queue"));
724 }
725 if(!(root == q->getTTH()))
726 {
727 throw QueueException(_("A file with different tth root already exists in the queue"));
728 }
729 }
730 }//NOTE: freedcpp
731
add(const string & aTarget,int64_t aSize,const TTHValue & root,const HintedUser & aUser,int aFlags,bool addBad)732 void QueueManager::add(const string& aTarget, int64_t aSize, const TTHValue& root, const HintedUser& aUser,
733 int aFlags /* = 0 */, bool addBad /* = true */)
734 {
735 bool wantConnection = true;
736
737 // Check that we're not downloading from ourselves...
738 if(aUser == ClientManager::getInstance()->getMe()) {
739 throw QueueException(_("You're trying to download from yourself!"));
740 }
741
742 // Check if we're not downloading something already in our share
743 if(BOOLSETTING(DONT_DL_ALREADY_SHARED)){
744 if (ShareManager::getInstance()->isTTHShared(root)){
745 throw QueueException(_("A file with the same hash already exists in your share"));
746 }
747 }
748
749 string target;
750 string tempTarget;
751 if((aFlags & QueueItem::FLAG_USER_LIST) == QueueItem::FLAG_USER_LIST) {
752 target = getListPath(aUser);
753 tempTarget = aTarget;
754 } else {
755 target = checkTarget(aTarget, /*checkExistence*/ true);
756 }
757
758 // Check if it's a zero-byte file, if so, create and return...
759 if(aSize == 0) {
760 if(!BOOLSETTING(SKIP_ZERO_BYTE)) {
761 File::ensureDirectory(target);
762 File f(target, File::WRITE, File::CREATE);
763 }
764 return;
765 }
766
767 {
768 Lock l(cs);
769
770 // This will be pretty slow on large queues...
771 if(BOOLSETTING(DONT_DL_ALREADY_QUEUED) && !(aFlags & QueueItem::FLAG_USER_LIST)) {
772 QueueItem::List ql;
773 fileQueue.find(ql, root);
774 if (!ql.empty()) {
775 // Found one or more existing queue items, lets see if we can add the source to them
776 bool sourceAdded = false;
777 for(QueueItem::Iter i = ql.begin(); i != ql.end(); ++i) {
778 if(!(*i)->isSource(aUser)) {
779 try {
780 wantConnection = addSource(*i, aUser, addBad ? QueueItem::Source::FLAG_MASK : 0);
781 sourceAdded = true;
782 } catch(...) { }
783 }
784 }
785
786 if(!sourceAdded) {
787 throw QueueException(_("This file is already queued"));
788 }
789 goto connect;
790 }
791 }
792
793 QueueItem* q = fileQueue.find(target);
794 if(q == NULL) {
795 q = fileQueue.add(target, aSize, aFlags, QueueItem::DEFAULT, tempTarget, GET_TIME(), root);
796 fire(QueueManagerListener::Added(), q);
797 } else {
798 if(q->getSize() != aSize) {
799 throw QueueException(_("A file with a different size already exists in the queue"));
800 }
801 if(!(root == q->getTTH())) {
802 throw QueueException(_("A file with a different TTH root already exists in the queue"));
803 }
804
805 if(q->isFinished()) {
806 throw QueueException(_("This file has already finished downloading"));
807 }
808
809 q->setFlag(aFlags);
810 }
811
812 wantConnection = addSource(q, aUser, addBad ? QueueItem::Source::FLAG_MASK : 0);
813 }
814
815 connect:
816 if(wantConnection && aUser.user->isOnline())
817 ConnectionManager::getInstance()->getDownloadConnection(aUser);
818 }
819
readd(const string & target,const HintedUser & aUser)820 void QueueManager::readd(const string& target, const HintedUser& aUser) {
821 bool wantConnection = false;
822 {
823 Lock l(cs);
824 QueueItem* q = fileQueue.find(target);
825 if(q && q->isBadSource(aUser)) {
826 wantConnection = addSource(q, aUser, QueueItem::Source::FLAG_MASK);
827 }
828 }
829 if(wantConnection && aUser.user->isOnline())
830 ConnectionManager::getInstance()->getDownloadConnection(aUser);
831 }
832
setDirty()833 void QueueManager::setDirty() {
834 if(!dirty) {
835 dirty = true;
836 lastSave = GET_TICK();
837 }
838 }
839
checkTarget(const string & aTarget,bool checkExistence)840 string QueueManager::checkTarget(const string& aTarget, bool checkExistence) {
841 #ifdef _WIN32
842 if(aTarget.length() > MAX_PATH) {
843 throw QueueException(_("Target filename too long"));
844 }
845 // Check that target starts with a drive or is an UNC path
846 if( (aTarget[1] != ':' || aTarget[2] != '\\') &&
847 (aTarget[0] != '\\' && aTarget[1] != '\\') ) {
848 throw QueueException(_("Invalid target file (missing directory, check default download directory setting)"));
849 }
850 #else
851 if(aTarget.length() > PATH_MAX) {
852 throw QueueException(_("Target filename too long"));
853 }
854 // Check that target contains at least one directory...we don't want headless files...
855 if(aTarget[0] != '/') {
856 throw QueueException(_("Invalid target file (missing directory, check default download directory setting)"));
857 }
858 #endif
859
860 string target = Util::validateFileName(aTarget);
861
862 // Check that the file doesn't already exist...
863 if(checkExistence && File::getSize(target) != -1) {
864 throw FileException(_("File already exists at the target location"));
865 }
866 return target;
867 }
868
869 /** Add a source to an existing queue item */
addSource(QueueItem * qi,const HintedUser & aUser,Flags::MaskType addBad)870 bool QueueManager::addSource(QueueItem* qi, const HintedUser& aUser, Flags::MaskType addBad) {
871 bool wantConnection = (qi->getPriority() != QueueItem::PAUSED) && !userQueue.getRunning(aUser);
872
873 if(qi->isSource(aUser)) {
874 if(qi->isSet(QueueItem::FLAG_USER_LIST)) {
875 return wantConnection;
876 }
877 throw QueueException(str(F_("Duplicate source: %1%") % Util::getFileName(qi->getTarget())));
878 }
879
880 if(qi->isBadSourceExcept(aUser, addBad)) {
881 throw QueueException(str(F_("Duplicate source: %1%") % Util::getFileName(qi->getTarget())));
882 }
883
884 qi->addSource(aUser);
885
886 if(aUser.user->isSet(User::PASSIVE) && !ClientManager::getInstance()->isActive() ) {
887 qi->removeSource(aUser, QueueItem::Source::FLAG_PASSIVE);
888 wantConnection = false;
889 } else if(qi->isFinished()) {
890 wantConnection = false;
891 } else {
892 userQueue.add(qi, aUser);
893 }
894
895 fire(QueueManagerListener::SourcesUpdated(), qi);
896 setDirty();
897
898 return wantConnection;
899 }
900
addDirectory(const string & aDir,const HintedUser & aUser,const string & aTarget,QueueItem::Priority p)901 void QueueManager::addDirectory(const string& aDir, const HintedUser& aUser, const string& aTarget, QueueItem::Priority p /* = QueueItem::DEFAULT */) noexcept {
902 bool needList;
903 {
904 Lock l(cs);
905
906 DirectoryItem::DirectoryPair dp = directories.equal_range(aUser);
907
908 for(DirectoryItem::DirectoryIter i = dp.first; i != dp.second; ++i) {
909 if(Util::stricmp(aTarget.c_str(), i->second->getName().c_str()) == 0)
910 return;
911 }
912
913 // Unique directory, fine...
914 directories.insert(make_pair(aUser, new DirectoryItem(aUser, aDir, aTarget, p)));
915 needList = (dp.first == dp.second);
916 setDirty();
917 }
918
919 if(needList) {
920 try {
921 addList(aUser, QueueItem::FLAG_DIRECTORY_DOWNLOAD);
922 } catch(const Exception&) {
923 // Ignore, we don't really care...
924 }
925 }
926 }
927
hasDownload(const UserPtr & aUser)928 QueueItem::Priority QueueManager::hasDownload(const UserPtr& aUser) noexcept {
929 Lock l(cs);
930 QueueItem* qi = userQueue.getNext(aUser, QueueItem::LOWEST);
931 if(!qi) {
932 return QueueItem::PAUSED;
933 }
934 return qi->getPriority();
935 }
936 namespace {
937 typedef unordered_map<TTHValue, const DirectoryListing::File*> TTHMap;
938
939 // *** WARNING ***
940 // Lock(cs) makes sure that there's only one thread accessing this
941 static TTHMap tthMap;
942
buildMap(const DirectoryListing::Directory * dir)943 void buildMap(const DirectoryListing::Directory* dir) noexcept {
944 for(DirectoryListing::Directory::List::const_iterator j = dir->directories.begin(); j != dir->directories.end(); ++j) {
945 if(!(*j)->getAdls())
946 buildMap(*j);
947 }
948
949 for(DirectoryListing::File::List::const_iterator i = dir->files.begin(); i != dir->files.end(); ++i) {
950 const DirectoryListing::File* df = *i;
951 tthMap.insert(make_pair(df->getTTH(), df));
952 }
953 }
954 }
955
matchListing(const DirectoryListing & dl)956 int QueueManager::matchListing(const DirectoryListing& dl) noexcept {
957 int matches = 0;
958 {
959 Lock l(cs);
960 tthMap.clear();
961 buildMap(dl.getRoot());
962
963 for(QueueItem::StringMap::const_iterator i = fileQueue.getQueue().begin(); i != fileQueue.getQueue().end(); ++i) {
964 QueueItem* qi = i->second;
965 if(qi->isFinished())
966 continue;
967 if(qi->isSet(QueueItem::FLAG_USER_LIST))
968 continue;
969 TTHMap::iterator j = tthMap.find(qi->getTTH());
970 if(j != tthMap.end() && i->second->getSize() == qi->getSize()) {
971 try {
972 addSource(qi, dl.getUser(), QueueItem::Source::FLAG_FILE_NOT_AVAILABLE);
973 } catch(...) {
974 // Ignore...
975 }
976 matches++;
977 }
978 }
979 }
980 if(matches > 0)
981 ConnectionManager::getInstance()->getDownloadConnection(dl.getUser());
982 return matches;
983 }
984
getPos(const string & target)985 int64_t QueueManager::getPos(const string& target) noexcept {
986 Lock l(cs);
987 QueueItem* qi = fileQueue.find(target);
988 if(qi) {
989 return qi->getDownloadedBytes();
990 }
991 return -1;
992 }
993
getSize(const string & target)994 int64_t QueueManager::getSize(const string& target) noexcept {
995 Lock l(cs);
996 QueueItem* qi = fileQueue.find(target);
997 if(qi) {
998 return qi->getSize();
999 }
1000 return -1;
1001 }
1002
1003
move(const string & aSource,const string & aTarget)1004 void QueueManager::move(const string& aSource, const string& aTarget) noexcept {
1005 string target = Util::validateFileName(aTarget);
1006 if(aSource == target)
1007 return;
1008
1009 bool delSource = false;
1010
1011 Lock l(cs);
1012 QueueItem* qs = fileQueue.find(aSource);
1013 if(qs) {
1014 // Don't move running downloads
1015 if(qs->isRunning()) {
1016 return;
1017 }
1018 // Don't move file lists
1019 if(qs->isSet(QueueItem::FLAG_USER_LIST))
1020 return;
1021
1022 // Let's see if the target exists...then things get complicated...
1023 QueueItem* qt = fileQueue.find(target);
1024 if(qt == NULL || Util::stricmp(aSource, target) == 0) {
1025 // Good, update the target and move in the queue...
1026 fileQueue.move(qs, target);
1027 fire(QueueManagerListener::Moved(), qs, aSource);
1028 setDirty();
1029 } else {
1030 // Don't move to target of different size
1031 if(qs->getSize() != qt->getSize() || qs->getTTH() != qt->getTTH())
1032 return;
1033
1034 for(QueueItem::SourceConstIter i = qs->getSources().begin(); i != qs->getSources().end(); ++i) {
1035 try {
1036 addSource(qt, i->getUser(), QueueItem::Source::FLAG_MASK);
1037 } catch(const Exception&) { }
1038 }
1039 delSource = true;
1040 }
1041 }
1042
1043 if(delSource) {
1044 remove(aSource);
1045 }
1046 }
1047
getTargets(const TTHValue & tth,StringList & sl)1048 void QueueManager::getTargets(const TTHValue& tth, StringList& sl) {
1049 Lock l(cs);
1050 QueueItem::List ql;
1051 fileQueue.find(ql, tth);
1052 for(QueueItem::Iter i = ql.begin(); i != ql.end(); ++i) {
1053 sl.push_back((*i)->getTarget());
1054 }
1055 }
1056
getDownload(UserConnection & aSource,bool supportsTrees)1057 Download* QueueManager::getDownload(UserConnection& aSource, bool supportsTrees) noexcept {
1058 Lock l(cs);
1059
1060 UserPtr& u = aSource.getUser();
1061 dcdebug("Getting download for %s...", u->getCID().toBase32().c_str());
1062
1063 QueueItem* q = userQueue.getNext(u, QueueItem::LOWEST, aSource.getChunkSize());
1064
1065 if(!q) {
1066 dcdebug("none\n");
1067 return 0;
1068 }
1069
1070 // Check that the file we will be downloading to exists
1071 if(q->getDownloadedBytes() > 0) {
1072 int64_t tempSize = File::getSize(q->getTempTarget());
1073 if(tempSize != q->getSize()) {
1074 // <= 0.706 added ".antifrag" to temporary download files if antifrag was enabled...
1075 // 0.705 added ".antifrag" even if antifrag was disabled
1076 std::string antifrag = q->getTempTarget() + ".antifrag";
1077 if(File::getSize(antifrag) > 0) {
1078 File::renameFile(antifrag, q->getTempTarget());
1079 tempSize = File::getSize(q->getTempTarget());
1080 }
1081 if(tempSize != q->getSize()) {
1082 if(tempSize > 0 && tempSize < q->getSize()) {
1083 // Probably started with <=0.699 or with 0.705 without antifrag enabled...
1084 try {
1085 File(q->getTempTarget(), File::WRITE, File::OPEN).setSize(q->getSize());
1086 } catch(const FileException&) { }
1087 } else {
1088 // Temp target gone?
1089 q->resetDownloaded();
1090 }
1091 }
1092 }
1093 }
1094 Download* d = new Download(aSource, *q, q->isSet(QueueItem::FLAG_PARTIAL_LIST) ? q->getTempTarget() : q->getTarget(), supportsTrees);
1095
1096 userQueue.addDownload(q, d);
1097
1098 fire(QueueManagerListener::StatusUpdated(), q);
1099 dcdebug("found %s\n", q->getTarget().c_str());
1100 return d;
1101 }
1102
1103 namespace {
1104 class TreeOutputStream : public OutputStream {
1105 public:
TreeOutputStream(TigerTree & aTree)1106 TreeOutputStream(TigerTree& aTree) : tree(aTree), bufPos(0) {
1107 }
1108
write(const void * xbuf,size_t len)1109 virtual size_t write(const void* xbuf, size_t len) {
1110 size_t pos = 0;
1111 uint8_t* b = (uint8_t*)xbuf;
1112 while(pos < len) {
1113 size_t left = len - pos;
1114 if(bufPos == 0 && left >= TigerTree::BYTES) {
1115 tree.getLeaves().push_back(TTHValue(b + pos));
1116 pos += TigerTree::BYTES;
1117 } else {
1118 size_t bytes = min(TigerTree::BYTES - bufPos, left);
1119 memcpy(buf + bufPos, b + pos, bytes);
1120 bufPos += bytes;
1121 pos += bytes;
1122 if(bufPos == TigerTree::BYTES) {
1123 tree.getLeaves().push_back(TTHValue(buf));
1124 bufPos = 0;
1125 }
1126 }
1127 }
1128 return len;
1129 }
1130
flush()1131 virtual size_t flush() {
1132 return 0;
1133 }
1134 private:
1135 TigerTree& tree;
1136 uint8_t buf[TigerTree::BYTES];
1137 size_t bufPos;
1138 };
1139
1140 }
1141
setFile(Download * d)1142 void QueueManager::setFile(Download* d) {
1143 if(d->getType() == Transfer::TYPE_FILE) {
1144 Lock l(cs);
1145
1146 QueueItem* qi = fileQueue.find(d->getPath());
1147 if(!qi) {
1148 throw QueueException(_("Target removed"));
1149 }
1150
1151 string target = d->getDownloadTarget();
1152
1153 if(d->getSegment().getStart() > 0) {
1154 if(File::getSize(target) != qi->getSize()) {
1155 // When trying the download the next time, the resume pos will be reset
1156 throw QueueException(_("Target file is missing or wrong size"));
1157 }
1158 } else {
1159 File::ensureDirectory(target);
1160 }
1161
1162 File* f = new File(target, File::WRITE, File::OPEN | File::CREATE | File::SHARED);
1163
1164 if(f->getSize() != qi->getSize()) {
1165 f->setSize(qi->getSize());
1166 }
1167
1168 f->setPos(d->getSegment().getStart());
1169 d->setFile(f);
1170 } else if(d->getType() == Transfer::TYPE_FULL_LIST) {
1171 string target = d->getPath();
1172 File::ensureDirectory(target);
1173
1174 if(d->isSet(Download::FLAG_XML_BZ_LIST)) {
1175 target += ".xml.bz2";
1176 } else {
1177 target += ".xml";
1178 }
1179 d->setFile(new File(target, File::WRITE, File::OPEN | File::TRUNCATE | File::CREATE));
1180 } else if(d->getType() == Transfer::TYPE_PARTIAL_LIST) {
1181 d->setFile(new StringOutputStream(d->getPFS()));
1182 } else if(d->getType() == Transfer::TYPE_TREE) {
1183 d->setFile(new TreeOutputStream(d->getTigerTree()));
1184 }
1185 }
1186
moveFile(const string & source,const string & target)1187 void QueueManager::moveFile(const string& source, const string& target) {
1188 File::ensureDirectory(target);
1189 if(File::getSize(source) > MOVER_LIMIT) {
1190 mover.moveFile(source, target);
1191 } else {
1192 moveFile_(source, target);
1193 }
1194 }
1195
moveFile_(const string & source,const string & target)1196 void QueueManager::moveFile_(const string& source, const string& target) {
1197 try {
1198 File::renameFile(source, target);
1199 getInstance()->fire(QueueManagerListener::FileMoved(), target);
1200 } catch(const FileException& e1) {
1201 // Try to just rename it to the correct name at least
1202 string newTarget = Util::getFilePath(source) + Util::getFileName(target);
1203 try {
1204 File::renameFile(source, newTarget);
1205 LogManager::getInstance()->message(str(F_("Unable to move %1% to %2% (%3%); renamed to %4%") %
1206 Util::addBrackets(source) % Util::addBrackets(target) % e1.getError() % Util::addBrackets(newTarget)));
1207 } catch(const FileException& e2) {
1208 LogManager::getInstance()->message(str(F_("Unable to move %1% to %2% (%3%) nor to rename to %4% (%5%)") %
1209 Util::addBrackets(source) % Util::addBrackets(target) % e1.getError() % Util::addBrackets(newTarget) % e2.getError()));
1210 }
1211 }
1212 }
1213
moveStuckFile(QueueItem * qi)1214 void QueueManager::moveStuckFile(QueueItem* qi) {
1215 moveFile(qi->getTempTarget(), qi->getTarget());
1216
1217 if(qi->isFinished()) {
1218 userQueue.remove(qi);
1219 }
1220
1221 string target = qi->getTarget();
1222
1223 if(!BOOLSETTING(KEEP_FINISHED_FILES)) {
1224 fire(QueueManagerListener::Removed(), qi);
1225 fileQueue.remove(qi);
1226 } else {
1227 qi->addSegment(Segment(0, qi->getSize()));
1228 fire(QueueManagerListener::StatusUpdated(), qi);
1229 }
1230
1231 fire(QueueManagerListener::RecheckAlreadyFinished(), target);
1232 }
1233
rechecked(QueueItem * qi)1234 void QueueManager::rechecked(QueueItem* qi) {
1235 fire(QueueManagerListener::RecheckDone(), qi->getTarget());
1236 fire(QueueManagerListener::StatusUpdated(), qi);
1237
1238 setDirty();
1239 }
1240
putDownload(Download * aDownload,bool finished)1241 void QueueManager::putDownload(Download* aDownload, bool finished) noexcept {
1242 HintedUserList getConn;
1243 string fl_fname;
1244 HintedUser fl_user(UserPtr(), Util::emptyString);
1245 int fl_flag = 0;
1246
1247 {
1248 Lock l(cs);
1249
1250 delete aDownload->getFile();
1251 aDownload->setFile(0);
1252
1253 if(aDownload->getType() == Transfer::TYPE_PARTIAL_LIST) {
1254 QueueItem* q = fileQueue.find(getListPath(aDownload->getHintedUser()));
1255 if(q) {
1256 if(!aDownload->getPFS().empty()) {
1257 if( (q->isSet(QueueItem::FLAG_DIRECTORY_DOWNLOAD) && directories.find(aDownload->getUser()) != directories.end()) ||
1258 (q->isSet(QueueItem::FLAG_MATCH_QUEUE)) )
1259 {
1260 dcassert(finished);
1261
1262 fl_fname = aDownload->getPFS();
1263 fl_user = aDownload->getHintedUser();
1264 fl_flag = (q->isSet(QueueItem::FLAG_DIRECTORY_DOWNLOAD) ? (QueueItem::FLAG_DIRECTORY_DOWNLOAD) : 0)
1265 | (q->isSet(QueueItem::FLAG_MATCH_QUEUE) ? QueueItem::FLAG_MATCH_QUEUE : 0) | QueueItem::FLAG_TEXT;
1266 } else {
1267 fire(QueueManagerListener::PartialList(), aDownload->getHintedUser(), aDownload->getPFS());
1268 }
1269 } else {
1270 // partial filelist probably failed, redownload full list
1271 dcassert(!finished);
1272 fl_flag = q->getFlags() & ~QueueItem::FLAG_PARTIAL_LIST;
1273 }
1274 fire(QueueManagerListener::Removed(), q);
1275
1276 userQueue.remove(q);
1277 fileQueue.remove(q);
1278 //} else {
1279 //userQueue.removeDownload(q, aDownload->getUser());
1280 //fire(QueueManagerListener::StatusUpdated(), q);
1281 //}
1282 }
1283 } else {
1284 QueueItem* q = fileQueue.find(aDownload->getPath());
1285
1286 if(q) {
1287 if(aDownload->getType() == Transfer::TYPE_FULL_LIST) {
1288 if(aDownload->isSet(Download::FLAG_XML_BZ_LIST)) {
1289 q->setFlag(QueueItem::FLAG_XML_BZLIST);
1290 } else {
1291 q->unsetFlag(QueueItem::FLAG_XML_BZLIST);
1292 }
1293 }
1294
1295 if(finished) {
1296 if(aDownload->getType() == Transfer::TYPE_TREE) {
1297 // Got a full tree, now add it to the HashManager
1298 dcassert(aDownload->getTreeValid());
1299 HashManager::getInstance()->addTree(aDownload->getTigerTree());
1300
1301 userQueue.removeDownload(q, aDownload->getUser());
1302 fire(QueueManagerListener::StatusUpdated(), q);
1303 } else {
1304 // Now, let's see if this was a directory download filelist...
1305 if( (q->isSet(QueueItem::FLAG_DIRECTORY_DOWNLOAD) && directories.find(aDownload->getUser()) != directories.end()) ||
1306 (q->isSet(QueueItem::FLAG_MATCH_QUEUE)) )
1307 {
1308 fl_fname = q->getListName();
1309 fl_user = aDownload->getHintedUser();
1310 fl_flag = (q->isSet(QueueItem::FLAG_DIRECTORY_DOWNLOAD) ? QueueItem::FLAG_DIRECTORY_DOWNLOAD : 0)
1311 | (q->isSet(QueueItem::FLAG_MATCH_QUEUE) ? QueueItem::FLAG_MATCH_QUEUE : 0);
1312 }
1313
1314 string dir;
1315 bool crcError = false;
1316 if(aDownload->getType() == Transfer::TYPE_FULL_LIST) {
1317 dir = q->getTempTarget();
1318 q->addSegment(Segment(0, q->getSize()));
1319 } else if(aDownload->getType() == Transfer::TYPE_FILE) {
1320 q->addSegment(aDownload->getSegment());
1321 }
1322
1323 if (q->isFinished() && BOOLSETTING(SFV_CHECK)) {
1324 crcError = checkSfv(q, aDownload);
1325 }
1326
1327 if(aDownload->getType() != Transfer::TYPE_FILE || q->isFinished()) {
1328 // Check if we need to move the file
1329 if( aDownload->getType() == Transfer::TYPE_FILE && !aDownload->getTempTarget().empty() && (Util::stricmp(aDownload->getPath().c_str(), aDownload->getTempTarget().c_str()) != 0) ) {
1330 moveFile(aDownload->getTempTarget(), aDownload->getPath());
1331 }
1332 if (BOOLSETTING(LOG_FINISHED_DOWNLOADS) && aDownload->getType() == Transfer::TYPE_FILE) {
1333 logFinishedDownload(q, aDownload, crcError);
1334 }
1335
1336 fire(QueueManagerListener::Finished(), q, dir, aDownload->getAverageSpeed());
1337
1338 userQueue.remove(q);
1339
1340 if(!BOOLSETTING(KEEP_FINISHED_FILES) || aDownload->getType() == Transfer::TYPE_FULL_LIST) {
1341 fire(QueueManagerListener::Removed(), q);
1342 fileQueue.remove(q);
1343 } else {
1344 fire(QueueManagerListener::StatusUpdated(), q);
1345 }
1346 } else {
1347 userQueue.removeDownload(q, aDownload->getUser());
1348 fire(QueueManagerListener::StatusUpdated(), q);
1349 }
1350 setDirty();
1351 }
1352 } else {
1353 if(aDownload->getType() != Transfer::TYPE_TREE) {
1354 if(q->getDownloadedBytes() == 0) {
1355 q->setTempTarget(Util::emptyString);
1356 }
1357 if(q->isSet(QueueItem::FLAG_USER_LIST)) {
1358 // Blah...no use keeping an unfinished file list...
1359 File::deleteFile(q->getListName());
1360 }
1361 if(aDownload->getType() == Transfer::TYPE_FILE) {
1362 // mark partially downloaded chunk, but align it to block size
1363 int64_t downloaded = aDownload->getPos();
1364 downloaded -= downloaded % aDownload->getTigerTree().getBlockSize();
1365
1366 if(downloaded > 0) {
1367 q->addSegment(Segment(aDownload->getStartPos(), downloaded));
1368 setDirty();
1369 }
1370 }
1371 }
1372
1373 if(q->getPriority() != QueueItem::PAUSED) {
1374 q->getOnlineUsers(getConn);
1375 }
1376
1377 userQueue.removeDownload(q, aDownload->getUser());
1378 fire(QueueManagerListener::StatusUpdated(), q);
1379 }
1380 } else if(aDownload->getType() != Transfer::TYPE_TREE) {
1381 if(!aDownload->getTempTarget().empty() && (aDownload->getType() == Transfer::TYPE_FULL_LIST || aDownload->getTempTarget() != aDownload->getPath())) {
1382 File::deleteFile(aDownload->getTempTarget());
1383 }
1384 }
1385 }
1386 delete aDownload;
1387 }
1388
1389 for(HintedUserList::iterator i = getConn.begin(); i != getConn.end(); ++i) {
1390 ConnectionManager::getInstance()->getDownloadConnection(*i);
1391 }
1392
1393 if(!fl_fname.empty()) {
1394 processList(fl_fname, fl_user, fl_flag);
1395 }
1396 }
1397
processList(const string & name,const HintedUser & user,int flags)1398 void QueueManager::processList(const string& name, const HintedUser& user, int flags) {
1399 DirectoryListing dirList(user);
1400 try {
1401 dirList.loadFile(name);
1402 } catch(const Exception&) {
1403 LogManager::getInstance()->message(str(F_("Unable to open filelist: %1%") % Util::addBrackets(name)));
1404 return;
1405 }
1406
1407 if(flags & QueueItem::FLAG_DIRECTORY_DOWNLOAD) {
1408 DirectoryItem::List dl;
1409 {
1410 Lock l(cs);
1411 DirectoryItem::DirectoryPair dp = directories.equal_range(user);
1412 for(DirectoryItem::DirectoryIter i = dp.first; i != dp.second; ++i) {
1413 dl.push_back(i->second);
1414 }
1415 directories.erase(user);
1416 }
1417
1418 for(DirectoryItem::Iter i = dl.begin(); i != dl.end(); ++i) {
1419 DirectoryItem* di = *i;
1420 dirList.download(di->getName(), di->getTarget(), false);
1421 delete di;
1422 }
1423 }
1424 if(flags & QueueItem::FLAG_MATCH_QUEUE) {
1425 size_t files = matchListing(dirList);
1426 LogManager::getInstance()->message(str(FN_("%1%: Matched %2% file", "%1%: Matched %2% files", files) %
1427 Util::toString(ClientManager::getInstance()->getNicks(user)) % files));
1428 }
1429 }
1430
recheck(const string & aTarget)1431 void QueueManager::recheck(const string& aTarget) {
1432 rechecker.add(aTarget);
1433 }
1434
remove(const string & aTarget)1435 void QueueManager::remove(const string& aTarget) noexcept {
1436 UserList x;
1437
1438 {
1439 Lock l(cs);
1440
1441 QueueItem* q = fileQueue.find(aTarget);
1442 if(!q)
1443 return;
1444
1445 if(q->isSet(QueueItem::FLAG_DIRECTORY_DOWNLOAD)) {
1446 dcassert(q->getSources().size() == 1);
1447 DirectoryItem::DirectoryPair dp = directories.equal_range(q->getSources()[0].getUser());
1448 for(DirectoryItem::DirectoryIter i = dp.first; i != dp.second; ++i) {
1449 delete i->second;
1450 }
1451 directories.erase(q->getSources()[0].getUser());
1452 }
1453
1454 if(q->isRunning()) {
1455 for(DownloadList::iterator i = q->getDownloads().begin(); i != q->getDownloads().end(); ++i) {
1456 x.push_back((*i)->getUser());
1457 }
1458 } else if(!q->getTempTarget().empty() && q->getTempTarget() != q->getTarget()) {
1459 File::deleteFile(q->getTempTarget());
1460 }
1461
1462 fire(QueueManagerListener::Removed(), q);
1463
1464 if(!q->isFinished()) {
1465 userQueue.remove(q);
1466 }
1467 fileQueue.remove(q);
1468
1469 setDirty();
1470 }
1471
1472 for(UserList::iterator i = x.begin(); i != x.end(); ++i) {
1473 ConnectionManager::getInstance()->disconnect(*i, true);
1474 }
1475 }
1476
removeSource(const string & aTarget,const UserPtr & aUser,int reason,bool removeConn)1477 void QueueManager::removeSource(const string& aTarget, const UserPtr& aUser, int reason, bool removeConn /* = true */) noexcept {
1478 bool isRunning = false;
1479 bool removeCompletely = false;
1480 {
1481 Lock l(cs);
1482 QueueItem* q = fileQueue.find(aTarget);
1483 if(!q)
1484 return;
1485
1486 if(!q->isSource(aUser))
1487 return;
1488
1489 if(q->isSet(QueueItem::FLAG_USER_LIST)) {
1490 removeCompletely = true;
1491 goto endCheck;
1492 }
1493
1494 if(reason == QueueItem::Source::FLAG_NO_TREE) {
1495 q->getSource(aUser)->setFlag(reason);
1496 return;
1497 }
1498
1499 if(q->isRunning() && userQueue.getRunning(aUser) == q) {
1500 isRunning = true;
1501 userQueue.removeDownload(q, aUser);
1502 fire(QueueManagerListener::StatusUpdated(), q);
1503 }
1504
1505 if(!q->isFinished()) {
1506 userQueue.remove(q, aUser);
1507 }
1508 q->removeSource(aUser, reason);
1509
1510 fire(QueueManagerListener::SourcesUpdated(), q);
1511 setDirty();
1512 }
1513 endCheck:
1514 if(isRunning && removeConn) {
1515 ConnectionManager::getInstance()->disconnect(aUser, true);
1516 }
1517 if(removeCompletely) {
1518 remove(aTarget);
1519 }
1520 }
1521
getQueued(const UserPtr & aUser) const1522 int64_t QueueManager::getQueued(const UserPtr& aUser) const {
1523 Lock l(cs);
1524 return userQueue.getQueued(aUser);
1525 }
1526
removeSource(const UserPtr & aUser,int reason)1527 void QueueManager::removeSource(const UserPtr& aUser, int reason) noexcept {
1528 // @todo remove from finished items
1529 bool isRunning = false;
1530 string removeRunning;
1531 {
1532 Lock l(cs);
1533 QueueItem* qi = NULL;
1534 while( (qi = userQueue.getNext(aUser, QueueItem::PAUSED)) != NULL) {
1535 if(qi->isSet(QueueItem::FLAG_USER_LIST)) {
1536 remove(qi->getTarget());
1537 } else {
1538 userQueue.remove(qi, aUser);
1539 qi->removeSource(aUser, reason);
1540 fire(QueueManagerListener::SourcesUpdated(), qi);
1541 setDirty();
1542 }
1543 }
1544
1545 qi = userQueue.getRunning(aUser);
1546 if(qi) {
1547 if(qi->isSet(QueueItem::FLAG_USER_LIST)) {
1548 removeRunning = qi->getTarget();
1549 } else {
1550 userQueue.removeDownload(qi, aUser);
1551 userQueue.remove(qi, aUser);
1552 isRunning = true;
1553 qi->removeSource(aUser, reason);
1554 fire(QueueManagerListener::StatusUpdated(), qi);
1555 fire(QueueManagerListener::SourcesUpdated(), qi);
1556 setDirty();
1557 }
1558 }
1559 }
1560
1561 if(isRunning) {
1562 ConnectionManager::getInstance()->disconnect(aUser, true);
1563 }
1564 if(!removeRunning.empty()) {
1565 remove(removeRunning);
1566 }
1567 }
1568
setPriority(const string & aTarget,QueueItem::Priority p)1569 void QueueManager::setPriority(const string& aTarget, QueueItem::Priority p) noexcept {
1570 HintedUserList getConn;
1571
1572 {
1573 Lock l(cs);
1574
1575 QueueItem* q = fileQueue.find(aTarget);
1576 if( (q != NULL) && (q->getPriority() != p) && !q->isFinished() ) {
1577 if(q->getPriority() == QueueItem::PAUSED || p == QueueItem::HIGHEST) {
1578 // Problem, we have to request connections to all these users...
1579 q->getOnlineUsers(getConn);
1580 }
1581 userQueue.setPriority(q, p);
1582 setDirty();
1583 fire(QueueManagerListener::StatusUpdated(), q);
1584 }
1585 }
1586
1587 for(HintedUserList::iterator i = getConn.begin(); i != getConn.end(); ++i) {
1588 ConnectionManager::getInstance()->getDownloadConnection(*i);
1589 }
1590 }
1591
saveQueue(bool force)1592 void QueueManager::saveQueue(bool force) noexcept {
1593 if(!dirty && !force)
1594 return;
1595
1596 std::vector<CID> cids;
1597
1598 try {
1599 Lock l(cs);
1600
1601 File ff(getQueueFile() + ".tmp", File::WRITE, File::CREATE | File::TRUNCATE);
1602 BufferedOutputStream<false> f(&ff);
1603
1604 f.write(SimpleXML::utf8Header);
1605 f.write(LIT("<Downloads Version=\"" VERSIONSTRING "\">\r\n"));
1606 string tmp;
1607 string b32tmp;
1608 for(QueueItem::StringIter i = fileQueue.getQueue().begin(); i != fileQueue.getQueue().end(); ++i) {
1609 QueueItem* qi = i->second;
1610 if(!qi->isSet(QueueItem::FLAG_USER_LIST)
1611 || (qi->isSet(QueueItem::FLAG_USER_LIST)
1612 && SETTING(KEEP_LISTS))) {
1613 f.write(LIT("\t<Download Target=\""));
1614 f.write(SimpleXML::escape(qi->getTarget(), tmp, true));
1615 f.write(LIT("\" Size=\""));
1616 f.write(Util::toString(qi->getSize()));
1617 f.write(LIT("\" Priority=\""));
1618 f.write(Util::toString((int)qi->getPriority()));
1619 f.write(LIT("\" Added=\""));
1620 f.write(Util::toString(qi->getAdded()));
1621 b32tmp.clear();
1622 f.write(LIT("\" TTH=\""));
1623 f.write(qi->getTTH().toBase32(b32tmp));
1624 if(!qi->getDone().empty()) {
1625 f.write(LIT("\" TempTarget=\""));
1626 f.write(SimpleXML::escape(qi->getTempTarget(), tmp, true));
1627 }
1628 f.write(LIT("\">\r\n"));
1629
1630 for(QueueItem::SegmentSet::const_iterator i = qi->getDone().begin(); i != qi->getDone().end(); ++i) {
1631 f.write(LIT("\t\t<Segment Start=\""));
1632 f.write(Util::toString(i->getStart()));
1633 f.write(LIT("\" Size=\""));
1634 f.write(Util::toString(i->getSize()));
1635 f.write(LIT("\"/>\r\n"));
1636 }
1637
1638 for(QueueItem::SourceConstIter j = qi->sources.begin(); j != qi->sources.end(); ++j) {
1639 if(j->isSet(QueueItem::Source::FLAG_PARTIAL)
1640 #ifdef WITH_DHT
1641 || j->getUser().hint == "DHT"
1642 #endif
1643 ) continue;
1644
1645 const CID& cid = j->getUser().user->getCID();
1646 const string& hint = j->getUser().hint;
1647
1648 f.write(LIT("\t\t<Source CID=\""));
1649 f.write(cid.toBase32());
1650 if(!hint.empty()) {
1651 f.write(LIT("\" Hub=\""));
1652 f.write(hint);
1653 }
1654 f.write(LIT("\"/>\r\n"));
1655
1656 cids.push_back(cid);
1657 }
1658
1659 f.write(LIT("\t</Download>\r\n"));
1660 }
1661 }
1662
1663 f.write("</Downloads>\r\n");
1664 f.flush();
1665 ff.close();
1666 File::deleteFile(getQueueFile());
1667 File::renameFile(getQueueFile() + ".tmp", getQueueFile());
1668
1669 dirty = false;
1670 } catch(const FileException&) {
1671 // ...
1672 }
1673 // Put this here to avoid very many saves tries when disk is full...
1674 lastSave = GET_TICK();
1675
1676 //NOTE: freedcpp, save user cids and nicks to Users.xml see dcplusplus revision 1771
1677 ClientManager* cm = ClientManager::getInstance();
1678 //#ifdef _WIN32
1679 // std::for_each(cids.begin(), cids.end(), std::bind(&ClientManager::saveUser, cm, std::placeholders::_1));
1680 //#else
1681 for (vector<CID>::const_iterator it = cids.begin(); it != cids.end(); ++it)
1682 {
1683 cm->saveUser(*it);
1684 }
1685 //#endif
1686 }
1687
1688 class QueueLoader : public SimpleXMLReader::CallBack {
1689 public:
QueueLoader()1690 QueueLoader() : cur(NULL), inDownloads(false) { }
~QueueLoader()1691 virtual ~QueueLoader() { }
1692 virtual void startTag(const string& name, StringPairList& attribs, bool simple);
1693 virtual void endTag(const string& name, const string& data);
1694 private:
1695 string target;
1696
1697 QueueItem* cur;
1698 bool inDownloads;
1699 };
1700
loadQueue()1701 void QueueManager::loadQueue() noexcept {
1702 try {
1703 QueueLoader l;
1704 Util::migrate(getQueueFile());
1705
1706 File f(getQueueFile(), File::READ, File::OPEN);
1707 SimpleXMLReader(&l).parse(f);
1708 dirty = false;
1709 } catch(const Exception&) {
1710 // ...
1711 }
1712 }
1713
countOnlineSources(const string & aTarget)1714 int QueueManager::countOnlineSources(const string& aTarget) {
1715 Lock l(cs);
1716
1717 QueueItem* qi = fileQueue.find(aTarget);
1718 if(!qi)
1719 return 0;
1720 int onlineSources = 0;
1721 for(QueueItem::SourceConstIter i = qi->getSources().begin(); i != qi->getSources().end(); ++i) {
1722 if(i->getUser().user->isOnline())
1723 onlineSources++;
1724 }
1725 return onlineSources;
1726 }
1727
1728 static const string sDownload = "Download";
1729 static const string sTempTarget = "TempTarget";
1730 static const string sTarget = "Target";
1731 static const string sSize = "Size";
1732 static const string sDownloaded = "Downloaded";
1733 static const string sPriority = "Priority";
1734 static const string sSource = "Source";
1735 static const string sNick = "Nick";
1736 static const string sDirectory = "Directory";
1737 static const string sAdded = "Added";
1738 static const string sTTH = "TTH";
1739 static const string sCID = "CID";
1740 static const string sHubHint = "Hub";
1741 static const string sSegment = "Segment";
1742 static const string sStart = "Start";
1743
startTag(const string & name,StringPairList & attribs,bool simple)1744 void QueueLoader::startTag(const string& name, StringPairList& attribs, bool simple) {
1745 QueueManager* qm = QueueManager::getInstance();
1746 if(!inDownloads && name == "Downloads") {
1747 inDownloads = true;
1748 } else if(inDownloads) {
1749 if(cur == NULL && name == sDownload) {
1750 int64_t size = Util::toInt64(getAttrib(attribs, sSize, 1));
1751 if(size == 0)
1752 return;
1753 try {
1754 const string& tgt = getAttrib(attribs, sTarget, 0);
1755 // @todo do something better about existing files
1756 target = QueueManager::checkTarget(tgt, /*checkExistence*/ false);
1757 if(target.empty())
1758 return;
1759 } catch(const Exception&) {
1760 return;
1761 }
1762 QueueItem::Priority p = (QueueItem::Priority)Util::toInt(getAttrib(attribs, sPriority, 3));
1763 time_t added = static_cast<time_t>(Util::toInt(getAttrib(attribs, sAdded, 4)));
1764 const string& tthRoot = getAttrib(attribs, sTTH, 5);
1765 if(tthRoot.empty())
1766 return;
1767
1768 string tempTarget = getAttrib(attribs, sTempTarget, 5);
1769 int64_t downloaded = Util::toInt64(getAttrib(attribs, sDownloaded, 5));
1770 if (downloaded > size || downloaded < 0)
1771 downloaded = 0;
1772
1773 if(added == 0)
1774 added = GET_TIME();
1775
1776 QueueItem* qi = qm->fileQueue.find(target);
1777
1778 if(qi == NULL) {
1779 qi = qm->fileQueue.add(target, size, 0, p, tempTarget, added, TTHValue(tthRoot));
1780 if(downloaded > 0) {
1781 qi->addSegment(Segment(0, downloaded));
1782 }
1783 qm->fire(QueueManagerListener::Added(), qi);
1784 }
1785 if(!simple)
1786 cur = qi;
1787 } else if(cur && name == sSegment) {
1788 int64_t start = Util::toInt64(getAttrib(attribs, sStart, 0));
1789 int64_t size = Util::toInt64(getAttrib(attribs, sSize, 1));
1790
1791 if(size > 0 && start >= 0 && (start + size) <= cur->getSize()) {
1792 cur->addSegment(Segment(start, size));
1793 }
1794 } else if (cur && !Util::fileExists(Util::getFilePath(cur->getTarget())) && BOOLSETTING(CHECK_TARGETS_PATHS_ON_START)) {
1795 QueueManager::getInstance()->setPriority(cur->getTarget(), QueueItem::PAUSED);
1796 LogManager::getInstance()->message(str(F_("Target path for this item is not available: %1%; pause this queue item.") % Util::addBrackets(cur->getTarget())));
1797 } else if(cur && name == sSource) {
1798 const string& cid = getAttrib(attribs, sCID, 0);
1799 if(cid.length() != 39) {
1800 // Skip loading this source - sorry old users
1801 return;
1802 }
1803 UserPtr user = ClientManager::getInstance()->getUser(CID(cid));
1804
1805 try {
1806 const string& hubHint = getAttrib(attribs, sHubHint, 1);
1807 HintedUser hintedUser(user, hubHint);
1808 if(qm->addSource(cur, hintedUser, 0) && user->isOnline())
1809 ConnectionManager::getInstance()->getDownloadConnection(hintedUser);
1810 } catch(const Exception&) {
1811 return;
1812 }
1813 }
1814 }
1815 }
1816
endTag(const string & name,const string &)1817 void QueueLoader::endTag(const string& name, const string&) {
1818 if(inDownloads) {
1819 if(name == sDownload) {
1820 cur = NULL;
1821 } else if(name == "Downloads") {
1822 inDownloads = false;
1823 }
1824 }
1825 }
1826
noDeleteFileList(const string & path)1827 void QueueManager::noDeleteFileList(const string& path) {
1828 if(!BOOLSETTING(KEEP_LISTS)) {
1829 protectedFileLists.push_back(path);
1830 }
1831 }
1832
1833 // SearchManagerListener
on(SearchManagerListener::SR,const SearchResultPtr & sr)1834 void QueueManager::on(SearchManagerListener::SR, const SearchResultPtr& sr) noexcept {
1835 bool added = false;
1836 bool wantConnection = false;
1837
1838 {
1839 Lock l(cs);
1840 QueueItem::List matches;
1841
1842 fileQueue.find(matches, sr->getTTH());
1843
1844 for(QueueItem::Iter i = matches.begin(); i != matches.end(); ++i) {
1845 QueueItem* qi = *i;
1846
1847 // Size compare to avoid popular spoof
1848 if(qi->getSize() == sr->getSize() && !qi->isSource(sr->getUser())) {
1849 try {
1850 if(!BOOLSETTING(AUTO_SEARCH_AUTO_MATCH))
1851 wantConnection = addSource(qi, HintedUser(sr->getUser(), sr->getHubURL()), 0);
1852 added = true;
1853 } catch(const Exception&) {
1854 // ...
1855 }
1856 break;
1857 }
1858 }
1859 }
1860
1861 if(added && BOOLSETTING(AUTO_SEARCH_AUTO_MATCH)) {
1862 try {
1863 addList(HintedUser(sr->getUser(), sr->getHubURL()), QueueItem::FLAG_MATCH_QUEUE);
1864 } catch(const Exception&) {
1865 // ...
1866 }
1867 }
1868 if(added && sr->getUser()->isOnline() && wantConnection) {
1869 ConnectionManager::getInstance()->getDownloadConnection(HintedUser(sr->getUser(), sr->getHubURL()));
1870 }
1871
1872 }
1873
1874 // ClientManagerListener
on(ClientManagerListener::UserConnected,const UserPtr & aUser)1875 void QueueManager::on(ClientManagerListener::UserConnected, const UserPtr& aUser) noexcept {
1876 bool hasDown = false;
1877 {
1878 Lock l(cs);
1879 for(int i = 0; i < QueueItem::LAST; ++i) {
1880 QueueItem::UserListIter j = userQueue.getList(i).find(aUser);
1881 if(j != userQueue.getList(i).end()) {
1882 for(QueueItem::Iter m = j->second.begin(); m != j->second.end(); ++m)
1883 fire(QueueManagerListener::StatusUpdated(), *m);
1884 if(i != QueueItem::PAUSED)
1885 hasDown = true;
1886 }
1887 }
1888 }
1889
1890 if(hasDown) {
1891 // the user just came on, so there's only 1 possible hub, no need for a hint
1892 ConnectionManager::getInstance()->getDownloadConnection(HintedUser(aUser, Util::emptyString));
1893 }
1894 }
1895
on(ClientManagerListener::UserDisconnected,const UserPtr & aUser)1896 void QueueManager::on(ClientManagerListener::UserDisconnected, const UserPtr& aUser) noexcept {
1897 Lock l(cs);
1898 for(int i = 0; i < QueueItem::LAST; ++i) {
1899 QueueItem::UserListIter j = userQueue.getList(i).find(aUser);
1900 if(j != userQueue.getList(i).end()) {
1901 for(QueueItem::Iter m = j->second.begin(); m != j->second.end(); ++m)
1902 fire(QueueManagerListener::StatusUpdated(), *m);
1903 }
1904 }
1905 }
1906
on(TimerManagerListener::Second,uint64_t aTick)1907 void QueueManager::on(TimerManagerListener::Second, uint64_t aTick) noexcept {
1908 if(dirty && ((lastSave + 10000) < aTick)) {
1909 saveQueue();
1910 }
1911 }
1912
handlePartialResult(const UserPtr & aUser,const string & hubHint,const TTHValue & tth,const QueueItem::PartialSource & partialSource,PartsInfo & outPartialInfo)1913 bool QueueManager::handlePartialResult(const UserPtr& aUser, const string& hubHint, const TTHValue& tth, const QueueItem::PartialSource& partialSource, PartsInfo& outPartialInfo) {
1914 bool wantConnection = false;
1915 dcassert(outPartialInfo.empty());
1916
1917 {
1918 Lock l(cs);
1919
1920 // Locate target QueueItem in download queue
1921 QueueItem::List ql;
1922 fileQueue.find(ql, tth);
1923
1924 if(ql.empty()){
1925 dcdebug("Not found in download queue\n");
1926 return false;
1927 }
1928
1929 QueueItem::Ptr qi = ql[0];
1930 // don't add sources to finished files
1931 // this could happen when "Keep finished files in queue" is enabled
1932 if(qi->isFinished())
1933 return false;
1934
1935 // Check min size
1936 if(qi->getSize() < PARTIAL_SHARE_MIN_SIZE){
1937 dcassert(0);
1938 return false;
1939 }
1940
1941 // Get my parts info
1942 int64_t blockSize = HashManager::getInstance()->getBlockSize(qi->getTTH());
1943 if(blockSize == 0)
1944 blockSize = qi->getSize();
1945 qi->getPartialInfo(outPartialInfo, blockSize);
1946
1947 // Any parts for me?
1948 wantConnection = qi->isNeededPart(partialSource.getPartialInfo(), blockSize);
1949
1950 // If this user isn't a source and has no parts needed, ignore it
1951 QueueItem::SourceIter si = qi->getSource(aUser);
1952 if(si == qi->getSources().end()){
1953 si = qi->getBadSource(aUser);
1954
1955 if(si != qi->getBadSources().end() && (si->isSet(QueueItem::Source::FLAG_BAD_TREE) || si->isSet(QueueItem::Source::FLAG_TTH_INCONSISTENCY)))
1956 return false;
1957
1958 if(!wantConnection){
1959 if(si == qi->getBadSources().end())
1960 return false;
1961 }else{
1962 // add this user as partial file sharing source
1963 qi->addSource(HintedUser(aUser,""));
1964 si = qi->getSource(aUser);
1965 si->setFlag(QueueItem::Source::FLAG_PARTIAL);
1966
1967 QueueItem::PartialSource* ps = new QueueItem::PartialSource(partialSource.getMyNick(),
1968 partialSource.getHubIpPort(), partialSource.getIp(), partialSource.getUdpPort());
1969 si->setPartialSource(ps);
1970
1971 userQueue.add(qi, aUser);
1972 dcassert(si != qi->getSources().end());
1973 fire(QueueManagerListener::SourcesUpdated(), qi);
1974 }
1975 }
1976
1977 // Update source's parts info
1978 if(si->getPartialSource()) {
1979 si->getPartialSource()->setPartialInfo(partialSource.getPartialInfo());
1980 }
1981 }
1982
1983 // Connect to this user
1984 if(wantConnection)
1985 ConnectionManager::getInstance()->getDownloadConnection(HintedUser(aUser, hubHint));
1986
1987 return true;
1988 }
1989
handlePartialSearch(const TTHValue & tth,PartsInfo & _outPartsInfo)1990 bool QueueManager::handlePartialSearch(const TTHValue& tth, PartsInfo& _outPartsInfo) {
1991 {
1992 Lock l(cs);
1993
1994 // Locate target QueueItem in download queue
1995 QueueItem::List ql;
1996 fileQueue.find(ql, tth);
1997
1998 if(ql.empty()){
1999 return false;
2000 }
2001
2002 QueueItem::Ptr qi = ql[0];
2003 if(qi->getSize() < PARTIAL_SHARE_MIN_SIZE){
2004 return false;
2005 }
2006
2007 int64_t blockSize = HashManager::getInstance()->getBlockSize(qi->getTTH());
2008 if(blockSize == 0)
2009 blockSize = qi->getSize();
2010 qi->getPartialInfo(_outPartsInfo, blockSize);
2011 }
2012
2013 return !_outPartsInfo.empty();
2014 }
2015
checkSfv(QueueItem * qi,Download * d)2016 bool QueueManager::checkSfv(QueueItem* qi, Download* d) {
2017 SFVReader sfv(qi->getTarget());
2018
2019 if(sfv.hasCRC()) {
2020 bool crcMatch = false;
2021 try {
2022 crcMatch = (calcCrc32(qi->getTempTarget()) == sfv.getCRC());
2023 } catch(const FileException& ) {
2024 // Couldn't read the file to get the CRC(!!!)
2025 }
2026
2027 if(!crcMatch) {
2028 /// @todo There is a slight chance that something happens with a file while it's being saved to disk
2029 /// maybe calculate tth along with crc and if tth is ok and crc is not flag the file as bad at once
2030 /// if tth mismatches (possible disk error) then repair / redownload the file
2031
2032 File::deleteFile(qi->getTempTarget());
2033 qi->resetDownloaded();
2034 dcdebug("QueueManager: CRC32 mismatch for %s\n", qi->getTarget().c_str());
2035 LogManager::getInstance()->message(_("CRC32 inconsistency (SFV-Check)") + ' ' + Util::addBrackets(qi->getTarget()));
2036
2037 setPriority(qi->getTarget(), QueueItem::PAUSED);
2038
2039 QueueItem::SourceList sources = qi->getSources();
2040 for(QueueItem::SourceConstIter i = sources.begin(); i != sources.end(); ++i) {
2041 removeSource(qi->getTarget(), i->getUser(), QueueItem::Source::FLAG_CRC_FAILED, false);
2042 }
2043
2044 fire(QueueManagerListener::CRCFailed(), d, _("CRC32 inconsistency (SFV-Check)"));
2045 return true;
2046 }
2047
2048 dcdebug("QueueManager: CRC32 match for %s\n", qi->getTarget().c_str());
2049 fire(QueueManagerListener::CRCChecked(), d);
2050 }
2051 return false;
2052 }
2053
calcCrc32(const string & file)2054 uint32_t QueueManager::calcCrc32(const string& file) {
2055 File ff(file, File::READ, File::OPEN);
2056 CalcInputStream<CRC32Filter, false> f(&ff);
2057
2058 const size_t BUF_SIZE = 1024*1024;
2059 boost::scoped_array<uint8_t> b(new uint8_t[BUF_SIZE]);
2060 size_t n = BUF_SIZE;
2061 while(f.read(&b[0], n) > 0)
2062 ; // Keep on looping...
2063
2064 return f.getFilter().getValue();
2065 }
2066
2067 // compare nextQueryTime, get the oldest ones
findPFSSources(PFSSourceList & sl)2068 void QueueManager::FileQueue::findPFSSources(PFSSourceList& sl)
2069 {
2070 typedef multimap<time_t, pair<QueueItem::SourceConstIter, const QueueItem*> > Buffer;
2071 Buffer buffer;
2072 uint64_t now = GET_TICK();
2073
2074 for(QueueItem::StringIter i = queue.begin(); i != queue.end(); ++i) {
2075 const QueueItem* q = i->second;
2076
2077 if(q->getSize() < PARTIAL_SHARE_MIN_SIZE) continue;
2078
2079 const QueueItem::SourceList& sources = q->getSources();
2080 const QueueItem::SourceList& badSources = q->getBadSources();
2081
2082 for(QueueItem::SourceConstIter j = sources.begin(); j != sources.end(); ++j) {
2083 if( (*j).isSet(QueueItem::Source::FLAG_PARTIAL) && (*j).getPartialSource()->getNextQueryTime() <= now &&
2084 (*j).getPartialSource()->getPendingQueryCount() < 10 && (*j).getPartialSource()->getUdpPort() > 0)
2085 {
2086 buffer.insert(make_pair((*j).getPartialSource()->getNextQueryTime(), make_pair(j, q)));
2087 }
2088 }
2089
2090 for(QueueItem::SourceConstIter j = badSources.begin(); j != badSources.end(); ++j) {
2091 if( (*j).isSet(QueueItem::Source::FLAG_TTH_INCONSISTENCY) == false && (*j).isSet(QueueItem::Source::FLAG_PARTIAL) &&
2092 (*j).getPartialSource()->getNextQueryTime() <= now && (*j).getPartialSource()->getPendingQueryCount() < 10 &&
2093 (*j).getPartialSource()->getUdpPort() > 0)
2094 {
2095 buffer.insert(make_pair((*j).getPartialSource()->getNextQueryTime(), make_pair(j, q)));
2096 }
2097 }
2098 }
2099
2100 // copy to results
2101 dcassert(sl.empty());
2102 const uint8_t maxElements = 10;
2103 sl.reserve(maxElements);
2104 for(Buffer::iterator i = buffer.begin(); i != buffer.end() && sl.size() < maxElements; ++i){
2105 sl.push_back(i->second);
2106 }
2107 }
2108
2109 #ifdef WITH_DHT
findPFSPubTTH()2110 TTHValue* QueueManager::FileQueue::findPFSPubTTH()
2111 {
2112 uint64_t now = GET_TICK();
2113 QueueItem::Ptr cand = NULL;
2114
2115 for(QueueItem::StringIter i = queue.begin(); i != queue.end(); ++i)
2116 {
2117 QueueItem::Ptr qi = i->second;
2118 if(qi && qi->getSize() >= PARTIAL_SHARE_MIN_SIZE && now >= qi->getNextPublishingTime() && qi->getPriority() > QueueItem::PAUSED && qi->isRunning())
2119 {
2120 if(cand == NULL || cand->getNextPublishingTime() > qi->getNextPublishingTime() || (cand->getNextPublishingTime() == qi->getNextPublishingTime() && cand->getPriority() < qi->getPriority()) )
2121 {
2122 if(qi->getDownloadedBytes() > HashManager::getInstance()->getBlockSize(qi->getTTH()))
2123 cand = qi;
2124 }
2125 }
2126 }
2127 if(cand)
2128 {
2129 cand->setNextPublishingTime(now + PFS_REPUBLISH_TIME); // one hour
2130 return new TTHValue(cand->getTTH());
2131 }
2132 return NULL;
2133 }
2134 #endif
2135
logFinishedDownload(QueueItem * qi,Download * d,bool crcError)2136 void QueueManager::logFinishedDownload(QueueItem* qi, Download* d, bool crcError)
2137 {
2138 StringMap params;
2139 params["target"] = qi->getTarget();
2140 params["fileSI"] = Util::toString(qi->getSize());
2141 params["fileSIshort"] = Util::formatBytes(qi->getSize());
2142 params["fileTR"] = qi->getTTH().toBase32();
2143 params["sfv"] = Util::toString(crcError ? 1 : 0);
2144
2145 {
2146 #ifdef DO_NOT_USE_MUTEX
2147 FinishedManager::getInstance()->lockLists();
2148 #else // DO_NOT_USE_MUTEX
2149 auto lock = FinishedManager::getInstance()->lockLists();
2150 #endif // DO_NOT_USE_MUTEX
2151 const FinishedManager::MapByFile& map = FinishedManager::getInstance()->getMapByFile(false);
2152 FinishedManager::MapByFile::const_iterator it = map.find(qi->getTarget());
2153 if(it != map.end()) {
2154 FinishedFileItemPtr entry = it->second;
2155 if (!entry->getUsers().empty()) {
2156 StringList nicks, cids, ips, hubNames, hubUrls, temp;
2157 string ip;
2158 for(HintedUserList::const_iterator i = entry->getUsers().begin(), iend = entry->getUsers().end(); i != iend; ++i) {
2159
2160 nicks.push_back(Util::toString(ClientManager::getInstance()->getNicks(*i)));
2161 cids.push_back(i->user->getCID().toBase32());
2162
2163 ip.clear();
2164 if (i->user->isOnline()) {
2165 OnlineUser* u = ClientManager::getInstance()->findOnlineUser(*i, false);
2166 if (u) {
2167 ip = u->getIdentity().getIp();
2168 }
2169 }
2170 if (ip.empty()) {
2171 ip = _("Offline");
2172 }
2173 ips.push_back(ip);
2174
2175 temp = ClientManager::getInstance()->getHubNames(*i);
2176 if(temp.empty()) {
2177 temp.push_back(_("Offline"));
2178 }
2179 hubNames.push_back(Util::toString(temp));
2180
2181 temp = ClientManager::getInstance()->getHubs(*i);
2182 if(temp.empty()) {
2183 temp.push_back(_("Offline"));
2184 }
2185 hubUrls.push_back(Util::toString(temp));
2186 }
2187
2188 params["userNI"] = Util::toString(nicks);
2189 params["userCID"] = Util::toString(cids);
2190 params["userI4"] = Util::toString(ips);
2191 params["hubNI"] = Util::toString(hubNames);
2192 params["hubURL"] = Util::toString(hubUrls);
2193 }
2194
2195 params["fileSIsession"] = Util::toString(entry->getTransferred());
2196 params["fileSIsessionshort"] = Util::formatBytes(entry->getTransferred());
2197 params["fileSIactual"] = Util::toString(entry->getActual());
2198 params["fileSIactualshort"] = Util::formatBytes(entry->getActual());
2199
2200 params["speed"] = str(F_("%1%/s") % Util::formatBytes(entry->getAverageSpeed()));
2201 params["time"] = Util::formatSeconds(entry->getMilliSeconds() / 1000);
2202 }
2203 }
2204
2205 LOG(LogManager::FINISHED_DOWNLOAD, params);
2206 #ifdef DO_NOT_USE_MUTEX
2207 FinishedManager::getInstance()->unlockLists();
2208 #endif // DO_NOT_USE_MUTEX
2209 }
2210
2211 class ListMatcher : public dcpp::Thread
2212 {
2213 public:
ListMatcher(const StringList & files_)2214 ListMatcher(const StringList& files_) : files(files_) { }
run()2215 int run() {
2216 for (auto i = files.cbegin(); i != files.cend(); ++i) {
2217 UserPtr u = DirectoryListing::getUserFromFilename(*i);
2218 if (!u)
2219 continue;
2220
2221 HintedUser user(u, Util::emptyString);
2222 DirectoryListing dl(user);
2223 try {
2224 dl.loadFile(*i);
2225 LogManager::getInstance()->message(str(F_("%1% : Matched %2% files") % Util::toString(ClientManager::getInstance()->getNicks(user)) % QueueManager::getInstance()->matchListing(dl)));
2226 } catch (const Exception&) { }
2227 }
2228 delete this;// Cleanup the thread object
2229 return 0;
2230 }
2231 private:
2232 StringList files;
2233 };
2234
matchAllListings()2235 void QueueManager::matchAllListings() {
2236 ListMatcher* matcher = new ListMatcher(File::findFiles(Util::getListPath(), "*.xml*"));
2237 try {
2238 matcher->start();
2239 } catch (const ThreadException&) {
2240 ///@todo add error message
2241 delete matcher;
2242 }
2243 }
2244
2245 } // namespace dcpp
2246