1 /*
2 * Copyright (C) 2001-2012 Jacek Sieka, arnetheduck on gmail point com
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17 */
18
19 #include "stdinc.h"
20
21 #include "DownloadManager.h"
22
23 #include "QueueManager.h"
24 #include "Download.h"
25 #include "LogManager.h"
26 #include "User.h"
27 #include "File.h"
28 #include "FilteredFile.h"
29 #include "MerkleCheckOutputStream.h"
30 #include "UserConnection.h"
31 #include "ZUtils.h"
32 #include "extra/ipfilter.h"
33 #include <limits>
34 #include <cmath>
35
36 // some strange mac definition
37 #ifdef ff
38 #undef ff
39 #endif
40
41 namespace dcpp {
42
43 static const string DOWNLOAD_AREA = "Downloads";
44
DownloadManager()45 DownloadManager::DownloadManager() {
46 TimerManager::getInstance()->addListener(this);
47 }
48
~DownloadManager()49 DownloadManager::~DownloadManager() {
50 TimerManager::getInstance()->removeListener(this);
51 while(true) {
52 {
53 Lock l(cs);
54 if(downloads.empty())
55 break;
56 }
57 Thread::sleep(100);
58 }
59 }
60
on(TimerManagerListener::Second,uint64_t aTick)61 void DownloadManager::on(TimerManagerListener::Second, uint64_t aTick) noexcept {
62 typedef vector<pair<string, UserPtr> > TargetList;
63 TargetList dropTargets;
64
65 {
66 Lock l(cs);
67
68 DownloadList tickList;
69 // Tick each ongoing download
70 for(auto i = downloads.begin(); i != downloads.end(); ++i) {
71 if((*i)->getPos() > 0) {
72 tickList.push_back(*i);
73 (*i)->tick();
74 }
75 }
76
77 if(!tickList.empty())
78 fire(DownloadManagerListener::Tick(), tickList);
79
80
81 // Automatically remove or disconnect slow sources
82 if((uint32_t)(aTick / 1000) % SETTING(AUTODROP_INTERVAL) == 0) {
83 for(auto i = downloads.begin(); i != downloads.end(); ++i) {
84 Download* d = *i;
85 uint64_t timeElapsed = aTick - d->getStart();
86 uint64_t timeInactive = aTick - d->getUserConnection().getLastActivity();
87 uint64_t bytesDownloaded = d->getPos();
88 bool timeElapsedOk = timeElapsed >= (uint32_t)SETTING(AUTODROP_ELAPSED) * 1000;
89 bool timeInactiveOk = timeInactive <= (uint32_t)SETTING(AUTODROP_INACTIVITY) * 1000;
90 bool speedTooLow = timeElapsedOk && timeInactiveOk && bytesDownloaded > 0 ?
91 bytesDownloaded / timeElapsed * 1000 < (uint32_t)SETTING(AUTODROP_SPEED) : false;
92 bool isUserList = d->getType() == Transfer::TYPE_FULL_LIST;
93 bool onlineSourcesOk = isUserList ?
94 true : QueueManager::getInstance()->countOnlineSources(d->getPath()) >= SETTING(AUTODROP_MINSOURCES);
95 bool filesizeOk = !isUserList && d->getSize() >= ((int64_t)SETTING(AUTODROP_FILESIZE)) * 1024;
96 bool dropIt = (isUserList && BOOLSETTING(AUTODROP_FILELISTS)) ||
97 (filesizeOk && BOOLSETTING(AUTODROP_ALL));
98 if(speedTooLow && onlineSourcesOk && dropIt) {
99 if(BOOLSETTING(AUTODROP_DISCONNECT) && isUserList) {
100 d->getUserConnection().disconnect();
101 } else {
102 dropTargets.push_back(make_pair(d->getPath(), d->getUser()));
103 }
104 }
105 }
106 }
107 }
108 for(auto i = dropTargets.begin(); i != dropTargets.end(); ++i) {
109 QueueManager::getInstance()->removeSource(i->first, i->second, QueueItem::Source::FLAG_SLOW_SOURCE);
110 }
111 }
112
checkIdle(const UserPtr & user)113 void DownloadManager::checkIdle(const UserPtr& user) {
114 Lock l(cs);
115 for(auto i = idlers.begin(); i != idlers.end(); ++i) {
116 UserConnection* uc = *i;
117 if(uc->getUser() == user) {
118 uc->updated();
119 return;
120 }
121 }
122 }
123
addConnection(UserConnectionPtr conn)124 void DownloadManager::addConnection(UserConnectionPtr conn) {
125
126 if(!conn->isSet(UserConnection::FLAG_SUPPORTS_TTHF) || !conn->isSet(UserConnection::FLAG_SUPPORTS_ADCGET)) {
127 // Can't download from these...
128 conn->getUser()->setFlag(User::OLD_CLIENT);
129 QueueManager::getInstance()->removeSource(conn->getUser(), QueueItem::Source::FLAG_NO_TTHF);
130 conn->disconnect();
131 return;
132 }
133 if (BOOLSETTING(IPFILTER) && !ipfilter::getInstance()->OK(conn->getRemoteIp(),eDIRECTION_IN)) {
134 conn->error("Your IP is Blocked!");
135 LogManager::getInstance()->message(_("IPFilter: Blocked outgoing connection to ") + conn->getRemoteIp());
136 QueueManager::getInstance()->removeSource(conn->getUser(), QueueItem::Source::FLAG_REMOVED);
137 conn->disconnect();
138 return;
139 }
140
141 if(SETTING(REQUIRE_TLS) && !conn->isSecure()) {
142 conn->error("Secure connection required!");
143 QueueManager::getInstance()->removeSource(conn->getUser(), QueueItem::Source::FLAG_UNENCRYPTED);
144 return;
145 }
146
147 conn->addListener(this);
148 checkDownloads(conn);
149 }
150
startDownload(QueueItem::Priority prio)151 bool DownloadManager::startDownload(QueueItem::Priority prio) {
152 size_t downloadCount = getDownloadCount();
153
154 bool full = (SETTING(DOWNLOAD_SLOTS) != 0) && (downloadCount >= (size_t)SETTING(DOWNLOAD_SLOTS));
155 full = full || ((SETTING(MAX_DOWNLOAD_SPEED) != 0) && (getRunningAverage() >= (SETTING(MAX_DOWNLOAD_SPEED)*1024)));
156
157 if(full) {
158 bool extraFull = (SETTING(DOWNLOAD_SLOTS) != 0) && (getDownloadCount() >= (size_t)(SETTING(DOWNLOAD_SLOTS)+3));
159 if(extraFull) {
160 return false;
161 }
162 return prio == QueueItem::HIGHEST;
163 }
164
165 if(downloadCount > 0) {
166 return prio != QueueItem::LOWEST;
167 }
168
169 return true;
170 }
171
checkDownloads(UserConnection * aConn)172 void DownloadManager::checkDownloads(UserConnection* aConn) {
173 dcassert(aConn->getDownload() == NULL);
174
175 QueueItem::Priority prio = QueueManager::getInstance()->hasDownload(aConn->getUser());
176 if(!startDownload(prio)) {
177 removeConnection(aConn);
178 return;
179 }
180
181 Download* d = QueueManager::getInstance()->getDownload(*aConn, aConn->isSet(UserConnection::FLAG_SUPPORTS_TTHL));
182
183 if(!d) {
184 Lock l(cs);
185 aConn->setState(UserConnection::STATE_IDLE);
186 idlers.push_back(aConn);
187 return;
188 }
189
190 aConn->setState(UserConnection::STATE_SND);
191
192 if(aConn->isSet(UserConnection::FLAG_SUPPORTS_XML_BZLIST) && d->getType() == Transfer::TYPE_FULL_LIST) {
193 d->setFlag(Download::FLAG_XML_BZ_LIST);
194 }
195
196 {
197 Lock l(cs);
198 downloads.push_back(d);
199 }
200 fire(DownloadManagerListener::Requesting(), d);
201
202 dcdebug("Requesting " I64_FMT "/" I64_FMT "\n", static_cast<long long int>(d->getStartPos()), static_cast<long long int>(d->getSize()));
203
204 aConn->send(d->getCommand(aConn->isSet(UserConnection::FLAG_SUPPORTS_ZLIB_GET)));
205 }
206
on(AdcCommand::SND,UserConnection * aSource,const AdcCommand & cmd)207 void DownloadManager::on(AdcCommand::SND, UserConnection* aSource, const AdcCommand& cmd) noexcept {
208 if(aSource->getState() != UserConnection::STATE_SND) {
209 dcdebug("DM::onFileLength Bad state, ignoring\n");
210 return;
211 }
212
213 const string& type = cmd.getParam(0);
214 int64_t start = Util::toInt64(cmd.getParam(2));
215 int64_t bytes = Util::toInt64(cmd.getParam(3));
216
217 if(type != Transfer::names[aSource->getDownload()->getType()]) {
218 // Uhh??? We didn't ask for this...
219 aSource->disconnect();
220 return;
221 }
222
223 startData(aSource, start, bytes, cmd.hasFlag("ZL", 4));
224 }
225
startData(UserConnection * aSource,int64_t start,int64_t bytes,bool z)226 void DownloadManager::startData(UserConnection* aSource, int64_t start, int64_t bytes, bool z) {
227 Download* d = aSource->getDownload();
228 dcassert(d != NULL);
229
230 dcdebug("Preparing " I64_FMT ":" I64_FMT ", " I64_FMT ":" I64_FMT"\n",
231 static_cast<long long int>(d->getStartPos()), static_cast<long long int>(start),
232 static_cast<long long int>(d->getSize()), static_cast<long long int>(bytes));
233 if(d->getSize() == -1) {
234 if(bytes >= 0) {
235 d->setSize(bytes);
236 } else {
237 failDownload(aSource, _("Invalid size"));
238 return;
239 }
240 } else if(d->getSize() != bytes || d->getStartPos() != start) {
241 // This is not what we requested...
242 failDownload(aSource, _("Response does not match request"));
243 return;
244 }
245
246 try {
247 QueueManager::getInstance()->setFile(d);
248 } catch(const FileException& e) {
249 failDownload(aSource, str(F_("Could not open target file: %1%") % e.getError()));
250 return;
251 } catch(const Exception& e) {
252 failDownload(aSource, e.getError());
253 return;
254 }
255
256 if((d->getType() == Transfer::TYPE_FILE || d->getType() == Transfer::TYPE_FULL_LIST) && SETTING(BUFFER_SIZE) > 0 ) {
257 d->setFile(new BufferedOutputStream<true>(d->getFile()));
258 }
259
260 if(d->getType() == Transfer::TYPE_FILE) {
261 typedef MerkleCheckOutputStream<TigerTree, true> MerkleStream;
262
263 d->setFile(new MerkleStream(d->getTigerTree(), d->getFile(), d->getStartPos()));
264 d->setFlag(Download::FLAG_TTH_CHECK);
265 }
266
267 // Check that we don't get too many bytes
268 d->setFile(new LimitedOutputStream<true>(d->getFile(), bytes));
269
270 if(z) {
271 d->setFlag(Download::FLAG_ZDOWNLOAD);
272 d->setFile(new FilteredOutputStream<UnZFilter, true>(d->getFile()));
273 }
274
275 d->setStart(GET_TICK());
276 d->tick();
277 aSource->setState(UserConnection::STATE_RUNNING);
278
279 fire(DownloadManagerListener::Starting(), d);
280
281 if(d->getPos() == d->getSize()) {
282 try {
283 // Already finished? A zero-byte file list could cause this...
284 endData(aSource);
285 } catch(const Exception& e) {
286 failDownload(aSource, e.getError());
287 }
288 } else {
289 aSource->setDataMode();
290 }
291 }
292
on(UserConnectionListener::Data,UserConnection * aSource,const uint8_t * aData,size_t aLen)293 void DownloadManager::on(UserConnectionListener::Data, UserConnection* aSource, const uint8_t* aData, size_t aLen) noexcept {
294 Download* d = aSource->getDownload();
295 dcassert(d != NULL);
296
297 try {
298 d->addPos(d->getFile()->write(aData, aLen), aLen);
299 d->tick();
300
301 if(d->getFile()->eof()) {
302 endData(aSource);
303 aSource->setLineMode(0);
304 }
305 } catch(const Exception& e) {
306 failDownload(aSource, e.getError());
307 }
308 }
309
310 /** Download finished! */
endData(UserConnection * aSource)311 void DownloadManager::endData(UserConnection* aSource) {
312 dcassert(aSource->getState() == UserConnection::STATE_RUNNING);
313 Download* d = aSource->getDownload();
314 dcassert(d != NULL);
315
316 if(d->getType() == Transfer::TYPE_TREE) {
317 d->getFile()->flush();
318
319 int64_t bl = 1024;
320 while(bl * (int64_t)d->getTigerTree().getLeaves().size() < d->getTigerTree().getFileSize())
321 bl *= 2;
322 d->getTigerTree().setBlockSize(bl);
323 d->getTigerTree().calcRoot();
324
325 if(!(d->getTTH() == d->getTigerTree().getRoot())) {
326 // This tree is for a different file, remove from queue...
327 removeDownload(d);
328 fire(DownloadManagerListener::Failed(), d, _("Full tree does not match TTH root"));
329
330 QueueManager::getInstance()->removeSource(d->getPath(), aSource->getUser(), QueueItem::Source::FLAG_BAD_TREE, false);
331
332 QueueManager::getInstance()->putDownload(d, false);
333
334 checkDownloads(aSource);
335 return;
336 }
337 d->setTreeValid(true);
338 } else {
339 // First, finish writing the file (flushing the buffers and closing the file...)
340 try {
341 d->getFile()->flush();
342 } catch(const Exception& e) {
343 d->resetPos();
344 failDownload(aSource, e.getError());
345 return;
346 }
347
348 aSource->setSpeed(d->getAverageSpeed());
349 aSource->updateChunkSize(d->getTigerTree().getBlockSize(), d->getSize(), GET_TICK() - d->getStart());
350
351 dcdebug("Download finished: %s, size " I64_FMT ", downloaded " I64_FMT "\n", d->getPath().c_str(),
352 static_cast<long long int>(d->getSize()), static_cast<long long int>(d->getPos()));
353
354 if(BOOLSETTING(LOG_DOWNLOADS) && (BOOLSETTING(LOG_FILELIST_TRANSFERS) || d->getType() == Transfer::TYPE_FILE)) {
355 logDownload(aSource, d);
356 }
357 }
358
359 removeDownload(d);
360 fire(DownloadManagerListener::Complete(), d);
361
362 QueueManager::getInstance()->putDownload(d, true);
363 checkDownloads(aSource);
364 }
365
getRunningAverage()366 int64_t DownloadManager::getRunningAverage() {
367 Lock l(cs);
368 int64_t avg = 0;
369 for(auto i = downloads.begin(); i != downloads.end(); ++i) {
370 Download* d = *i;
371 avg += d->getAverageSpeed();
372 }
373 return avg;
374 }
375
logDownload(UserConnection * aSource,Download * d)376 void DownloadManager::logDownload(UserConnection* aSource, Download* d) {
377 StringMap params;
378 d->getParams(*aSource, params);
379 LOG(LogManager::DOWNLOAD, params);
380 }
381
on(UserConnectionListener::MaxedOut,UserConnection * aSource)382 void DownloadManager::on(UserConnectionListener::MaxedOut, UserConnection* aSource) noexcept {
383 noSlots(aSource);
384 }
385
noSlots(UserConnection * aSource)386 void DownloadManager::noSlots(UserConnection* aSource) {
387 if(aSource->getState() != UserConnection::STATE_SND) {
388 dcdebug("DM::noSlots Bad state, disconnecting\n");
389 aSource->disconnect();
390 return;
391 }
392
393 failDownload(aSource, _("No slots available"));
394 }
395
onFailed(UserConnection * aSource,const string & aError)396 void DownloadManager::onFailed(UserConnection* aSource, const string& aError) {
397 {
398 Lock l(cs);
399 idlers.erase(remove(idlers.begin(), idlers.end(), aSource), idlers.end());
400 }
401 failDownload(aSource, aError);
402 }
403
failDownload(UserConnection * aSource,const string & reason)404 void DownloadManager::failDownload(UserConnection* aSource, const string& reason) {
405
406 Download* d = aSource->getDownload();
407
408 if(d) {
409 removeDownload(d);
410 fire(DownloadManagerListener::Failed(), d, reason);
411
412 QueueManager::getInstance()->putDownload(d, false);
413 }
414
415 removeConnection(aSource);
416 }
417
removeConnection(UserConnectionPtr aConn)418 void DownloadManager::removeConnection(UserConnectionPtr aConn) {
419 dcassert(aConn->getDownload() == NULL);
420 aConn->removeListener(this);
421 aConn->disconnect();
422 }
423
removeDownload(Download * d)424 void DownloadManager::removeDownload(Download* d) {
425 if(d->getFile()) {
426 if(d->getActual() > 0) {
427 try {
428 d->getFile()->flush();
429 } catch(const Exception&) {
430 }
431 }
432 }
433
434 {
435 Lock l(cs);
436 dcassert(find(downloads.begin(), downloads.end(), d) != downloads.end());
437
438 downloads.erase(remove(downloads.begin(), downloads.end(), d), downloads.end());
439 }
440 }
441
on(UserConnectionListener::FileNotAvailable,UserConnection * aSource)442 void DownloadManager::on(UserConnectionListener::FileNotAvailable, UserConnection* aSource) noexcept {
443 fileNotAvailable(aSource);
444 }
445
446 /** @todo Handle errors better */
on(AdcCommand::STA,UserConnection * aSource,const AdcCommand & cmd)447 void DownloadManager::on(AdcCommand::STA, UserConnection* aSource, const AdcCommand& cmd) noexcept {
448 if(cmd.getParameters().size() < 2) {
449 aSource->disconnect();
450 return;
451 }
452
453 const string& err = cmd.getParameters()[0];
454 if(err.length() != 3) {
455 aSource->disconnect();
456 return;
457 }
458
459 switch(Util::toInt(err.substr(0, 1))) {
460 case AdcCommand::SEV_FATAL:
461 aSource->disconnect();
462 return;
463 case AdcCommand::SEV_RECOVERABLE:
464 switch(Util::toInt(err.substr(1))) {
465 case AdcCommand::ERROR_FILE_NOT_AVAILABLE:
466 fileNotAvailable(aSource);
467 return;
468 case AdcCommand::ERROR_SLOTS_FULL:
469 noSlots(aSource);
470 return;
471 }
472 case AdcCommand::SEV_SUCCESS:
473 // We don't know any messages that would give us these...
474 dcdebug("Unknown success message %s %s", err.c_str(), cmd.getParam(1).c_str());
475 return;
476 }
477 aSource->disconnect();
478 }
479
on(UserConnectionListener::Updated,UserConnection * aSource)480 void DownloadManager::on(UserConnectionListener::Updated, UserConnection* aSource) noexcept {
481 {
482 Lock l(cs);
483 auto i = find(idlers.begin(), idlers.end(), aSource);
484 if(i == idlers.end())
485 return;
486 idlers.erase(i);
487 }
488
489 checkDownloads(aSource);
490 }
491
fileNotAvailable(UserConnection * aSource)492 void DownloadManager::fileNotAvailable(UserConnection* aSource) {
493 if(aSource->getState() != UserConnection::STATE_SND) {
494 dcdebug("DM::fileNotAvailable Invalid state, disconnecting");
495 aSource->disconnect();
496 return;
497 }
498
499 Download* d = aSource->getDownload();
500 dcassert(d != NULL);
501 dcdebug("File Not Available: %s\n", d->getPath().c_str());
502
503 removeDownload(d);
504 fire(DownloadManagerListener::Failed(), d, str(F_("%1%: File not available") % d->getTargetFileName()));
505
506 QueueManager::getInstance()->removeSource(d->getPath(), aSource->getUser(), d->getType() == Transfer::TYPE_TREE ? QueueItem::Source::FLAG_NO_TREE : QueueItem::Source::FLAG_FILE_NOT_AVAILABLE, false);
507
508 QueueManager::getInstance()->putDownload(d, false);
509
510 checkDownloads(aSource);
511 }
512
513 } // namespace dcpp
514