1 /* <!-- copyright */
2 /*
3 * aria2 - The high speed download utility
4 *
5 * Copyright (C) 2006 Tatsuhiro Tsujikawa
6 * Copyright (C) 2008 Aurelien Lefebvre, Mandriva
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 *
22 * In addition, as a special exception, the copyright holders give
23 * permission to link the code of portions of this program with the
24 * OpenSSL library under certain conditions as described in each
25 * individual source file, and distribute linked combinations
26 * including the two.
27 * You must obey the GNU General Public License in all respects
28 * for all of the code used other than OpenSSL. If you modify
29 * file(s) with this exception, you may extend this exception to your
30 * version of the file(s), but you are not obligated to do so. If you
31 * do not wish to do so, delete this exception statement from your
32 * version. If you delete this exception statement from all source
33 * files in the program, then also delete it here.
34 */
35 /* copyright --> */
36 #include "AdaptiveURISelector.h"
37
38 #include <cstdlib>
39 #include <cmath>
40 #include <algorithm>
41
42 #include "DownloadCommand.h"
43 #include "DownloadContext.h"
44 #include "ServerStatMan.h"
45 #include "ServerStat.h"
46 #include "RequestGroup.h"
47 #include "Logger.h"
48 #include "LogFactory.h"
49 #include "A2STR.h"
50 #include "prefs.h"
51 #include "Option.h"
52 #include "SimpleRandomizer.h"
53 #include "SocketCore.h"
54 #include "FileEntry.h"
55 #include "uri.h"
56 #include "fmt.h"
57 #include "SocketRecvBuffer.h"
58
59 namespace aria2 {
60
61 /* In that URI Selector, select method returns one of the bests
62 * mirrors for first and reserved connections. For supplementary
63 * ones, it returns mirrors which has not been tested yet, and
64 * if each of them already tested, returns mirrors which has to
65 * be tested again. Otherwise, it doesn't return anymore mirrors.
66 */
67
AdaptiveURISelector(std::shared_ptr<ServerStatMan> serverStatMan,RequestGroup * requestGroup)68 AdaptiveURISelector::AdaptiveURISelector(
69 std::shared_ptr<ServerStatMan> serverStatMan, RequestGroup* requestGroup)
70 : serverStatMan_(std::move(serverStatMan)), requestGroup_(requestGroup)
71 {
72 resetCounters();
73 }
74
75 AdaptiveURISelector::~AdaptiveURISelector() = default;
76
select(FileEntry * fileEntry,const std::vector<std::pair<size_t,std::string>> & usedHosts)77 std::string AdaptiveURISelector::select(
78 FileEntry* fileEntry,
79 const std::vector<std::pair<size_t, std::string>>& usedHosts)
80 {
81 A2_LOG_DEBUG(
82 fmt("AdaptiveURISelector: called %d", requestGroup_->getNumConnection()));
83 std::deque<std::string>& uris = fileEntry->getRemainingUris();
84 if (uris.empty() && requestGroup_->getNumConnection() <= 1) {
85 // here we know the download will fail, trying to find previously
86 // failed uris that may succeed with more permissive values
87 mayRetryWithIncreasedTimeout(fileEntry);
88 }
89
90 std::string selected = selectOne(uris);
91
92 if (selected != A2STR::NIL) {
93 uris.erase(std::find(std::begin(uris), std::end(uris), selected));
94 }
95 return selected;
96 }
97
98 namespace {
99 constexpr auto MAX_TIMEOUT = 60_s;
100 } // namespace
101
mayRetryWithIncreasedTimeout(FileEntry * fileEntry)102 void AdaptiveURISelector::mayRetryWithIncreasedTimeout(FileEntry* fileEntry)
103 {
104 if (requestGroup_->getTimeout() * 2 >= MAX_TIMEOUT)
105 return;
106 requestGroup_->setTimeout(requestGroup_->getTimeout() * 2);
107
108 std::deque<std::string>& uris = fileEntry->getRemainingUris();
109 // looking for retries
110 std::deque<URIResult> timeouts;
111 fileEntry->extractURIResult(timeouts, error_code::TIME_OUT);
112 std::transform(std::begin(timeouts), std::end(timeouts),
113 std::back_inserter(uris), std::mem_fn(&URIResult::getURI));
114
115 if (A2_LOG_DEBUG_ENABLED) {
116 for (const auto& uri : uris) {
117 A2_LOG_DEBUG(
118 fmt("AdaptiveURISelector: will retry server with increased"
119 " timeout (%ld s): %s",
120 static_cast<long int>(requestGroup_->getTimeout().count()),
121 uri.c_str()));
122 }
123 }
124 }
125
selectOne(const std::deque<std::string> & uris)126 std::string AdaptiveURISelector::selectOne(const std::deque<std::string>& uris)
127 {
128
129 if (uris.empty()) {
130 return A2STR::NIL;
131 }
132 else {
133 const size_t numPieces =
134 requestGroup_->getDownloadContext()->getNumPieces();
135
136 bool reservedContext =
137 numPieces > 0 &&
138 static_cast<size_t>(nbConnections_) >
139 std::min(numPieces, static_cast<size_t>(
140 requestGroup_->getNumConcurrentCommand()));
141 bool selectBest = numPieces == 0 || reservedContext;
142
143 if (numPieces > 0)
144 ++nbConnections_;
145
146 /* At least, 3 mirrors must be tested */
147 if (getNbTestedServers(uris) < 3) {
148 std::string notTested = getFirstNotTestedUri(uris);
149 if (notTested != A2STR::NIL) {
150 A2_LOG_DEBUG(fmt("AdaptiveURISelector: choosing the first non tested"
151 " mirror: %s",
152 notTested.c_str()));
153 --nbServerToEvaluate_;
154 return notTested;
155 }
156 }
157
158 if (!selectBest && nbConnections_ > 1 && nbServerToEvaluate_ > 0) {
159 nbServerToEvaluate_--;
160 std::string notTested = getFirstNotTestedUri(uris);
161 if (notTested != A2STR::NIL) {
162 /* Here we return the first untested mirror */
163 A2_LOG_DEBUG(fmt("AdaptiveURISelector: choosing non tested mirror %s"
164 " for connection #%d",
165 notTested.c_str(), nbConnections_));
166 return notTested;
167 }
168 else {
169 /* Here we return a mirror which need to be tested again */
170 std::string toReTest = getFirstToTestUri(uris);
171 if (toReTest != A2STR::NIL) {
172 A2_LOG_DEBUG(fmt("AdaptiveURISelector: choosing mirror %s which has"
173 " not been tested recently for connection #%d",
174 toReTest.c_str(), nbConnections_));
175 return toReTest;
176 }
177 else {
178 return getBestMirror(uris);
179 }
180 }
181 }
182 else {
183 return getBestMirror(uris);
184 }
185 }
186 }
187
188 std::string
getBestMirror(const std::deque<std::string> & uris) const189 AdaptiveURISelector::getBestMirror(const std::deque<std::string>& uris) const
190 {
191 /* Here we return one of the bests mirrors */
192 int max = getMaxDownloadSpeed(uris);
193 int min = max - (int)(max * 0.25);
194 std::deque<std::string> bests = getUrisBySpeed(uris, min);
195
196 if (bests.size() < 2) {
197 std::string uri = getMaxDownloadSpeedUri(uris);
198 A2_LOG_DEBUG(fmt("AdaptiveURISelector: choosing the best mirror :"
199 " %.2fKB/s %s (other mirrors are at least 25%% slower)",
200 (float)max / 1024, uri.c_str()));
201 return uri;
202 }
203 else {
204 std::string uri = selectRandomUri(bests);
205 A2_LOG_DEBUG(fmt("AdaptiveURISelector: choosing randomly one of the best"
206 " mirrors (range [%.2fKB/s, %.2fKB/s]): %s",
207 (float)min / 1024, (float)max / 1024, uri.c_str()));
208 return uri;
209 }
210 }
211
resetCounters()212 void AdaptiveURISelector::resetCounters()
213 {
214 nbConnections_ = 1;
215 nbServerToEvaluate_ = requestGroup_->getOption()->getAsInt(PREF_SPLIT) - 1;
216 }
217
tuneDownloadCommand(const std::deque<std::string> & uris,DownloadCommand * command)218 void AdaptiveURISelector::tuneDownloadCommand(
219 const std::deque<std::string>& uris, DownloadCommand* command)
220 {
221 adjustLowestSpeedLimit(uris, command);
222 }
223
adjustLowestSpeedLimit(const std::deque<std::string> & uris,DownloadCommand * command) const224 void AdaptiveURISelector::adjustLowestSpeedLimit(
225 const std::deque<std::string>& uris, DownloadCommand* command) const
226 {
227 int lowest = requestGroup_->getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT);
228 if (lowest > 0) {
229 int low_lowest = 4_k;
230 int max = getMaxDownloadSpeed(uris);
231 if (max > 0 && lowest > max / 4) {
232 A2_LOG_NOTICE(fmt(_("Lowering lowest-speed-limit since known max speed is"
233 " too near (new:%d was:%d max:%d)"),
234 max / 4, lowest, max));
235 command->setLowestDownloadSpeedLimit(max / 4);
236 }
237 else if (max == 0 && lowest > low_lowest) {
238 A2_LOG_NOTICE(fmt(_("Lowering lowest-speed-limit since we have no clue"
239 " about available speed (now:%d was:%d)"),
240 low_lowest, lowest));
241 command->setLowestDownloadSpeedLimit(low_lowest);
242 }
243 }
244 }
245
246 namespace {
getUriMaxSpeed(std::shared_ptr<ServerStat> ss)247 int getUriMaxSpeed(std::shared_ptr<ServerStat> ss)
248 {
249 return std::max(ss->getSingleConnectionAvgSpeed(),
250 ss->getMultiConnectionAvgSpeed());
251 }
252 } // namespace
253
getMaxDownloadSpeed(const std::deque<std::string> & uris) const254 int AdaptiveURISelector::getMaxDownloadSpeed(
255 const std::deque<std::string>& uris) const
256 {
257 std::string uri = getMaxDownloadSpeedUri(uris);
258 if (uri == A2STR::NIL)
259 return 0;
260 return getUriMaxSpeed(getServerStats(uri));
261 }
262
getMaxDownloadSpeedUri(const std::deque<std::string> & uris) const263 std::string AdaptiveURISelector::getMaxDownloadSpeedUri(
264 const std::deque<std::string>& uris) const
265 {
266 int max = -1;
267 std::string uri = A2STR::NIL;
268 for (auto& u : uris) {
269 std::shared_ptr<ServerStat> ss = getServerStats(u);
270 if (!ss)
271 continue;
272
273 if ((int)ss->getSingleConnectionAvgSpeed() > max) {
274 max = ss->getSingleConnectionAvgSpeed();
275 uri = u;
276 }
277 if ((int)ss->getMultiConnectionAvgSpeed() > max) {
278 max = ss->getMultiConnectionAvgSpeed();
279 uri = u;
280 }
281 }
282 return uri;
283 }
284
285 std::deque<std::string>
getUrisBySpeed(const std::deque<std::string> & uris,int min) const286 AdaptiveURISelector::getUrisBySpeed(const std::deque<std::string>& uris,
287 int min) const
288 {
289 std::deque<std::string> bests;
290 for (auto& uri : uris) {
291 std::shared_ptr<ServerStat> ss = getServerStats(uri);
292 if (!ss)
293 continue;
294 if (ss->getSingleConnectionAvgSpeed() > min ||
295 ss->getMultiConnectionAvgSpeed() > min) {
296 bests.push_back(uri);
297 }
298 }
299 return bests;
300 }
301
302 std::string
selectRandomUri(const std::deque<std::string> & uris) const303 AdaptiveURISelector::selectRandomUri(const std::deque<std::string>& uris) const
304 {
305 int pos = SimpleRandomizer::getInstance()->getRandomNumber(uris.size());
306 auto i = std::begin(uris);
307 i = i + pos;
308 return *i;
309 }
310
getFirstNotTestedUri(const std::deque<std::string> & uris) const311 std::string AdaptiveURISelector::getFirstNotTestedUri(
312 const std::deque<std::string>& uris) const
313 {
314 for (const auto& i : uris) {
315 std::shared_ptr<ServerStat> ss = getServerStats(i);
316 if (!ss)
317 return i;
318 }
319 return A2STR::NIL;
320 }
321
getFirstToTestUri(const std::deque<std::string> & uris) const322 std::string AdaptiveURISelector::getFirstToTestUri(
323 const std::deque<std::string>& uris) const
324 {
325 int counter;
326 int power;
327 for (const auto& u : uris) {
328 std::shared_ptr<ServerStat> ss = getServerStats(u);
329 if (!ss)
330 continue;
331 counter = ss->getCounter();
332 if (counter > 8)
333 continue;
334 power = (int)pow(2.0, (float)counter);
335 /* We test the mirror another time if it has not been
336 * tested since 2^counter days */
337 if (ss->getLastUpdated().difference() > std::chrono::hours(power * 24)) {
338 return u;
339 }
340 }
341 return A2STR::NIL;
342 }
343
344 std::shared_ptr<ServerStat>
getServerStats(const std::string & uri) const345 AdaptiveURISelector::getServerStats(const std::string& uri) const
346 {
347 uri_split_result us;
348 if (uri_split(&us, uri.c_str()) == 0) {
349 std::string host = uri::getFieldString(us, USR_HOST, uri.c_str());
350 std::string protocol = uri::getFieldString(us, USR_SCHEME, uri.c_str());
351 return serverStatMan_->find(host, protocol);
352 }
353 else {
354 return nullptr;
355 }
356 }
357
getNbTestedServers(const std::deque<std::string> & uris) const358 int AdaptiveURISelector::getNbTestedServers(
359 const std::deque<std::string>& uris) const
360 {
361 int counter = 0;
362 for (const auto& u : uris) {
363 std::shared_ptr<ServerStat> ss = getServerStats(u);
364 if (!ss)
365 ++counter;
366 }
367 return uris.size() - counter;
368 }
369
370 } // namespace aria2
371