1 /* <!-- copyright */
2 /*
3 * aria2 - The high speed download utility
4 *
5 * Copyright (C) 2013 Tatsuhiro Tsujikawa
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 * In addition, as a special exception, the copyright holders give
22 * permission to link the code of portions of this program with the
23 * OpenSSL library under certain conditions as described in each
24 * individual source file, and distribute linked combinations
25 * including the two.
26 * You must obey the GNU General Public License in all respects
27 * for all of the code used other than OpenSSL. If you modify
28 * file(s) with this exception, you may extend this exception to your
29 * version of the file(s), but you are not obligated to do so. If you
30 * do not wish to do so, delete this exception statement from your
31 * version. If you delete this exception statement from all source
32 * files in the program, then also delete it here.
33 */
34 /* copyright --> */
35 #include "CookieStorage.h"
36
37 #include <cstring>
38 #include <cstdio>
39 #include <algorithm>
40
41 #include "util.h"
42 #include "LogFactory.h"
43 #include "Logger.h"
44 #include "DlAbortEx.h"
45 #include "fmt.h"
46 #include "NsCookieParser.h"
47 #include "File.h"
48 #include "a2functional.h"
49 #include "A2STR.h"
50 #include "message.h"
51 #include "cookie_helper.h"
52 #include "BufferedFile.h"
53 #ifdef HAVE_SQLITE3
54 # include "Sqlite3CookieParserImpl.h"
55 #endif // HAVE_SQLITE3
56
57 namespace aria2 {
58
DomainNode(std::string label,DomainNode * parent)59 DomainNode::DomainNode(std::string label, DomainNode* parent)
60 : label_{std::move(label)},
61 parent_{parent},
62 lastAccessTime_{0},
63 lruAccessTime_{0},
64 inLru_{false}
65 {
66 }
67
findCookie(std::vector<const Cookie * > & out,const std::string & requestHost,const std::string & requestPath,time_t now,bool secure)68 void DomainNode::findCookie(std::vector<const Cookie*>& out,
69 const std::string& requestHost,
70 const std::string& requestPath, time_t now,
71 bool secure)
72 {
73 if (cookies_) {
74 for (auto& c : *cookies_) {
75 if (c->match(requestHost, requestPath, now, secure)) {
76 c->setLastAccessTime(now);
77 out.push_back(c.get());
78 }
79 }
80 }
81 }
82
addCookie(std::unique_ptr<Cookie> cookie,time_t now)83 bool DomainNode::addCookie(std::unique_ptr<Cookie> cookie, time_t now)
84 {
85 using namespace std::placeholders;
86 setLastAccessTime(now);
87 if (!cookies_) {
88 if (cookie->isExpired(now)) {
89 return false;
90 }
91 else {
92 cookies_ = make_unique<std::deque<std::unique_ptr<Cookie>>>();
93 cookies_->push_back(std::move(cookie));
94 return true;
95 }
96 }
97
98 auto i = std::find_if(
99 std::begin(*cookies_), std::end(*cookies_),
100 [&](const std::unique_ptr<Cookie>& c) { return *c == *cookie; });
101 if (i == std::end(*cookies_)) {
102 if (cookie->isExpired(now)) {
103 return false;
104 }
105 else {
106 if (cookies_->size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) {
107 cookies_->erase(std::remove_if(std::begin(*cookies_),
108 std::end(*cookies_),
109 std::bind(&Cookie::isExpired, _1, now)),
110 std::end(*cookies_));
111 if (cookies_->size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) {
112 auto m = std::min_element(std::begin(*cookies_), std::end(*cookies_),
113 [](const std::unique_ptr<Cookie>& lhs,
114 const std::unique_ptr<Cookie>& rhs) {
115 return lhs->getLastAccessTime() <
116 rhs->getLastAccessTime();
117 });
118 *m = std::move(cookie);
119 }
120 else {
121 cookies_->push_back(std::move(cookie));
122 }
123 }
124 else {
125 cookies_->push_back(std::move(cookie));
126 }
127 return true;
128 }
129 }
130 else if (cookie->isExpired(now)) {
131 cookies_->erase(i);
132 return false;
133 }
134 else {
135 cookie->setCreationTime((*i)->getCreationTime());
136 *i = std::move(cookie);
137 return true;
138 }
139 }
140
contains(const Cookie & cookie) const141 bool DomainNode::contains(const Cookie& cookie) const
142 {
143 if (cookies_) {
144 for (auto& i : *cookies_) {
145 if (*i == cookie) {
146 return true;
147 }
148 }
149 }
150 return false;
151 }
152
writeCookie(BufferedFile & fp) const153 bool DomainNode::writeCookie(BufferedFile& fp) const
154 {
155 if (cookies_) {
156 for (const auto& c : *cookies_) {
157 std::string data = c->toNsCookieFormat();
158 data += "\n";
159 if (fp.write(data.data(), data.size()) != data.size()) {
160 return false;
161 }
162 }
163 }
164 return true;
165 }
166
countCookie() const167 size_t DomainNode::countCookie() const
168 {
169 if (cookies_) {
170 return cookies_->size();
171 }
172 else {
173 return 0;
174 }
175 }
176
clearCookie()177 void DomainNode::clearCookie() { cookies_->clear(); }
178
setLastAccessTime(time_t lastAccessTime)179 void DomainNode::setLastAccessTime(time_t lastAccessTime)
180 {
181 lastAccessTime_ = lastAccessTime;
182 }
183
getLastAccessTime() const184 time_t DomainNode::getLastAccessTime() const { return lastAccessTime_; }
185
setLruAccessTime(time_t t)186 void DomainNode::setLruAccessTime(time_t t) { lruAccessTime_ = t; }
187
getLruAccessTime() const188 time_t DomainNode::getLruAccessTime() const { return lruAccessTime_; }
189
empty() const190 bool DomainNode::empty() const { return !cookies_ || cookies_->empty(); }
191
hasNext() const192 bool DomainNode::hasNext() const { return !next_.empty(); }
193
getParent() const194 DomainNode* DomainNode::getParent() const { return parent_; }
195
removeNode(DomainNode * node)196 void DomainNode::removeNode(DomainNode* node) { next_.erase(node->getLabel()); }
197
findNext(const std::string & label) const198 DomainNode* DomainNode::findNext(const std::string& label) const
199 {
200 auto i = next_.find(label);
201 if (i == std::end(next_)) {
202 return nullptr;
203 }
204 else {
205 return (*i).second.get();
206 }
207 }
208
addNext(std::string label,std::unique_ptr<DomainNode> node)209 DomainNode* DomainNode::addNext(std::string label,
210 std::unique_ptr<DomainNode> node)
211 {
212 auto& res = next_[std::move(label)] = std::move(node);
213 return res.get();
214 }
215
getLabel() const216 const std::string& DomainNode::getLabel() const { return label_; }
217
getInLru() const218 bool DomainNode::getInLru() const { return inLru_; }
219
setInLru(bool f)220 void DomainNode::setInLru(bool f) { inLru_ = f; }
221
CookieStorage()222 CookieStorage::CookieStorage() : rootNode_{make_unique<DomainNode>("", nullptr)}
223 {
224 }
225
226 namespace {
227 // See CookieStorageTest::testDomainIsFull() in CookieStorageTest.cc
228 const size_t DOMAIN_EVICTION_TRIGGER = 2000;
229
230 const double DOMAIN_EVICTION_RATE = 0.1;
231 } // namespace
232
233 namespace {
splitDomainLabel(const std::string & domain)234 std::vector<std::string> splitDomainLabel(const std::string& domain)
235 {
236 auto labels = std::vector<std::string>{};
237 if (util::isNumericHost(domain)) {
238 labels.push_back(domain);
239 }
240 else {
241 util::split(std::begin(domain), std::end(domain),
242 std::back_inserter(labels), '.');
243 }
244 return labels;
245 }
246 } // namespace
247
getLruTrackerSize() const248 size_t CookieStorage::getLruTrackerSize() const { return lruTracker_.size(); }
249
evictNode(size_t delnum)250 void CookieStorage::evictNode(size_t delnum)
251 {
252 for (; delnum > 0 && !lruTracker_.empty(); --delnum) {
253 auto node = (*lruTracker_.begin()).second;
254 lruTracker_.erase(lruTracker_.begin());
255 node->setInLru(false);
256 node->clearCookie();
257 while (node->empty() && !node->hasNext()) {
258 auto parent = node->getParent();
259 parent->removeNode(node);
260 if (!parent->empty() || parent->hasNext() || parent == rootNode_.get()) {
261 break;
262 }
263 node = parent;
264 if (node->getInLru()) {
265 lruTracker_.erase({node->getLruAccessTime(), node});
266 node->setInLru(false);
267 }
268 }
269 }
270 }
271
getRootNode() const272 const DomainNode* CookieStorage::getRootNode() const { return rootNode_.get(); }
273
store(std::unique_ptr<Cookie> cookie,time_t now)274 bool CookieStorage::store(std::unique_ptr<Cookie> cookie, time_t now)
275 {
276 if (lruTracker_.size() >= DOMAIN_EVICTION_TRIGGER) {
277 auto delnum = size_t(lruTracker_.size() * DOMAIN_EVICTION_RATE);
278 evictNode(delnum);
279 }
280 auto labels = splitDomainLabel(cookie->getDomain());
281 auto node = rootNode_.get();
282 for (auto i = labels.rbegin(), eoi = labels.rend(); i != eoi; ++i) {
283 auto nextNode = node->findNext(*i);
284 if (nextNode) {
285 node = nextNode;
286 }
287 else {
288 node = node->addNext(*i, make_unique<DomainNode>(*i, node));
289 }
290 }
291 bool ok = node->addCookie(std::move(cookie), now);
292 if (ok) {
293 updateLru(node, now);
294 }
295 return ok;
296 }
297
updateLru(DomainNode * node,time_t now)298 void CookieStorage::updateLru(DomainNode* node, time_t now)
299 {
300 if (node->getInLru()) {
301 lruTracker_.erase({node->getLruAccessTime(), node});
302 }
303 else {
304 node->setInLru(true);
305 }
306 node->setLruAccessTime(now);
307 lruTracker_.insert({node->getLruAccessTime(), node});
308 }
309
parseAndStore(const std::string & setCookieString,const std::string & requestHost,const std::string & defaultPath,time_t now)310 bool CookieStorage::parseAndStore(const std::string& setCookieString,
311 const std::string& requestHost,
312 const std::string& defaultPath, time_t now)
313 {
314 auto cookie = cookie::parse(setCookieString, requestHost, defaultPath, now);
315 return cookie && store(std::move(cookie), now);
316 }
317
318 namespace {
319 struct CookiePathDivider {
320 const Cookie* cookie_;
321 int pathDepth_;
CookiePathDivideraria2::__anon586b6b870511::CookiePathDivider322 CookiePathDivider(const Cookie* cookie) : cookie_(cookie), pathDepth_(0)
323 {
324 const std::string& path = cookie_->getPath();
325 if (!path.empty()) {
326 for (size_t i = 1, len = path.size(); i < len; ++i) {
327 if (path[i] == '/' && path[i - 1] != '/') {
328 ++pathDepth_;
329 }
330 }
331 if (path[path.size() - 1] != '/') {
332 ++pathDepth_;
333 }
334 }
335 }
336 };
337 } // namespace
338
339 namespace {
340 class CookiePathDividerConverter {
341 public:
operator ()(const Cookie * cookie) const342 CookiePathDivider operator()(const Cookie* cookie) const
343 {
344 return CookiePathDivider(cookie);
345 }
346
operator ()(const CookiePathDivider & cookiePathDivider) const347 const Cookie* operator()(const CookiePathDivider& cookiePathDivider) const
348 {
349 return cookiePathDivider.cookie_;
350 }
351 };
352 } // namespace
353
354 namespace {
355 class OrderByPathDepthDesc : public std::binary_function<Cookie, Cookie, bool> {
356 public:
operator ()(const CookiePathDivider & lhs,const CookiePathDivider & rhs) const357 bool operator()(const CookiePathDivider& lhs,
358 const CookiePathDivider& rhs) const
359 {
360 // From http://tools.ietf.org/html/rfc6265#section-5.4:
361 // 2. The user agent SHOULD sort the cookie-list in the following
362 // order:
363 //
364 // * Cookies with longer paths are listed before cookies with
365 // shorter paths.
366 //
367 // * Among cookies that have equal-length path fields, cookies with
368 // earlier creation-times are listed before cookies with later
369 // creation-times.
370 return lhs.pathDepth_ > rhs.pathDepth_ ||
371 (!(rhs.pathDepth_ > lhs.pathDepth_) &&
372 lhs.cookie_->getCreationTime() < rhs.cookie_->getCreationTime());
373 }
374 };
375 } // namespace
376
377 namespace {
findNode(const std::string & domain,DomainNode * node)378 DomainNode* findNode(const std::string& domain, DomainNode* node)
379 {
380 auto labels = splitDomainLabel(domain);
381 for (auto i = labels.rbegin(), eoi = labels.rend(); i != eoi && node; ++i) {
382 node = node->findNext(*i);
383 }
384 return node;
385 }
386 } // namespace
387
contains(const Cookie & cookie) const388 bool CookieStorage::contains(const Cookie& cookie) const
389 {
390 auto node = findNode(cookie.getDomain(), rootNode_.get());
391 return node && node->contains(cookie);
392 }
393
394 std::vector<const Cookie*>
criteriaFind(const std::string & requestHost,const std::string & requestPath,time_t now,bool secure)395 CookieStorage::criteriaFind(const std::string& requestHost,
396 const std::string& requestPath, time_t now,
397 bool secure)
398 {
399 auto res = std::vector<const Cookie*>{};
400 if (requestPath.empty()) {
401 return res;
402 }
403 auto labels = splitDomainLabel(requestHost);
404 auto node = rootNode_.get();
405 for (auto i = labels.rbegin(), eoi = labels.rend(); i != eoi; ++i) {
406 auto nextNode = node->findNext(*i);
407 if (!nextNode) {
408 break;
409 }
410 nextNode->setLastAccessTime(now);
411 if (nextNode->getInLru()) {
412 updateLru(nextNode, now);
413 }
414 nextNode->findCookie(res, requestHost, requestPath, now, secure);
415 node = nextNode;
416 }
417 auto divs = std::vector<CookiePathDivider>{};
418 std::transform(std::begin(res), std::end(res), std::back_inserter(divs),
419 CookiePathDividerConverter{});
420 std::sort(std::begin(divs), std::end(divs), OrderByPathDepthDesc{});
421 std::transform(std::begin(divs), std::end(divs), std::begin(res),
422 CookiePathDividerConverter{});
423 return res;
424 }
425
size() const426 size_t CookieStorage::size() const
427 {
428 size_t n = 0;
429 for (auto& p : lruTracker_) {
430 n += p.second->countCookie();
431 }
432 return n;
433 }
434
load(const std::string & filename,time_t now)435 bool CookieStorage::load(const std::string& filename, time_t now)
436 {
437 char header[16]; // "SQLite format 3" plus \0
438 size_t headlen;
439 {
440 BufferedFile fp{filename.c_str(), BufferedFile::READ};
441 if (!fp) {
442 A2_LOG_ERROR(fmt("Failed to open cookie file %s", filename.c_str()));
443 return false;
444 }
445 headlen = fp.read(header, sizeof(header));
446 }
447 try {
448 if (headlen == 16 && memcmp(header, "SQLite format 3\0", 16) == 0) {
449 #ifdef HAVE_SQLITE3
450 try {
451 auto cookies = Sqlite3MozCookieParser(filename).parse();
452 storeCookies(std::make_move_iterator(std::begin(cookies)),
453 std::make_move_iterator(std::end(cookies)), now);
454 }
455 catch (RecoverableException& e) {
456 A2_LOG_INFO_EX(EX_EXCEPTION_CAUGHT, e);
457 A2_LOG_INFO("This does not look like Firefox3 cookie file."
458 " Retrying, assuming it is Chromium cookie file.");
459 // Try chrome cookie format
460 auto cookies = Sqlite3ChromiumCookieParser(filename).parse();
461 storeCookies(std::make_move_iterator(std::begin(cookies)),
462 std::make_move_iterator(std::end(cookies)), now);
463 }
464 #else // !HAVE_SQLITE3
465 throw DL_ABORT_EX(
466 "Cannot read SQLite3 database because SQLite3 support is disabled by"
467 " configuration.");
468 #endif // !HAVE_SQLITE3
469 }
470 else {
471 auto cookies = NsCookieParser().parse(filename, now);
472 storeCookies(std::make_move_iterator(std::begin(cookies)),
473 std::make_move_iterator(std::end(cookies)), now);
474 }
475 return true;
476 }
477 catch (RecoverableException& e) {
478 A2_LOG_ERROR(fmt("Failed to load cookies from %s", filename.c_str()));
479 return false;
480 }
481 }
482
saveNsFormat(const std::string & filename)483 bool CookieStorage::saveNsFormat(const std::string& filename)
484 {
485 auto tempfilename = filename;
486 tempfilename += "__temp";
487 {
488 BufferedFile fp{tempfilename.c_str(), BufferedFile::WRITE};
489 if (!fp) {
490 A2_LOG_ERROR(fmt("Cannot create cookie file %s", filename.c_str()));
491 return false;
492 }
493 for (auto& p : lruTracker_) {
494 if (!p.second->writeCookie(fp)) {
495 A2_LOG_ERROR(fmt("Failed to save cookies to %s", filename.c_str()));
496 return false;
497 }
498 }
499 if (fp.close() == EOF) {
500 A2_LOG_ERROR(fmt("Failed to save cookies to %s", filename.c_str()));
501 return false;
502 }
503 }
504 if (File(tempfilename).renameTo(filename)) {
505 return true;
506 }
507 else {
508 A2_LOG_ERROR(fmt("Could not rename file %s as %s", tempfilename.c_str(),
509 filename.c_str()));
510 return false;
511 }
512 }
513
514 } // namespace aria2
515