1 /**
2 * @file posix/net.cpp
3 * @brief POSIX network access layer (using cURL + c-ares)
4 *
5 * (c) 2013-2017 by Mega Limited, Auckland, New Zealand
6 *
7 * This file is part of the MEGA SDK - Client Access Engine.
8 *
9 * Applications using the MEGA API must present a valid application key
10 * and comply with the the rules set forth in the Terms of Service.
11 *
12 * The MEGA SDK is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
15 *
16 * @copyright Simplified (2-clause) BSD License.
17 *
18 * You should have received a copy of the license along with this
19 * program.
20 */
21
22 #include "mega.h"
23 #include "mega/posix/meganet.h"
24 #include "mega/logging.h"
25
26 #if defined(__ANDROID__) && ARES_VERSION >= 0x010F00
27 #include <jni.h>
28 extern JavaVM *MEGAjvm;
29 #endif
30
31 #define IPV6_RETRY_INTERVAL_DS 72000
32 #define DNS_CACHE_TIMEOUT_DS 18000
33 #define DNS_CACHE_EXPIRES 0
34 #define MAX_SPEED_CONTROL_TIMEOUT_MS 500
35
36 namespace mega {
37
38
39 #if defined(_WIN32)
40
sharedEventHandle()41 HANDLE SockInfo::sharedEventHandle()
42 {
43 return mSharedEvent;
44 }
45
createAssociateEvent()46 bool SockInfo::createAssociateEvent()
47 {
48 int events = (mode & SockInfo::READ ? FD_READ : 0) | (mode & SockInfo::WRITE ? FD_WRITE : 0);
49
50 if (associatedHandleEvents != events)
51 {
52 if (WSAEventSelect(fd, mSharedEvent, events))
53 {
54 auto err = WSAGetLastError();
55 LOG_err << "WSAEventSelect failed " << fd << " " << mSharedEvent << " " << events << " " << err;
56 closeEvent();
57 return false;
58 }
59 associatedHandleEvents = events;
60 }
61 return true;
62 }
63
checkEvent(bool & read,bool & write)64 bool SockInfo::checkEvent(bool& read, bool& write)
65 {
66 WSANETWORKEVENTS wne;
67 memset(&wne, 0, sizeof(wne));
68 auto err = WSAEnumNetworkEvents(fd, NULL, &wne);
69 if (err)
70 {
71 auto e = WSAGetLastError();
72 LOG_err << "WSAEnumNetworkEvents error " << e;
73 return false;
74 }
75
76 read = 0 != (FD_READ & wne.lNetworkEvents);
77 write = 0 != (FD_WRITE & wne.lNetworkEvents);
78
79 // Even though the writeable network event occurred, double check there is no space available in the write buffer
80 // Otherwise curl can report a spurious timeout error
81
82 if (FD_WRITE & associatedHandleEvents)
83 {
84 // per https://curl.haxx.se/mail/lib-2009-10/0313.html check if the socket has any buffer space
85
86 // The trick is that we want to wait on the event handle to know when we can read and write
87 // that works fine for read, however for write the event is not signalled in the normal case
88 // where curl wrote to the socket, but not enough to cause it to become unwriteable for now.
89 // So, we need to signal curl to write again if it has more data to write, if the socket can take
90 // more data. This trick with WSASend for 0 bytes enables that - if it fails with would-block
91 // then we can stop asking curl to write to the socket, and start waiting on the handle to
92 // know when to try again.
93 // If curl has finished writing to the socket, it will call us back to change the mode to read only.
94
95 WSABUF buf{ 0, (CHAR*)&buf };
96 DWORD bSent = 0;
97 auto writeResult = WSASend(fd, &buf, 1, &bSent, 0, NULL, NULL);
98 auto writeError = WSAGetLastError();
99 write = writeResult == 0 || (writeError != WSAEWOULDBLOCK && writeError != WSAENOTCONN);
100 if (writeResult != 0 && writeError != WSAEWOULDBLOCK && writeError != WSAENOTCONN)
101 {
102 LOG_err << "Unexpected WSASend check error: " << writeError;
103 }
104 }
105
106 if (read || write)
107 {
108 signalledWrite = signalledWrite || write;
109 return true; // if we return true, both read and write must have been set.
110 }
111 return false;
112 }
113
closeEvent(bool adjustSocket)114 void SockInfo::closeEvent(bool adjustSocket)
115 {
116 if (adjustSocket)
117 {
118 #ifdef DEBUG
119 int result =
120 #endif
121 WSAEventSelect(fd, NULL, 0); // cancel association by specifying lNetworkEvents = 0
122 assert(result == 0);
123 }
124 associatedHandleEvents = 0;
125 signalledWrite = false;
126 }
127
SockInfo(SockInfo && o)128 SockInfo::SockInfo(SockInfo&& o)
129 : fd(o.fd)
130 , mode(o.mode)
131 , signalledWrite(o.signalledWrite)
132 , mSharedEvent(o.mSharedEvent)
133 , associatedHandleEvents(o.associatedHandleEvents)
134 {
135 }
136
~SockInfo()137 SockInfo::~SockInfo()
138 {
139 }
140 #endif
141
142 std::mutex CurlHttpIO::curlMutex;
143
144 #ifdef LIBRESSL_VERSION_NUMBER
145 /* missing RSA_get0_n() RSA_get0_e() */
146 #undef USE_OPENSSL
147 #endif
148
149 #if defined(USE_OPENSSL) && !defined(OPENSSL_IS_BORINGSSL)
150
151 std::recursive_mutex **CurlHttpIO::sslMutexes = NULL;
152 static std::mutex lock_init_mutex;
locking_function(int mode,int lockNumber,const char *,int)153 void CurlHttpIO::locking_function(int mode, int lockNumber, const char *, int)
154 {
155 std::recursive_mutex *mutex = sslMutexes[lockNumber];
156 if (mutex == NULL)
157 {
158 // we still have to be careful about multiple threads getting to this point simultaneously
159 lock_init_mutex.lock();
160 if (!(mutex = sslMutexes[lockNumber]))
161 {
162 mutex = sslMutexes[lockNumber] = new std::recursive_mutex;
163 }
164 lock_init_mutex.unlock();
165 }
166
167 if (mode & CRYPTO_LOCK)
168 {
169 mutex->lock();
170 }
171 else
172 {
173 mutex->unlock();
174 }
175 }
176
177 #if OPENSSL_VERSION_NUMBER >= 0x10000000 || defined (LIBRESSL_VERSION_NUMBER)
id_function(CRYPTO_THREADID * id)178 void CurlHttpIO::id_function(CRYPTO_THREADID* id)
179 {
180 CRYPTO_THREADID_set_pointer(id, (void *)THREAD_CLASS::currentThreadId());
181 }
182 #else
id_function()183 unsigned long CurlHttpIO::id_function()
184 {
185 return THREAD_CLASS::currentThreadId();
186 }
187 #endif
188
189 #endif
190
CurlHttpIO()191 CurlHttpIO::CurlHttpIO()
192 {
193 #ifdef WIN32
194 mSocketsWaitEvent = WSACreateEvent();
195 if (mSocketsWaitEvent == WSA_INVALID_EVENT)
196 {
197 LOG_err << "Failed to create WSA event for cURL";
198 }
199 #endif
200
201 curl_version_info_data* data = curl_version_info(CURLVERSION_NOW);
202 if (data->version)
203 {
204 LOG_debug << "cURL version: " << data->version;
205 }
206
207 if (data->ssl_version)
208 {
209 LOG_debug << "SSL version: " << data->ssl_version;
210
211 string curlssl = data->ssl_version;
212 tolower_string(curlssl);
213 if (strstr(curlssl.c_str(), "gskit"))
214 {
215 LOG_fatal << "Unsupported SSL backend (GSKit). Aborting.";
216 throw std::runtime_error("Unsupported SSL backend (GSKit). Aborting.");
217 }
218
219 if (data->version_num < 0x072c00 // At least cURL 7.44.0
220 #ifdef USE_OPENSSL
221 && !(strstr(curlssl.c_str(), "openssl") && data->version_num > 0x070b00)
222 // or cURL 7.11.0 with OpenSSL
223 #endif
224 )
225 {
226 LOG_fatal << "cURL built without public key pinning support. Aborting.";
227 throw std::runtime_error("cURL built without public key pinning support. Aborting.");
228 }
229 }
230
231 if (data->libz_version)
232 {
233 LOG_debug << "libz version: " << data->libz_version;
234 }
235
236 int i;
237 for (i = 0; data->protocols[i]; i++)
238 {
239 if (strstr(data->protocols[i], "http"))
240 {
241 break;
242 }
243 }
244
245 if (!data->protocols[i] || !(data->features & CURL_VERSION_SSL))
246 {
247 LOG_fatal << "cURL built without HTTP/HTTPS support. Aborting.";
248 throw std::runtime_error("cURL built without HTTP/HTTPS support. Aborting.");
249 }
250
251 curlipv6 = data->features & CURL_VERSION_IPV6;
252 LOG_debug << "IPv6 enabled: " << curlipv6;
253
254 dnsok = false;
255 reset = false;
256 statechange = false;
257 disconnecting = false;
258 maxspeed[GET] = 0;
259 maxspeed[PUT] = 0;
260 pkpErrors = 0;
261
262 WAIT_CLASS::bumpds();
263 lastdnspurge = Waiter::ds + DNS_CACHE_TIMEOUT_DS / 2;
264
265 curlMutex.lock();
266
267 #if defined(USE_OPENSSL) && !defined(OPENSSL_IS_BORINGSSL)
268
269 if (!CRYPTO_get_locking_callback()
270 #if OPENSSL_VERSION_NUMBER >= 0x10000000 || defined (LIBRESSL_VERSION_NUMBER)
271 && !CRYPTO_THREADID_get_callback())
272 #else
273 && !CRYPTO_get_id_callback())
274 #endif
275 {
276 LOG_debug << "Initializing OpenSSL locking callbacks";
277 int numLocks = CRYPTO_num_locks();
278 sslMutexes = new std::recursive_mutex*[numLocks];
279 memset(sslMutexes, 0, numLocks * sizeof(std::recursive_mutex*));
280 #if OPENSSL_VERSION_NUMBER >= 0x10000000 || defined (LIBRESSL_VERSION_NUMBER)
281 CRYPTO_THREADID_set_callback(CurlHttpIO::id_function);
282 #else
283 CRYPTO_set_id_callback(CurlHttpIO::id_function);
284 #endif
285 CRYPTO_set_locking_callback(CurlHttpIO::locking_function);
286 }
287
288 #endif
289
290 if (++instanceCount == 1)
291 {
292 curl_global_init(CURL_GLOBAL_DEFAULT);
293 ares_library_init(ARES_LIB_INIT_ALL);
294 }
295
296 #if defined(__ANDROID__) && ARES_VERSION >= 0x010F00
297 initialize_android();
298 #endif
299
300 curlMutex.unlock();
301
302 curlm[API] = curl_multi_init();
303 curlm[GET] = curl_multi_init();
304 curlm[PUT] = curl_multi_init();
305 numconnections[API] = 0;
306 numconnections[GET] = 0;
307 numconnections[PUT] = 0;
308 curlsocketsprocessed = true;
309
310 struct ares_options options;
311 options.tries = 2;
312 ares_init_options(&ares, &options, ARES_OPT_TRIES);
313 arestimeout = -1;
314 filterDNSservers();
315
316 curl_multi_setopt(curlm[API], CURLMOPT_SOCKETFUNCTION, api_socket_callback);
317 curl_multi_setopt(curlm[API], CURLMOPT_SOCKETDATA, this);
318 curl_multi_setopt(curlm[API], CURLMOPT_TIMERFUNCTION, api_timer_callback);
319 curl_multi_setopt(curlm[API], CURLMOPT_TIMERDATA, this);
320 curltimeoutreset[API] = -1;
321 arerequestspaused[API] = false;
322
323 curl_multi_setopt(curlm[GET], CURLMOPT_SOCKETFUNCTION, download_socket_callback);
324 curl_multi_setopt(curlm[GET], CURLMOPT_SOCKETDATA, this);
325 curl_multi_setopt(curlm[GET], CURLMOPT_TIMERFUNCTION, download_timer_callback);
326 curl_multi_setopt(curlm[GET], CURLMOPT_TIMERDATA, this);
327 #ifdef _WIN32
328 curl_multi_setopt(curlm[GET], CURLMOPT_MAXCONNECTS, 200);
329 #endif
330 curltimeoutreset[GET] = -1;
331 arerequestspaused[GET] = false;
332
333 curl_multi_setopt(curlm[PUT], CURLMOPT_SOCKETFUNCTION, upload_socket_callback);
334 curl_multi_setopt(curlm[PUT], CURLMOPT_SOCKETDATA, this);
335 curl_multi_setopt(curlm[PUT], CURLMOPT_TIMERFUNCTION, upload_timer_callback);
336 curl_multi_setopt(curlm[PUT], CURLMOPT_TIMERDATA, this);
337 #ifdef _WIN32
338 curl_multi_setopt(curlm[PUT], CURLMOPT_MAXCONNECTS, 200);
339 #endif
340
341 curltimeoutreset[PUT] = -1;
342 arerequestspaused[PUT] = false;
343
344 curlsh = curl_share_init();
345 curl_share_setopt(curlsh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
346 curl_share_setopt(curlsh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
347
348 contenttypejson = curl_slist_append(NULL, "Content-Type: application/json");
349 contenttypejson = curl_slist_append(contenttypejson, "Expect:");
350
351 contenttypebinary = curl_slist_append(NULL, "Content-Type: application/octet-stream");
352 contenttypebinary = curl_slist_append(contenttypebinary, "Expect:");
353
354 proxyinflight = 0;
355 ipv6requestsenabled = false;
356 ipv6proxyenabled = ipv6requestsenabled;
357 ipv6deactivationtime = Waiter::ds;
358 waiter = NULL;
359 proxyport = 0;
360 }
361
ipv6available()362 bool CurlHttpIO::ipv6available()
363 {
364 static int ipv6_works = -1;
365
366 if (ipv6_works != -1)
367 {
368 return ipv6_works;
369 }
370
371 curl_socket_t s = socket(PF_INET6, SOCK_DGRAM, 0);
372
373 if (s == -1)
374 {
375 ipv6_works = 0;
376 }
377 else
378 {
379 ipv6_works = curlipv6;
380 #ifdef _WIN32
381 closesocket(s);
382 #else
383 close(s);
384 #endif
385 }
386
387 return ipv6_works;
388 }
389
filterDNSservers()390 void CurlHttpIO::filterDNSservers()
391 {
392 string newservers;
393 string serverlist;
394 set<string> serverset;
395 vector<string> filteredservers;
396 ares_addr_node *servers;
397 ares_addr_node *server;
398 if (ares_get_servers(ares, &servers) == ARES_SUCCESS)
399 {
400 bool first = true;
401 bool filtered = false;
402 server = servers;
403 while (server)
404 {
405 char straddr[INET6_ADDRSTRLEN];
406 straddr[0] = 0;
407
408 if (server->family == AF_INET6)
409 {
410 mega_inet_ntop(PF_INET6, &server->addr, straddr, sizeof(straddr));
411 }
412 else if (server->family == AF_INET)
413 {
414 mega_inet_ntop(PF_INET, &server->addr, straddr, sizeof(straddr));
415 }
416 else
417 {
418 LOG_warn << "Unknown IP address family: " << server->family;
419 }
420
421 if (straddr[0])
422 {
423 serverlist.append(straddr);
424 serverlist.append(",");
425 }
426
427 if (straddr[0]
428 && serverset.find(straddr) == serverset.end()
429 && strncasecmp(straddr, "fec0:", 5)
430 && strncasecmp(straddr, "169.254.", 8))
431 {
432 if (!first)
433 {
434 newservers.append(",");
435 }
436
437 newservers.append(straddr);
438 serverset.insert(straddr);
439 first = false;
440 }
441 else
442 {
443 filtered = true;
444 if (!straddr[0])
445 {
446 LOG_debug << "Filtering unkwnown address of DNS server";
447 }
448 else if (serverset.find(straddr) == serverset.end())
449 {
450 serverset.insert(straddr);
451 filteredservers.push_back(straddr);
452 }
453 }
454
455 server = server->next;
456 }
457
458 if (serverlist.size())
459 {
460 serverlist.resize(serverlist.size() - 1);
461 }
462 LOG_debug << "DNS servers: " << serverlist;
463
464 if (filtered && (newservers.size() || filteredservers.size()))
465 {
466 for (unsigned int i = 0; i < filteredservers.size(); i++)
467 {
468 if (newservers.size())
469 {
470 newservers.append(",");
471 }
472
473 newservers.append(filteredservers[i]);
474 }
475
476 LOG_debug << "Setting filtered DNS servers: " << newservers;
477 ares_set_servers_csv(ares, newservers.c_str());
478 }
479
480 ares_free_data(servers);
481 }
482 }
483
addaresevents(Waiter * waiter)484 void CurlHttpIO::addaresevents(Waiter *waiter)
485 {
486 CodeCounter::ScopeTimer ccst(countAddAresEventsCode);
487
488 SockInfoMap prevAressockets; // if there are SockInfo records that were in use, and won't be anymore, they will be deleted with this
489 prevAressockets.swap(aressockets);
490
491 ares_socket_t socks[ARES_GETSOCK_MAXNUM];
492 int bitmask = ares_getsock(ares, socks, ARES_GETSOCK_MAXNUM);
493 for (int i = 0; i < ARES_GETSOCK_MAXNUM; i++)
494 {
495 bool readable = ARES_GETSOCK_READABLE(bitmask, i);
496 bool writeable = ARES_GETSOCK_WRITABLE(bitmask, i);
497
498 if (readable || writeable)
499 {
500 // take the old record from the prior version of the map, if there is one, and then we will update it
501 auto it = prevAressockets.find(socks[i]);
502 if (it == prevAressockets.end())
503 {
504 #ifdef WIN32
505 auto pair = aressockets.emplace(socks[i], SockInfo(mSocketsWaitEvent));
506 #else
507 auto pair = aressockets.emplace(socks[i], SockInfo());
508 #endif
509 it = pair.first;
510 }
511 else
512 {
513 auto pair = aressockets.emplace(socks[i], std::move(it->second));
514 prevAressockets.erase(it);
515 it = pair.first;
516 }
517 SockInfo& info = it->second;
518 info.mode = 0;
519
520 if (readable)
521 {
522 info.fd = socks[i];
523 info.mode |= SockInfo::READ;
524 }
525
526 if (writeable)
527 {
528 info.fd = socks[i];
529 info.mode |= SockInfo::WRITE;
530 }
531
532 #if defined(_WIN32)
533 info.createAssociateEvent();
534 #else
535 if (readable)
536 {
537 FD_SET(info.fd, &((PosixWaiter *)waiter)->rfds);
538 ((PosixWaiter *)waiter)->bumpmaxfd(info.fd);
539 }
540 if (writeable)
541 {
542 FD_SET(info.fd, &((PosixWaiter *)waiter)->wfds);
543 ((PosixWaiter *)waiter)->bumpmaxfd(info.fd);
544 }
545 #endif
546 }
547 }
548
549 #if defined(_WIN32)
550 for (auto& mapPair : prevAressockets)
551 {
552 // We pass false for c-ares becase we can't be sure if c-ares closed the socket or not
553 // If it's not using the socket, the event should not be triggered, and even if it is
554 // then we just do one extra loop.
555 mapPair.second.closeEvent(false);
556 }
557 #endif
558 }
559
addcurlevents(Waiter * waiter,direction_t d)560 void CurlHttpIO::addcurlevents(Waiter *waiter, direction_t d)
561 {
562 CodeCounter::ScopeTimer ccst(countAddCurlEventsCode);
563
564 #if defined(_WIN32)
565 bool anyWriters = false;
566 #endif
567
568 SockInfoMap &socketmap = curlsockets[d];
569 for (SockInfoMap::iterator it = socketmap.begin(); it != socketmap.end(); it++)
570 {
571 SockInfo &info = it->second;
572 if (!info.mode)
573 {
574 continue;
575 }
576
577 #if defined(_WIN32)
578 anyWriters = anyWriters || info.signalledWrite;
579 info.signalledWrite = false;
580 info.createAssociateEvent();
581 #else
582
583 if (info.mode & SockInfo::READ)
584 {
585 FD_SET(info.fd, &((PosixWaiter *)waiter)->rfds);
586 ((PosixWaiter *)waiter)->bumpmaxfd(info.fd);
587 }
588
589 if (info.mode & SockInfo::WRITE)
590 {
591 FD_SET(info.fd, &((PosixWaiter *)waiter)->wfds);
592 ((PosixWaiter *)waiter)->bumpmaxfd(info.fd);
593 }
594 #endif
595 }
596
597 #if defined(_WIN32)
598 if (anyWriters)
599 {
600 // so long as we are writing at least one socket, keep looping until the socket is full, then start waiting on its associated event
601 static_cast<WinWaiter*>(waiter)->maxds = 0;
602 }
603 #endif
604 }
605
checkevents(Waiter *)606 int CurlHttpIO::checkevents(Waiter*)
607 {
608 #ifdef WIN32
609 ResetEvent(mSocketsWaitEvent);
610 #endif
611 return 0;
612 }
613
closearesevents()614 void CurlHttpIO::closearesevents()
615 {
616 #if defined(_WIN32)
617 for (auto& mapPair : aressockets)
618 {
619 mapPair.second.closeEvent();
620 }
621 #endif
622 aressockets.clear();
623 }
624
closecurlevents(direction_t d)625 void CurlHttpIO::closecurlevents(direction_t d)
626 {
627 SockInfoMap &socketmap = curlsockets[d];
628 #if defined(_WIN32)
629 for (SockInfoMap::iterator it = socketmap.begin(); it != socketmap.end(); it++)
630 {
631 it->second.closeEvent(false);
632 }
633 #endif
634 socketmap.clear();
635 }
636
processaresevents()637 void CurlHttpIO::processaresevents()
638 {
639 CodeCounter::ScopeTimer ccst(countProcessAresEventsCode);
640
641 #ifndef _WIN32
642 fd_set *rfds = &((PosixWaiter *)waiter)->rfds;
643 fd_set *wfds = &((PosixWaiter *)waiter)->wfds;
644 #endif
645
646 for (auto& mapPair : aressockets)
647 {
648 SockInfo &info = mapPair.second;
649 if (!info.mode)
650 {
651 continue;
652 }
653
654 #if defined(_WIN32)
655 bool read, write;
656 if (info.checkEvent(read, write)) // if checkEvent returns true, both `read` and `write` have been set.
657 {
658 ares_process_fd(ares, read ? info.fd : ARES_SOCKET_BAD, write ? info.fd : ARES_SOCKET_BAD);
659 }
660 #else
661 if (((info.mode & SockInfo::READ) && FD_ISSET(info.fd, rfds)) || ((info.mode & SockInfo::WRITE) && FD_ISSET(info.fd, wfds)))
662 {
663 ares_process_fd(ares,
664 ((info.mode & SockInfo::READ) && FD_ISSET(info.fd, rfds)) ? info.fd : ARES_SOCKET_BAD,
665 ((info.mode & SockInfo::WRITE) && FD_ISSET(info.fd, wfds)) ? info.fd : ARES_SOCKET_BAD);
666 }
667 #endif
668 }
669
670 if (arestimeout >= 0 && arestimeout <= Waiter::ds)
671 {
672 arestimeout = -1;
673 ares_process_fd(ares, ARES_SOCKET_BAD, ARES_SOCKET_BAD);
674 }
675 }
676
processcurlevents(direction_t d)677 void CurlHttpIO::processcurlevents(direction_t d)
678 {
679 CodeCounter::ScopeTimer ccst(countProcessCurlEventsCode);
680
681 #ifndef _WIN32
682 fd_set *rfds = &((PosixWaiter *)waiter)->rfds;
683 fd_set *wfds = &((PosixWaiter *)waiter)->wfds;
684 #endif
685
686 int dummy = 0;
687 SockInfoMap *socketmap = &curlsockets[d];
688 bool *paused = &arerequestspaused[d];
689
690 for (SockInfoMap::iterator it = socketmap->begin(); !(*paused) && it != socketmap->end();)
691 {
692 SockInfo &info = (it++)->second;
693 if (!info.mode)
694 {
695 continue;
696 }
697
698 #if defined(_WIN32)
699 bool read, write;
700 if (info.checkEvent(read, write)) // if checkEvent returns true, both `read` and `write` have been set.
701 {
702 curl_multi_socket_action(curlm[d], info.fd,
703 (read ? CURL_CSELECT_IN : 0)
704 | (write ? CURL_CSELECT_OUT : 0), &dummy);
705 }
706 #else
707 if (((info.mode & SockInfo::READ) && FD_ISSET(info.fd, rfds)) || ((info.mode & SockInfo::WRITE) && FD_ISSET(info.fd, wfds)))
708 {
709 curl_multi_socket_action(curlm[d], info.fd,
710 (((info.mode & SockInfo::READ) && FD_ISSET(info.fd, rfds)) ? CURL_CSELECT_IN : 0)
711 | (((info.mode & SockInfo::WRITE) && FD_ISSET(info.fd, wfds)) ? CURL_CSELECT_OUT : 0),
712 &dummy);
713 }
714 #endif
715 }
716
717 if (curltimeoutreset[d] >= 0 && curltimeoutreset[d] <= Waiter::ds)
718 {
719 curltimeoutreset[d] = -1;
720 LOG_debug << "Informing cURL of timeout reached for " << d << " at " << Waiter::ds;
721 curl_multi_socket_action(curlm[d], CURL_SOCKET_TIMEOUT, 0, &dummy);
722 }
723
724 for (SockInfoMap::iterator it = socketmap->begin(); it != socketmap->end();)
725 {
726 SockInfo &info = it->second;
727 if (!info.mode)
728 {
729 socketmap->erase(it++);
730 }
731 else
732 {
733 it++;
734 }
735 }
736 }
737
~CurlHttpIO()738 CurlHttpIO::~CurlHttpIO()
739 {
740 disconnecting = true;
741 ares_destroy(ares);
742 curl_multi_cleanup(curlm[API]);
743 curl_multi_cleanup(curlm[GET]);
744 curl_multi_cleanup(curlm[PUT]);
745 curl_share_cleanup(curlsh);
746
747 closearesevents();
748 closecurlevents(API);
749 closecurlevents(GET);
750 closecurlevents(PUT);
751
752 #ifdef WIN32
753 WSACloseEvent(mSocketsWaitEvent);
754 #endif
755
756 curlMutex.lock();
757 if (--instanceCount == 0)
758 {
759 ares_library_cleanup();
760 curl_global_cleanup();
761 }
762 curlMutex.unlock();
763
764 curl_slist_free_all(contenttypejson);
765 curl_slist_free_all(contenttypebinary);
766 }
767
768 int CurlHttpIO::instanceCount = 0;
769
setuseragent(string * u)770 void CurlHttpIO::setuseragent(string* u)
771 {
772 useragent = *u;
773 }
774
setdnsservers(const char * servers)775 void CurlHttpIO::setdnsservers(const char* servers)
776 {
777 if (servers)
778 {
779 lastdnspurge = Waiter::ds + DNS_CACHE_TIMEOUT_DS / 2;
780 if (DNS_CACHE_EXPIRES)
781 {
782 dnscache.clear();
783 }
784
785 dnsservers = servers;
786
787 LOG_debug << "Using custom DNS servers: " << dnsservers;
788 ares_set_servers_csv(ares, servers);
789 }
790 }
791
disconnect()792 void CurlHttpIO::disconnect()
793 {
794 LOG_debug << "Reinitializing the network layer";
795 disconnecting = true;
796 assert(!numconnections[API] && !numconnections[GET] && !numconnections[PUT]);
797
798 ares_destroy(ares);
799 curl_multi_cleanup(curlm[API]);
800 curl_multi_cleanup(curlm[GET]);
801 curl_multi_cleanup(curlm[PUT]);
802
803 if (numconnections[API] || numconnections[GET] || numconnections[PUT])
804 {
805 LOG_err << "Disconnecting without cancelling all requests first";
806 numconnections[API] = 0;
807 numconnections[GET] = 0;
808 numconnections[PUT] = 0;
809 }
810
811 closearesevents();
812 closecurlevents(API);
813 closecurlevents(GET);
814 closecurlevents(PUT);
815
816 lastdnspurge = Waiter::ds + DNS_CACHE_TIMEOUT_DS / 2;
817 if (DNS_CACHE_EXPIRES)
818 {
819 dnscache.clear();
820 }
821 else
822 {
823 for (auto &dnsPair: dnscache)
824 {
825 dnsPair.second.mNeedsResolvingAgain= true;
826 }
827 }
828
829 curlm[API] = curl_multi_init();
830 curlm[GET] = curl_multi_init();
831 curlm[PUT] = curl_multi_init();
832 struct ares_options options;
833 options.tries = 2;
834 ares_init_options(&ares, &options, ARES_OPT_TRIES);
835 arestimeout = -1;
836
837 curl_multi_setopt(curlm[API], CURLMOPT_SOCKETFUNCTION, api_socket_callback);
838 curl_multi_setopt(curlm[API], CURLMOPT_SOCKETDATA, this);
839 curl_multi_setopt(curlm[API], CURLMOPT_TIMERFUNCTION, api_timer_callback);
840 curl_multi_setopt(curlm[API], CURLMOPT_TIMERDATA, this);
841 curltimeoutreset[API] = -1;
842 arerequestspaused[API] = false;
843
844 curl_multi_setopt(curlm[GET], CURLMOPT_SOCKETFUNCTION, download_socket_callback);
845 curl_multi_setopt(curlm[GET], CURLMOPT_SOCKETDATA, this);
846 curl_multi_setopt(curlm[GET], CURLMOPT_TIMERFUNCTION, download_timer_callback);
847 curl_multi_setopt(curlm[GET], CURLMOPT_TIMERDATA, this);
848 #ifdef _WIN32
849 curl_multi_setopt(curlm[GET], CURLMOPT_MAXCONNECTS, 200);
850 #endif
851 curltimeoutreset[GET] = -1;
852 arerequestspaused[GET] = false;
853
854
855 curl_multi_setopt(curlm[PUT], CURLMOPT_SOCKETFUNCTION, upload_socket_callback);
856 curl_multi_setopt(curlm[PUT], CURLMOPT_SOCKETDATA, this);
857 curl_multi_setopt(curlm[PUT], CURLMOPT_TIMERFUNCTION, upload_timer_callback);
858 curl_multi_setopt(curlm[PUT], CURLMOPT_TIMERDATA, this);
859 #ifdef _WIN32
860 curl_multi_setopt(curlm[PUT], CURLMOPT_MAXCONNECTS, 200);
861 #endif
862 curltimeoutreset[PUT] = -1;
863 arerequestspaused[PUT] = false;
864
865 disconnecting = false;
866 if (dnsservers.size())
867 {
868 LOG_debug << "Using custom DNS servers: " << dnsservers;
869 ares_set_servers_csv(ares, dnsservers.c_str());
870 }
871 else
872 {
873 filterDNSservers();
874 }
875
876 if (proxyurl.size() && !proxyip.size())
877 {
878 LOG_debug << "Unresolved proxy name. Resolving...";
879 request_proxy_ip();
880 }
881 }
882
setmaxdownloadspeed(m_off_t bpslimit)883 bool CurlHttpIO::setmaxdownloadspeed(m_off_t bpslimit)
884 {
885 maxspeed[GET] = bpslimit;
886 return true;
887 }
888
setmaxuploadspeed(m_off_t bpslimit)889 bool CurlHttpIO::setmaxuploadspeed(m_off_t bpslimit)
890 {
891 maxspeed[PUT] = bpslimit;
892 return true;
893 }
894
getmaxdownloadspeed()895 m_off_t CurlHttpIO::getmaxdownloadspeed()
896 {
897 return maxspeed[GET];
898 }
899
getmaxuploadspeed()900 m_off_t CurlHttpIO::getmaxuploadspeed()
901 {
902 return maxspeed[PUT];
903 }
904
905 // wake up from cURL I/O
addevents(Waiter * w,int)906 void CurlHttpIO::addevents(Waiter* w, int)
907 {
908 CodeCounter::ScopeTimer ccst(countCurlHttpIOAddevents);
909
910 waiter = (WAIT_CLASS*)w;
911 long curltimeoutms = -1;
912
913 addaresevents(waiter);
914 addcurlevents(waiter, API);
915
916 #ifdef WIN32
917 ((WinWaiter *)waiter)->addhandle(mSocketsWaitEvent, Waiter::NEEDEXEC);
918 #endif
919
920 if (curltimeoutreset[API] >= 0)
921 {
922 m_time_t ds = curltimeoutreset[API] - Waiter::ds;
923 if (ds <= 0)
924 {
925 curltimeoutms = 0;
926 }
927 else
928 {
929 if (curltimeoutms < 0 || curltimeoutms > ds * 100)
930 {
931 curltimeoutms = long(ds * 100);
932 }
933 }
934 }
935
936 for (int d = GET; d == GET || d == PUT; d += PUT - GET)
937 {
938 if (arerequestspaused[d])
939 {
940 if (curltimeoutms < 0 || curltimeoutms > 100)
941 {
942 curltimeoutms = 100;
943 }
944 }
945 else
946 {
947 addcurlevents(waiter, (direction_t)d);
948 if (curltimeoutreset[d] >= 0)
949 {
950 m_time_t ds = curltimeoutreset[d] - Waiter::ds;
951 if (ds <= 0)
952 {
953 curltimeoutms = 0;
954 }
955 else
956 {
957 if (curltimeoutms < 0 || curltimeoutms > ds * 100)
958 {
959 curltimeoutms = long(ds * 100);
960 }
961 }
962 }
963 }
964 }
965
966 if ((curltimeoutms < 0 || curltimeoutms > MAX_SPEED_CONTROL_TIMEOUT_MS)
967 && (downloadSpeed || uploadSpeed))
968 {
969 curltimeoutms = MAX_SPEED_CONTROL_TIMEOUT_MS;
970 }
971
972 if (curltimeoutms >= 0)
973 {
974 m_time_t timeoutds = curltimeoutms / 100;
975 if (curltimeoutms % 100)
976 {
977 timeoutds++;
978 }
979
980 if ((unsigned long)timeoutds < waiter->maxds)
981 {
982 waiter->maxds = dstime(timeoutds);
983 }
984 }
985 curlsocketsprocessed = false;
986
987 timeval tv;
988 if (ares_timeout(ares, NULL, &tv))
989 {
990 arestimeout = tv.tv_sec * 10 + tv.tv_usec / 100000;
991 if (!arestimeout && tv.tv_usec)
992 {
993 arestimeout = 1;
994 }
995
996 if (arestimeout < waiter->maxds)
997 {
998 waiter->maxds = dstime(arestimeout);
999 }
1000 arestimeout += Waiter::ds;
1001 }
1002 else
1003 {
1004 arestimeout = -1;
1005 }
1006 }
1007
proxy_ready_callback(void * arg,int status,int,hostent * host)1008 void CurlHttpIO::proxy_ready_callback(void* arg, int status, int, hostent* host)
1009 {
1010 // the name of a proxy has been resolved
1011 CurlHttpContext* httpctx = (CurlHttpContext*)arg;
1012 CurlHttpIO* httpio = httpctx->httpio;
1013
1014 LOG_debug << "c-ares info received (proxy)";
1015
1016 httpctx->ares_pending--;
1017 if (!httpctx->ares_pending)
1018 {
1019 httpio->proxyinflight--;
1020 }
1021
1022 if (!httpio->proxyhost.size() // the proxy was disabled during the name resolution.
1023 || httpio->proxyip.size()) // or we already have the correct ip
1024 {
1025 if (!httpctx->ares_pending)
1026 {
1027 LOG_debug << "Proxy ready";
1028
1029 // name resolution finished.
1030 // nothing more to do.
1031 // free resources and continue sending requests.
1032 delete httpctx;
1033 httpio->send_pending_requests();
1034 }
1035 else
1036 {
1037 LOG_debug << "Proxy ready. Waiting for c-ares";
1038 }
1039
1040 return;
1041 }
1042
1043 // check if result is valid
1044 // IPv6 takes precedence over IPv4
1045 // discard the IP if it's IPv6 and IPv6 isn't available
1046 if (status == ARES_SUCCESS && host && host->h_addr_list[0]
1047 && httpio->proxyhost == httpctx->hostname
1048 && (!httpctx->hostip.size() || host->h_addrtype == PF_INET6)
1049 && (host->h_addrtype != PF_INET6 || httpio->ipv6available()))
1050 {
1051 LOG_debug << "Received a valid IP for the proxy";
1052
1053 // save the IP of the proxy
1054 char ip[INET6_ADDRSTRLEN];
1055
1056 mega_inet_ntop(host->h_addrtype, host->h_addr_list[0], ip, sizeof ip);
1057 httpctx->hostip = ip;
1058 httpctx->isIPv6 = host->h_addrtype == PF_INET6;
1059
1060 if (httpctx->isIPv6 && ip[0] != '[')
1061 {
1062 httpctx->hostip.insert(0, "[");
1063 httpctx->hostip.append("]");
1064 }
1065 }
1066 else if (status != ARES_SUCCESS)
1067 {
1068 LOG_warn << "c-ares error (proxy) " << status;
1069 }
1070
1071 if (!httpctx->ares_pending)
1072 {
1073 LOG_debug << "c-ares request finished (proxy)";
1074
1075 // name resolution finished
1076 // if the IP is valid, use it and continue sending requests.
1077 if (httpio->proxyhost == httpctx->hostname && httpctx->hostip.size())
1078 {
1079 std::ostringstream oss;
1080
1081 oss << httpctx->hostip << ":" << httpio->proxyport;
1082 httpio->proxyip = oss.str();
1083
1084 LOG_info << "Updated proxy URL: " << httpio->proxyip;
1085
1086 httpio->inetstatus(true);
1087
1088 httpio->send_pending_requests();
1089 }
1090 else if (!httpio->proxyinflight)
1091 {
1092 LOG_err << "Invalid proxy IP";
1093
1094 httpio->inetstatus(false);
1095
1096 // the IP isn't up to date and there aren't pending
1097 // name resolutions for proxies. Abort requests.
1098 httpio->drop_pending_requests();
1099
1100 if (status != ARES_EDESTRUCTION)
1101 {
1102 // reinitialize c-ares to prevent persistent hangs
1103 httpio->reset = true;
1104 }
1105 }
1106 else
1107 {
1108 LOG_debug << "Waiting for the IP of the proxy";
1109 }
1110
1111 // nothing more to do - free resources
1112 delete httpctx;
1113 }
1114 else
1115 {
1116 LOG_debug << "Waiting for the completion of the c-ares request (proxy)";
1117 }
1118 }
1119
ares_completed_callback(void * arg,int status,int,struct hostent * host)1120 void CurlHttpIO::ares_completed_callback(void* arg, int status, int, struct hostent* host)
1121 {
1122 CurlHttpContext* httpctx = (CurlHttpContext*)arg;
1123 CurlHttpIO* httpio = httpctx->httpio;
1124 HttpReq* req = httpctx->req;
1125 bool invalidcache = false;
1126 httpctx->ares_pending--;
1127
1128 LOG_debug << "c-ares info received";
1129
1130 // check if result is valid
1131 if (status == ARES_SUCCESS && host && host->h_addr_list[0])
1132 {
1133 char ip[INET6_ADDRSTRLEN];
1134 mega_inet_ntop(host->h_addrtype, host->h_addr_list[0], ip, sizeof(ip));
1135
1136 LOG_debug << "Received a valid IP for "<< httpctx->hostname << ": " << ip;
1137
1138 httpio->inetstatus(true);
1139
1140 // add to DNS cache
1141 CurlDNSEntry& dnsEntry = httpio->dnscache[httpctx->hostname];
1142
1143 int i = 0;
1144 bool incache = false;
1145 if ((host->h_addrtype == PF_INET6 && dnsEntry.ipv6.size())
1146 || (host->h_addrtype != PF_INET6 && dnsEntry.ipv4.size()))
1147 {
1148 invalidcache = true;
1149 while (host->h_addr_list[i] != NULL)
1150 {
1151 char checkip[INET6_ADDRSTRLEN];
1152 mega_inet_ntop(host->h_addrtype, host->h_addr_list[i], checkip, sizeof(checkip));
1153 if (host->h_addrtype == PF_INET6)
1154 {
1155 if (!strcmp(dnsEntry.ipv6.c_str(), checkip))
1156 {
1157 incache = true;
1158 invalidcache = false;
1159 break;
1160 }
1161 }
1162 else
1163 {
1164 if (!strcmp(dnsEntry.ipv4.c_str(), checkip))
1165 {
1166 incache = true;
1167 invalidcache = false;
1168 break;
1169 }
1170 }
1171 i++;
1172 }
1173 }
1174
1175 if (incache)
1176 {
1177 LOG_debug << "The current DNS cache record is still valid";
1178 }
1179 else if (invalidcache)
1180 {
1181 LOG_warn << "The current DNS cache record is invalid";
1182 }
1183
1184 if (host->h_addrtype == PF_INET6)
1185 {
1186 if (!incache)
1187 {
1188 dnsEntry.ipv6 = ip;
1189 }
1190 dnsEntry.ipv6timestamp = Waiter::ds;
1191 }
1192 else
1193 {
1194 if (!incache)
1195 {
1196 dnsEntry.ipv4 = ip;
1197 }
1198 dnsEntry.ipv4timestamp = Waiter::ds;
1199 }
1200
1201 // IPv6 takes precedence over IPv4
1202 if (!httpctx->hostip.size() || (host->h_addrtype == PF_INET6 && !httpctx->curl))
1203 {
1204 httpctx->isIPv6 = host->h_addrtype == PF_INET6;
1205
1206 //save the IP for this request
1207 std::ostringstream oss;
1208 if (httpctx->isIPv6)
1209 {
1210 oss << "[" << ip << "]";
1211 }
1212 else
1213 {
1214 oss << ip;
1215 }
1216
1217 httpctx->hostip = oss.str();
1218 }
1219 }
1220 else if (status != ARES_SUCCESS)
1221 {
1222 LOG_warn << "c-ares error. code: " << status;
1223 }
1224 else
1225 {
1226 LOG_err << "Unknown c-ares error";
1227 }
1228
1229 if (!req) // the request was cancelled
1230 {
1231 if (!httpctx->ares_pending)
1232 {
1233 LOG_debug << "Request cancelled";
1234 delete httpctx;
1235 }
1236
1237 return;
1238 }
1239
1240 if (httpctx->curl)
1241 {
1242 LOG_debug << "Request already sent using a previous DNS response";
1243 if (invalidcache && httpctx->isIPv6 == (host->h_addrtype == PF_INET6))
1244 {
1245 LOG_warn << "Cancelling request due to the detection of an invalid DNS cache record";
1246 httpio->cancel(req);
1247 }
1248 return;
1249 }
1250
1251 // check for fatal errors
1252 if ((httpio->proxyurl.size() && !httpio->proxyhost.size() && req->method != METHOD_NONE) //malformed proxy string
1253 || (!httpctx->ares_pending && !httpctx->hostip.size())) // or unable to get the IP for this request
1254 {
1255 if (!httpio->proxyinflight || req->method == METHOD_NONE)
1256 {
1257 req->status = REQ_FAILURE;
1258 httpio->statechange = true;
1259
1260 if (!httpctx->ares_pending && !httpctx->hostip.size())
1261 {
1262 LOG_debug << "Unable to get the IP for " << httpctx->hostname;
1263
1264 // unable to get the IP.
1265 httpio->inetstatus(false);
1266
1267 if (status != ARES_EDESTRUCTION)
1268 {
1269 // reinitialize c-ares to prevent permanent hangs
1270 httpio->reset = true;
1271 }
1272 }
1273
1274 req->httpiohandle = NULL;
1275
1276 httpctx->req = NULL;
1277 if (!httpctx->ares_pending)
1278 {
1279 delete httpctx;
1280 }
1281 }
1282 else if(!httpctx->ares_pending)
1283 {
1284 httpio->pendingrequests.push(httpctx);
1285 LOG_debug << "Waiting for the IP of the proxy (1)";
1286 }
1287
1288 return;
1289 }
1290
1291 bool ares_pending = httpctx->ares_pending;
1292 if (httpctx->hostip.size())
1293 {
1294 LOG_debug << "Name resolution finished";
1295
1296 // if there is no proxy or we already have the IP of the proxy, send the request.
1297 // otherwise, queue the request until we get the IP of the proxy
1298 if (!httpio->proxyurl.size() || httpio->proxyip.size() || req->method == METHOD_NONE)
1299 {
1300 send_request(httpctx);
1301 }
1302 else if (!httpctx->ares_pending)
1303 {
1304 httpio->pendingrequests.push(httpctx);
1305
1306 if (!httpio->proxyinflight)
1307 {
1308 LOG_err << "Unable to get the IP of the proxy";
1309
1310 // c-ares failed to get the IP of the proxy.
1311 // queue this request and retry.
1312 httpio->ipv6proxyenabled = !httpio->ipv6proxyenabled && httpio->ipv6available();
1313 httpio->request_proxy_ip();
1314 return;
1315 }
1316 else
1317 {
1318 LOG_debug << "Waiting for the IP of the proxy (2)";
1319 }
1320 }
1321 }
1322
1323 if (ares_pending)
1324 {
1325 LOG_debug << "Waiting for the completion of the c-ares request";
1326 }
1327 }
1328
clone_curl_slist(struct curl_slist * inlist)1329 struct curl_slist* CurlHttpIO::clone_curl_slist(struct curl_slist* inlist)
1330 {
1331 struct curl_slist* outlist = NULL;
1332 struct curl_slist* tmp;
1333
1334 while (inlist)
1335 {
1336 tmp = curl_slist_append(outlist, inlist->data);
1337
1338 if (!tmp)
1339 {
1340 curl_slist_free_all(outlist);
1341 return NULL;
1342 }
1343
1344 outlist = tmp;
1345 inlist = inlist->next;
1346 }
1347
1348 return outlist;
1349 }
1350
send_request(CurlHttpContext * httpctx)1351 void CurlHttpIO::send_request(CurlHttpContext* httpctx)
1352 {
1353 CurlHttpIO* httpio = httpctx->httpio;
1354 HttpReq* req = httpctx->req;
1355 int len = httpctx->len;
1356 const char* data = httpctx->data;
1357
1358 if (SimpleLogger::logCurrentLevel >= logDebug)
1359 {
1360 string safeurl = req->posturl;
1361 size_t sid = safeurl.find("sid=");
1362 if (sid != string::npos)
1363 {
1364 sid += 4;
1365 size_t end = safeurl.find("&", sid);
1366 if (end == string::npos)
1367 {
1368 end = safeurl.size();
1369 }
1370 memset((char *)safeurl.data() + sid, 'X', end - sid);
1371 }
1372 LOG_debug << httpctx->req->logname << "POST target URL: " << safeurl;
1373 }
1374
1375 if (req->binary)
1376 {
1377 LOG_debug << httpctx->req->logname << "[sending " << (data ? len : req->out->size()) << " bytes of raw data]";
1378 }
1379 else
1380 {
1381 if (req->out->size() < size_t(SimpleLogger::maxPayloadLogSize))
1382 {
1383 LOG_debug << httpctx->req->logname << "Sending " << req->out->size() << ": " << DirectMessage(req->out->c_str(), req->out->size());
1384 }
1385 else
1386 {
1387 LOG_debug << httpctx->req->logname << "Sending " << req->out->size() << ": "
1388 << DirectMessage(req->out->c_str(), static_cast<size_t>(SimpleLogger::maxPayloadLogSize / 2))
1389 << " [...] "
1390 << DirectMessage(req->out->c_str() + req->out->size() - SimpleLogger::maxPayloadLogSize / 2, static_cast<size_t>(SimpleLogger::maxPayloadLogSize / 2));
1391 }
1392 }
1393
1394 httpctx->headers = clone_curl_slist(req->type == REQ_JSON ? httpio->contenttypejson : httpio->contenttypebinary);
1395 httpctx->posturl = req->posturl;
1396
1397 if(httpio->proxyip.size())
1398 {
1399 LOG_debug << "Using the hostname instead of the IP";
1400 }
1401 else if(httpctx->hostip.size())
1402 {
1403 LOG_debug << "Using the IP of the hostname: " << httpctx->hostip;
1404 httpctx->posturl.replace(httpctx->posturl.find(httpctx->hostname), httpctx->hostname.size(), httpctx->hostip);
1405 httpctx->headers = curl_slist_append(httpctx->headers, httpctx->hostheader.c_str());
1406 }
1407 else
1408 {
1409 LOG_err << "No IP nor proxy available";
1410 req->status = REQ_FAILURE;
1411 req->httpiohandle = NULL;
1412 curl_slist_free_all(httpctx->headers);
1413
1414 httpctx->req = NULL;
1415 if (!httpctx->ares_pending)
1416 {
1417 delete httpctx;
1418 }
1419 httpio->statechange = true;
1420 return;
1421 }
1422
1423 CURL* curl;
1424 if ((curl = curl_easy_init()))
1425 {
1426 switch (req->method)
1427 {
1428 case METHOD_POST:
1429 curl_easy_setopt(curl, CURLOPT_POST, 1L);
1430 curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data ? len : req->out->size());
1431 break;
1432 case METHOD_GET:
1433 curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
1434 break;
1435 case METHOD_NONE:
1436 curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
1437 break;
1438 }
1439
1440 if (req->timeoutms)
1441 {
1442 curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, req->timeoutms);
1443 }
1444
1445 curl_easy_setopt(curl, CURLOPT_URL, httpctx->posturl.c_str());
1446 curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_data);
1447 curl_easy_setopt(curl, CURLOPT_READDATA, (void*)req);
1448 curl_easy_setopt(curl, CURLOPT_SEEKFUNCTION, seek_data);
1449 curl_easy_setopt(curl, CURLOPT_SEEKDATA, (void*)req);
1450 curl_easy_setopt(curl, CURLOPT_USERAGENT, httpio->useragent.c_str());
1451 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpctx->headers);
1452 curl_easy_setopt(curl, CURLOPT_ENCODING, "");
1453 curl_easy_setopt(curl, CURLOPT_SHARE, httpio->curlsh);
1454 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
1455 curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)req);
1456 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, check_header);
1457 curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)req);
1458 curl_easy_setopt(curl, CURLOPT_PRIVATE, (void*)req);
1459 curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
1460 curl_easy_setopt(curl, CURLOPT_NOSIGNAL, true);
1461 curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, HttpIO::CONNECTTIMEOUT / 10);
1462 curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
1463 curl_easy_setopt(curl, CURLOPT_TCP_KEEPIDLE, 90L);
1464 curl_easy_setopt(curl, CURLOPT_TCP_KEEPINTVL, 60L);
1465 curl_easy_setopt(curl, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
1466 curl_easy_setopt(curl, CURLOPT_SOCKOPTDATA, (void*)req);
1467
1468 if (httpio->maxspeed[GET] && httpio->maxspeed[GET] <= 102400)
1469 {
1470 curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 4096L);
1471 }
1472
1473 if (req->minspeed)
1474 {
1475 curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 60L);
1476 curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 30L);
1477 }
1478
1479 if (!MegaClient::disablepkp && req->protect)
1480 {
1481 #if LIBCURL_VERSION_NUM >= 0x072c00 // At least cURL 7.44.0
1482 if (curl_easy_setopt(curl, CURLOPT_PINNEDPUBLICKEY,
1483 !memcmp(req->posturl.data(), MegaClient::APIURL.data(), MegaClient::APIURL.size())
1484 ? "sha256//0W38e765pAfPqS3DqSVOrPsC4MEOvRBaXQ7nY1AJ47E=;" //API 1
1485 "sha256//gSRHRu1asldal0HP95oXM/5RzBfP1OIrPjYsta8og80=" //API 2
1486 : (!memcmp(req->posturl.data(), MegaClient::CHATSTATSURL.data(), MegaClient::CHATSTATSURL.size())
1487 || !memcmp(req->posturl.data(), MegaClient::GELBURL.data(), MegaClient::GELBURL.size()))
1488 ? "sha256//a1vEOQRTsb7jMsyAhr4X/6YSF774gWlht8JQZ58DHlQ=" //CHAT
1489 : nullptr) == CURLE_OK)
1490 {
1491 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
1492 if (httpio->pkpErrors)
1493 {
1494 curl_easy_setopt(curl, CURLOPT_CERTINFO, 1L);
1495 }
1496 }
1497 else
1498 #endif
1499 {
1500 #ifdef USE_OPENSSL
1501 curl_easy_setopt(curl, CURLOPT_SSL_CTX_FUNCTION, ssl_ctx_function);
1502 curl_easy_setopt(curl, CURLOPT_SSL_CTX_DATA, (void*)req);
1503 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
1504 #else
1505 LOG_fatal << "cURL built without support for public key pinning. Aborting.";
1506 throw std::runtime_error("ccURL built without support for public key pinning. Aborting.");
1507 #endif
1508 }
1509 }
1510 else
1511 {
1512 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
1513 if (MegaClient::disablepkp)
1514 {
1515 LOG_warn << "Public key pinning disabled.";
1516 }
1517 }
1518
1519 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
1520 curl_easy_setopt(curl, CURLOPT_CAINFO, NULL);
1521 curl_easy_setopt(curl, CURLOPT_CAPATH, NULL);
1522
1523 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, debug_callback);
1524 curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)req);
1525 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
1526
1527 if (httpio->proxyip.size())
1528 {
1529 if(!httpio->proxyscheme.size() || !httpio->proxyscheme.compare(0, 4, "http"))
1530 {
1531 LOG_debug << "Using HTTP proxy";
1532 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
1533 }
1534 else if(!httpio->proxyscheme.compare(0, 5, "socks"))
1535 {
1536 LOG_debug << "Using SOCKS proxy";
1537 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
1538 }
1539 else
1540 {
1541 LOG_warn << "Unknown proxy type";
1542 }
1543
1544 curl_easy_setopt(curl, CURLOPT_PROXY, httpio->proxyip.c_str());
1545 curl_easy_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
1546
1547 if (httpio->proxyusername.size())
1548 {
1549 LOG_debug << "Using proxy authentication " << httpio->proxyusername.size() << " " << httpio->proxypassword.size();
1550 curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, httpio->proxyusername.c_str());
1551 curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, httpio->proxypassword.c_str());
1552 }
1553 else
1554 {
1555 LOG_debug << "NOT using proxy authentication";
1556 }
1557
1558 if(httpctx->port == 443)
1559 {
1560 curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L);
1561 }
1562 }
1563
1564 httpio->numconnections[httpctx->d]++;
1565 curl_multi_add_handle(httpio->curlm[httpctx->d], curl);
1566 httpctx->curl = curl;
1567 }
1568 else
1569 {
1570 req->status = REQ_FAILURE;
1571 req->httpiohandle = NULL;
1572 curl_slist_free_all(httpctx->headers);
1573
1574 httpctx->req = NULL;
1575 if (!httpctx->ares_pending)
1576 {
1577 delete httpctx;
1578 }
1579 }
1580
1581 httpio->statechange = true;
1582 }
1583
request_proxy_ip()1584 void CurlHttpIO::request_proxy_ip()
1585 {
1586 if (!proxyhost.size())
1587 {
1588 return;
1589 }
1590
1591 proxyinflight++;
1592 proxyip.clear();
1593
1594 CurlHttpContext* httpctx = new CurlHttpContext;
1595 httpctx->httpio = this;
1596 httpctx->hostname = proxyhost;
1597 httpctx->ares_pending = 1;
1598
1599 if (ipv6proxyenabled)
1600 {
1601 httpctx->ares_pending++;
1602 LOG_debug << "Resolving IPv6 address for proxy: " << proxyhost;
1603 ares_gethostbyname(ares, proxyhost.c_str(), PF_INET6, proxy_ready_callback, httpctx);
1604 }
1605
1606 LOG_debug << "Resolving IPv4 address for proxy: " << proxyhost;
1607 ares_gethostbyname(ares, proxyhost.c_str(), PF_INET, proxy_ready_callback, httpctx);
1608 }
1609
crackurl(string * url,string * scheme,string * hostname,int * port)1610 bool CurlHttpIO::crackurl(string* url, string* scheme, string* hostname, int* port)
1611 {
1612 if (!url || !url->size() || !scheme || !hostname || !port)
1613 {
1614 return false;
1615 }
1616
1617 *port = 0;
1618 scheme->clear();
1619 hostname->clear();
1620
1621 size_t starthost, endhost = 0, startport, endport;
1622
1623 starthost = url->find("://");
1624
1625 if (starthost != string::npos)
1626 {
1627 *scheme = url->substr(0, starthost);
1628 starthost += 3;
1629 }
1630 else
1631 {
1632 starthost = 0;
1633 }
1634
1635 if ((*url)[starthost] == '[' && url->size() > 0)
1636 {
1637 starthost++;
1638 }
1639
1640 startport = url->find("]:", starthost);
1641
1642 if (startport == string::npos)
1643 {
1644 startport = url->find(":", starthost);
1645
1646 if (startport != string::npos)
1647 {
1648 endhost = startport;
1649 }
1650 }
1651 else
1652 {
1653 endhost = startport;
1654 startport++;
1655 }
1656
1657 if (startport != string::npos)
1658 {
1659 startport++;
1660
1661 endport = url->find("/", startport);
1662
1663 if (endport == string::npos)
1664 {
1665 endport = url->size();
1666 }
1667
1668 if (endport <= startport || endport - startport > 5)
1669 {
1670 *port = -1;
1671 }
1672 else
1673 {
1674 for (size_t i = startport; i < endport; i++)
1675 {
1676 int c = url->data()[i];
1677
1678 if (c < '0' || c > '9')
1679 {
1680 *port = -1;
1681 break;
1682 }
1683 }
1684 }
1685
1686 if (!*port)
1687 {
1688 *port = atoi(url->data() + startport);
1689
1690 if (*port > 65535)
1691 {
1692 *port = -1;
1693 }
1694 }
1695 }
1696 else
1697 {
1698 endhost = url->find("]/", starthost);
1699
1700 if (endhost == string::npos)
1701 {
1702 endhost = url->find("/", starthost);
1703
1704 if (endhost == string::npos)
1705 {
1706 endhost = url->size();
1707 }
1708 }
1709 }
1710
1711 if (!*port)
1712 {
1713 if (!scheme->compare("https"))
1714 {
1715 *port = 443;
1716 }
1717 else if (!scheme->compare("http"))
1718 {
1719 *port = 80;
1720 }
1721 else if (!scheme->compare(0, 5, "socks"))
1722 {
1723 *port = 1080;
1724 }
1725 else
1726 {
1727 *port = -1;
1728 }
1729 }
1730
1731 *hostname = url->substr(starthost, endhost - starthost);
1732
1733 if (*port <= 0 || starthost == string::npos || starthost >= endhost)
1734 {
1735 *port = 0;
1736 scheme->clear();
1737 hostname->clear();
1738 return false;
1739 }
1740
1741 return true;
1742 }
1743
debug_callback(CURL *,curl_infotype type,char * data,size_t size,void * debugdata)1744 int CurlHttpIO::debug_callback(CURL*, curl_infotype type, char* data, size_t size, void* debugdata)
1745 {
1746 if (type == CURLINFO_TEXT && size)
1747 {
1748 data[size - 1] = 0;
1749 LOG_verbose << (debugdata ? static_cast<HttpReq*>(debugdata)->logname : string()) << "cURL: " << data;
1750 }
1751
1752 return 0;
1753 }
1754
1755 // POST request to URL
post(HttpReq * req,const char * data,unsigned len)1756 void CurlHttpIO::post(HttpReq* req, const char* data, unsigned len)
1757 {
1758 CurlHttpContext* httpctx = new CurlHttpContext;
1759 httpctx->curl = NULL;
1760 httpctx->httpio = this;
1761 httpctx->req = req;
1762 httpctx->len = len;
1763 httpctx->data = data;
1764 httpctx->headers = NULL;
1765 httpctx->isIPv6 = false;
1766 httpctx->isCachedIp = false;
1767 httpctx->ares_pending = 0;
1768 httpctx->d = (req->type == REQ_JSON || req->method == METHOD_NONE) ? API : ((data ? len : req->out->size()) ? PUT : GET);
1769 req->httpiohandle = (void*)httpctx;
1770
1771 bool validrequest = true;
1772 if ((proxyurl.size() && !proxyhost.size()) // malformed proxy string
1773 || !(validrequest = crackurl(&req->posturl, &httpctx->scheme, &httpctx->hostname, &httpctx->port))) // invalid request
1774 {
1775 if (validrequest)
1776 {
1777 LOG_err << "Malformed proxy string: " << proxyurl;
1778 }
1779 else
1780 {
1781 LOG_err << "Invalid request: " << req->posturl;
1782 }
1783
1784 delete httpctx;
1785 req->httpiohandle = NULL;
1786 req->status = REQ_FAILURE;
1787 statechange = true;
1788 return;
1789 }
1790
1791 if (!ipv6requestsenabled && ipv6available() && Waiter::ds - ipv6deactivationtime > IPV6_RETRY_INTERVAL_DS)
1792 {
1793 ipv6requestsenabled = true;
1794 }
1795
1796 if (reset)
1797 {
1798 LOG_debug << "Error in c-ares. Reinitializing...";
1799 reset = false;
1800 ares_destroy(ares);
1801 struct ares_options options;
1802 options.tries = 2;
1803 ares_init_options(&ares, &options, ARES_OPT_TRIES);
1804
1805 if (dnsservers.size())
1806 {
1807 LOG_info << "Using custom DNS servers: " << dnsservers;
1808 ares_set_servers_csv(ares, dnsservers.c_str());
1809 }
1810 else if (!dnsok)
1811 {
1812 getMEGADNSservers(&dnsservers, false);
1813 ares_set_servers_csv(ares, dnsservers.c_str());
1814 }
1815
1816 if (proxyurl.size() && !proxyip.size())
1817 {
1818 LOG_debug << "Unresolved proxy name. Resolving...";
1819 request_proxy_ip();
1820 }
1821 }
1822
1823 // purge DNS cache if needed
1824 if (DNS_CACHE_EXPIRES && (Waiter::ds - lastdnspurge) > DNS_CACHE_TIMEOUT_DS)
1825 {
1826 std::map<string, CurlDNSEntry>::iterator it = dnscache.begin();
1827
1828 while (it != dnscache.end())
1829 {
1830 CurlDNSEntry& entry = it->second;
1831
1832 if (entry.ipv6.size() && entry.isIPv6Expired())
1833 {
1834 entry.ipv6timestamp = 0;
1835 entry.ipv6.clear();
1836 }
1837
1838 if (entry.ipv4.size() && entry.isIPv4Expired())
1839 {
1840 entry.ipv4timestamp = 0;
1841 entry.ipv4.clear();
1842 }
1843
1844 if (!entry.ipv6.size() && !entry.ipv4.size())
1845 {
1846 LOG_debug << "DNS cache record expired for " << it->first;
1847 dnscache.erase(it++);
1848 }
1849 else
1850 {
1851 it++;
1852 }
1853 }
1854
1855 lastdnspurge = Waiter::ds;
1856 }
1857
1858 req->in.clear();
1859 req->status = REQ_INFLIGHT;
1860
1861 if (proxyip.size() && req->method != METHOD_NONE)
1862 {
1863 // we are using a proxy, don't resolve the IP
1864 LOG_debug << "Sending the request through the proxy";
1865 send_request(httpctx);
1866 return;
1867 }
1868
1869 if (proxyurl.size() && proxyinflight)
1870 {
1871 // we are waiting for a proxy, queue the request
1872 pendingrequests.push(httpctx);
1873 LOG_debug << "Queueing request for the proxy";
1874 return;
1875 }
1876
1877 httpctx->hostheader = "Host: ";
1878 httpctx->hostheader.append(httpctx->hostname);
1879 httpctx->ares_pending = 1;
1880
1881 CurlDNSEntry* dnsEntry = NULL;
1882 map<string, CurlDNSEntry>::iterator it = dnscache.find(httpctx->hostname);
1883 if (it != dnscache.end())
1884 {
1885 dnsEntry = &it->second;
1886 }
1887
1888 if (ipv6requestsenabled)
1889 {
1890 if (dnsEntry && dnsEntry->ipv6.size() && !dnsEntry->isIPv6Expired())
1891 {
1892 LOG_debug << "DNS cache hit for " << httpctx->hostname << " (IPv6) " << dnsEntry->ipv6;
1893 std::ostringstream oss;
1894 httpctx->isIPv6 = true;
1895 httpctx->isCachedIp = true;
1896 oss << "[" << dnsEntry->ipv6 << "]";
1897 httpctx->hostip = oss.str();
1898 httpctx->ares_pending = 0;
1899 send_request(httpctx);
1900 return;
1901 }
1902 }
1903
1904 if (dnsEntry && dnsEntry->ipv4.size() && !dnsEntry->isIPv4Expired())
1905 {
1906 LOG_debug << "DNS cache hit for " << httpctx->hostname << " (IPv4) " << dnsEntry->ipv4;
1907 httpctx->isIPv6 = false;
1908 httpctx->isCachedIp = true;
1909 httpctx->hostip = dnsEntry->ipv4;
1910 httpctx->ares_pending = 0;
1911 send_request(httpctx);
1912 return;
1913 }
1914
1915 if (ipv6requestsenabled)
1916 {
1917 httpctx->ares_pending++;
1918 LOG_debug << "Resolving IPv6 address for " << httpctx->hostname;
1919 ares_gethostbyname(ares, httpctx->hostname.c_str(), PF_INET6, ares_completed_callback, httpctx);
1920 }
1921
1922 LOG_debug << "Resolving IPv4 address for " << httpctx->hostname;
1923 ares_gethostbyname(ares, httpctx->hostname.c_str(), PF_INET, ares_completed_callback, httpctx);
1924 }
1925
setproxy(Proxy * proxy)1926 void CurlHttpIO::setproxy(Proxy* proxy)
1927 {
1928 // clear the previous proxy IP
1929 proxyip.clear();
1930
1931 if (proxy->getProxyType() != Proxy::CUSTOM || !proxy->getProxyURL().size())
1932 {
1933 // automatic proxy is not supported
1934 // invalidate inflight proxy changes
1935 proxyscheme.clear();
1936 proxyhost.clear();
1937
1938 // don't use a proxy
1939 proxyurl.clear();
1940
1941 // send pending requests without a proxy
1942 send_pending_requests();
1943 return;
1944 }
1945
1946 proxyurl = proxy->getProxyURL();
1947 proxyusername = proxy->getUsername();
1948 proxypassword = proxy->getPassword();
1949
1950 LOG_debug << "Setting proxy: " << proxyurl;
1951
1952 if (!crackurl(&proxyurl, &proxyscheme, &proxyhost, &proxyport))
1953 {
1954 LOG_err << "Malformed proxy string: " << proxyurl;
1955
1956 // invalidate inflight proxy changes
1957
1958 // mark the proxy as invalid (proxyurl set but proxyhost not set)
1959 proxyhost.clear();
1960 proxyscheme.clear();
1961
1962 // drop all pending requests
1963 drop_pending_requests();
1964 return;
1965 }
1966
1967 ipv6requestsenabled = false;
1968 ipv6proxyenabled = ipv6requestsenabled;
1969 request_proxy_ip();
1970 }
1971
1972 // cancel pending HTTP request
cancel(HttpReq * req)1973 void CurlHttpIO::cancel(HttpReq* req)
1974 {
1975 if (req->httpiohandle)
1976 {
1977 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
1978 if (httpctx->curl)
1979 {
1980 numconnections[httpctx->d]--;
1981 pausedrequests[httpctx->d].erase(httpctx->curl);
1982 curl_multi_remove_handle(curlm[httpctx->d], httpctx->curl);
1983 curl_easy_cleanup(httpctx->curl);
1984 curl_slist_free_all(httpctx->headers);
1985 }
1986
1987 httpctx->req = NULL;
1988
1989 if ((req->status == REQ_FAILURE || httpctx->curl) && !httpctx->ares_pending)
1990 {
1991 delete httpctx;
1992 }
1993
1994 req->httpstatus = 0;
1995
1996 if (req->status != REQ_FAILURE)
1997 {
1998 req->status = REQ_FAILURE;
1999 statechange = true;
2000 }
2001
2002 req->httpiohandle = NULL;
2003 }
2004 }
2005
2006 // real-time progress information on POST data
postpos(void * handle)2007 m_off_t CurlHttpIO::postpos(void* handle)
2008 {
2009 double bytes = 0;
2010 CurlHttpContext* httpctx = (CurlHttpContext*)handle;
2011
2012 if (httpctx->curl)
2013 {
2014 curl_easy_getinfo(httpctx->curl, CURLINFO_SIZE_UPLOAD, &bytes);
2015 }
2016
2017 return (m_off_t)bytes;
2018 }
2019
2020 // process events
doio()2021 bool CurlHttpIO::doio()
2022 {
2023 bool result;
2024 statechange = false;
2025
2026 processaresevents();
2027
2028 result = statechange;
2029 statechange = false;
2030
2031 if (curlsocketsprocessed)
2032 {
2033 return result;
2034 }
2035
2036 processcurlevents(API);
2037 result |= multidoio(curlm[API]);
2038
2039 for (int d = GET; d == GET || d == PUT; d += PUT - GET)
2040 {
2041 partialdata[d] = 0;
2042 if (arerequestspaused[d])
2043 {
2044 arerequestspaused[d] = false;
2045 set<CURL *>::iterator it = pausedrequests[d].begin();
2046 while (!arerequestspaused[d] && it != pausedrequests[d].end())
2047 {
2048 CURL *easy_handle = *it;
2049 pausedrequests[d].erase(it++);
2050 curl_easy_pause(easy_handle, CURLPAUSE_CONT);
2051 }
2052
2053 if (!arerequestspaused[d])
2054 {
2055 int dummy;
2056 curl_multi_socket_action(curlm[d], CURL_SOCKET_TIMEOUT, 0, &dummy);
2057 }
2058 }
2059
2060 if (!arerequestspaused[d])
2061 {
2062 processcurlevents((direction_t)d);
2063 result |= multidoio(curlm[d]);
2064 }
2065 }
2066
2067 curlsocketsprocessed = true;
2068 return result;
2069 }
2070
multidoio(CURLM * curlmhandle)2071 bool CurlHttpIO::multidoio(CURLM *curlmhandle)
2072 {
2073 int dummy = 0;
2074 CURLMsg* msg;
2075 bool result;
2076
2077 while ((msg = curl_multi_info_read(curlmhandle, &dummy)))
2078 {
2079 HttpReq* req = NULL;
2080 if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&req) == CURLE_OK && req)
2081 {
2082 req->httpio = NULL;
2083
2084 if (msg->msg == CURLMSG_DONE)
2085 {
2086 CURLcode errorCode = msg->data.result;
2087 if (errorCode != CURLE_OK)
2088 {
2089 LOG_debug << req->logname << "CURLMSG_DONE with error " << errorCode << ": " << curl_easy_strerror(errorCode);
2090
2091 #if LIBCURL_VERSION_NUM >= 0x072c00 // At least cURL 7.44.0
2092 if (errorCode == CURLE_SSL_PINNEDPUBKEYNOTMATCH)
2093 {
2094 pkpErrors++;
2095 LOG_warn << "Invalid public key?";
2096
2097 if (pkpErrors == 3)
2098 {
2099 pkpErrors = 0;
2100
2101 LOG_err << "Invalid public key. Possible MITM attack!!";
2102 req->sslcheckfailed = true;
2103
2104 struct curl_certinfo *ci;
2105 if (curl_easy_getinfo(msg->easy_handle, CURLINFO_CERTINFO, &ci) == CURLE_OK)
2106 {
2107 LOG_warn << "Fake SSL certificate data:";
2108 for (int i = 0; i < ci->num_of_certs; i++)
2109 {
2110 struct curl_slist *slist = ci->certinfo[i];
2111 while (slist)
2112 {
2113 LOG_warn << i << ": " << slist->data;
2114 if (i == 0 && !memcmp("Issuer:", slist->data, 7))
2115 {
2116 const char *issuer = NULL;
2117 if ((issuer = strstr(slist->data, "CN = ")))
2118 {
2119 issuer += 5;
2120 }
2121 else if ((issuer = strstr(slist->data, "CN=")))
2122 {
2123 issuer += 3;
2124 }
2125
2126 if (issuer)
2127 {
2128 req->sslfakeissuer = issuer;
2129 }
2130 }
2131 slist = slist->next;
2132 }
2133 }
2134
2135 if (req->sslfakeissuer.size())
2136 {
2137 LOG_debug << "Fake certificate issuer: " << req->sslfakeissuer;
2138 }
2139 }
2140 }
2141 }
2142 #endif
2143 }
2144 else if (req->protect)
2145 {
2146 pkpErrors = 0;
2147 }
2148
2149 long httpstatus;
2150 curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &httpstatus);
2151 req->httpstatus = int(httpstatus);
2152
2153 LOG_debug << "CURLMSG_DONE with HTTP status: " << req->httpstatus << " from "
2154 << (req->httpiohandle ? (((CurlHttpContext*)req->httpiohandle)->hostname + " - " + ((CurlHttpContext*)req->httpiohandle)->hostip) : "(unknown) ");
2155 if (req->httpstatus)
2156 {
2157 if (req->method == METHOD_NONE)
2158 {
2159 char *ip = NULL;
2160 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2161 if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIMARY_IP, &ip) == CURLE_OK
2162 && ip && !strstr(httpctx->hostip.c_str(), ip))
2163 {
2164 LOG_err << "cURL has changed the original IP! " << httpctx ->hostip << " -> " << ip;
2165 req->in = strstr(ip, ":") ? (string("[") + ip + "]") : string(ip);
2166 }
2167 else
2168 {
2169 req->in = httpctx->hostip;
2170 }
2171 req->httpstatus = 200;
2172 }
2173
2174 if (req->binary)
2175 {
2176 LOG_debug << "[received " << (req->buf ? req->bufpos : (int)req->in.size()) << " bytes of raw data]";
2177 }
2178 else
2179 {
2180 if (req->in.size() < size_t(SimpleLogger::maxPayloadLogSize))
2181 {
2182 LOG_debug << req->logname << "Received " << req->in.size() << ": " << DirectMessage(req->in.c_str(), req->in.size());
2183 }
2184 else
2185 {
2186 LOG_debug << req->logname << "Received " << req->in.size() << ": "
2187 << DirectMessage(req->in.c_str(), static_cast<size_t>(SimpleLogger::maxPayloadLogSize / 2))
2188 << " [...] "
2189 << DirectMessage(req->in.c_str() + req->in.size() - SimpleLogger::maxPayloadLogSize / 2, static_cast<size_t>(SimpleLogger::maxPayloadLogSize / 2));
2190 }
2191 }
2192 }
2193
2194 // check httpstatus and response length
2195 req->status = (req->httpstatus == 200
2196 && (req->contentlength < 0
2197 || req->contentlength == (req->buf ? req->bufpos : (int)req->in.size())))
2198 ? REQ_SUCCESS : REQ_FAILURE;
2199
2200 if (req->status == REQ_SUCCESS)
2201 {
2202 dnsok = true;
2203 lastdata = Waiter::ds;
2204 req->lastdata = Waiter::ds;
2205 }
2206 else
2207 {
2208 LOG_warn << req->logname << "REQ_FAILURE. Status: " << req->httpstatus << " Content-Length: " << req->contentlength
2209 << " buffer? " << (req->buf != NULL) << " bufferSize: " << (req->buf ? req->bufpos : (int)req->in.size());
2210 }
2211
2212 if (req->httpstatus)
2213 {
2214 success = true;
2215 }
2216 }
2217 else
2218 {
2219 req->status = REQ_FAILURE;
2220 }
2221
2222 statechange = true;
2223
2224 if (req->status == REQ_FAILURE && !req->httpstatus)
2225 {
2226 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2227 if (httpctx)
2228 {
2229 // remove the IP from the DNS cache
2230 CurlDNSEntry &dnsEntry = dnscache[httpctx->hostname];
2231
2232 if (httpctx->isIPv6)
2233 {
2234 dnsEntry.ipv6.clear();
2235 dnsEntry.ipv6timestamp = 0;
2236 }
2237 else
2238 {
2239 dnsEntry.ipv4.clear();
2240 dnsEntry.ipv4timestamp = 0;
2241 }
2242
2243 ipv6requestsenabled = !httpctx->isIPv6 && ipv6available();
2244
2245 if (ipv6requestsenabled)
2246 {
2247 // change the protocol of the proxy after fails contacting
2248 // MEGA servers with both protocols (IPv4 and IPv6)
2249 ipv6proxyenabled = !ipv6proxyenabled && ipv6available();
2250 request_proxy_ip();
2251 }
2252 else if (httpctx->isIPv6)
2253 {
2254 ipv6deactivationtime = Waiter::ds;
2255
2256 // for IPv6 errors, try IPv4 before sending an error to the engine
2257 if ((dnsEntry.ipv4.size() && !dnsEntry.isIPv4Expired())
2258 || (!httpctx->isCachedIp && httpctx->ares_pending))
2259 {
2260 numconnections[httpctx->d]--;
2261 pausedrequests[httpctx->d].erase(msg->easy_handle);
2262 curl_multi_remove_handle(curlmhandle, msg->easy_handle);
2263 curl_easy_cleanup(msg->easy_handle);
2264 curl_slist_free_all(httpctx->headers);
2265 httpctx->isCachedIp = false;
2266 httpctx->headers = NULL;
2267 httpctx->curl = NULL;
2268 req->httpio = this;
2269 req->in.clear();
2270 req->status = REQ_INFLIGHT;
2271
2272 if (dnsEntry.ipv4.size() && !dnsEntry.isIPv4Expired())
2273 {
2274 LOG_debug << "Retrying using IPv4 from cache";
2275 httpctx->isIPv6 = false;
2276 httpctx->hostip = dnsEntry.ipv4;
2277 send_request(httpctx);
2278 }
2279 else
2280 {
2281 httpctx->hostip.clear();
2282 LOG_debug << "Retrying with the pending DNS response";
2283 }
2284 return true;
2285 }
2286 }
2287 }
2288 }
2289 }
2290 else
2291 {
2292 req = NULL;
2293 }
2294
2295 curl_multi_remove_handle(curlmhandle, msg->easy_handle);
2296 curl_easy_cleanup(msg->easy_handle);
2297
2298 if (req)
2299 {
2300 inetstatus(req->httpstatus);
2301
2302 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2303 if (httpctx)
2304 {
2305 numconnections[httpctx->d]--;
2306 pausedrequests[httpctx->d].erase(httpctx->curl);
2307
2308 curl_slist_free_all(httpctx->headers);
2309 req->httpiohandle = NULL;
2310
2311 httpctx->req = NULL;
2312 if (!httpctx->ares_pending)
2313 {
2314 delete httpctx;
2315 }
2316 }
2317 }
2318 }
2319
2320 result = statechange;
2321 statechange = false;
2322 return result;
2323 }
2324
2325 // callback for incoming HTTP payload
send_pending_requests()2326 void CurlHttpIO::send_pending_requests()
2327 {
2328 while (pendingrequests.size())
2329 {
2330 CurlHttpContext* httpctx = pendingrequests.front();
2331 if (httpctx->req)
2332 {
2333 send_request(httpctx);
2334 }
2335 else
2336 {
2337 delete httpctx;
2338 }
2339
2340 pendingrequests.pop();
2341 }
2342 }
2343
drop_pending_requests()2344 void CurlHttpIO::drop_pending_requests()
2345 {
2346 while (pendingrequests.size())
2347 {
2348 CurlHttpContext* httpctx = pendingrequests.front();
2349 if (httpctx->req)
2350 {
2351 httpctx->req->status = REQ_FAILURE;
2352 httpctx->req->httpiohandle = NULL;
2353 statechange = true;
2354 }
2355
2356 httpctx->req = NULL;
2357 if (!httpctx->ares_pending)
2358 {
2359 delete httpctx;
2360 }
2361 pendingrequests.pop();
2362 }
2363 }
2364
read_data(void * ptr,size_t size,size_t nmemb,void * source)2365 size_t CurlHttpIO::read_data(void* ptr, size_t size, size_t nmemb, void* source)
2366 {
2367 const char *buf;
2368 size_t totalsize;
2369 HttpReq *req = (HttpReq*)source;
2370 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2371 size_t len = size * nmemb;
2372 CurlHttpIO* httpio = (CurlHttpIO*)req->httpio;
2373
2374 if (httpctx->data)
2375 {
2376 buf = httpctx->data;
2377 totalsize = httpctx->len;
2378 }
2379 else
2380 {
2381 buf = req->out->data();
2382 totalsize = req->out->size();
2383 }
2384
2385 buf += req->outpos;
2386 size_t nread = totalsize - req->outpos;
2387 if (nread > len)
2388 {
2389 nread = len;
2390 }
2391
2392 if (!nread)
2393 {
2394 return 0;
2395 }
2396
2397 req->lastdata = Waiter::ds;
2398
2399 if (httpio->maxspeed[PUT])
2400 {
2401 bool isApi = (req->type == REQ_JSON);
2402 if (!isApi)
2403 {
2404 long maxbytes = long( (httpio->maxspeed[PUT] - httpio->uploadSpeed) * (SpeedController::SPEED_MEAN_INTERVAL_DS / 10) - httpio->partialdata[PUT] );
2405 if (maxbytes <= 0)
2406 {
2407 httpio->pausedrequests[PUT].insert(httpctx->curl);
2408 httpio->arerequestspaused[PUT] = true;
2409 return CURL_READFUNC_PAUSE;
2410 }
2411
2412 if (nread > (size_t)maxbytes)
2413 {
2414 nread = maxbytes;
2415 }
2416 httpio->partialdata[PUT] += nread;
2417 }
2418 }
2419
2420 memcpy(ptr, buf, nread);
2421 req->outpos += nread;
2422 //LOG_debug << req->logname << "Supplying " << nread << " bytes to cURL to send";
2423 return nread;
2424 }
2425
write_data(void * ptr,size_t size,size_t nmemb,void * target)2426 size_t CurlHttpIO::write_data(void* ptr, size_t size, size_t nmemb, void* target)
2427 {
2428 int len = int(size * nmemb);
2429 HttpReq *req = (HttpReq*)target;
2430 CurlHttpIO* httpio = (CurlHttpIO*)req->httpio;
2431 if (httpio)
2432 {
2433 if (httpio->maxspeed[GET])
2434 {
2435 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2436 bool isUpload = httpctx->data ? httpctx->len : req->out->size();
2437 bool isApi = (req->type == REQ_JSON);
2438 if (!isApi && !isUpload)
2439 {
2440 if ((httpio->downloadSpeed + 10 * (httpio->partialdata[GET] + len) / SpeedController::SPEED_MEAN_INTERVAL_DS) > httpio->maxspeed[GET])
2441 {
2442 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2443 httpio->pausedrequests[GET].insert(httpctx->curl);
2444 httpio->arerequestspaused[GET] = true;
2445 return CURL_WRITEFUNC_PAUSE;
2446 }
2447 httpio->partialdata[GET] += len;
2448 }
2449 }
2450
2451 if (len)
2452 {
2453 req->put(ptr, len, true);
2454 }
2455
2456 httpio->lastdata = Waiter::ds;
2457 req->lastdata = Waiter::ds;
2458 }
2459
2460 return len;
2461 }
2462
2463 // set contentlength according to Original-Content-Length header
check_header(void * ptr,size_t size,size_t nmemb,void * target)2464 size_t CurlHttpIO::check_header(void* ptr, size_t size, size_t nmemb, void* target)
2465 {
2466 HttpReq *req = (HttpReq*)target;
2467 size_t len = size * nmemb;
2468 if (len > 2)
2469 {
2470 LOG_verbose << "Header: " << string((const char *)ptr, len - 2);
2471 }
2472
2473 if (len > 5 && !memcmp(ptr, "HTTP/", 5))
2474 {
2475 if (req->contentlength >= 0)
2476 {
2477 // For authentication with some proxies, cURL sends two requests in the context of a single one
2478 // Content-Length is reset here to not take into account the header from the first response
2479
2480 LOG_warn << "Receiving a second response. Resetting Content-Length";
2481 req->contentlength = -1;
2482 }
2483
2484 return size * nmemb;
2485 }
2486 else if (len > 15 && !memcmp(ptr, "Content-Length:", 15))
2487 {
2488 if (req->contentlength < 0)
2489 {
2490 req->setcontentlength(atoll((char*)ptr + 15));
2491 }
2492 }
2493 else if (len > 24 && !memcmp(ptr, "Original-Content-Length:", 24))
2494 {
2495 req->setcontentlength(atoll((char*)ptr + 24));
2496 }
2497 else if (len > 17 && !memcmp(ptr, "X-MEGA-Time-Left:", 17))
2498 {
2499 req->timeleft = atol((char*)ptr + 17);
2500 }
2501 else if (len > 15 && !memcmp(ptr, "Content-Type:", 13))
2502 {
2503 req->contenttype.assign((char *)ptr + 13, len - 15);
2504 }
2505 else
2506 {
2507 return len;
2508 }
2509
2510 if (req->httpio)
2511 {
2512 req->httpio->lastdata = Waiter::ds;
2513 req->lastdata = Waiter::ds;
2514 }
2515
2516 return len;
2517 }
2518
seek_data(void * userp,curl_off_t offset,int origin)2519 int CurlHttpIO::seek_data(void *userp, curl_off_t offset, int origin)
2520 {
2521 HttpReq *req = (HttpReq*)userp;
2522 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2523 curl_off_t newoffset;
2524 size_t totalsize;
2525
2526 if (httpctx->data)
2527 {
2528 totalsize = httpctx->len;
2529 }
2530 else
2531 {
2532 totalsize = req->out->size();
2533 }
2534
2535 switch (origin)
2536 {
2537 case SEEK_SET:
2538 newoffset = offset;
2539 break;
2540 case SEEK_CUR:
2541 newoffset = req->outpos + offset;
2542 break;
2543 case SEEK_END:
2544 newoffset = totalsize + offset;
2545 break;
2546 default:
2547 LOG_err << "Invalid origin in seek function: " << origin;
2548 return CURL_SEEKFUNC_FAIL;
2549 }
2550
2551 if (newoffset > (int) totalsize || newoffset < 0)
2552 {
2553 LOG_err << "Invalid offset " << origin << " " << offset << " " << totalsize
2554 << " " << req->outbuf << " " << newoffset;
2555 return CURL_SEEKFUNC_FAIL;
2556 }
2557 req->outpos = size_t(newoffset);
2558 LOG_debug << "Successful seek to position " << newoffset << " of " << totalsize;
2559 return CURL_SEEKFUNC_OK;
2560 }
2561
socket_callback(CURL *,curl_socket_t s,int what,void * userp,void *,direction_t d)2562 int CurlHttpIO::socket_callback(CURL *, curl_socket_t s, int what, void *userp, void *, direction_t d)
2563 {
2564 CurlHttpIO *httpio = (CurlHttpIO *)userp;
2565 SockInfoMap &socketmap = httpio->curlsockets[d];
2566
2567 if (what == CURL_POLL_REMOVE)
2568 {
2569 auto it = socketmap.find(s);
2570 if (it != socketmap.end())
2571 {
2572 LOG_debug << "Removing socket " << s;
2573
2574 #if defined(_WIN32)
2575 it->second.closeEvent();
2576 #endif
2577 it->second.mode = 0;
2578 }
2579 }
2580 else
2581 {
2582 auto it = socketmap.find(s);
2583 if (it == socketmap.end())
2584 {
2585 LOG_debug << "Adding curl socket " << s << " to " << what;
2586 #ifdef WIN32
2587 auto pair = socketmap.emplace(s, SockInfo(httpio->mSocketsWaitEvent));
2588 #else
2589 auto pair = socketmap.emplace(s, SockInfo());
2590 #endif
2591 it = pair.first;
2592 }
2593 else
2594 {
2595 LOG_debug << "Setting curl socket " << s << " to " << what;
2596 }
2597
2598 auto& info = it->second;
2599 info.fd = s;
2600 info.mode = what;
2601 #if defined(_WIN32)
2602 info.createAssociateEvent();
2603 #endif
2604 }
2605
2606 return 0;
2607 }
2608
2609 // This one was causing us to issue additional c-ares requests, when normal usage already sends those requests
2610 // CURL doco: When set, this callback function gets called by libcurl when the socket has been created, but before the connect call to allow applications to change specific socket options.The callback's purpose argument identifies the exact purpose for this particular socket:
2611
sockopt_callback(void * clientp,curl_socket_t,curlsocktype)2612 int CurlHttpIO::sockopt_callback(void *clientp, curl_socket_t, curlsocktype)
2613 {
2614 HttpReq *req = (HttpReq*)clientp;
2615 CurlHttpIO* httpio = (CurlHttpIO*)req->httpio;
2616 CurlHttpContext* httpctx = (CurlHttpContext*)req->httpiohandle;
2617 if (httpio && !httpio->disconnecting
2618 && httpctx && httpctx->isCachedIp && !httpctx->ares_pending && httpio->dnscache[httpctx->hostname].mNeedsResolvingAgain)
2619 {
2620 httpio->dnscache[httpctx->hostname].mNeedsResolvingAgain = false;
2621 httpctx->ares_pending = 1;
2622 if (httpio->ipv6requestsenabled)
2623 {
2624 httpctx->ares_pending++;
2625 LOG_debug << "Resolving IPv6 address for " << httpctx->hostname << " during connection";
2626 ares_gethostbyname(httpio->ares, httpctx->hostname.c_str(), PF_INET6, ares_completed_callback, httpctx);
2627 }
2628
2629 LOG_debug << "Resolving IPv4 address for " << httpctx->hostname << " during connection";
2630 ares_gethostbyname(httpio->ares, httpctx->hostname.c_str(), PF_INET, ares_completed_callback, httpctx);
2631 }
2632
2633 return CURL_SOCKOPT_OK;
2634 }
2635
api_socket_callback(CURL * e,curl_socket_t s,int what,void * userp,void * socketp)2636 int CurlHttpIO::api_socket_callback(CURL *e, curl_socket_t s, int what, void *userp, void *socketp)
2637 {
2638 return socket_callback(e, s, what, userp, socketp, API);
2639 }
2640
download_socket_callback(CURL * e,curl_socket_t s,int what,void * userp,void * socketp)2641 int CurlHttpIO::download_socket_callback(CURL *e, curl_socket_t s, int what, void *userp, void *socketp)
2642 {
2643 return socket_callback(e, s, what, userp, socketp, GET);
2644 }
2645
upload_socket_callback(CURL * e,curl_socket_t s,int what,void * userp,void * socketp)2646 int CurlHttpIO::upload_socket_callback(CURL *e, curl_socket_t s, int what, void *userp, void *socketp)
2647 {
2648 return socket_callback(e, s, what, userp, socketp, PUT);
2649 }
2650
timer_callback(CURLM *,long timeout_ms,void * userp,direction_t d)2651 int CurlHttpIO::timer_callback(CURLM *, long timeout_ms, void *userp, direction_t d)
2652 {
2653 CurlHttpIO *httpio = (CurlHttpIO *)userp;
2654 auto oldValue = httpio->curltimeoutreset[d];
2655 if (timeout_ms < 0)
2656 {
2657 httpio->curltimeoutreset[d] = -1;
2658 }
2659 else
2660 {
2661 m_time_t timeoutds = timeout_ms / 100;
2662 if (timeout_ms % 100)
2663 {
2664 timeoutds++;
2665 }
2666
2667 httpio->curltimeoutreset[d] = Waiter::ds + timeoutds;
2668 }
2669
2670 if (oldValue != httpio->curltimeoutreset[d])
2671 {
2672 LOG_debug << "Set cURL timeout[" << d << "] to " << httpio->curltimeoutreset[d] << " from " << timeout_ms << "(ms) at ds: " << Waiter::ds;
2673 }
2674 return 0;
2675 }
2676
api_timer_callback(CURLM * multi,long timeout_ms,void * userp)2677 int CurlHttpIO::api_timer_callback(CURLM *multi, long timeout_ms, void *userp)
2678 {
2679 return timer_callback(multi, timeout_ms, userp, API);
2680 }
2681
download_timer_callback(CURLM * multi,long timeout_ms,void * userp)2682 int CurlHttpIO::download_timer_callback(CURLM *multi, long timeout_ms, void *userp)
2683 {
2684 return timer_callback(multi, timeout_ms, userp, GET);
2685 }
2686
upload_timer_callback(CURLM * multi,long timeout_ms,void * userp)2687 int CurlHttpIO::upload_timer_callback(CURLM *multi, long timeout_ms, void *userp)
2688 {
2689 return timer_callback(multi, timeout_ms, userp, PUT);
2690 }
2691
2692 #ifdef USE_OPENSSL
ssl_ctx_function(CURL *,void * sslctx,void * req)2693 CURLcode CurlHttpIO::ssl_ctx_function(CURL*, void* sslctx, void*req)
2694 {
2695 SSL_CTX_set_cert_verify_callback((SSL_CTX*)sslctx, cert_verify_callback, req);
2696 return CURLE_OK;
2697 }
2698
2699 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined (LIBRESSL_VERSION_NUMBER) || defined (OPENSSL_IS_BORINGSSL)
2700 #define X509_STORE_CTX_get0_cert(ctx) (ctx->cert)
2701 #define X509_STORE_CTX_get0_untrusted(ctx) (ctx->untrusted)
2702 #define EVP_PKEY_get0_DSA(_pkey_) ((_pkey_)->pkey.dsa)
2703 #define EVP_PKEY_get0_RSA(_pkey_) ((_pkey_)->pkey.rsa)
2704 #endif
2705
2706 #if (OPENSSL_VERSION_NUMBER < 0x1010100fL) || defined (LIBRESSL_VERSION_NUMBER)
RSA_get0_n(const RSA * rsa)2707 const BIGNUM *RSA_get0_n(const RSA *rsa)
2708 {
2709 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined (LIBRESSL_VERSION_NUMBER)
2710 return rsa->n;
2711 #else
2712 const BIGNUM *result;
2713 RSA_get0_key(rsa, &result, NULL, NULL);
2714 return result;
2715 #endif
2716 }
2717
RSA_get0_e(const RSA * rsa)2718 const BIGNUM *RSA_get0_e(const RSA *rsa)
2719 {
2720 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined (LIBRESSL_VERSION_NUMBER)
2721 return rsa->e;
2722 #else
2723 const BIGNUM *result;
2724 RSA_get0_key(rsa, NULL, &result, NULL);
2725 return result;
2726 #endif
2727 }
2728
RSA_get0_d(const RSA * rsa)2729 const BIGNUM *RSA_get0_d(const RSA *rsa)
2730 {
2731 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined (LIBRESSL_VERSION_NUMBER)
2732 return rsa->d;
2733 #else
2734 const BIGNUM *result;
2735 RSA_get0_key(rsa, NULL, NULL, &result);
2736 return result;
2737 #endif
2738 }
2739 #endif
2740
2741 // SSL public key pinning
cert_verify_callback(X509_STORE_CTX * ctx,void * req)2742 int CurlHttpIO::cert_verify_callback(X509_STORE_CTX* ctx, void* req)
2743 {
2744 HttpReq *request = (HttpReq *)req;
2745 CurlHttpIO *httpio = (CurlHttpIO *)request->httpio;
2746 unsigned char buf[sizeof(APISSLMODULUS1) - 1];
2747 EVP_PKEY* evp = nullptr;
2748 int ok = 0;
2749
2750 if (MegaClient::disablepkp)
2751 {
2752 LOG_warn << "Public key pinning disabled.";
2753 return 1;
2754 }
2755
2756 if (EVP_PKEY_id(evp) == EVP_PKEY_RSA
2757 && (evp = X509_PUBKEY_get(X509_get_X509_PUBKEY(X509_STORE_CTX_get0_cert(ctx)))))
2758 {
2759 if (BN_num_bytes(RSA_get0_n(EVP_PKEY_get0_RSA(evp))) == sizeof APISSLMODULUS1 - 1
2760 && BN_num_bytes(RSA_get0_e(EVP_PKEY_get0_RSA(evp))) == sizeof APISSLEXPONENT - 1)
2761 {
2762 BN_bn2bin(RSA_get0_n(EVP_PKEY_get0_RSA(evp)), buf);
2763
2764 if ((!memcmp(request->posturl.data(), MegaClient::APIURL.data(), MegaClient::APIURL.size())
2765 && (!memcmp(buf, APISSLMODULUS1, sizeof APISSLMODULUS1 - 1) || !memcmp(buf, APISSLMODULUS2, sizeof APISSLMODULUS2 - 1)))
2766 || ((!memcmp(request->posturl.data(), MegaClient::CHATSTATSURL.data(), MegaClient::CHATSTATSURL.size())
2767 || !memcmp(request->posturl.data(), MegaClient::GELBURL.data(), MegaClient::GELBURL.size()))
2768 && !memcmp(buf, CHATSSLMODULUS, sizeof CHATSSLMODULUS - 1)))
2769 {
2770 BN_bn2bin(RSA_get0_e(EVP_PKEY_get0_RSA(evp)), buf);
2771
2772 if (!memcmp(buf, APISSLEXPONENT, sizeof APISSLEXPONENT - 1))
2773 {
2774 LOG_debug << "SSL public key OK";
2775 ok = 1;
2776 }
2777 }
2778 else
2779 {
2780 LOG_warn << "Public key mismatch for " << request->posturl;
2781 }
2782 }
2783 else
2784 {
2785 LOG_warn << "Public key size mismatch " << BN_num_bytes(RSA_get0_n(EVP_PKEY_get0_RSA(evp))) << " " << BN_num_bytes(RSA_get0_e(EVP_PKEY_get0_RSA(evp)));
2786 }
2787
2788 EVP_PKEY_free(evp);
2789 }
2790 else
2791 {
2792 LOG_warn << "Public key not found";
2793 }
2794
2795 if (!ok)
2796 {
2797 httpio->pkpErrors++;
2798 LOG_warn << "Invalid public key?";
2799
2800 if (httpio->pkpErrors == 3)
2801 {
2802 httpio->pkpErrors = 0;
2803
2804 LOG_err << "Invalid public key. Possible MITM attack!!";
2805 request->sslcheckfailed = true;
2806 request->sslfakeissuer.resize(256);
2807 int len = X509_NAME_get_text_by_NID (X509_get_issuer_name (X509_STORE_CTX_get0_cert(ctx)),
2808 NID_commonName,
2809 (char *)request->sslfakeissuer.data(),
2810 int(request->sslfakeissuer.size()));
2811 request->sslfakeissuer.resize(len > 0 ? len : 0);
2812 LOG_debug << "Fake certificate issuer: " << request->sslfakeissuer;
2813 }
2814 }
2815
2816 return ok;
2817 }
2818 #endif
2819
CurlDNSEntry()2820 CurlDNSEntry::CurlDNSEntry()
2821 {
2822 ipv4timestamp = 0;
2823 ipv6timestamp = 0;
2824 }
2825
isIPv4Expired()2826 bool CurlDNSEntry::isIPv4Expired()
2827 {
2828 return (DNS_CACHE_EXPIRES && (Waiter::ds - ipv4timestamp) >= DNS_CACHE_TIMEOUT_DS);
2829 }
2830
isIPv6Expired()2831 bool CurlDNSEntry::isIPv6Expired()
2832 {
2833 return (DNS_CACHE_EXPIRES && (Waiter::ds - ipv6timestamp) >= DNS_CACHE_TIMEOUT_DS);
2834 }
2835
2836 #if defined(__ANDROID__) && ARES_VERSION >= 0x010F00
initialize_android()2837 void CurlHttpIO::initialize_android()
2838 {
2839 if (!MEGAjvm)
2840 {
2841 LOG_err << "No JVM found";
2842 return;
2843 }
2844
2845 bool detach = false;
2846 try
2847 {
2848 JNIEnv *env;
2849 int result = MEGAjvm->GetEnv((void **)&env, JNI_VERSION_1_6);
2850 if (result == JNI_EDETACHED)
2851 {
2852 if (MEGAjvm->AttachCurrentThread(&env, NULL) != JNI_OK)
2853 {
2854 LOG_err << "Unable to attach the current thread";
2855 return;
2856 }
2857 detach = true;
2858 }
2859 else if (result != JNI_OK)
2860 {
2861 LOG_err << "Unable to get JNI environment";
2862 return;
2863 }
2864
2865 jclass appGlobalsClass = env->FindClass("android/app/AppGlobals");
2866 if (!appGlobalsClass)
2867 {
2868 env->ExceptionClear();
2869 LOG_err << "Failed to get android/app/AppGlobals";
2870 if (detach)
2871 {
2872 MEGAjvm->DetachCurrentThread();
2873 }
2874 return;
2875 }
2876
2877 jmethodID getInitialApplicationMID = env->GetStaticMethodID(appGlobalsClass,"getInitialApplication","()Landroid/app/Application;");
2878 if (!getInitialApplicationMID)
2879 {
2880 env->ExceptionClear();
2881 LOG_err << "Failed to get getInitialApplication()";
2882 if (detach)
2883 {
2884 MEGAjvm->DetachCurrentThread();
2885 }
2886 return;
2887 }
2888
2889 jobject context = env->CallStaticObjectMethod(appGlobalsClass, getInitialApplicationMID);
2890 if (!context)
2891 {
2892 env->ExceptionClear();
2893 LOG_err << "Failed to get context";
2894 if (detach)
2895 {
2896 MEGAjvm->DetachCurrentThread();
2897 }
2898 return;
2899 }
2900
2901 jclass contextClass = env->FindClass("android/content/Context");
2902 if (!contextClass)
2903 {
2904 env->ExceptionClear();
2905 LOG_err << "Failed to get android/content/Context";
2906 if (detach)
2907 {
2908 MEGAjvm->DetachCurrentThread();
2909 }
2910 return;
2911 }
2912
2913 jmethodID getSystemServiceMID = env->GetMethodID(contextClass, "getSystemService", "(Ljava/lang/String;)Ljava/lang/Object;");
2914 if (!getSystemServiceMID)
2915 {
2916 env->ExceptionClear();
2917 LOG_err << "Failed to get getSystemService()";
2918 if (detach)
2919 {
2920 MEGAjvm->DetachCurrentThread();
2921 }
2922 return;
2923 }
2924
2925 jfieldID fid = env->GetStaticFieldID(contextClass, "CONNECTIVITY_SERVICE", "Ljava/lang/String;");
2926 if (!fid)
2927 {
2928 env->ExceptionClear();
2929 LOG_err << "Failed to get CONNECTIVITY_SERVICE";
2930 if (detach)
2931 {
2932 MEGAjvm->DetachCurrentThread();
2933 }
2934 return;
2935 }
2936
2937 jstring str = (jstring)env->GetStaticObjectField(contextClass, fid);
2938 if (!str)
2939 {
2940 env->ExceptionClear();
2941 LOG_err << "Failed to get CONNECTIVITY_SERVICE value";
2942 if (detach)
2943 {
2944 MEGAjvm->DetachCurrentThread();
2945 }
2946 return;
2947 }
2948
2949 jobject connectivityManager = env->CallObjectMethod(context, getSystemServiceMID, str);
2950 if (!connectivityManager)
2951 {
2952 env->ExceptionClear();
2953 LOG_err << "Failed to get connectivityManager";
2954 if (detach)
2955 {
2956 MEGAjvm->DetachCurrentThread();
2957 }
2958 return;
2959 }
2960
2961 ares_library_init_jvm(MEGAjvm);
2962 ares_library_init_android(connectivityManager);
2963 assert(ares_library_android_initialized() == ARES_SUCCESS);
2964
2965 if (detach)
2966 {
2967 MEGAjvm->DetachCurrentThread();
2968 }
2969 }
2970 catch (...)
2971 {
2972 try
2973 {
2974 if (detach)
2975 {
2976 MEGAjvm->DetachCurrentThread();
2977 }
2978 }
2979 catch (...) { }
2980 }
2981 }
2982 #endif
2983
2984 } // namespace
2985