1 /**
2 * \file HTTPClient.cxx - simple HTTP client engine for SimHear
3 */
4
5 // Written by James Turner
6 //
7 // Copyright (C) 2013 James Turner <zakalawe@mac.com>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Library General Public
11 // License as published by the Free Software Foundation; either
12 // version 2 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Library General Public License for more details.
18 //
19 // You should have received a copy of the GNU General Public License
20 // along with this program; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
22 //
23
24 #include <simgear_config.h>
25
26 #include "HTTPClient.hxx"
27 #include "HTTPFileRequest.hxx"
28
29 #include <sstream>
30 #include <cassert>
31 #include <cstdlib> // rand()
32 #include <list>
33 #include <errno.h>
34 #include <map>
35 #include <stdexcept>
36 #include <mutex>
37
38 #include <simgear/simgear_config.h>
39
40
41 #include <simgear/io/sg_netChat.hxx>
42
43 #include <simgear/misc/strutils.hxx>
44 #include <simgear/compiler.h>
45 #include <simgear/debug/logstream.hxx>
46 #include <simgear/timing/timestamp.hxx>
47 #include <simgear/structure/exception.hxx>
48
49 #include "HTTPClient_private.hxx"
50 #include "HTTPTestApi_private.hxx"
51
52 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
53 #include "version.h"
54 #else
55 # if !defined(SIMGEAR_VERSION)
56 # define SIMGEAR_VERSION "simgear-development"
57 # endif
58 #endif
59
60 namespace simgear
61 {
62
63 namespace HTTP
64 {
65
66 extern const int DEFAULT_HTTP_PORT = 80;
67 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
68
createCurlMulti()69 void Client::ClientPrivate::createCurlMulti() {
70 curlMulti = curl_multi_init();
71 // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
72 // we request HTTP 1.1 pipelining
73 curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
74 #if (LIBCURL_VERSION_MINOR >= 30)
75 curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS,
76 (long)maxConnections);
77 curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
78 (long)maxPipelineDepth);
79 curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
80 (long)maxHostConnections);
81 #endif
82 }
83
Client()84 Client::Client()
85 {
86 static bool didInitCurlGlobal = false;
87 static std::mutex initMutex;
88
89 std::lock_guard<std::mutex> g(initMutex);
90 if (!didInitCurlGlobal) {
91 curl_global_init(CURL_GLOBAL_ALL);
92 didInitCurlGlobal = true;
93 }
94
95 reset();
96 }
97
~Client()98 Client::~Client()
99 {
100 curl_multi_cleanup(d->curlMulti);
101 }
102
setMaxConnections(unsigned int maxCon)103 void Client::setMaxConnections(unsigned int maxCon)
104 {
105 d->maxConnections = maxCon;
106 #if (LIBCURL_VERSION_MINOR >= 30)
107 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon);
108 #endif
109 }
110
setMaxHostConnections(unsigned int maxHostCon)111 void Client::setMaxHostConnections(unsigned int maxHostCon)
112 {
113 d->maxHostConnections = maxHostCon;
114 #if (LIBCURL_VERSION_MINOR >= 30)
115 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, (long) maxHostCon);
116 #endif
117 }
118
setMaxPipelineDepth(unsigned int depth)119 void Client::setMaxPipelineDepth(unsigned int depth)
120 {
121 d->maxPipelineDepth = depth;
122 #if (LIBCURL_VERSION_MINOR >= 30)
123 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) depth);
124 #endif
125 }
126
reset()127 void Client::reset()
128 {
129 if (d.get()) {
130 curl_multi_cleanup(d->curlMulti);
131 }
132
133 d.reset(new ClientPrivate);
134
135 d->proxyPort = 0;
136 d->maxConnections = 4;
137 d->maxHostConnections = 4;
138 d->bytesTransferred = 0;
139 d->lastTransferRate = 0;
140 d->timeTransferSample.stamp();
141 d->totalBytesDownloaded = 0;
142 d->maxPipelineDepth = 5;
143 setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
144 d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
145 d->createCurlMulti();
146 }
147
update(int waitTimeout)148 void Client::update(int waitTimeout)
149 {
150 if (d->requests.empty()) {
151 // curl_multi_wait returns immediately if there's no requests active,
152 // but that can cause high CPU usage for us.
153 SGTimeStamp::sleepForMSec(waitTimeout);
154 return;
155 }
156
157 int remainingActive, messagesInQueue;
158 int numFds;
159 CURLMcode mc = curl_multi_wait(d->curlMulti, NULL, 0, waitTimeout, &numFds);
160 if (mc != CURLM_OK) {
161 SG_LOG(SG_IO, SG_WARN, "curl_multi_wait failed:" << curl_multi_strerror(mc));
162 return;
163 }
164
165 mc = curl_multi_perform(d->curlMulti, &remainingActive);
166 if (mc == CURLM_CALL_MULTI_PERFORM) {
167 // we could loop here, but don't want to get blocked
168 // also this shouldn't ocurr in any modern libCurl
169 curl_multi_perform(d->curlMulti, &remainingActive);
170 } else if (mc != CURLM_OK) {
171 SG_LOG(SG_IO, SG_WARN, "curl_multi_perform failed:" << curl_multi_strerror(mc));
172 return;
173 }
174
175 CURLMsg* msg;
176 while ((msg = curl_multi_info_read(d->curlMulti, &messagesInQueue))) {
177 if (msg->msg == CURLMSG_DONE) {
178 Request* rawReq = 0;
179 CURL *e = msg->easy_handle;
180 curl_easy_getinfo(e, CURLINFO_PRIVATE, &rawReq);
181
182 // ensure request stays valid for the moment
183 // eg if responseComplete cancels us
184 Request_ptr req(rawReq);
185
186 long responseCode;
187 curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &responseCode);
188
189 // remove from the requests map now,
190 // in case the callbacks perform a cancel. We'll use
191 // the absence from the request dict in cancel to avoid
192 // a double remove
193 ClientPrivate::RequestCurlMap::iterator it = d->requests.find(req);
194 assert(it != d->requests.end());
195 assert(it->second == e);
196 d->requests.erase(it);
197
198 bool doProcess = true;
199 if (d->testsuiteResponseDoneCallback) {
200 doProcess =
201 !d->testsuiteResponseDoneCallback(msg->data.result, req);
202 }
203
204 if (doProcess) {
205 if (msg->data.result == 0) {
206 req->responseComplete();
207 } else {
208 SG_LOG(SG_IO, SG_WARN,
209 "CURL Result:" << msg->data.result << " "
210 << curl_easy_strerror(msg->data.result));
211 req->setFailure(msg->data.result,
212 curl_easy_strerror(msg->data.result));
213 }
214 }
215
216 curl_multi_remove_handle(d->curlMulti, e);
217 curl_easy_cleanup(e);
218 } else {
219 // should never happen since CURLMSG_DONE is the only code
220 // defined!
221 SG_LOG(SG_IO, SG_ALERT, "unknown CurlMSG:" << msg->msg);
222 }
223 } // of curl message processing loop
224 }
225
makeRequest(const Request_ptr & r)226 void Client::makeRequest(const Request_ptr& r)
227 {
228 if( r->isComplete() )
229 return;
230
231 if (r->url().empty()) {
232 r->setFailure(EINVAL, "no URL specified on request");
233 return;
234 }
235
236 if( r->url().find("://") == std::string::npos ) {
237 r->setFailure(EINVAL, "malformed URL: '" + r->url() + "'");
238 return;
239 }
240
241 r->_client = this;
242
243 assert(d->requests.find(r) == d->requests.end());
244
245 CURL* curlRequest = curl_easy_init();
246 curl_easy_setopt(curlRequest, CURLOPT_URL, r->url().c_str());
247
248 d->requests[r] = curlRequest;
249
250 curl_easy_setopt(curlRequest, CURLOPT_PRIVATE, r.get());
251 // disable built-in libCurl progress feedback
252 curl_easy_setopt(curlRequest, CURLOPT_NOPROGRESS, 1);
253
254 curl_easy_setopt(curlRequest, CURLOPT_WRITEFUNCTION, requestWriteCallback);
255 curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
256 curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
257 curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
258
259 #if !defined(CURL_MAX_READ_SIZE)
260 const int CURL_MAX_READ_SIZE = 512 * 1024;
261 #endif
262
263 curl_easy_setopt(curlRequest, CURLOPT_BUFFERSIZE, CURL_MAX_READ_SIZE);
264 curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
265 curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
266
267 if (sglog().would_log(SG_TERRASYNC, SG_DEBUG)) {
268 curl_easy_setopt(curlRequest, CURLOPT_VERBOSE, 1);
269 }
270
271 curl_easy_setopt(curlRequest, CURLOPT_FOLLOWLOCATION, 1);
272
273 if (!d->tlsCertificatePath.isNull()) {
274 const auto utf8 = d->tlsCertificatePath.utf8Str();
275 curl_easy_setopt(curlRequest, CURLOPT_CAINFO, utf8.c_str());
276 }
277
278 if (!d->proxy.empty()) {
279 curl_easy_setopt(curlRequest, CURLOPT_PROXY, d->proxy.c_str());
280 curl_easy_setopt(curlRequest, CURLOPT_PROXYPORT, d->proxyPort);
281
282 if (!d->proxyAuth.empty()) {
283 curl_easy_setopt(curlRequest, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
284 curl_easy_setopt(curlRequest, CURLOPT_PROXYUSERPWD, d->proxyAuth.c_str());
285 }
286 }
287
288 const std::string method = strutils::lowercase (r->method());
289 if (method == "get") {
290 curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);
291 } else if (method == "put") {
292 curl_easy_setopt(curlRequest, CURLOPT_PUT, 1);
293 curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
294 } else if (method == "post") {
295 // see http://curl.haxx.se/libcurl/c/CURLOPT_POST.html
296 curl_easy_setopt(curlRequest, CURLOPT_HTTPPOST, 1);
297
298 std::string q = r->query().substr(1);
299 curl_easy_setopt(curlRequest, CURLOPT_COPYPOSTFIELDS, q.c_str());
300
301 // reset URL to exclude query pieces
302 std::string urlWithoutQuery = r->url();
303 std::string::size_type queryPos = urlWithoutQuery.find('?');
304 urlWithoutQuery.resize(queryPos);
305 curl_easy_setopt(curlRequest, CURLOPT_URL, urlWithoutQuery.c_str());
306 } else {
307 curl_easy_setopt(curlRequest, CURLOPT_CUSTOMREQUEST, r->method().c_str());
308 }
309
310 struct curl_slist* headerList = NULL;
311 if (r->hasBodyData() && (method != "post")) {
312 curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
313 curl_easy_setopt(curlRequest, CURLOPT_INFILESIZE, r->bodyLength());
314 curl_easy_setopt(curlRequest, CURLOPT_READFUNCTION, requestReadCallback);
315 curl_easy_setopt(curlRequest, CURLOPT_READDATA, r.get());
316 std::string h = "Content-Type:" + r->bodyType();
317 headerList = curl_slist_append(headerList, h.c_str());
318 }
319
320 StringMap::const_iterator it;
321 for (it = r->requestHeaders().begin(); it != r->requestHeaders().end(); ++it) {
322 std::string h = it->first + ": " + it->second;
323 headerList = curl_slist_append(headerList, h.c_str());
324 }
325
326 if (headerList != NULL) {
327 curl_easy_setopt(curlRequest, CURLOPT_HTTPHEADER, headerList);
328 }
329
330 curl_multi_add_handle(d->curlMulti, curlRequest);
331
332 // this seems premature, but we don't have a callback from Curl we could
333 // use to trigger when the requst is actually sent.
334 r->requestStart();
335 }
336
cancelRequest(const Request_ptr & r,std::string reason)337 void Client::cancelRequest(const Request_ptr &r, std::string reason)
338 {
339 ClientPrivate::RequestCurlMap::iterator it = d->requests.find(r);
340 if(it == d->requests.end()) {
341 // already being removed, presumably inside ::update()
342 // nothing more to do
343 return;
344 }
345
346 CURLMcode err = curl_multi_remove_handle(d->curlMulti, it->second);
347 if (err != CURLM_OK) {
348 SG_LOG(SG_IO, SG_WARN, "curl_multi_remove_handle failed:" << err);
349 }
350
351 // clear the request pointer form the curl-easy object
352 curl_easy_setopt(it->second, CURLOPT_PRIVATE, 0);
353
354 curl_easy_cleanup(it->second);
355 d->requests.erase(it);
356
357 r->setFailure(-1, reason);
358 }
359
360 //------------------------------------------------------------------------------
save(const std::string & url,const std::string & filename)361 FileRequestRef Client::save( const std::string& url,
362 const std::string& filename )
363 {
364 FileRequestRef req = new FileRequest(url, filename);
365 makeRequest(req);
366 return req;
367 }
368
369 //------------------------------------------------------------------------------
load(const std::string & url)370 MemoryRequestRef Client::load(const std::string& url)
371 {
372 MemoryRequestRef req = new MemoryRequest(url);
373 makeRequest(req);
374 return req;
375 }
376
requestFinished(Connection * con)377 void Client::requestFinished(Connection* con)
378 {
379
380 }
381
setUserAgent(const std::string & ua)382 void Client::setUserAgent(const std::string& ua)
383 {
384 d->userAgent = ua;
385 }
386
userAgent() const387 const std::string& Client::userAgent() const
388 {
389 return d->userAgent;
390 }
391
proxyHost() const392 const std::string& Client::proxyHost() const
393 {
394 return d->proxy;
395 }
396
proxyAuth() const397 const std::string& Client::proxyAuth() const
398 {
399 return d->proxyAuth;
400 }
401
setProxy(const std::string & proxy,int port,const std::string & auth)402 void Client::setProxy( const std::string& proxy,
403 int port,
404 const std::string& auth )
405 {
406 d->proxy = proxy;
407 d->proxyPort = port;
408 d->proxyAuth = auth;
409 }
410
hasActiveRequests() const411 bool Client::hasActiveRequests() const
412 {
413 return !d->requests.empty();
414 }
415
receivedBytes(unsigned int count)416 void Client::receivedBytes(unsigned int count)
417 {
418 d->bytesTransferred += count;
419 d->totalBytesDownloaded += count;
420 }
421
transferRateBytesPerSec() const422 unsigned int Client::transferRateBytesPerSec() const
423 {
424 unsigned int e = d->timeTransferSample.elapsedMSec();
425 if (e > 400) {
426 // too long a window, ignore
427 d->timeTransferSample.stamp();
428 d->bytesTransferred = 0;
429 d->lastTransferRate = 0;
430 return 0;
431 }
432
433 if (e < 100) { // avoid really narrow windows
434 return d->lastTransferRate;
435 }
436
437 unsigned int ratio = (d->bytesTransferred * 1000) / e;
438 // run a low-pass filter
439 unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
440 smoothed /= 400;
441
442 d->timeTransferSample.stamp();
443 d->bytesTransferred = 0;
444 d->lastTransferRate = smoothed;
445 return smoothed;
446 }
447
totalBytesDownloaded() const448 uint64_t Client::totalBytesDownloaded() const
449 {
450 return d->totalBytesDownloaded;
451 }
452
requestWriteCallback(char * ptr,size_t size,size_t nmemb,void * userdata)453 size_t Client::requestWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
454 {
455 size_t byteSize = size * nmemb;
456 Request* req = static_cast<Request*>(userdata);
457 req->processBodyBytes(ptr, byteSize);
458
459 Client* cl = req->http();
460 if (cl) {
461 cl->receivedBytes(byteSize);
462 }
463
464 return byteSize;
465 }
466
requestReadCallback(char * ptr,size_t size,size_t nmemb,void * userdata)467 size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
468 {
469 size_t maxBytes = size * nmemb;
470 Request* req = static_cast<Request*>(userdata);
471 size_t actualBytes = req->getBodyData(ptr, 0, maxBytes);
472 return actualBytes;
473 }
474
isRedirectStatus(int code)475 bool isRedirectStatus(int code)
476 {
477 return ((code >= 300) && (code < 400));
478 }
479
requestHeaderCallback(char * rawBuffer,size_t size,size_t nitems,void * userdata)480 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
481 {
482 size_t byteSize = size * nitems;
483 Request* req = static_cast<Request*>(userdata);
484 std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
485
486 if (req->readyState() >= HTTP::Request::HEADERS_RECEIVED) {
487 // this can happen with chunked transfers (secondary chunks)
488 // or redirects
489 if (isRedirectStatus(req->responseCode())) {
490 req->responseStart(h);
491 return byteSize;
492 }
493 }
494
495 if (req->readyState() == HTTP::Request::OPENED) {
496 req->responseStart(h);
497 return byteSize;
498 }
499
500 if (h.empty()) {
501 // got a 100-continue reponse; restart
502 if (req->responseCode() == 100) {
503 req->setReadyState(HTTP::Request::OPENED);
504 return byteSize;
505 }
506
507 req->responseHeadersComplete();
508 return byteSize;
509 }
510
511 if (req->responseCode() == 100) {
512 return byteSize; // skip headers associated with 100-continue status
513 }
514
515 size_t colonPos = h.find(':');
516 if (colonPos == std::string::npos) {
517 SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
518 return byteSize;
519 }
520
521 const std::string key = strutils::simplify(h.substr(0, colonPos));
522 const std::string lkey = strutils::lowercase (key);
523 std::string value = strutils::strip(h.substr(colonPos + 1));
524
525 req->responseHeader(lkey, value);
526 return byteSize;
527 }
528
debugDumpRequests()529 void Client::debugDumpRequests()
530 {
531 SG_LOG(SG_IO, SG_INFO, "== HTTP request dump");
532 ClientPrivate::RequestCurlMap::iterator it = d->requests.begin();
533 for (; it != d->requests.end(); ++it) {
534 SG_LOG(SG_IO, SG_INFO, "\t" << it->first->url());
535 }
536 SG_LOG(SG_IO, SG_INFO, "==");
537 }
538
clearAllConnections()539 void Client::clearAllConnections()
540 {
541 curl_multi_cleanup(d->curlMulti);
542 d->createCurlMulti();
543 }
544
545 /////////////////////////////////////////////////////////////////////
546
setResponseDoneCallback(Client * cl,ResponseDoneCallback cb)547 void TestApi::setResponseDoneCallback(Client *cl, ResponseDoneCallback cb) {
548 cl->d->testsuiteResponseDoneCallback = cb;
549 }
550
markRequestAsFailed(Request_ptr req,int curlCode,const std::string & message)551 void TestApi::markRequestAsFailed(Request_ptr req, int curlCode,
552 const std::string &message) {
553 req->setFailure(curlCode, message);
554 }
555
556 } // of namespace HTTP
557
558 } // of namespace simgear
559