1 /* This file is part of pr-downloader (GPL v2 or later), see the LICENSE file */
2
3 #include "HttpDownloader.h"
4 #include "DownloadData.h"
5 #include "FileSystem/FileSystem.h"
6 #include "FileSystem/File.h"
7 #include "FileSystem/HashMD5.h"
8 #include "FileSystem/HashSHA1.h"
9 #include "Util.h"
10 #include "Logger.h"
11 #include "Downloader/Mirror.h"
12 #include "Downloader/CurlWrapper.h"
13 #include "lib/xmlrpc++/src/XmlRpcCurlClient.h"
14 #include "lib/xmlrpc++/src/XmlRpcValue.h"
15
16 #ifdef WIN32
17 #include <winsock2.h>
18 #else
19 #include <sys/select.h>
20 #endif
21
22 #include <stdio.h>
23 #include <curl/curl.h>
24 #include <string>
25 #include <sstream>
26 #include <stdlib.h>
27
CHttpDownloader()28 CHttpDownloader::CHttpDownloader()
29 {
30 }
31
~CHttpDownloader()32 CHttpDownloader::~CHttpDownloader()
33 {
34 }
35
search(std::list<IDownload * > & res,const std::string & name,IDownload::category cat)36 bool CHttpDownloader::search(std::list<IDownload*>& res, const std::string& name, IDownload::category cat)
37 {
38 CURL* curl = CurlWrapper::CurlInit();
39 LOG_DEBUG("%s", name.c_str() );
40
41 const std::string method(XMLRPC_METHOD);
42 //std::string category;
43 XmlRpc::XmlRpcCurlClient client(curl, XMLRPC_HOST,XMLRPC_PORT, XMLRPC_URI);
44 XmlRpc::XmlRpcValue arg;
45 arg["springname"]=name;
46 arg["torrent"]=true;
47 switch(cat) {
48 case IDownload::CAT_MAPS:
49 arg["category"]="map";
50 break;
51 case IDownload::CAT_GAMES:
52 arg["category"]="game";
53 break;
54 case IDownload::CAT_ENGINE_LINUX:
55 arg["category"]="engine_linux";
56 break;
57 case IDownload::CAT_ENGINE_LINUX64:
58 arg["category"]="engine_linux64";
59 break;
60 case IDownload::CAT_ENGINE_WINDOWS:
61 arg["category"]="engine_windows";
62 break;
63 case IDownload::CAT_ENGINE_MACOSX:
64 arg["category"]="engine_macosx";
65 break;
66 default:
67 break;
68 }
69
70 XmlRpc::XmlRpcValue result;
71 client.execute(method.c_str(),arg, result);
72
73
74 if (result.getType()!=XmlRpc::XmlRpcValue::TypeArray) {
75 return false;
76 }
77
78 for(int i=0; i<result.size(); i++) {
79 XmlRpc::XmlRpcValue resfile = result[i];
80
81 if (resfile.getType()!=XmlRpc::XmlRpcValue::TypeStruct) {
82 return false;
83 }
84 if (resfile["category"].getType()!=XmlRpc::XmlRpcValue::TypeString) {
85 LOG_ERROR("No category in result");
86 return false;
87 }
88 std::string filename=fileSystem->getSpringDir();
89 std::string category=resfile["category"];
90 filename+=PATH_DELIMITER;
91 if (category=="map")
92 filename+="maps";
93 else if (category=="game")
94 filename+="games";
95 else if (category.find("engine")==0) // engine_windows, engine_linux, engine_macosx
96 filename+="engine";
97 else
98 LOG_ERROR("Unknown Category %s", category.c_str());
99 filename+=PATH_DELIMITER;
100 if ((resfile["mirrors"].getType()!=XmlRpc::XmlRpcValue::TypeArray) ||
101 (resfile["filename"].getType()!=XmlRpc::XmlRpcValue::TypeString)) {
102 LOG_ERROR("Invalid type in result");
103 return false;
104 }
105 filename.append(resfile["filename"]);
106 IDownload* dl=new IDownload(filename,name, cat);
107 XmlRpc::XmlRpcValue mirrors = resfile["mirrors"];
108 for(int j=0; j<mirrors.size(); j++) {
109 if (mirrors[j].getType()!=XmlRpc::XmlRpcValue::TypeString) {
110 LOG_ERROR("Invalid type in result");
111 } else {
112 dl->addMirror(mirrors[j]);
113 }
114 }
115
116 if(resfile["torrent"].getType()==XmlRpc::XmlRpcValue::TypeBase64) {
117 const std::vector<char> torrent = resfile["torrent"];
118 fileSystem->parseTorrent(&torrent[0], torrent.size(), dl);
119 }
120 if (resfile["version"].getType()==XmlRpc::XmlRpcValue::TypeString) {
121 const std::string& version = resfile["version"];
122 dl->version = version;
123 }
124 if (resfile["md5"].getType()==XmlRpc::XmlRpcValue::TypeString) {
125 dl->hash=new HashMD5();
126 dl->hash->Set(resfile["md5"]);
127 }
128 if (resfile["size"].getType()==XmlRpc::XmlRpcValue::TypeInt) {
129 dl->size=resfile["size"];
130 }
131 if (resfile["depends"].getType() == XmlRpc::XmlRpcValue::TypeArray) {
132 for(int i=0; i<resfile["depends"].size(); i++) {
133 if (resfile["depends"][i].getType() == XmlRpc::XmlRpcValue::TypeString) {
134 const std::string &dep = resfile["depends"][i];
135 dl->addDepend(dep);
136 }
137 }
138 }
139 res.push_back(dl);
140 }
141 return true;
142 }
143
multi_write_data(void * ptr,size_t size,size_t nmemb,DownloadData * data)144 size_t multi_write_data(void *ptr, size_t size, size_t nmemb, DownloadData* data)
145 {
146 //LOG_DEBUG("%d %d",size, nmemb);
147 if (!data->got_ranges) {
148 LOG_INFO("Server refused ranges"); // The server refused ranges , download only from this piece , overwrite from 0 , and drop everything else
149
150 data->download->write_only_from = data;
151 data->got_ranges = true; //Silence the error
152 }
153 if ( data->download->write_only_from != NULL && data->download->write_only_from != data )
154 return size*nmemb;
155 else if ( data->download->write_only_from != NULL ) {
156 return data->download->file->Write((const char*)ptr, size*nmemb, 0);
157 }
158 return data->download->file->Write((const char*)ptr, size*nmemb, data->start_piece);
159 }
160
multiHeader(void * ptr,size_t size,size_t nmemb,DownloadData * data)161 size_t multiHeader(void *ptr, size_t size, size_t nmemb, DownloadData *data)
162 {
163 if(data->download->pieces.empty()) { //no chunked transfer, don't check headers
164 LOG_DEBUG("Unchunked transfer!");
165 data->got_ranges = true;
166 return size*nmemb;
167 }
168 const std::string buf((char*)ptr, size*nmemb-1);
169 int start, end, total;
170 int count = sscanf(buf.c_str(), "Content-Range: bytes %d-%d/%d", &start, &end, &total);
171 if(count == 3) {
172 int piecesize = data->download->file->GetPiecesSize(data->pieces);
173 if (end-start+1!=piecesize ) {
174 LOG_DEBUG("piecesize %d doesn't match server size: %d", piecesize, end-start+1);
175 return -1;
176 }
177 data->got_ranges = true;
178 }
179 LOG_DEBUG("%s", buf.c_str());
180 return size*nmemb;
181 }
182
getRange(std::string & range,int start_piece,int num_pieces,int piecesize)183 bool CHttpDownloader::getRange(std::string& range, int start_piece, int num_pieces, int piecesize)
184 {
185 std::ostringstream s;
186 s << (int)(piecesize*start_piece) <<"-"<< (piecesize*start_piece) + piecesize*num_pieces-1;
187 range=s.str();
188 LOG_DEBUG("%s", range.c_str());
189 return true;
190 }
191
showProcess(IDownload * download,bool force)192 void CHttpDownloader::showProcess(IDownload* download, bool force)
193 {
194 int done = download->getProgress();
195 int size = download->size;
196 LOG_PROGRESS(done, size, force);
197 }
198
verifyAndGetNextPieces(CFile & file,IDownload * download)199 std::vector< unsigned int > CHttpDownloader::verifyAndGetNextPieces(CFile& file, IDownload* download)
200 {
201 std::vector< unsigned int > pieces;
202 //verify file by md5 if pieces.size == 0
203 if((download->pieces.empty()) && (download->hash!=NULL) && (download->hash->isSet())) {
204 HashMD5 md5=HashMD5();
205 file.Hash(md5);
206 if (md5.compare(download->hash)) {
207 LOG_INFO("md5 correct: %s", md5.toString().c_str());
208 download->state=IDownload::STATE_FINISHED;
209 showProcess(download, true);
210 return pieces;
211 } else {
212 LOG_ERROR("md5 sum missmatch %s %s", download->hash->toString().c_str(), md5.toString().c_str());
213 }
214 }
215
216 HashSHA1 sha1=HashSHA1();
217 unsigned alreadyDl=0;
218 for(unsigned i=0; i<download->pieces.size(); i++ ) { //find first not downloaded piece
219 showProcess(download, false);
220 if (download->pieces[i].state==IDownload::STATE_FINISHED) {
221 alreadyDl++;
222 LOG_DEBUG("piece %d marked as downloaded", i);
223 if ( pieces.size() > 0 )
224 break;//Contiguos non-downloaded area finished
225 continue;
226 } else if (download->pieces[i].state==IDownload::STATE_NONE) {
227 if ((download->pieces[i].sha->isSet()) && (!file.IsNewFile())) { //reuse piece, if checksum is fine
228 file.Hash(sha1, i);
229 // LOG("bla %s %s", sha1.toString().c_str(), download.pieces[i].sha->toString().c_str());
230 if (sha1.compare(download->pieces[i].sha)) {
231 LOG_DEBUG("piece %d has already correct checksum, reusing", i);
232 download->pieces[i].state=IDownload::STATE_FINISHED;
233 showProcess(download, true);
234 alreadyDl++;
235 if ( pieces.size() > 0 )
236 break;//Contiguos non-downloaded area finished
237 continue;
238 }
239 }
240 pieces.push_back(i);
241 if ( pieces.size() == download->pieces.size()/download->parallel_downloads )
242 break;
243 }
244 }
245 if (pieces.size() == 0 && download->pieces.size() != 0) {
246 LOG_DEBUG("Finished\n");
247 download->state=IDownload::STATE_FINISHED;
248 showProcess(download, true);
249 }
250 LOG_DEBUG("Pieces to download: %d\n",pieces.size());
251 return pieces;
252 }
253
progress_func(DownloadData * data,double total,double done,double,double)254 static int progress_func(DownloadData* data, double total, double done, double, double)
255 {
256 data->download->progress = done;
257 if (data->got_ranges) {
258 LOG_PROGRESS(done, total, done >= total);
259 }
260 return 0;
261 }
262
setupDownload(DownloadData * piece)263 bool CHttpDownloader::setupDownload(DownloadData* piece)
264 {
265 std::vector<unsigned int> pieces = verifyAndGetNextPieces(*(piece->download->file), piece->download);
266 if (piece->download->state==IDownload::STATE_FINISHED)
267 return false;
268 if ( piece->download->file ) {
269 piece->download->size = piece->download->file->GetPieceSize(-1);
270 LOG_DEBUG("Size is %d",piece->download->size);
271 }
272 piece->start_piece=pieces.size() > 0 ? pieces[0] : -1;
273 assert(piece->download->pieces.size()<=0 || piece->start_piece >=0);
274 piece->pieces = pieces;
275 if (piece->easy_handle==NULL) {
276 piece->easy_handle=CurlWrapper::CurlInit();
277 } else {
278 curl_easy_cleanup(piece->easy_handle);
279 piece->easy_handle=CurlWrapper::CurlInit();
280 }
281
282 CURL* curle= piece->easy_handle;
283 piece->mirror=piece->download->getFastestMirror();
284 if (piece->mirror==NULL) {
285 LOG_ERROR("No mirror found");
286 return false;
287 }
288 std::string escaped;
289 piece->mirror->escapeUrl(escaped);
290 curl_easy_setopt(curle, CURLOPT_WRITEFUNCTION, multi_write_data);
291 curl_easy_setopt(curle, CURLOPT_WRITEDATA, piece);
292 curl_easy_setopt(curle, CURLOPT_NOPROGRESS, 0L);
293 curl_easy_setopt(curle, CURLOPT_PROGRESSDATA, piece);
294 curl_easy_setopt(curle, CURLOPT_PROGRESSFUNCTION, progress_func);
295 curl_easy_setopt(curle, CURLOPT_URL, escaped.c_str());
296
297 if ((piece->download->size>0) && (piece->start_piece>=0) && piece->download->pieces.size() > 0) { //don't set range, if size unknown
298 std::string range;
299 if (!getRange(range, piece->start_piece, piece->pieces.size() , piece->download->piecesize)) {
300 LOG_ERROR("Error getting range for download");
301 return false;
302 }
303 //set range for request, format is <start>-<end>
304 if ( !(piece->start_piece == 0 && piece->pieces.size() == piece->download->pieces.size()) )
305 curl_easy_setopt(curle, CURLOPT_RANGE, range.c_str());
306 //parse server response header as well
307 curl_easy_setopt(curle, CURLOPT_HEADERFUNCTION, multiHeader);
308 curl_easy_setopt(curle, CURLOPT_WRITEHEADER, piece);
309 for ( std::vector<unsigned int>::iterator it = piece->pieces.begin(); it != piece->pieces.end(); it++ )
310 piece->download->pieces[*it].state=IDownload::STATE_DOWNLOADING;
311 } else { //
312 LOG_DEBUG("single piece transfer");
313 piece->got_ranges = true;
314
315 //this sets the header If-Modified-Since -> downloads only when remote file is newer than local file
316 const long timestamp = piece->download->file->GetTimestamp();
317 curl_easy_setopt(curle, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
318 curl_easy_setopt(curle, CURLOPT_TIMEVALUE, timestamp );
319 curl_easy_setopt(curle, CURLOPT_FILETIME, 1);
320 }
321 return true;
322 }
323
getDataByHandle(const std::vector<DownloadData * > & downloads,const CURL * easy_handle) const324 DownloadData* CHttpDownloader::getDataByHandle(const std::vector <DownloadData*>& downloads, const CURL* easy_handle) const
325 {
326 for(int i=0; i<(int)downloads.size(); i++) { //search corresponding data structure
327 if (downloads[i]->easy_handle == easy_handle) {
328 return downloads[i];
329 }
330 }
331 return NULL;
332 }
333
processMessages(CURLM * curlm,std::vector<DownloadData * > & downloads)334 bool CHttpDownloader::processMessages(CURLM* curlm, std::vector <DownloadData*>& downloads)
335 {
336 int msgs_left;
337 HashSHA1 sha1;
338 bool aborted=false;
339 while(struct CURLMsg* msg=curl_multi_info_read(curlm, &msgs_left)) {
340 switch(msg->msg) {
341 case CURLMSG_DONE: { //a piece has been downloaded, verify it
342 DownloadData* data=getDataByHandle(downloads, msg->easy_handle);
343 switch(msg->data.result) {
344 case CURLE_OK:
345 break;
346 case CURLE_HTTP_RETURNED_ERROR: //some 4* HTTP-Error (file not found, access denied,...)
347 default:
348 long http_code = 0;
349 curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &http_code);
350 LOG_ERROR("CURL error(%d:%d): %s %d (%s)",msg->msg, msg->data.result, curl_easy_strerror(msg->data.result), http_code, data->mirror->url.c_str());
351 if (data->start_piece>=0) {
352 data->download->pieces[data->start_piece].state=IDownload::STATE_NONE;
353 }
354 data->mirror->status=Mirror::STATUS_BROKEN;
355 //FIXME: cleanup curl handle here + process next dl
356 }
357 if (data==NULL) {
358 LOG_ERROR("Couldn't find download in download list");
359 return false;
360 }
361 if (data->start_piece<0) { //download without pieces
362 return false;
363 }
364 assert(data->download->file!=NULL);
365 assert(data->start_piece< (int)data->download->pieces.size());
366 for ( std::vector<unsigned int>::iterator it = data->pieces.begin(); it != data->pieces.end(); it++ ) {
367 if (data->download->pieces[*it].sha->isSet()) {
368 data->download->file->Hash(sha1, *it);
369
370 if (sha1.compare(data->download->pieces[*it].sha)) { //piece valid
371
372 data->download->pieces[*it].state=IDownload::STATE_FINISHED;
373 showProcess(data->download, true);
374 // LOG("piece %d verified!", data->piece);
375 } else { // piece download broken, mark mirror as broken (for this file)
376 data->download->pieces[*it].state=IDownload::STATE_NONE;
377 data->mirror->status=Mirror::STATUS_BROKEN;
378 //FIXME: cleanup curl handle here + process next dl
379 LOG_ERROR("Piece %d is invalid",*it);
380 }
381 } else {
382 LOG_INFO("sha1 checksum seems to be not set, can't check received piece %d-%d", data->start_piece,data->pieces.size());
383 }
384 }
385 //get speed at which this piece was downloaded + update mirror info
386 double dlSpeed;
387 curl_easy_getinfo(data->easy_handle, CURLINFO_SPEED_DOWNLOAD, &dlSpeed);
388 data->mirror->UpdateSpeed(dlSpeed);
389 if (data->mirror->status == Mirror::STATUS_UNKNOWN) //set mirror status only when unset
390 data->mirror->status=Mirror::STATUS_OK;
391
392 //remove easy handle, as its finished
393 curl_multi_remove_handle(curlm, data->easy_handle);
394 curl_easy_cleanup(data->easy_handle);
395 data->easy_handle=NULL;
396 LOG_INFO("piece finished");
397 //piece finished / failed, try a new one
398 if (!setupDownload(data)) {
399 LOG_DEBUG("No piece found, all pieces finished / currently downloading");
400 break;
401 }
402 int ret=curl_multi_add_handle(curlm, data->easy_handle);
403 if (ret!=CURLM_OK) {
404 LOG_ERROR("curl_multi_perform_error: %d %d", ret, CURLM_BAD_EASY_HANDLE);
405 }
406 break;
407 }
408 default:
409 LOG_ERROR("Unhandled message %d", msg->msg);
410 }
411 }
412 return aborted;
413 }
414
download(std::list<IDownload * > & download,int max_parallel)415 bool CHttpDownloader::download(std::list<IDownload*>& download, int max_parallel)
416 {
417
418 std::list<IDownload*>::iterator it;
419 std::vector <DownloadData*> downloads;
420 CURLM* curlm=curl_multi_init();
421 for(it=download.begin(); it!=download.end(); ++it) {
422 if ((*it)->dltype != IDownload::TYP_HTTP) {
423 LOG_DEBUG("skipping non http-dl")
424 continue;
425 }
426 const int count=std::min(max_parallel, std::max(1, std::min((int)(*it)->pieces.size(), (*it)->getMirrorCount()))); //count of parallel downloads
427 if((*it)->getMirrorCount()<=0) {
428 LOG_ERROR("No mirrors found");
429 return false;
430 }
431 LOG_DEBUG("Using %d parallel downloads", count);
432 (*it)->parallel_downloads = count;
433 CFile* file=new CFile();
434 if(!file->Open((*it)->name, (*it)->size, (*it)->piecesize)) {
435 delete file;
436 return false;
437 }
438 (*it)->file = file;
439 for(int i=0; i<count; i++) {
440 DownloadData* dlData=new DownloadData();
441 dlData->download=*it;
442 if (!setupDownload(dlData)) { //no piece found (all pieces already downloaded), skip
443 delete dlData;
444 if ((*it)->state!=IDownload::STATE_FINISHED) {
445 LOG_ERROR("no piece found");
446 return false;
447 }
448 } else {
449 downloads.push_back(dlData);
450 curl_multi_add_handle(curlm, dlData->easy_handle);
451 }
452 }
453 }
454
455 bool aborted=false;
456 int running=1, last=-1;
457 while((running>0)&&(!aborted)) {
458 CURLMcode ret = CURLM_CALL_MULTI_PERFORM;
459 while(ret == CURLM_CALL_MULTI_PERFORM) {
460 ret=curl_multi_perform(curlm, &running);
461 }
462 if ( ret == CURLM_OK ) {
463 // showProcess(download, file);
464 if (last!=running) { //count of running downloads changed
465 aborted=processMessages(curlm, downloads);
466 last=running++;
467 }
468 } else {
469 LOG_ERROR("curl_multi_perform_error: %d", ret);
470 aborted=true;
471 }
472
473 fd_set rSet;
474 fd_set wSet;
475 fd_set eSet;
476
477 FD_ZERO(&rSet);
478 FD_ZERO(&wSet);
479 FD_ZERO(&eSet);
480 int count=0;
481 struct timeval t;
482 t.tv_sec = 1;
483 t.tv_usec = 0;
484 curl_multi_fdset(curlm, &rSet, &wSet, &eSet, &count);
485 //sleep for one sec / until something happened
486 select(count+1, &rSet, &wSet, &eSet, &t);
487 }
488 if (!aborted) { // if download didn't fail, get file size reported in http-header
489 double size=-1;
490 for (unsigned i=0; i<downloads.size(); i++) {
491 double tmp;
492 curl_easy_getinfo(downloads[i]->easy_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &tmp);
493 if (tmp>size) {
494 size=tmp;
495 }
496 }
497 //set download size if isn't set and we have a valid number
498 // if ((size>0) && (download->size<0)) {
499 // download->size = size;
500 // }
501
502 }
503 // showProcess(download, file);
504 LOG("\n");
505
506 if (!aborted) {
507 LOG_DEBUG("download complete");
508 }
509
510 //close all open files
511 for(it=download.begin(); it!=download.end(); ++it) {
512 if ((*it)->file!=NULL)
513 (*it)->file->Close();
514 }
515 for (unsigned i=0; i<downloads.size(); i++) {
516 long timestamp;
517 if (curl_easy_getinfo(downloads[i]->easy_handle, CURLINFO_FILETIME, ×tamp) == CURLE_OK) {
518 if (downloads[i]->download->state != IDownload::STATE_FINISHED) //decrease local timestamp if download failed to force redownload next time
519 timestamp--;
520 downloads[i]->download->file->SetTimestamp(timestamp);
521 }
522 delete downloads[i];
523 }
524
525 downloads.clear();
526 curl_multi_cleanup(curlm);
527 return !aborted;
528 }
529
530
531