1 /* librepo - A library providing (libcURL like) API to downloading repository
2 * Copyright (C) 2013 Tomas Mlcoch
3 *
4 * Licensed under the GNU Lesser General Public License Version 2.1
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #if __FreeBSD__
22 //#define _XOPEN_SOURCE 500 // Because of fdopen() and ftruncate()
23 //#define _DEFAULT_SOURCE // Because of futimes()
24 //#define _BSD_SOURCE // Because of futimes()
25 #endif
26
27 #include <glib.h>
28 #include <assert.h>
29 #include <stdio.h>
30 #include <time.h>
31 #include <unistd.h>
32 #include <string.h>
33 #include <errno.h>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 #include <sys/time.h>
37 #include "xattr.h"
38 #include <fcntl.h>
39 #include <curl/curl.h>
40
41 #ifdef WITH_ZCHUNK
42 #include <zck.h>
43 #endif /* WITH_ZCHUNK */
44
45 #include "downloader.h"
46 #include "downloader_internal.h"
47 #include "rcodes.h"
48 #include "util.h"
49 #include "downloadtarget.h"
50 #include "downloadtarget_internal.h"
51 #include "handle.h"
52 #include "handle_internal.h"
53 #include "cleanup.h"
54 #include "url_substitution.h"
55 #include "yum_internal.h"
56 #include "xattr_internal.h"
57
58
59 volatile sig_atomic_t lr_interrupt = 0;
60
61 void
lr_sigint_handler(G_GNUC_UNUSED int sig)62 lr_sigint_handler(G_GNUC_UNUSED int sig)
63 {
64 lr_interrupt = 1;
65 }
66
67 typedef enum {
68 LR_DS_WAITING, /*!<
69 The target is waiting to be processed. */
70 LR_DS_RUNNING, /*!<
71 The transfer is running. */
72 LR_DS_FINISHED, /*!<
73 The transfer is successfully finished. */
74 LR_DS_FAILED, /*!<
75 The transfer is finished without success. */
76 } LrDownloadState;
77
78 typedef enum {
79 LR_HCS_DEFAULT, /*!<
80 Default state */
81 LR_HCS_HTTP_STATE_OK, /*!<
82 HTTP headers with OK state */
83 LR_HCS_INTERRUPTED, /*!<
84 Download was interrupted (e.g. Content-Length doesn't match
85 expected size etc.) */
86 LR_HCS_DONE, /*!<
87 All headers which we were looking for are already found*/
88 } LrHeaderCbState;
89
90 /** Enum with zchunk file status */
91 typedef enum {
92 LR_ZCK_DL_HEADER_CK, /*!<
93 The zchunk file is waiting to check whether the header is available
94 locally. */
95 LR_ZCK_DL_HEADER, /*!<
96 The zchunk file is waiting to download the header */
97 LR_ZCK_DL_BODY_CK, /*!<
98 The zchunk file is waiting to check what chunks are available locally */
99 LR_ZCK_DL_BODY, /*!<
100 The zchunk file is waiting for its body to be downloaded. */
101 LR_ZCK_DL_FINISHED /*!<
102 The zchunk file is finished being downloaded. */
103 } LrZckState;
104
105 typedef struct {
106 LrHandle *handle; /*!<
107 Handle (could be NULL) */
108 GSList *lrmirrors; /*!<
109 List of LrMirrors created from the handle internal mirrorlist
110 (could be NULL) */
111 } LrHandleMirrors;
112
113 typedef struct {
114 LrInternalMirror *mirror; /*!<
115 Mirror */
116 int allowed_parallel_connections; /*!<
117 Maximum number of allowed parallel connections to this mirror. -1 means no limit.
118 Dynamically adjusted (decreased) if no fatal (temporary) error will occur. */
119 int max_tried_parallel_connections; /*!<
120 The maximum number of tried parallel connections to this mirror
121 (including unsuccessful). */
122 int running_transfers; /*!<
123 How many transfers from this mirror are currently in progress. */
124 int successful_transfers; /*!<
125 How many transfers was finished successfully from the mirror. */
126 int failed_transfers; /*!<
127 How many transfers failed. */
128 int max_ranges; /*!<
129 Maximum ranges supported in a single request. This will be automatically
130 adjusted when mirrors respond with 200 to a range request */
131 } LrMirror;
132
133 typedef struct {
134 LrDownloadState state; /*!<
135 State of the download (transfer). */
136 LrDownloadTarget *target; /*!<
137 Download target */
138 LrMirror *mirror; /*!<
139 Mirror is:
140 If a base_location is used then NULL.
141 If state is LR_DS_WAITING then could be NULL (if no download
142 was tried yet) or notNULL if previous download failed, in that case
143 the mirror is the last used mirror (the one that failed).
144 If state is LR_DS_RUNNING then currently used mirror.
145 If state is LR_DS_FINISHED then mirror from which download was
146 successfully performed.
147 If state is LR_DS_FAILED then mirror from which last try
148 was done. */
149 LrProtocol protocol; /*!<
150 Current protocol */
151 CURL *curl_handle; /*!<
152 Used curl handle or NULL */
153 FILE *f; /*!<
154 fdopened file descriptor from LrDownloadTarget and used
155 in curl_handle. */
156 char errorbuffer[CURL_ERROR_SIZE]; /*!<
157 Error buffer used in curl handle */
158 GSList *tried_mirrors; /*!<
159 List of already tried mirrors (LrMirror *).
160 This mirrors won't be tried again. */
161 gboolean resume; /*!<
162 Is resume enabled? Download target may state that resume is True
163 but Librepo can decide that resuming won't be done.
164 This variable states if the resume is enabled or not. */
165 gint64 original_offset; /*!<
166 If resume is enabled, this is the specified offset where to resume
167 the downloading. If resume is not enabled, then value is -1. */
168 gint resume_count; /*!<
169 How many resumes were done */
170 GSList *lrmirrors; /*!<
171 List of all available mirors (LrMirror *).
172 This list is generated from LrHandle related to this target
173 and is common for all targets that uses the handle. */
174 LrHandle *handle; /*!<
175 LrHandle associated with this target */
176 LrHeaderCbState headercb_state; /*!<
177 State of the header callback for current transfer */
178 gchar *headercb_interrupt_reason; /*!<
179 Reason why was the transfer interrupted */
180 gint64 writecb_recieved; /*!<
181 Total number of bytes received by the write function
182 during the current transfer. */
183 gboolean writecb_required_range_written; /*!<
184 If a byte range was specified to download and the
185 range was downloaded, it is TRUE. Otherwise FALSE. */
186 LrCbReturnCode cb_return_code; /*!<
187 Last cb return code. */
188 struct curl_slist *curl_rqheaders; /*!<
189 Extra headers for request. */
190
191 #ifdef WITH_ZCHUNK
192 LrZckState zck_state; /*!<
193 Zchunk download status */
194 #endif /* WITH_ZCHUNK */
195
196 gboolean range_fail; /*!<
197 Whether range request failed. */
198 } LrTarget;
199
200 typedef struct {
201
202 // Configuration
203
204 gboolean failfast; /*!<
205 Fail fast */
206
207 int max_parallel_connections; /*!<
208 Maximal number of parallel downloads. */
209
210 int max_connection_per_host; /*!<
211 Maximal number of connections per host. -1 means no limit. */
212
213 int max_mirrors_to_try; /*!<
214 Maximal number of mirrors to try. Number <= 0 means no limit. */
215
216 long allowed_mirror_failures; /*!<
217 See LRO_ALLOWEDMIRRORFAILURES */
218
219 long adaptivemirrorsorting; /*!<
220 See LRO_ADAPTIVEMIRRORSORTING */
221
222 // Data
223
224 CURLM *multi_handle; /*!<
225 Curl Multi handle */
226
227 GSList *handle_mirrors; /*!<
228 All mirrors (list of pointers to LrHandleMirrors structures) */
229
230 GSList *targets; /*!<
231 List of all targets (list of pointers to LrTarget stuctures) */
232
233 GSList *running_transfers; /*!<
234 List of running transfers (list of pointer to LrTarget structures) */
235
236 } LrDownload;
237
238 /** Schema of structures as used in downloader module:
239 *
240 * +------------------------------+
241 * | LrDownload |
242 * +------------------------------+
243 * | int max_parallel_connections |
244 * | int max_connection_par_host |
245 * | int max_mirrors_to_try | +-------------------+
246 * | | /->| LrHandleMirrors |
247 * | | | +-------------------+
248 * | CURLM *multi_handle | | | LrHandle *handle |
249 * | | | | GSList *lrmirrors --\
250 * | GSList *handle_mirrors ---/ +-------------------+ |
251 * | GSList *targets --\ |
252 * | GSList *running_transfers ---\ |
253 * +------------------------------+ | |
254 * | |
255 * /------------------------------/ |
256 * | |
257 * | /--------------------------------/
258 * | \/
259 * | +------------------------------------+
260 * | | LrMirror |
261 * | +------------------------------------+-|
262 * | | LrMirror | |
263 * | +------------------------------------+-| |
264 * | | LrMirror | | |
265 * | +------------------------------------+ | | +---------------------+
266 * | | LrInternalMirror *mirror ------------------>| LrInternalMirror |
267 * | | int allowed_parallel_connections | | +---------------------+
268 * | | int max_tried_parallel_connections |-+ | char *url |
269 * | | int running_transfersers | | int preference |
270 * | | int successful_transfers | | LrProtocol protocol |
271 * | + int failed_transfers +<--\ +---------------------+
272 * | +------------------------------------+ |
273 * | |
274 * | +----------------------------+ /-----/
275 * | | LrTarget | |
276 * | +----------------------------+-| |
277 * | | LrTarget | | | +--------------------------+
278 * | +----------------------------+-| | | /->| LrDownloadTarget |
279 * \-> | LrTarget | | | | | +--------------------------+
280 * +----------------------------+ | | | | | char *path |
281 * | LrDownloadState state | | | | | | char *baseurl |
282 * | LrDownloadTarget *target -----------/ | int fd |
283 * | LrMirror *mirror --------/ | LrChecksumType checks.. |
284 * | CURL *curl_handle |-+ | char *checksum |
285 * | FILE *f | | int resume |
286 * | GSList *tried_mirrors | | LrProgressCb progresscb |
287 * | gint64 original_offset | | void *cbdata |
288 * | GSlist *lrmirrors ---\ | GStringChunk *chunk |
289 * +----------------------------+ | | int rcode |
290 * | | char *err |
291 * Points to list of LrMirrors <---/ +--------------------------+
292 */
293
294 static gboolean
is_max_mirrors_unlimited(const LrDownload * download)295 is_max_mirrors_unlimited(const LrDownload *download)
296 {
297 return download->max_mirrors_to_try <= 0;
298 }
299
300 static gboolean
301 /**
302 * @brief Returns whether the download can be retried, using the same URL in case of baseurl or full
303 * path, or using another mirror in case of using mirrors.
304 *
305 * @param complete_path_or_baseurl determine type of download - mirrors or baseurl/fullpath
306 * @return gboolean Return TRUE when another chance to download is allowed.
307 */
can_retry_download(const LrDownload * download,int num_of_tried_mirrors,const char * complete_path_or_baseurl)308 can_retry_download(const LrDownload *download, int num_of_tried_mirrors,
309 const char * complete_path_or_baseurl)
310 {
311 if (complete_path_or_baseurl) {
312 if (g_str_has_prefix(complete_path_or_baseurl, "file:/")) {
313 return FALSE;
314 }
315 return download->allowed_mirror_failures > num_of_tried_mirrors;
316 }
317 return is_max_mirrors_unlimited(download) ||
318 num_of_tried_mirrors < download->max_mirrors_to_try;
319 }
320
321 static gboolean
has_running_transfers(const LrMirror * mirror)322 has_running_transfers(const LrMirror *mirror)
323 {
324 return mirror->running_transfers > 0;
325 }
326
327 static void
init_once_allowed_parallel_connections(LrMirror * mirror,int max_allowed_parallel_connections)328 init_once_allowed_parallel_connections(LrMirror *mirror, int max_allowed_parallel_connections)
329 {
330 if (mirror->allowed_parallel_connections == 0) {
331 mirror->allowed_parallel_connections = max_allowed_parallel_connections;
332 }
333 }
334
335 static void
increase_running_transfers(LrMirror * mirror)336 increase_running_transfers(LrMirror *mirror)
337 {
338 mirror->running_transfers++;
339 if (mirror->max_tried_parallel_connections < mirror->running_transfers)
340 mirror->max_tried_parallel_connections = mirror->running_transfers;
341 }
342
343 static gboolean
is_parallel_connections_limited_and_reached(const LrMirror * mirror)344 is_parallel_connections_limited_and_reached(const LrMirror *mirror)
345 {
346 return mirror->allowed_parallel_connections != -1 &&
347 mirror->running_transfers >= mirror->allowed_parallel_connections;
348 }
349
350 static void
mirror_update_statistics(LrMirror * mirror,gboolean transfer_success)351 mirror_update_statistics(LrMirror *mirror, gboolean transfer_success)
352 {
353 mirror->running_transfers--;
354 if (transfer_success)
355 mirror->successful_transfers++;
356 else
357 mirror->failed_transfers++;
358 }
359
360 /** Create GSList of LrMirrors (if it doesn't exist) for a handle.
361 * If the list already exists (if more targets use the same handle)
362 * then just set the list to the current target.
363 * If the list doesn't exist yet, create it then create a mapping between
364 * the list and the handle (LrHandleMirrors) and set the list to
365 * the current target.
366 */
367 static GSList *
lr_prepare_lrmirrors(GSList * list,LrTarget * target)368 lr_prepare_lrmirrors(GSList *list, LrTarget *target)
369 {
370 LrHandle *handle = target->handle;
371
372 for (GSList *elem = list; elem; elem = g_slist_next(elem)) {
373 LrHandleMirrors *handle_mirrors = elem->data;
374 if (handle_mirrors->handle == handle) {
375 // List of LrMirrors for this handle is already created
376 target->lrmirrors = handle_mirrors->lrmirrors;
377 return list;
378 }
379 }
380
381 GSList *lrmirrors = NULL;
382
383 if (handle && handle->internal_mirrorlist) {
384 g_debug("%s: Preparing internal mirror list for handle id: %p", __func__, handle);
385 for (GSList *elem = handle->internal_mirrorlist;
386 elem;
387 elem = g_slist_next(elem))
388 {
389 LrInternalMirror *imirror = elem->data;
390
391 assert(imirror);
392 assert(imirror->url);
393
394 if (!imirror || !imirror->url || !strlen(imirror->url))
395 continue;
396
397 g_debug("%s: Mirror: %s", __func__, imirror->url);
398
399 LrMirror *mirror = lr_malloc0(sizeof(*mirror));
400 mirror->mirror = imirror;
401 mirror->max_ranges = 256;
402 lrmirrors = g_slist_append(lrmirrors, mirror);
403 }
404 }
405
406 LrHandleMirrors *handle_mirrors = lr_malloc0(sizeof(*handle_mirrors));
407 handle_mirrors->handle = handle;
408 handle_mirrors->lrmirrors = lrmirrors;
409
410 target->lrmirrors = lrmirrors;
411 list = g_slist_append(list, handle_mirrors);
412
413 return list;
414 }
415
416
417 /** Progress callback for CURL handles.
418 * progress callback set by the user of librepo.
419 */
420 static int
lr_progresscb(void * ptr,double total_to_download,double now_downloaded,G_GNUC_UNUSED double total_to_upload,G_GNUC_UNUSED double now_uploaded)421 lr_progresscb(void *ptr,
422 double total_to_download,
423 double now_downloaded,
424 G_GNUC_UNUSED double total_to_upload,
425 G_GNUC_UNUSED double now_uploaded)
426 {
427 int ret = LR_CB_OK;
428 LrTarget *target = ptr;
429
430 assert(target);
431 assert(target->target);
432
433 if (target->state != LR_DS_RUNNING)
434 return ret;
435
436 if (!target->target->progresscb)
437 return ret;
438
439 #ifdef WITH_ZCHUNK
440 if (target->target->is_zchunk) {
441 total_to_download = target->target->total_to_download;
442 now_downloaded = now_downloaded + target->target->downloaded;
443 }
444 #endif /* WITH_ZCHUNK */
445
446 ret = target->target->progresscb(target->target->cbdata,
447 total_to_download,
448 now_downloaded);
449
450 target->cb_return_code = ret;
451
452 return ret;
453 }
454
455 #define STRLEN(s) (sizeof(s)/sizeof(s[0]) - 1)
456
457 #ifdef WITH_ZCHUNK
458 /* Fail if dl_ctx->fail_no_ranges is set and we get a 200 response */
lr_zckheadercb(char * b,size_t l,size_t c,void * dl_v)459 size_t lr_zckheadercb(char *b, size_t l, size_t c, void *dl_v) {
460 LrTarget *target = (LrTarget *)dl_v;
461 assert(target && target->target);
462
463 long code = -1;
464 curl_easy_getinfo(target->curl_handle, CURLINFO_RESPONSE_CODE, &code);
465 if(code == 200) {
466 g_debug("%s: Too many ranges were attempted in one download", __func__);
467 target->range_fail = 1;
468 return 0;
469 }
470 return zck_header_cb(b, l, c, target->target->zck_dl);
471 }
472 #endif /* WITH_ZCHUNK */
473
474 /** Header callback for CURL handles.
475 * It parses HTTP and FTP headers and try to find length of the content
476 * (file size of the target). If the size is different then the expected
477 * size, then the transfer is interrupted.
478 * This callback is used only if the expected size is specified.
479 */
480 static size_t
lr_headercb(void * ptr,size_t size,size_t nmemb,void * userdata)481 lr_headercb(void *ptr, size_t size, size_t nmemb, void *userdata)
482 {
483 assert(userdata);
484
485 size_t ret = size * nmemb;
486 LrTarget *lrtarget = userdata;
487 LrHeaderCbState state = lrtarget->headercb_state;
488
489 if (state == LR_HCS_DONE || state == LR_HCS_INTERRUPTED) {
490 // Nothing to do
491 return ret;
492 }
493
494 #ifdef WITH_ZCHUNK
495 if(lrtarget->target->is_zchunk && !lrtarget->range_fail && lrtarget->mirror->mirror->protocol == LR_PROTOCOL_HTTP)
496 return lr_zckheadercb(ptr, size, nmemb, userdata);
497 #endif /* WITH_ZCHUNK */
498
499 char *header = g_strndup(ptr, size*nmemb);
500 // strips in place
501 g_strstrip(header);
502
503 gint64 expected = lrtarget->target->expectedsize;
504
505 if (state == LR_HCS_DEFAULT) {
506 if (lrtarget->protocol == LR_PROTOCOL_HTTP
507 && g_str_has_prefix(header, "HTTP/")) {
508 // Header of a HTTP protocol
509 if ((g_strrstr(header, "200") ||
510 g_strrstr(header, "206")) && !(
511 g_strrstr(header, "connection established") ||
512 g_strrstr(header, "Connection established") ||
513 g_strrstr(header, "Connection Established")
514 )) {
515 lrtarget->headercb_state = LR_HCS_HTTP_STATE_OK;
516 } else {
517 // Do nothing (do not change the state)
518 // in case of redirection, 200 OK still could come
519 g_debug("%s: Non OK HTTP header status: %s", __func__, header);
520 }
521 } else if (lrtarget->protocol == LR_PROTOCOL_FTP) {
522 // Headers of a FTP protocol
523 if (g_str_has_prefix(header, "213 ")) {
524 // Code 213 should keep the file size
525 gint64 content_length = g_ascii_strtoll(header+4, NULL, 0);
526
527 g_debug("%s: Server returned size: \"%s\" "
528 "(converted %"G_GINT64_FORMAT"/%"G_GINT64_FORMAT
529 " expected)",
530 __func__, header+4, content_length, expected);
531
532 // Compare expected size and size reported by a FTP server
533 if (content_length > 0 && content_length != expected) {
534 g_debug("%s: Size doesn't match (%"G_GINT64_FORMAT
535 " != %"G_GINT64_FORMAT")",
536 __func__, content_length, expected);
537 lrtarget->headercb_state = LR_HCS_INTERRUPTED;
538 lrtarget->headercb_interrupt_reason = g_strdup_printf(
539 "FTP server reports size: %"G_GINT64_FORMAT" "
540 "via 213 code, but expected size is: %"G_GINT64_FORMAT,
541 content_length, expected);
542 ret++; // Return error value
543 } else {
544 lrtarget->headercb_state = LR_HCS_DONE;
545 }
546 } else if (g_str_has_prefix(header, "150")) {
547 // Code 150 should keep the file size
548 // TODO: See parse150 in /usr/lib64/python2.7/ftplib.py
549 }
550 }
551 }
552
553 if (state == LR_HCS_HTTP_STATE_OK) {
554 if (g_str_has_prefix(header, "Content-Length: ")) {
555 // Content-Length header found
556 char *content_length_str = header + STRLEN("Content-Length: ");
557 gint64 content_length = g_ascii_strtoll(content_length_str,
558 NULL, 0);
559 g_debug("%s: Server returned Content-Length: \"%s\" "
560 "(converted %"G_GINT64_FORMAT"/%"G_GINT64_FORMAT" expected)",
561 __func__, content_length_str, content_length, expected);
562
563 // Compare expected size and size reported by a HTTP server
564 if (content_length > 0 && content_length != expected) {
565 g_debug("%s: Size doesn't match (%"G_GINT64_FORMAT
566 " != %"G_GINT64_FORMAT")",
567 __func__, content_length, expected);
568 lrtarget->headercb_state = LR_HCS_INTERRUPTED;
569 lrtarget->headercb_interrupt_reason = g_strdup_printf(
570 "Server reports Content-Length: %"G_GINT64_FORMAT" but "
571 "expected size is: %"G_GINT64_FORMAT,
572 content_length, expected);
573 ret++; // Return error value
574 } else {
575 lrtarget->headercb_state = LR_HCS_DONE;
576 }
577 }
578 }
579
580 g_free(header);
581
582 return ret;
583 }
584
585
586 #ifdef WITH_ZCHUNK
587 /** Zchunk write callback for CURL handles.
588 */
589 size_t
lr_zck_writecb(char * ptr,size_t size,size_t nmemb,void * userdata)590 lr_zck_writecb(char *ptr, size_t size, size_t nmemb, void *userdata)
591 {
592 LrTarget *target = (LrTarget *) userdata;
593 if(target->zck_state == LR_ZCK_DL_HEADER)
594 return zck_write_zck_header_cb(ptr, size, nmemb, target->target->zck_dl);
595 else
596 return zck_write_chunk_cb(ptr, size, nmemb, target->target->zck_dl);
597 }
598 #endif /* WITH_ZCHUNK */
599
600 /** Write callback for CURL handles.
601 * This callback handles situation when an user wants only specified
602 * byte range of the target file.
603 */
604 size_t
lr_writecb(char * ptr,size_t size,size_t nmemb,void * userdata)605 lr_writecb(char *ptr, size_t size, size_t nmemb, void *userdata)
606 {
607 size_t cur_written_expected = nmemb;
608 size_t cur_written;
609 LrTarget *target = (LrTarget *) userdata;
610 #ifdef WITH_ZCHUNK
611 if(target->target->is_zchunk && !target->range_fail && target->mirror->mirror->protocol == LR_PROTOCOL_HTTP)
612 return lr_zck_writecb(ptr, size, nmemb, userdata);
613 #endif /* WITH_ZCHUNK */
614
615 gint64 all = size * nmemb; // Total number of bytes from curl
616 gint64 range_start = target->target->byterangestart;
617 gint64 range_end = target->target->byterangeend;
618
619 if (range_start <= 0 && range_end <= 0) {
620 // Write everything curl give to you
621 target->writecb_recieved += all;
622 return fwrite(ptr, size, nmemb, target->f);
623 }
624
625 /* Deal with situation when user wants only specific byte range of the
626 * target file, and write only the range.
627 */
628
629 gint64 cur_range_start = target->writecb_recieved;
630 gint64 cur_range_end = cur_range_start + all;
631
632 target->writecb_recieved += all;
633
634 if (target->target->byterangestart > 0) {
635 // If byterangestart is specified, then CURLOPT_RESUME_FROM_LARGE
636 // is used by default
637 cur_range_start += target->target->byterangestart;
638 cur_range_end += target->target->byterangestart;
639 } else if (target->original_offset > 0) {
640 cur_range_start += target->original_offset;
641 cur_range_end += target->original_offset;
642 }
643
644 if (cur_range_end < range_start)
645 // The wanted byte range doesn't start yet
646 return nmemb;
647
648 if (range_end != 0 && cur_range_start > range_end) {
649 // The wanted byte range is over
650 // Return zero that will lead to transfer abortion
651 // with error code CURLE_WRITE_ERROR
652 target->writecb_required_range_written = TRUE;
653 return 0;
654 }
655
656 size = 1;
657 nmemb = all;
658
659 if (cur_range_start >= range_start) {
660 // Write the current curl passed range from the start
661 ;
662 } else {
663 // Find the right starting offset
664 gint64 offset = range_start - cur_range_start;
665 assert(offset > 0);
666 ptr += offset;
667 // Corret the length appropriately
668 nmemb = all - offset;
669 }
670
671 if (range_end != 0) {
672 // End range is specified
673
674 if (cur_range_end <= range_end) {
675 // Write the current curl passed range to the end
676 ;
677 } else {
678 // Find the length of the new sequence
679 gint64 offset = cur_range_end - range_end;
680 assert(offset > 0);
681 // Corret the length appropriately
682 nmemb -= (offset - 1);
683 }
684 }
685
686 assert(nmemb > 0);
687 cur_written = fwrite(ptr, size, nmemb, target->f);
688 if (cur_written != nmemb) {
689 g_warning("Error while writing file: %s", g_strerror(errno));
690 return 0; // There was an error
691 }
692
693 return cur_written_expected;
694 }
695
696 /** Select a suitable mirror
697 */
698 static gboolean
select_suitable_mirror(LrDownload * dd,LrTarget * target,LrMirror ** selected_mirror,GError ** err)699 select_suitable_mirror(LrDownload *dd,
700 LrTarget *target,
701 LrMirror **selected_mirror,
702 GError **err)
703 {
704
705 gboolean at_least_one_suitable_mirror_found = FALSE;
706 // ^^^ This variable is used to indentify that all possible mirrors
707 // were already tried and the transfer should be marked as failed.
708
709 assert(dd);
710 assert(target);
711 assert(selected_mirror);
712 assert(!err || *err == NULL);
713
714 *selected_mirror = NULL;
715 // mirrors_iterated is used to allow to use mirrors multiple times for a target
716 unsigned mirrors_iterated = 0;
717 // retry local paths have no reason
718 gboolean reiterate = FALSE;
719 // Iterate over mirrors for the target. If no suitable mirror is found on
720 // the first iteration, relax the conditions (by allowing previously
721 // failing mirrors to be used again) and do additional iterations up to
722 // number of allowed failures equal to dd->allowed_mirror_failures.
723 do {
724 // Iterate over mirror for the target
725 for (GSList *elem = target->lrmirrors; elem; elem = g_slist_next(elem)) {
726 LrMirror *c_mirror = elem->data;
727 gchar *mirrorurl = c_mirror->mirror->url;
728
729 // first iteration, filter out mirrors that failed previously
730 if (mirrors_iterated == 0) {
731 if (c_mirror->mirror->protocol != LR_PROTOCOL_FILE)
732 reiterate = TRUE;
733 if (g_slist_find(target->tried_mirrors, c_mirror)) {
734 // This mirror was already tried for this target
735 continue;
736 }
737 if (c_mirror->successful_transfers == 0 &&
738 dd->allowed_mirror_failures > 0 &&
739 c_mirror->failed_transfers >= dd->allowed_mirror_failures)
740 {
741 // Skip bad mirrors
742 g_debug("%s: Skipping bad mirror (%d failures and no success): %s",
743 __func__, c_mirror->failed_transfers, mirrorurl);
744 continue;
745 }
746 // retry of local paths have no reason
747 } else if (c_mirror->mirror->protocol == LR_PROTOCOL_FILE) {
748 continue;
749 // On subsequent iterations, only skip mirrors that failed proportionally to the number
750 // of iterations. It allows to reuse mirrors with low number of failures first.
751 } else if (mirrors_iterated < c_mirror->failed_transfers) {
752 continue;
753 }
754
755 if (mirrors_iterated == 0 && c_mirror->mirror->protocol == LR_PROTOCOL_FTP && target->target->is_zchunk) {
756 continue;
757 }
758
759 if (c_mirror->mirror->protocol == LR_PROTOCOL_RSYNC) {
760 if (mirrors_iterated == 0) {
761 // Skip rsync mirrors
762 g_debug("%s: Skipping rsync url: %s", __func__, mirrorurl);
763 }
764 continue;
765 }
766
767 if (target->handle
768 && target->handle->offline
769 && c_mirror->mirror->protocol != LR_PROTOCOL_FILE)
770 {
771 if (mirrors_iterated == 0) {
772 // Skip each url that doesn't have "file://" or "file:" prefix
773 g_debug("%s: Skipping mirror %s - Offline mode enabled", __func__, mirrorurl);
774 }
775 continue;
776 }
777
778 at_least_one_suitable_mirror_found = TRUE;
779
780 // Number of transfers which are downloading from the mirror
781 // should always be lower or equal than maximum allowed number
782 // of connection to a single host.
783 assert(dd->max_connection_per_host == -1 ||
784 c_mirror->running_transfers <= dd->max_connection_per_host);
785
786 // Init max of allowed parallel connections from config
787 init_once_allowed_parallel_connections(c_mirror, dd->max_connection_per_host);
788
789 // Check number of connections to the mirror
790 if (is_parallel_connections_limited_and_reached(c_mirror))
791 {
792 continue;
793 }
794
795 // This mirror looks suitable - use it
796 *selected_mirror = c_mirror;
797 return TRUE;
798 }
799 } while (reiterate && g_slist_length(target->tried_mirrors) < dd->allowed_mirror_failures &&
800 ++mirrors_iterated < dd->allowed_mirror_failures);
801
802 if (!at_least_one_suitable_mirror_found) {
803 // No suitable mirror even exists => Set transfer as failed
804 g_debug("%s: All mirrors were tried without success", __func__);
805 target->state = LR_DS_FAILED;
806
807 lr_downloadtarget_set_error(target->target, LRE_NOURL,
808 "Cannot download, all mirrors were already tried "
809 "without success");
810
811
812 // Call end callback
813 LrEndCb end_cb = target->target->endcb;
814 if (end_cb) {
815 int ret = end_cb(target->target->cbdata,
816 LR_TRANSFER_ERROR,
817 "No more mirrors to try - All mirrors "
818 "were already tried without success");
819 if (ret == LR_CB_ERROR) {
820 target->cb_return_code = LR_CB_ERROR;
821 g_debug("%s: Downloading was aborted by LR_CB_ERROR "
822 "from end callback", __func__);
823 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CBINTERRUPTED,
824 "Interrupted by LR_CB_ERROR from end callback");
825 return FALSE;
826 }
827 }
828
829 if (dd->failfast) {
830 // Fail immediately
831 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_NOURL,
832 "Cannot download %s: All mirrors were tried",
833 target->target->path);
834 return FALSE;
835 }
836 }
837
838 return TRUE;
839 }
840
841
842 /** Select next target
843 */
844 static gboolean
select_next_target(LrDownload * dd,LrTarget ** selected_target,char ** selected_full_url,GError ** err)845 select_next_target(LrDownload *dd,
846 LrTarget **selected_target,
847 char **selected_full_url,
848 GError **err)
849 {
850 assert(dd);
851 assert(selected_target);
852 assert(selected_full_url);
853 assert(!err || *err == NULL);
854
855 *selected_target = NULL;
856 *selected_full_url = NULL;
857
858 for (GSList *elem = dd->targets; elem; elem = g_slist_next(elem)) {
859 LrTarget *target = elem->data;
860 LrMirror *mirror = NULL;
861 char *full_url = NULL;
862 int complete_url_in_path = 0;
863
864 if (target->state != LR_DS_WAITING) // Pick only waiting targets
865 continue;
866
867 // Determine if path is a complete URL
868
869 complete_url_in_path = strstr(target->target->path, "://") ? 1 : 0;
870
871 // Sanity check
872
873 if (!target->target->baseurl
874 && !target->lrmirrors
875 && !complete_url_in_path)
876 {
877 // Used relative path with empty internal mirrorlist
878 // and no basepath specified!
879 g_warning("Empty mirrorlist and no basepath specified");
880 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_NOURL,
881 "Empty mirrorlist and no basepath specified!");
882 return FALSE;
883 }
884
885 g_debug("Selecting mirror for: %s", target->target->path);
886
887 // Prepare full target URL
888
889 if (complete_url_in_path) {
890 // Path is a complete URL (do not use mirror nor base URL)
891 full_url = g_strdup(target->target->path);
892 } else if (target->target->baseurl) {
893 // Base URL is specified
894 full_url = lr_pathconcat(target->target->baseurl,
895 target->target->path,
896 NULL);
897 } else {
898 // Find a suitable mirror
899 if (!select_suitable_mirror(dd, target, &mirror , err))
900 return FALSE;
901
902 if (mirror) {
903 // A mirror was found
904 full_url = lr_pathconcat(mirror->mirror->url,
905 target->target->path,
906 NULL);
907 } else {
908 // No free mirror
909 g_debug("%s: Currently there is no free mirror for: %s",
910 __func__, target->target->path);
911 }
912 }
913
914 // If LRO_OFFLINE is specified, check if the obtained full_url
915 // is local or not
916 // This condition should never be true for a full_url built
917 // from a mirror, because select_suitable_mirror() checks if
918 // the URL is local if LRO_OFFLINE is enabled by itself.
919 if (full_url
920 && target->handle
921 && target->handle->offline
922 && !lr_is_local_path(full_url))
923 {
924 g_debug("%s: Skipping %s because LRO_OFFLINE is specified",
925 __func__, full_url);
926
927 // Mark the target as failed
928 target->state = LR_DS_FAILED;
929 lr_downloadtarget_set_error(target->target, LRE_NOURL,
930 "Cannot download, offline mode is specified and no "
931 "local URL is available");
932
933 // Call end callback
934 LrEndCb end_cb = target->target->endcb;
935 if (end_cb) {
936 int ret = end_cb(target->target->cbdata,
937 LR_TRANSFER_ERROR,
938 "Cannot download: Offline mode is specified "
939 "and no local URL is available");
940 if (ret == LR_CB_ERROR) {
941 target->cb_return_code = LR_CB_ERROR;
942 g_debug("%s: Downloading was aborted by LR_CB_ERROR "
943 "from end callback", __func__);
944 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CBINTERRUPTED,
945 "Interrupted by LR_CB_ERROR from end callback");
946 return FALSE;
947 }
948 }
949
950 if (dd->failfast) {
951 // Fail immediately
952 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_NOURL,
953 "Cannot download %s: Offline mode is specified "
954 "and no local URL is available",
955 target->target->path);
956 return FALSE;
957 }
958 }
959
960 if (full_url) { // A waiting target found
961 target->mirror = mirror; // Note: mirror is NULL if baseurl is used
962
963 *selected_target = target;
964 *selected_full_url = full_url;
965
966 return TRUE;
967 }
968 }
969
970 // No suitable target found
971 return TRUE;
972 }
973
974
975 #define XATTR_LIBREPO "user.Librepo.DownloadInProgress"
976
977 /** Add an extended attribute that indicates that
978 * the file is being/was downloaded by Librepo
979 */
980 static void
add_librepo_xattr(int fd,const gchar * fn)981 add_librepo_xattr(int fd, const gchar *fn)
982 {
983 _cleanup_free_ gchar *dst = NULL;
984
985 if (!fn)
986 dst = g_strdup_printf("fd: %d", fd);
987 else
988 dst = g_strdup(fn);
989
990 int attr_ret = FSETXATTR(fd, XATTR_LIBREPO, "1", 1, 0);
991 if (attr_ret == -1) {
992 g_debug("%s: Cannot set xattr %s (%s): %s",
993 __func__, XATTR_LIBREPO, dst, g_strerror(errno));
994 }
995 }
996
997
998 /** Check if the file was downloaded by Librepo
999 */
1000 static gboolean
has_librepo_xattr(int fd)1001 has_librepo_xattr(int fd)
1002 {
1003 ssize_t attr_ret = FGETXATTR(fd, XATTR_LIBREPO, NULL, 0);
1004 if (attr_ret == -1) {
1005 //g_debug("%s: Cannot get xattr %s: %s",
1006 // __func__, XATTR_LIBREPO, g_strerror(errno));
1007 return FALSE;
1008 }
1009 return TRUE;
1010 }
1011
1012
1013 /** Remove Librepo extended attribute
1014 */
1015 static void
remove_librepo_xattr(LrDownloadTarget * target)1016 remove_librepo_xattr(LrDownloadTarget * target)
1017 {
1018 int fd = target->fd;
1019 if (fd != -1) {
1020 FREMOVEXATTR(fd, XATTR_LIBREPO);
1021 return;
1022 }
1023 // If file descriptor wasn't set, file name was, and we need to open it
1024 fd = open(target->fn, O_RDWR, 0666);
1025 if (fd == -1) {
1026 return;
1027 }
1028 FREMOVEXATTR(fd, XATTR_LIBREPO);
1029 close(fd);
1030 }
1031
1032 #ifdef WITH_ZCHUNK
1033 gboolean
lr_zck_clear_header(LrTarget * target,GError ** err)1034 lr_zck_clear_header(LrTarget *target, GError **err)
1035 {
1036 assert(target && target->f && target->target && target->target->path);
1037
1038 int fd = fileno(target->f);
1039 lseek(fd, 0, SEEK_END);
1040 if(ftruncate(fd, 0) < 0) {
1041 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1042 "Unable to truncate %s", target->target->path);
1043 return FALSE;
1044 } else {
1045 return TRUE;
1046 }
1047 }
1048
1049 static gboolean
find_local_zck_header(LrTarget * target,GError ** err)1050 find_local_zck_header(LrTarget *target, GError **err)
1051 {
1052 zckCtx *zck = NULL;
1053 gboolean found = FALSE;
1054 int fd = fileno(target->f);
1055
1056 if(target->target->handle->cachedir) {
1057 g_debug("%s: Cache directory: %s\n", __func__,
1058 target->handle->cachedir);
1059 GError *tmp_err = NULL;
1060 GSList *filelist =
1061 lr_get_recursive_files(target->handle->cachedir, ".zck",
1062 &tmp_err);
1063 if(tmp_err) {
1064 g_warning("Error reading cache directory %s: %s",
1065 target->handle->cachedir, tmp_err->message);
1066 g_clear_error(&tmp_err);
1067 }
1068
1069 char *uf = g_build_path("/", target->handle->destdir,
1070 target->target->path, NULL);
1071
1072 for(GSList *file = filelist; file && !found; file = g_slist_next(file)) {
1073 char *cf = (char *)file->data;
1074
1075 /* Don't try to read from self */
1076 if (strcmp(cf, uf) == 0)
1077 continue;
1078
1079 int chk_fd = open(file->data, O_RDONLY);
1080 if (chk_fd < 0) {
1081 g_warning("Unable to open %s: %s", cf, g_strerror(errno));
1082 continue;
1083 }
1084 if(lr_zck_valid_header(target->target, (char *)file->data, chk_fd,
1085 &tmp_err)) {
1086 g_debug("%s: Found file with same header at %s", __func__,
1087 (char *)file->data);
1088 lseek(fd, 0, SEEK_SET);
1089 lseek(chk_fd, 0, SEEK_SET);
1090 if(lr_copy_content(chk_fd, fd) == 0 &&
1091 ftruncate(fd, lseek(chk_fd, 0, SEEK_END)) >= 0 &&
1092 lseek(fd, 0, SEEK_SET) == 0 &&
1093 (zck = lr_zck_init_read(target->target, (char *)file->data,
1094 chk_fd, &tmp_err))) {
1095 found = TRUE;
1096 break;
1097 } else {
1098 g_warning("Error copying file");
1099 g_clear_error(&tmp_err);
1100 }
1101 } else {
1102 g_clear_error(&tmp_err);
1103 }
1104 close(chk_fd);
1105 }
1106 g_slist_free_full(filelist, free);
1107 free(uf);
1108 } else {
1109 g_debug("%s: No cache directory set", __func__);
1110 }
1111
1112 if(found) {
1113 zckCtx *old_zck = zck_dl_get_zck(target->target->zck_dl);
1114 zck_free(&old_zck);
1115 if(!zck_dl_set_zck(target->target->zck_dl, zck)) {
1116 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1117 "Unable to setup zchunk download context for %s",
1118 target->target->path);
1119 return FALSE;
1120 }
1121 target->zck_state = LR_ZCK_DL_BODY_CK;
1122 return TRUE;
1123 }
1124 target->zck_state = LR_ZCK_DL_HEADER;
1125 return TRUE;
1126 }
1127
1128 static gboolean
prep_zck_header(LrTarget * target,GError ** err)1129 prep_zck_header(LrTarget *target, GError **err)
1130 {
1131 zckCtx *zck = NULL;
1132 int fd = fileno(target->f);
1133 GError *tmp_err = NULL;
1134
1135 if(lr_zck_valid_header(target->target, target->target->path, fd,
1136 &tmp_err)) {
1137 zck = lr_zck_init_read(target->target, target->target->path,
1138 fd, &tmp_err);
1139 if(zck) {
1140 if(!zck_dl_set_zck(target->target->zck_dl, zck)) {
1141 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1142 "Unable to setup zchunk download context");
1143 return FALSE;
1144 }
1145 target->zck_state = LR_ZCK_DL_BODY_CK;
1146 return TRUE;
1147 } else {
1148 g_warning("Error reading validated header: %s", tmp_err->message);
1149 g_clear_error(&tmp_err);
1150 }
1151 } else {
1152 g_clear_error(&tmp_err);
1153 }
1154
1155 lseek(fd, 0, SEEK_SET);
1156 zck = zck_create();
1157 if(!zck_init_adv_read(zck, fd)) {
1158 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1159 "Unable to initialize zchunk file %s for reading",
1160 target->target->path);
1161 return FALSE;
1162 }
1163
1164 if(target->target->zck_dl) {
1165 zckCtx *old_zck = zck_dl_get_zck(target->target->zck_dl);
1166 zck_free(&old_zck);
1167 if(!zck_dl_set_zck(target->target->zck_dl, zck)) {
1168 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1169 "Unable to setup zchunk download context for %s",
1170 target->target->path);
1171 return FALSE;
1172 }
1173 } else {
1174 target->target->zck_dl = zck_dl_init(zck);
1175 }
1176 target->target->range = zck_get_range(0, target->target->zck_header_size-1);
1177 target->target->total_to_download = target->target->zck_header_size;
1178 target->target->resume = 0;
1179 target->zck_state = LR_ZCK_DL_HEADER;
1180 return lr_zck_clear_header(target, err);
1181 }
1182
1183 static gboolean
find_local_zck_chunks(LrTarget * target,GError ** err)1184 find_local_zck_chunks(LrTarget *target, GError **err)
1185 {
1186 assert(!err || *err == NULL);
1187 assert(target && target->target && target->target->zck_dl);
1188
1189 zckCtx *zck = zck_dl_get_zck(target->target->zck_dl);
1190 int fd = fileno(target->f);
1191 if(zck && fd != zck_get_fd(zck) && !zck_set_fd(zck, fd)) {
1192 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1193 "Unable to set zchunk file descriptor for %s: %s",
1194 target->target->path, zck_get_error(zck));
1195 return FALSE;
1196 }
1197
1198 if(target->target->handle->cachedir) {
1199 g_debug("%s: Cache directory: %s\n", __func__,
1200 target->handle->cachedir);
1201 GError *tmp_err = NULL;
1202 GSList *filelist =
1203 lr_get_recursive_files(target->handle->cachedir, ".zck",
1204 &tmp_err);
1205 if(tmp_err) {
1206 g_warning("Error reading cache directory %s: %s",
1207 target->handle->cachedir, tmp_err->message);
1208 g_clear_error(&tmp_err);
1209 }
1210
1211 gboolean found = FALSE;
1212 char *uf = g_build_path("/", target->handle->destdir,
1213 target->target->path, NULL);
1214 for(GSList *file = filelist; file && !found; file = g_slist_next(file)) {
1215 char *cf = (char *)file->data;
1216
1217 /* Don't try to read from self */
1218 if (strcmp(cf, uf) == 0)
1219 continue;
1220
1221 int chk_fd = open(file->data, O_RDONLY);
1222 if (chk_fd < 0) {
1223 g_warning("Unable to open %s: %s", cf, g_strerror(errno));
1224 continue;
1225 }
1226
1227 zckCtx *zck_src = zck_create();
1228 if(!zck_init_read(zck_src, chk_fd)) {
1229 close(chk_fd);
1230 continue;
1231 }
1232
1233 if(!zck_copy_chunks(zck_src, zck)) {
1234 g_warning("Error copying chunks from %s to %s: %s", cf, uf, zck_get_error(zck));
1235 zck_free(&zck_src);
1236 close(chk_fd);
1237 continue;
1238 }
1239 zck_free(&zck_src);
1240 close(chk_fd);
1241 }
1242 g_slist_free_full(filelist, free);
1243 free(uf);
1244 }
1245 target->target->downloaded = target->target->total_to_download;
1246 /* Calculate how many bytes need to be downloaded */
1247 for(zckChunk *idx = zck_get_first_chunk(zck); idx != NULL; idx = zck_get_next_chunk(idx))
1248 if(zck_get_chunk_valid(idx) != 1)
1249 target->target->total_to_download += zck_get_chunk_comp_size(idx) + 92; /* Estimate of multipart overhead */
1250 target->zck_state = LR_ZCK_DL_BODY;
1251 return TRUE;
1252 }
1253
1254 static gboolean
prep_zck_body(LrTarget * target,GError ** err)1255 prep_zck_body(LrTarget *target, GError **err)
1256 {
1257 zckCtx *zck = zck_dl_get_zck(target->target->zck_dl);
1258 int fd = fileno(target->f);
1259 if(zck && fd != zck_get_fd(zck) && !zck_set_fd(zck, fd)) {
1260 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1261 "Unable to set zchunk file descriptor for %s: %s",
1262 target->target->path, zck_get_error(zck));
1263 return FALSE;
1264 }
1265
1266 zck_reset_failed_chunks(zck);
1267 if(zck_missing_chunks(zck) == 0) {
1268 target->zck_state = LR_ZCK_DL_FINISHED;
1269 return TRUE;
1270 }
1271
1272 lseek(fd, 0, SEEK_SET);
1273
1274 g_debug("%s: Chunks that still need to be downloaded: %i", __func__,
1275 zck_missing_chunks(zck));
1276 zck_dl_reset(target->target->zck_dl);
1277 zckRange *range = zck_get_missing_range(zck, target->mirror->max_ranges);
1278 zckRange *old_range = zck_dl_get_range(target->target->zck_dl);
1279 if(old_range)
1280 zck_range_free(&old_range);
1281 if(!zck_dl_set_range(target->target->zck_dl, range)) {
1282 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1283 "Unable to set range for zchunk downloader");
1284 return FALSE;
1285 }
1286 if(target->target->range)
1287 free(target->target->range);
1288 target->target->range = zck_get_range_char(zck, range);
1289 target->target->expectedsize = 1;
1290 target->zck_state = LR_ZCK_DL_BODY;
1291 return TRUE;
1292 }
1293
1294 static gboolean
check_zck(LrTarget * target,GError ** err)1295 check_zck(LrTarget *target, GError **err)
1296 {
1297 assert(!err || *err == NULL);
1298 assert(target && target->f && target->target);
1299
1300 if(target->mirror->max_ranges == 0 || target->mirror->mirror->protocol != LR_PROTOCOL_HTTP) {
1301 target->zck_state = LR_ZCK_DL_BODY;
1302 target->target->expectedsize = target->target->origsize;
1303 free(target->target->range);
1304 target->target->range = NULL;
1305 return TRUE;
1306 }
1307
1308 if(target->target->zck_dl == NULL) {
1309 target->target->zck_dl = zck_dl_init(NULL);
1310 if(target->target->zck_dl == NULL) {
1311 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK, "%s",
1312 zck_get_error(NULL));
1313 return FALSE;
1314 }
1315 target->zck_state = LR_ZCK_DL_HEADER_CK;
1316 }
1317
1318 /* Reset range fail flag */
1319 target->range_fail = FALSE;
1320
1321 /* If we've finished, then there's no point in checking any further */
1322 if(target->zck_state == LR_ZCK_DL_FINISHED)
1323 return TRUE;
1324
1325 zckCtx *zck = zck_dl_get_zck(target->target->zck_dl);
1326 if (!zck) {
1327 target->zck_state = LR_ZCK_DL_HEADER_CK;
1328 g_debug("%s: Unable to read zchunk header: %s", __func__, target->target->path);
1329 if(!find_local_zck_header(target, err))
1330 return FALSE;
1331 }
1332 zck = zck_dl_get_zck(target->target->zck_dl);
1333
1334 if(target->zck_state == LR_ZCK_DL_HEADER) {
1335 if(!prep_zck_header(target, err))
1336 return FALSE;
1337 if(target->zck_state == LR_ZCK_DL_HEADER)
1338 return TRUE;
1339 }
1340 zck = zck_dl_get_zck(target->target->zck_dl);
1341
1342 if(target->zck_state == LR_ZCK_DL_BODY_CK) {
1343 g_debug("%s: Checking zchunk data checksum: %s", __func__, target->target->path);
1344 // Check whether file has been fully downloaded
1345 int cks_good = zck_find_valid_chunks(zck);
1346 if(!cks_good) { // Error while validating checksums
1347 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1348 "%s: Error validating zchunk file: %s", __func__,
1349 zck_get_error(zck));
1350 return FALSE;
1351 }
1352
1353 if(cks_good == 1) { // All checksums good
1354 g_debug("%s: File is complete", __func__);
1355 if(target->target->zck_dl)
1356 zck_dl_free(&(target->target->zck_dl));
1357 target->zck_state = LR_ZCK_DL_FINISHED;
1358 return TRUE;
1359 }
1360
1361 g_debug("%s: Downloading rest of zchunk body: %s", __func__, target->target->path);
1362 // Download the remaining checksums
1363 zck_reset_failed_chunks(zck);
1364 if(!find_local_zck_chunks(target, err))
1365 return FALSE;
1366
1367 cks_good = zck_find_valid_chunks(zck);
1368 if(!cks_good) { // Error while validating checksums
1369 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1370 "%s: Error validating zchunk file: %s", __func__,
1371 zck_get_error(zck));
1372 return FALSE;
1373 }
1374
1375 if(cks_good == 1) { // All checksums good
1376 if(target->target->zck_dl)
1377 zck_dl_free(&(target->target->zck_dl));
1378 target->zck_state = LR_ZCK_DL_FINISHED;
1379 return TRUE;
1380 }
1381 }
1382 zck_reset_failed_chunks(zck);
1383 /* Recalculate how many bytes remain to be downloaded by subtracting from total_to_download */
1384 target->target->downloaded = target->target->total_to_download;
1385 for(zckChunk *idx = zck_get_first_chunk(zck); idx != NULL; idx = zck_get_next_chunk(idx))
1386 if(zck_get_chunk_valid(idx) != 1)
1387 target->target->downloaded -= zck_get_chunk_comp_size(idx) + 92;
1388 return prep_zck_body(target, err);
1389 }
1390 #endif /* WITH_ZCHUNK */
1391
1392 /** Open the file to write to
1393 */
1394 static FILE*
open_target_file(LrTarget * target,GError ** err)1395 open_target_file(LrTarget *target, GError **err)
1396 {
1397 int fd;
1398 FILE *f;
1399
1400 if (target->target->fd != -1) {
1401 // Use supplied filedescriptor
1402 fd = dup(target->target->fd);
1403 if (fd == -1) {
1404 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1405 "dup(%d) failed: %s",
1406 target->target->fd, g_strerror(errno));
1407 return NULL;
1408 }
1409 } else {
1410 // Use supplied filename
1411 int open_flags = O_CREAT|O_TRUNC|O_RDWR;
1412 if (target->resume || target->target->is_zchunk)
1413 open_flags &= ~O_TRUNC;
1414
1415 fd = open(target->target->fn, open_flags, 0666);
1416 if (fd == -1) {
1417 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1418 "Cannot open %s: %s",
1419 target->target->fn, g_strerror(errno));
1420 return NULL;
1421 }
1422 }
1423
1424 f = fdopen(fd, "w+b");
1425 if (!f) {
1426 close(fd);
1427 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1428 "fdopen(%d) failed: %s",
1429 fd, g_strerror(errno));
1430 return NULL;
1431 }
1432
1433 return f;
1434 }
1435
1436 /** Prepare next transfer
1437 */
1438 static gboolean
prepare_next_transfer(LrDownload * dd,gboolean * candidatefound,GError ** err)1439 prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err)
1440 {
1441 LrTarget *target;
1442 _cleanup_free_ char *full_url = NULL;
1443 LrProtocol protocol = LR_PROTOCOL_OTHER;
1444 gboolean ret;
1445
1446 assert(dd);
1447 assert(!err || *err == NULL);
1448
1449 *candidatefound = FALSE;
1450
1451 ret = select_next_target(dd, &target, &full_url, err);
1452 if (!ret) // Error
1453 return FALSE;
1454
1455 if (!target) // Nothing to do
1456 return TRUE;
1457
1458 *candidatefound = TRUE;
1459
1460 // Append the LRO_ONETIMEFLAG if instructed to do so
1461 LrHandle *handle = target->handle;
1462 if (handle && handle->onetimeflag && handle->onetimeflag_apply) {
1463 const char *sep = "?";
1464 if (g_strrstr(full_url, sep) != NULL)
1465 sep = "&";
1466 char *new_url = g_strjoin(sep, full_url, handle->onetimeflag, NULL);
1467 g_free(full_url);
1468 full_url = new_url;
1469 // No other CURL handle on this LrHandle shall apply the flag again
1470 free(handle->onetimeflag);
1471 handle->onetimeflag = NULL;
1472 handle->onetimeflag_apply = FALSE;
1473 }
1474
1475 g_info("Downloading: %s", full_url);
1476
1477 protocol = lr_detect_protocol(full_url);
1478
1479 // Prepare CURL easy handle
1480 CURLcode c_rc;
1481 CURL *h;
1482 if (target->handle)
1483 h = curl_easy_duphandle(target->handle->curl_handle);
1484 else
1485 h = lr_get_curl_handle();
1486 if (!h) {
1487 // Something went wrong
1488 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURL,
1489 "curl_easy_duphandle() call failed");
1490 goto fail;
1491 }
1492 target->curl_handle = h;
1493
1494 // Set URL
1495 c_rc = curl_easy_setopt(h, CURLOPT_URL, full_url);
1496 if (c_rc != CURLE_OK) {
1497 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURL,
1498 "curl_easy_setopt(h, CURLOPT_URL, %s) failed: %s",
1499 full_url, curl_easy_strerror(c_rc));
1500 goto fail;
1501 }
1502
1503 // Set error buffer
1504 target->errorbuffer[0] = '\0';
1505 c_rc = curl_easy_setopt(h, CURLOPT_ERRORBUFFER, target->errorbuffer);
1506 if (c_rc != CURLE_OK) {
1507 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURL,
1508 "curl_easy_setopt(h, CURLOPT_ERRORBUFFER, target->errorbuffer) failed: %s",
1509 curl_easy_strerror(c_rc));
1510 goto fail;
1511 }
1512
1513 // Prepare FILE
1514 target->f = open_target_file(target, err);
1515 if (!target->f)
1516 goto fail;
1517 target->writecb_recieved = 0;
1518 target->writecb_required_range_written = FALSE;
1519
1520 #ifdef WITH_ZCHUNK
1521 // If file is zchunk, prep it
1522 if(target->target->is_zchunk) {
1523 GError *tmp_err = NULL;
1524
1525 if (!check_zck(target, &tmp_err)) {
1526 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1527 "Unable to initialize zchunk file %s: %s",
1528 target->target->path,
1529 tmp_err->message);
1530 goto fail;
1531 }
1532
1533 // If zchunk is finished, we're done, so move to next target
1534 if(target->zck_state == LR_ZCK_DL_FINISHED) {
1535 g_debug("%s: Target already fully downloaded: %s", __func__, target->target->path);
1536 target->state = LR_DS_FINISHED;
1537 curl_easy_cleanup(target->curl_handle);
1538 target->curl_handle = NULL;
1539 g_free(target->headercb_interrupt_reason);
1540 target->headercb_interrupt_reason = NULL;
1541 fclose(target->f);
1542 target->f = NULL;
1543 lr_downloadtarget_set_error(target->target, LRE_OK, NULL);
1544 return prepare_next_transfer(dd, candidatefound, err);
1545 }
1546 }
1547 # endif /* WITH_ZCHUNK */
1548
1549 int fd = fileno(target->f);
1550
1551 // Allow resume only for files that were originally being
1552 // downloaded by librepo
1553 if (target->resume && !has_librepo_xattr(fd)) {
1554 target->resume = FALSE;
1555 g_debug("%s: Resume ignored, existing file was not originally "
1556 "being downloaded by Librepo", __func__);
1557 if (ftruncate(fd, 0) == -1) {
1558 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1559 "ftruncate() failed: %s", g_strerror(errno));
1560 goto fail;
1561 }
1562 }
1563
1564 if (target->resume && target->resume_count >= LR_DOWNLOADER_MAXIMAL_RESUME_COUNT) {
1565 target->resume = FALSE;
1566 g_debug("%s: Download resume ignored, maximal number of attempts (%d)"
1567 " has been reached", __func__, LR_DOWNLOADER_MAXIMAL_RESUME_COUNT);
1568 }
1569
1570 // Resume - set offset to resume incomplete download
1571 if (target->resume) {
1572 target->resume_count++;
1573
1574 if (target->original_offset == -1) {
1575 // Determine offset
1576 fseek(target->f, 0L, SEEK_END);
1577 gint64 determined_offset = ftell(target->f);
1578 if (determined_offset == -1) {
1579 // An error while determining offset =>
1580 // Download the whole file again
1581 determined_offset = 0;
1582 }
1583 target->original_offset = determined_offset;
1584 }
1585
1586 gint64 used_offset = target->original_offset;
1587 g_debug("%s: Used offset for download resume: %"G_GINT64_FORMAT,
1588 __func__, used_offset);
1589
1590 c_rc = curl_easy_setopt(h, CURLOPT_RESUME_FROM_LARGE,
1591 (curl_off_t) used_offset);
1592 assert(c_rc == CURLE_OK);
1593 }
1594
1595 // Add librepo extended attribute to the file
1596 // This xattr states that file is being downloaded by librepo
1597 // This xattr is removed once the file is completely downloaded
1598 // If librepo tries to resume a download, it checks if the xattr is present.
1599 // If it isn't the download is not resumed, but whole file is
1600 // downloaded again.
1601 add_librepo_xattr(fd, target->target->fn);
1602
1603 if (target->target->byterangestart > 0) {
1604 assert(!target->target->resume && !target->target->range);
1605 g_debug("%s: byterangestart is specified -> resume is set to %"
1606 G_GINT64_FORMAT, __func__, target->target->byterangestart);
1607 c_rc = curl_easy_setopt(h, CURLOPT_RESUME_FROM_LARGE,
1608 (curl_off_t) target->target->byterangestart);
1609 assert(c_rc == CURLE_OK);
1610 }
1611
1612 // Set range if user specified one
1613 if (target->target->range) {
1614 assert(!target->target->resume && !target->target->byterangestart);
1615 c_rc = curl_easy_setopt(h, CURLOPT_RANGE, target->target->range);
1616 assert(c_rc == CURLE_OK);
1617 }
1618
1619 // Prepare progress callback
1620 target->cb_return_code = LR_CB_OK;
1621 if (target->target->progresscb) {
1622 c_rc = curl_easy_setopt(h, CURLOPT_PROGRESSFUNCTION, lr_progresscb) ||
1623 curl_easy_setopt(h, CURLOPT_NOPROGRESS, 0) ||
1624 curl_easy_setopt(h, CURLOPT_PROGRESSDATA, target);
1625 assert(c_rc == CURLE_OK);
1626 }
1627
1628 // Prepare header callback
1629 if (target->target->expectedsize > 0) {
1630 c_rc = curl_easy_setopt(h, CURLOPT_HEADERFUNCTION, lr_headercb) ||
1631 curl_easy_setopt(h, CURLOPT_HEADERDATA, target);
1632 assert(c_rc == CURLE_OK);
1633 }
1634
1635 // Prepare write callback
1636 c_rc = curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, lr_writecb) ||
1637 curl_easy_setopt(h, CURLOPT_WRITEDATA, target);
1638 assert(c_rc == CURLE_OK);
1639
1640 // Set extra HTTP headers
1641 struct curl_slist *headers = NULL;
1642 if (target->handle && target->handle->httpheader) {
1643 // Fill in headers specified by user in LrHandle via LRO_HTTPHEADER
1644 for (int x=0; target->handle->httpheader[x]; x++) {
1645 headers = curl_slist_append(headers, target->handle->httpheader[x]);
1646 if (!headers)
1647 lr_out_of_memory();
1648 }
1649 }
1650 if (target->target->no_cache) {
1651 // Add headers that tell proxy to serve us fresh data
1652 headers = curl_slist_append(headers, "Cache-Control: no-cache");
1653 headers = curl_slist_append(headers, "Pragma: no-cache");
1654 if (!headers)
1655 lr_out_of_memory();
1656 }
1657 target->curl_rqheaders = headers;
1658 c_rc = curl_easy_setopt(h, CURLOPT_HTTPHEADER, headers);
1659 assert(c_rc == CURLE_OK);
1660
1661 // Add the new handle to the curl multi handle
1662 CURLMcode cm_rc = curl_multi_add_handle(dd->multi_handle, h);
1663 assert(cm_rc == CURLM_OK);
1664
1665 // Set the state of transfer as running
1666 target->state = LR_DS_RUNNING;
1667
1668 // Increase running transfers counter for mirror
1669 if (target->mirror) {
1670 increase_running_transfers(target->mirror);
1671 }
1672
1673 // Set the state of header callback for this transfer
1674 target->headercb_state = LR_HCS_DEFAULT;
1675 g_free(target->headercb_interrupt_reason);
1676 target->headercb_interrupt_reason = NULL;
1677
1678 // Set protocol of the target
1679 target->protocol = protocol;
1680
1681 // Add the transfer to the list of running transfers
1682 dd->running_transfers = g_slist_append(dd->running_transfers, target);
1683
1684 return TRUE;
1685
1686 fail:
1687 // Cleanup target
1688 if (target->curl_handle) {
1689 curl_easy_cleanup(target->curl_handle);
1690 target->curl_handle = NULL;
1691 }
1692 if (target->f != NULL) {
1693 fclose(target->f);
1694 target->f = NULL;
1695 }
1696
1697 return FALSE;
1698 }
1699
1700 static gboolean
set_max_speeds_to_transfers(LrDownload * dd,GError ** err)1701 set_max_speeds_to_transfers(LrDownload *dd, GError **err)
1702 {
1703 assert(!err || *err == NULL);
1704
1705 if (!g_slist_length(dd->running_transfers)) // Nothing to do
1706 return TRUE;
1707
1708 // Compute number of running downloads from repos with limited speed
1709 GHashTable *num_running_downloads_per_repo = g_hash_table_new(NULL, NULL);
1710 for (GSList *elem = dd->running_transfers; elem; elem = g_slist_next(elem)) {
1711 const LrTarget *ltarget = elem->data;
1712
1713 if (!ltarget->handle || !ltarget->handle->maxspeed) // Skip repos with unlimited speed or without handle
1714 continue;
1715
1716 guint num_running_downloads_from_repo =
1717 GPOINTER_TO_UINT(g_hash_table_lookup(num_running_downloads_per_repo, ltarget->handle));
1718 if (num_running_downloads_from_repo)
1719 ++num_running_downloads_from_repo;
1720 else
1721 num_running_downloads_from_repo = 1;
1722 g_hash_table_insert(num_running_downloads_per_repo, ltarget->handle,
1723 GUINT_TO_POINTER(num_running_downloads_from_repo));
1724 }
1725
1726 // Set max speed to transfers
1727 GHashTableIter iter;
1728 gpointer key, value;
1729 g_hash_table_iter_init(&iter, num_running_downloads_per_repo);
1730 while (g_hash_table_iter_next(&iter, &key, &value)) {
1731 const LrHandle *repo = key;
1732 const guint num_running_downloads_from_repo = GPOINTER_TO_UINT(value);
1733
1734 // Calculate a max speed (rounded up) per target (for repo)
1735 const gint64 single_target_speed =
1736 (repo->maxspeed + (num_running_downloads_from_repo - 1)) / num_running_downloads_from_repo;
1737
1738 for (GSList *elem = dd->running_transfers; elem; elem = g_slist_next(elem)) {
1739 LrTarget *ltarget = elem->data;
1740 if (ltarget->handle == repo) {
1741 CURL *curl_handle = ltarget->curl_handle;
1742 CURLcode code = curl_easy_setopt(curl_handle,
1743 CURLOPT_MAX_RECV_SPEED_LARGE,
1744 (curl_off_t)single_target_speed);
1745 if (code != CURLE_OK) {
1746 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURLSETOPT,
1747 "Cannot set CURLOPT_MAX_RECV_SPEED_LARGE option: %s",
1748 curl_easy_strerror(code));
1749 g_hash_table_destroy(num_running_downloads_per_repo);
1750 return FALSE;
1751 }
1752 }
1753 }
1754 }
1755
1756 g_hash_table_destroy(num_running_downloads_per_repo);
1757
1758 return TRUE;
1759 }
1760
1761 static gboolean
prepare_next_transfers(LrDownload * dd,GError ** err)1762 prepare_next_transfers(LrDownload *dd, GError **err)
1763 {
1764 guint length = g_slist_length(dd->running_transfers);
1765 guint free_slots = dd->max_parallel_connections - length;
1766
1767 assert(!err || *err == NULL);
1768
1769 while (free_slots > 0) {
1770 gboolean candidatefound;
1771 if (!prepare_next_transfer(dd, &candidatefound, err))
1772 return FALSE;
1773 if (!candidatefound)
1774 break;
1775 free_slots--;
1776 }
1777
1778 // Set maximal speed for each target
1779 if (!set_max_speeds_to_transfers(dd, err))
1780 return FALSE;
1781
1782 return TRUE;
1783 }
1784
1785
1786 /** Check the finished transfer
1787 * Evaluate CURL return code and status code of protocol if needed.
1788 * @param serious_error Serious error is an error that isn't fatal,
1789 * but mirror that generate it should be penalized.
1790 * E.g.: Connection timeout - a mirror we are unable
1791 * to connect at is pretty useless for us, but
1792 * this could be only temporary state.
1793 * No fatal but also no good.
1794 * @param fatal_error An error that cannot be recovered - e.g.
1795 * we cannot write to a socket, we cannot write
1796 * data to disk, bad function argument, ...
1797 */
1798 static gboolean
check_finished_transfer_status(CURLMsg * msg,LrTarget * target,gboolean * serious_error,gboolean * fatal_error,GError ** transfer_err,GError ** err)1799 check_finished_transfer_status(CURLMsg *msg,
1800 LrTarget *target,
1801 gboolean *serious_error,
1802 gboolean *fatal_error,
1803 GError **transfer_err,
1804 GError **err)
1805 {
1806 long code = 0;
1807 char *effective_url = NULL;
1808
1809 assert(msg);
1810 assert(target);
1811 assert(!transfer_err || *transfer_err == NULL);
1812 assert(!err || *err == NULL);
1813
1814 *fatal_error = FALSE;
1815
1816 curl_easy_getinfo(msg->easy_handle,
1817 CURLINFO_EFFECTIVE_URL,
1818 &effective_url);
1819
1820 if (msg->data.result != CURLE_OK) {
1821 // There was an error that is reported by CURLcode
1822
1823 if (msg->data.result == CURLE_WRITE_ERROR &&
1824 target->writecb_required_range_written)
1825 {
1826 // Download was interrupted by writecb because
1827 // user want only specified byte range of the
1828 // target and the range was already downloaded
1829 g_debug("%s: Transfer was interrupted by writecb() "
1830 "because the required range "
1831 "(%"G_GINT64_FORMAT"-%"G_GINT64_FORMAT") "
1832 "was downloaded.", __func__,
1833 target->target->byterangestart,
1834 target->target->byterangeend);
1835 } else if (target->headercb_state == LR_HCS_INTERRUPTED) {
1836 // Download was interrupted by header callback
1837 g_set_error(transfer_err, LR_DOWNLOADER_ERROR, LRE_CURL,
1838 "Interrupted by header callback: %s",
1839 target->headercb_interrupt_reason);
1840 }
1841 #ifdef WITH_ZCHUNK
1842 else if (target->range_fail) {
1843 zckRange *range = zck_dl_get_range(target->target->zck_dl);
1844 int range_count = zck_get_range_count(range);
1845 if(target->mirror->max_ranges >= range_count) {
1846 target->mirror->max_ranges = range_count / 2;
1847 g_debug("%s: Setting mirror's max_ranges to %i", __func__,
1848 target->mirror->max_ranges);
1849 }
1850 return TRUE;
1851 } else if (target->target->zck_dl != NULL && zck_is_error(zck_dl_get_zck(target->target->zck_dl)) > 0) {
1852 zckCtx *zck = zck_dl_get_zck(target->target->zck_dl);
1853
1854 // Something went wrong while writing the zchunk file
1855 g_set_error(transfer_err, LR_DOWNLOADER_ERROR, LRE_ZCK,
1856 "Zchunk error: %s",
1857 zck_get_error(zck));
1858 if (zck_is_error(zck) == 1) {
1859 // Non-fatal zchunk error
1860 g_info("Serious zchunk error: %s",
1861 zck_get_error(zck));
1862 *serious_error = TRUE;
1863 } else { // zck_is_error(zck) == 2
1864 // Fatal zchunk error
1865 g_info("Fatal zchunk error: %s",
1866 zck_get_error(zck));
1867 *fatal_error = TRUE;
1868 }
1869 return TRUE;
1870 }
1871 #endif /* WITH_ZCHUNK */
1872 else {
1873 // There was a CURL error
1874 g_set_error(transfer_err, LR_DOWNLOADER_ERROR, LRE_CURL,
1875 "Curl error (%d): %s for %s [%s]",
1876 msg->data.result,
1877 curl_easy_strerror(msg->data.result),
1878 effective_url,
1879 target->errorbuffer);
1880
1881 switch (msg->data.result) {
1882 case CURLE_ABORTED_BY_CALLBACK:
1883 case CURLE_BAD_FUNCTION_ARGUMENT:
1884 case CURLE_CONV_REQD:
1885 case CURLE_COULDNT_RESOLVE_PROXY:
1886 case CURLE_FILESIZE_EXCEEDED:
1887 case CURLE_INTERFACE_FAILED:
1888 #if LR_CURL_VERSION_CHECK(7, 21, 5)
1889 case CURLE_NOT_BUILT_IN:
1890 #endif
1891 case CURLE_OUT_OF_MEMORY:
1892 //case CURLE_RECV_ERROR: // See RhBug: 1219817
1893 //case CURLE_SEND_ERROR:
1894 case CURLE_SSL_CACERT_BADFILE:
1895 case CURLE_SSL_CRL_BADFILE:
1896 case CURLE_WRITE_ERROR:
1897 // Fatal error
1898 g_info("Fatal error - Curl code (%d): %s for %s [%s]",
1899 msg->data.result,
1900 curl_easy_strerror(msg->data.result),
1901 effective_url,
1902 target->errorbuffer);
1903 *fatal_error = TRUE;
1904 break;
1905 case CURLE_OPERATION_TIMEDOUT:
1906 // Serious error
1907 g_info("Serious error - Curl code (%d): %s for %s [%s]",
1908 msg->data.result,
1909 curl_easy_strerror(msg->data.result),
1910 effective_url,
1911 target->errorbuffer);
1912 *serious_error = TRUE;
1913 break;
1914 default:
1915 // Other error are not considered fatal
1916 break;
1917 }
1918 }
1919
1920 return TRUE;
1921 }
1922
1923 // curl return code is CURLE_OK but we need to check status code
1924 curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1925 if (code) {
1926 char * effective_ip = NULL;
1927 curl_easy_getinfo(msg->easy_handle,
1928 CURLINFO_PRIMARY_IP,
1929 &effective_ip);
1930 // Check status codes for some protocols
1931 if (effective_url && g_str_has_prefix(effective_url, "http")) {
1932 // Check HTTP(S) code
1933 if (code/100 != 2) {
1934 g_set_error(transfer_err,
1935 LR_DOWNLOADER_ERROR,
1936 LRE_BADSTATUS,
1937 "Status code: %ld for %s (IP: %s)", code, effective_url, effective_ip);
1938 }
1939 } else if (effective_url) {
1940 // Check FTP
1941 if (code/100 != 2) {
1942 g_set_error(transfer_err,
1943 LR_DOWNLOADER_ERROR,
1944 LRE_BADSTATUS,
1945 "Status code: %ld for %s (IP: %s)", code, effective_url, effective_ip);
1946 }
1947 } else {
1948 // Other protocols
1949 g_set_error(transfer_err,
1950 LR_DOWNLOADER_ERROR,
1951 LRE_BADSTATUS,
1952 "Status code: %ld for %s (IP: %s)", code, effective_url, effective_ip);
1953 }
1954 }
1955
1956 return TRUE;
1957 }
1958
1959
1960 static gchar *
list_of_checksums_to_str(GSList * checksums)1961 list_of_checksums_to_str(GSList *checksums)
1962 {
1963 if (!checksums)
1964 return NULL;
1965
1966 gchar *expected = g_strdup("");
1967
1968 // Prepare pretty messages with list of expected checksums
1969 for (GSList *elem = checksums; elem; elem = g_slist_next(elem)) {
1970 gchar *tmp = NULL;
1971 LrDownloadTargetChecksum *chksum = elem->data;
1972 if (!chksum || !chksum->value || chksum->type == LR_CHECKSUM_UNKNOWN)
1973 continue; // Bad checksum
1974
1975 const gchar *chtype_str = lr_checksum_type_to_str(chksum->type);
1976 tmp = g_strconcat(expected, chksum->value, "(",
1977 chtype_str ? chtype_str : "UNKNOWN",
1978 ") ", NULL);
1979 free(expected);
1980 expected = tmp;
1981 }
1982
1983 return expected;
1984 }
1985
1986
1987 static gboolean
check_finished_transfer_checksum(int fd,GSList * checksums,gboolean * checksum_matches,GError ** transfer_err,GError ** err)1988 check_finished_transfer_checksum(int fd,
1989 GSList *checksums,
1990 gboolean *checksum_matches,
1991 GError **transfer_err,
1992 GError **err)
1993 {
1994 gboolean ret = TRUE;
1995 gboolean matches = TRUE;
1996 GSList *calculated_chksums = NULL;
1997
1998 for (GSList *elem = checksums; elem; elem = g_slist_next(elem)) {
1999 LrDownloadTargetChecksum *chksum = elem->data;
2000 LrDownloadTargetChecksum *calculated_chksum = NULL;
2001 gchar *calculated = NULL;
2002
2003 if (!chksum || !chksum->value || chksum->type == LR_CHECKSUM_UNKNOWN)
2004 continue; // Bad checksum
2005
2006 lseek(fd, 0, SEEK_SET);
2007 ret = lr_checksum_fd_compare(chksum->type,
2008 fd,
2009 chksum->value,
2010 1,
2011 &matches,
2012 &calculated,
2013 err);
2014 if (!ret)
2015 goto cleanup;
2016
2017 // Store calculated checksum
2018 calculated_chksum = lr_downloadtargetchecksum_new(chksum->type,
2019 calculated);
2020 g_free(calculated);
2021 calculated_chksums = g_slist_append(calculated_chksums,
2022 calculated_chksum);
2023
2024 if (matches) {
2025 // At least one checksum matches
2026 g_debug("%s: Checksum (%s) %s is OK", __func__,
2027 lr_checksum_type_to_str(chksum->type),
2028 chksum->value);
2029 break;
2030 }
2031 }
2032
2033 *checksum_matches = matches;
2034
2035 if (!matches) {
2036 // Checksums doesn't match
2037 _cleanup_free_ gchar *calculated = NULL;
2038 _cleanup_free_ gchar *expected = NULL;
2039
2040 calculated = list_of_checksums_to_str(calculated_chksums);
2041 expected = list_of_checksums_to_str(checksums);
2042
2043 // Set error message
2044 g_set_error(transfer_err,
2045 LR_DOWNLOADER_ERROR,
2046 LRE_BADCHECKSUM,
2047 "Downloading successful, but checksum doesn't match. "
2048 "Calculated: %s Expected: %s", calculated, expected);
2049 }
2050
2051 cleanup:
2052 g_slist_free_full(calculated_chksums,
2053 (GDestroyNotify) lr_downloadtargetchecksum_free);
2054
2055 return ret;
2056 }
2057
2058
2059 /** Truncate file - Used to remove downloaded garbage (error html pages, etc.)
2060 */
2061 static gboolean
truncate_transfer_file(LrTarget * target,GError ** err)2062 truncate_transfer_file(LrTarget *target, GError **err)
2063 {
2064 off_t original_offset = 0; // Truncate whole file by default
2065 int rc;
2066
2067 assert(!err || *err == NULL);
2068
2069 if (target->original_offset > -1)
2070 // If resume is enabled -> truncate file to its original position
2071 original_offset = target->original_offset;
2072
2073 if (target->target->fn) // Truncate by filename
2074 rc = truncate(target->target->fn, original_offset);
2075 else // Truncate by file descriptor number
2076 rc = ftruncate(target->target->fd, original_offset);
2077
2078 if (rc == -1) {
2079 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
2080 "ftruncate() failed: %s", g_strerror(errno));
2081 return FALSE;
2082 }
2083
2084 if (!target->target->fn) {
2085 // In case fd is used, seek to the original offset
2086 if (lseek(target->target->fd, original_offset, SEEK_SET) == -1) {
2087 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
2088 "lseek() failed: %s", g_strerror(errno));
2089 return FALSE;
2090 }
2091 }
2092
2093 return TRUE;
2094 }
2095
2096
2097 /** Return mirror rank or -1.0 if the rank cannot be determined
2098 * (e.g. when is too early)
2099 * Rank is currently just success rate for the mirror
2100 */
2101 static gdouble
mirror_rank(LrMirror * mirror)2102 mirror_rank(LrMirror *mirror)
2103 {
2104 gdouble rank = -1.0;
2105
2106 int successful = mirror->successful_transfers;
2107 int failed = mirror->failed_transfers;
2108 int finished_transfers = successful + failed;
2109
2110 if (finished_transfers < 3)
2111 return rank; // Do not judge too early
2112
2113 rank = successful / (double) finished_transfers;
2114
2115 return rank;
2116 }
2117
2118
2119 /** Sort mirrors. Penalize the error ones.
2120 * In fact only move the current finished mirror forward or backward
2121 * by one position.
2122 * @param mirrors GSList of mirrors (order of list elements won't be
2123 * changed, only data pointers)
2124 * @param mirror Mirror of just finished transfer
2125 * @param success Was download from the mirror successful
2126 * @param serious If success is FALSE, serious mean that error was serious
2127 * (like connection timeout), and the mirror should be
2128 * penalized more that usual.
2129 */
2130 static gboolean
sort_mirrors(GSList * mirrors,LrMirror * mirror,gboolean success,gboolean serious)2131 sort_mirrors(GSList *mirrors, LrMirror *mirror, gboolean success, gboolean serious)
2132 {
2133 GSList *elem = mirrors;
2134 GSList *prev = NULL;
2135 GSList *next = NULL;
2136 gdouble rank_cur;
2137
2138 assert(mirrors);
2139 assert(mirror);
2140
2141 for (; elem && elem->data != mirror; elem = g_slist_next(elem))
2142 prev = elem;
2143
2144 assert(elem); // Mirror should always exists in the list of mirrors
2145
2146 next = g_slist_next(elem);
2147
2148 if (!success && !next)
2149 goto exit; // Penalization not needed - Mirror is already the last one
2150 if (success && !prev)
2151 goto exit; // Bonus not needed - Mirror is already the first one
2152
2153 // Serious errors
2154 if (serious && mirror->successful_transfers == 0) {
2155 // Mirror that encounter a serious error and has no successful
2156 // transfers should be moved at the end of the list
2157 // (such mirror is probably down/broken/buggy)
2158 GSList *last = g_slist_last(elem);
2159 elem->data = last->data;
2160 last->data = (gpointer) mirror;
2161 g_debug("%s: Mirror %s was moved at the end", __func__, mirror->mirror->url);
2162 goto exit; // No more hadling needed
2163 }
2164
2165 // Calculate ranks
2166 rank_cur = mirror_rank(mirror);
2167 if (rank_cur < 0.0)
2168 goto exit; // Too early to judge
2169
2170
2171 if (!success) {
2172 // Penalize
2173 gdouble rank_next = mirror_rank(next->data);
2174 if (rank_next < 0.0 || rank_next > rank_cur) {
2175 elem->data = next->data;
2176 next->data = (gpointer) mirror;
2177 g_debug("%s: Mirror %s was penalized", __func__, mirror->mirror->url);
2178 }
2179 } else {
2180 // Bonus
2181 gdouble rank_prev = mirror_rank(prev->data);
2182 if (rank_prev < rank_cur) {
2183 elem->data = prev->data;
2184 prev->data = mirror;
2185 g_debug("%s: Mirror %s was awarded", __func__, mirror->mirror->url);
2186 }
2187 }
2188
2189 exit:
2190 if (g_getenv("LIBREPO_DEBUG_ADAPTIVEMIRRORSORTING")) {
2191 // Debug
2192 g_debug("%s: Updated order of mirrors (for %p):", __func__, mirrors);
2193 for (GSList *elem = mirrors; elem; elem = g_slist_next(elem)) {
2194 LrMirror *m = elem->data;
2195 g_debug(" %s (s: %d f: %d)", m->mirror->url,
2196 m->successful_transfers, m->failed_transfers);
2197 }
2198 }
2199
2200 return TRUE;
2201 }
2202
2203
2204 static gboolean
check_transfer_statuses(LrDownload * dd,GError ** err)2205 check_transfer_statuses(LrDownload *dd, GError **err)
2206 {
2207 assert(dd);
2208 assert(!err || *err == NULL);
2209
2210 int msgs_in_queue;
2211 CURLMsg *msg;
2212
2213 while ((msg = curl_multi_info_read(dd->multi_handle, &msgs_in_queue))) {
2214 LrTarget *target = NULL;
2215 _cleanup_free_ char *effective_url = NULL;
2216 int fd;
2217 gboolean matches = TRUE;
2218 GError *transfer_err = NULL;
2219 GError *tmp_err = NULL;
2220 gboolean ret;
2221 gboolean serious_error = FALSE;
2222 gboolean fatal_error = FALSE;
2223 GError *fail_fast_error = NULL;
2224
2225 if (msg->msg != CURLMSG_DONE) {
2226 // We are only interested in messages about finished transfers
2227 continue;
2228 }
2229
2230 // Find the target with this curl easy handle
2231 for (GSList *elem = dd->running_transfers; elem; elem = g_slist_next(elem)) {
2232 LrTarget *ltarget = elem->data;
2233 if (ltarget->curl_handle == msg->easy_handle)
2234 target = ltarget;
2235 }
2236
2237 assert(target); // Each easy handle used in the multi handle
2238 // should always belong to some target from
2239 // the running_transfers list
2240
2241 curl_easy_getinfo(msg->easy_handle,
2242 CURLINFO_EFFECTIVE_URL,
2243 &effective_url);
2244
2245 effective_url = g_strdup(effective_url); // Make the effective url
2246 // persistent to survive
2247 // the curl_easy_cleanup()
2248
2249 g_debug("Transfer finished: %s (Effective url: %s)", target->target->path, effective_url);
2250
2251 //
2252 // Check status of finished transfer
2253 //
2254 ret = check_finished_transfer_status(msg, target, &serious_error,
2255 &fatal_error, &transfer_err, err);
2256 if (!ret) // Error
2257 return FALSE;
2258
2259 if (transfer_err) // Transfer was unsuccessful
2260 goto transfer_error;
2261
2262 //
2263 // Checksum checking
2264 //
2265 fflush(target->f);
2266 fd = fileno(target->f);
2267
2268 // Preserve timestamp of downloaded file if requested
2269 if (target->target->handle && target->target->handle->preservetime) {
2270 CURLcode c_rc;
2271 long remote_filetime = -1;
2272 c_rc = curl_easy_getinfo(target->curl_handle, CURLINFO_FILETIME, &remote_filetime);
2273 if (c_rc == CURLE_OK && remote_filetime >= 0) {
2274 const struct timeval tv[] = {{remote_filetime, 0}, {remote_filetime, 0}};
2275 if (futimes(fd, tv) == -1)
2276 g_debug("%s: Failed to change timestamps of downloaded file \"%s\"",
2277 __func__, target->target->path);
2278 } else {
2279 g_debug("%s: Unable to get remote time of retrieved document \"%s\"",
2280 __func__, target->target->path);
2281 }
2282 }
2283
2284 #ifdef WITH_ZCHUNK
2285 if (target->target->is_zchunk) {
2286 zckCtx *zck = NULL;
2287 if (target->zck_state == LR_ZCK_DL_HEADER) {
2288 if(target->mirror->max_ranges > 0 && target->mirror->mirror->protocol == LR_PROTOCOL_HTTP &&
2289 !lr_zck_valid_header(target->target, target->target->path,
2290 fd, &transfer_err))
2291 goto transfer_error;
2292 } else if(target->zck_state == LR_ZCK_DL_BODY) {
2293 if(target->mirror->max_ranges > 0 && target->mirror->mirror->protocol == LR_PROTOCOL_HTTP) {
2294 zckCtx *zck = zck_dl_get_zck(target->target->zck_dl);
2295 if(zck == NULL) {
2296 g_set_error(&transfer_err, LR_DOWNLOADER_ERROR, LRE_ZCK,
2297 "Unable to get zchunk file from download context");
2298 goto transfer_error;
2299 }
2300 if(zck_failed_chunks(zck) == 0 && zck_missing_chunks(zck) == 0)
2301 target->zck_state = LR_ZCK_DL_FINISHED;
2302 } else {
2303 target->zck_state = LR_ZCK_DL_FINISHED;
2304 }
2305 }
2306 if(target->zck_state == LR_ZCK_DL_FINISHED) {
2307 zck = lr_zck_init_read(target->target, target->target->path, fd,
2308 &transfer_err);
2309 if(!zck)
2310 goto transfer_error;
2311 if(zck_validate_checksums(zck) < 1) {
2312 zck_free(&zck);
2313 g_set_error(&transfer_err, LR_DOWNLOADER_ERROR, LRE_BADCHECKSUM,
2314 "At least one of the zchunk checksums doesn't match in %s",
2315 effective_url);
2316 goto transfer_error;
2317 }
2318 zck_free(&zck);
2319 }
2320 } else {
2321 #endif /* WITH_ZCHUNK */
2322 // New file was downloaded - clear checksums cached in extended attributes
2323 lr_checksum_clear_cache(fd);
2324
2325 ret = check_finished_transfer_checksum(fd,
2326 target->target->checksums,
2327 &matches,
2328 &transfer_err,
2329 &tmp_err);
2330 if (!ret) { // Error
2331 g_propagate_prefixed_error(err, tmp_err, "Downloading from %s"
2332 "was successful but error encountered while "
2333 "checksumming: ", effective_url);
2334 return FALSE;
2335 }
2336 #ifdef WITH_ZCHUNK
2337 }
2338 #endif /* WITH_ZCHUNK */
2339 if (transfer_err) // Checksum doesn't match
2340 goto transfer_error;
2341
2342 //
2343 // Any other checks should go here
2344 //
2345
2346 transfer_error:
2347
2348 //
2349 // Cleanup
2350 //
2351 curl_multi_remove_handle(dd->multi_handle, target->curl_handle);
2352 curl_easy_cleanup(target->curl_handle);
2353 target->curl_handle = NULL;
2354 g_free(target->headercb_interrupt_reason);
2355 target->headercb_interrupt_reason = NULL;
2356 fclose(target->f);
2357 target->f = NULL;
2358 if (target->curl_rqheaders) {
2359 curl_slist_free_all(target->curl_rqheaders);
2360 target->curl_rqheaders = NULL;
2361 }
2362
2363 dd->running_transfers = g_slist_remove(dd->running_transfers,
2364 (gconstpointer) target);
2365 target->tried_mirrors = g_slist_append(target->tried_mirrors,
2366 target->mirror);
2367
2368 if (target->mirror) {
2369 gboolean success = transfer_err == NULL;
2370 mirror_update_statistics(target->mirror, success);
2371 if (dd->adaptivemirrorsorting)
2372 sort_mirrors(target->lrmirrors, target->mirror, success, serious_error);
2373 }
2374
2375 if (transfer_err) { // There was an error during transfer
2376 int complete_url_in_path = strstr(target->target->path, "://") ? 1 : 0;
2377 guint num_of_tried_mirrors = g_slist_length(target->tried_mirrors);
2378 gboolean retry = FALSE;
2379
2380 g_info("Error during transfer: %s", transfer_err->message);
2381
2382 // Call mirrorfailure callback
2383 LrMirrorFailureCb mf_cb = target->target->mirrorfailurecb;
2384 if (mf_cb) {
2385 int rc = mf_cb(target->target->cbdata,
2386 transfer_err->message,
2387 effective_url);
2388 if (rc == LR_CB_ABORT) {
2389 // User wants to abort this download, so make the error fatal
2390 fatal_error = TRUE;
2391 } else if (rc == LR_CB_ERROR) {
2392 gchar *original_err_msg = g_strdup(transfer_err->message);
2393 g_clear_error(&transfer_err);
2394 g_info("Downloading was aborted by LR_CB_ERROR from "
2395 "mirror failure callback. Original error was: %s", original_err_msg);
2396 g_set_error(&transfer_err, LR_DOWNLOADER_ERROR, LRE_CBINTERRUPTED,
2397 "Downloading was aborted by LR_CB_ERROR from "
2398 "mirror failure callback. Original error was: "
2399 "%s", original_err_msg);
2400 g_free(original_err_msg);
2401 fatal_error = TRUE;
2402 target->cb_return_code = LR_CB_ERROR;
2403 }
2404 }
2405
2406 if (!fatal_error)
2407 {
2408 // Temporary error (serious_error) during download occurred and
2409 // another transfers are running or there are successful transfers
2410 // and fewer failed transfers than tried parallel connections. It may be mirror is OK
2411 // but accepts fewer parallel connections.
2412 if (serious_error && target->mirror &&
2413 (has_running_transfers(target->mirror) ||
2414 (target->mirror->successful_transfers > 0 &&
2415 target->mirror->failed_transfers < target->mirror->max_tried_parallel_connections)))
2416 {
2417 g_debug("%s: Lower maximum of allowed parallel connections for this mirror", __func__);
2418 if (has_running_transfers(target->mirror))
2419 target->mirror->allowed_parallel_connections = target->mirror->running_transfers;
2420 else
2421 target->mirror->allowed_parallel_connections = 1;
2422
2423 // Give used mirror another chance
2424 target->tried_mirrors = g_slist_remove(target->tried_mirrors, target->mirror);
2425 num_of_tried_mirrors = g_slist_length(target->tried_mirrors);
2426 }
2427 // complete_url_in_path and target->baseurl doesn't have an alternatives like using
2428 // mirrors, therefore they are handled differently
2429 const char * complete_url_or_baseurl = complete_url_in_path ? target->target->path : target->target->baseurl;
2430 if (can_retry_download(dd, num_of_tried_mirrors, complete_url_or_baseurl))
2431 {
2432 // Try another mirror or retry
2433 if (complete_url_or_baseurl) {
2434 g_debug("%s: Ignore error - Retry download", __func__);
2435 } else {
2436 g_debug("%s: Ignore error - Try another mirror", __func__);
2437 }
2438 target->state = LR_DS_WAITING;
2439 retry = TRUE;
2440 g_error_free(transfer_err); // Ignore the error
2441
2442 // Truncate file - remove downloaded garbage (error html page etc.)
2443 #ifdef WITH_ZCHUNK
2444 if (!target->target->is_zchunk || target->zck_state == LR_ZCK_DL_HEADER) {
2445 #endif
2446 if (!truncate_transfer_file(target, err))
2447 return FALSE;
2448 #ifdef WITH_ZCHUNK
2449 }
2450 #endif
2451 }
2452 }
2453
2454 if (!retry) {
2455 // No more mirrors to try or baseurl used or fatal error
2456 g_debug("%s: No more retries (tried: %d)",
2457 __func__, num_of_tried_mirrors);
2458 target->state = LR_DS_FAILED;
2459
2460 // Call end callback
2461 LrEndCb end_cb = target->target->endcb;
2462 if (end_cb) {
2463 int rc = end_cb(target->target->cbdata,
2464 LR_TRANSFER_ERROR,
2465 transfer_err->message);
2466 if (rc == LR_CB_ERROR) {
2467 target->cb_return_code = LR_CB_ERROR;
2468 g_debug("%s: Downloading was aborted by LR_CB_ERROR "
2469 "from end callback", __func__);
2470 }
2471 }
2472
2473 lr_downloadtarget_set_error(target->target,
2474 transfer_err->code,
2475 "Download failed: %s",
2476 transfer_err->message);
2477 if (dd->failfast) {
2478 // Fail fast is enabled, fail on any error
2479 g_propagate_error(&fail_fast_error, transfer_err);
2480 } else if (target->cb_return_code == LR_CB_ERROR) {
2481 // Callback returned LR_CB_ERROR, abort the downloading
2482 g_debug("%s: Downloading was aborted by LR_CB_ERROR", __func__);
2483 g_propagate_error(&fail_fast_error, transfer_err);
2484 } else {
2485 // Fail fast is disabled and callback doesn't repor serious
2486 // error, so this download is aborted, but other download
2487 // can continue (do not abort whole downloading)
2488 g_error_free(transfer_err);
2489 }
2490 }
2491
2492 } else {
2493 #ifdef WITH_ZCHUNK
2494 // No error encountered, transfer finished successfully
2495 if(target->target->is_zchunk &&
2496 target->zck_state != LR_ZCK_DL_FINISHED) {
2497 // If we haven't finished downloading zchunk file, setup next
2498 // download
2499 target->state = LR_DS_WAITING;
2500 target->original_offset = -1;
2501 target->target->rcode = LRE_UNFINISHED;
2502 target->target->err = "Not finished";
2503 target->handle = target->target->handle;
2504 target->tried_mirrors = g_slist_remove(target->tried_mirrors, target->mirror);
2505 } else {
2506 #endif /* WITH_ZCHUNK */
2507 target->state = LR_DS_FINISHED;
2508
2509 // Remove xattr that states that the file is being downloaded
2510 // by librepo, because the file is now completely downloaded
2511 // and the xattr is not needed (is is useful only for resuming)
2512 remove_librepo_xattr(target->target);
2513
2514 // Call end callback
2515 LrEndCb end_cb = target->target->endcb;
2516 if (end_cb) {
2517 int rc = end_cb(target->target->cbdata,
2518 LR_TRANSFER_SUCCESSFUL,
2519 NULL);
2520 if (rc == LR_CB_ERROR) {
2521 target->cb_return_code = LR_CB_ERROR;
2522 g_debug("%s: Downloading was aborted by LR_CB_ERROR "
2523 "from end callback", __func__);
2524 g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR,
2525 LRE_CBINTERRUPTED,
2526 "Interrupted by LR_CB_ERROR from end callback");
2527 }
2528 }
2529 if (target->mirror)
2530 lr_downloadtarget_set_usedmirror(target->target,
2531 target->mirror->mirror->url);
2532 #ifdef WITH_ZCHUNK
2533 }
2534 #endif /* WITH_ZCHUNK */
2535
2536 lr_downloadtarget_set_error(target->target, LRE_OK, NULL);
2537 lr_downloadtarget_set_effectiveurl(target->target,
2538 effective_url);
2539 }
2540
2541 if (fail_fast_error) {
2542 // Interrupt whole downloading
2543 // A fatal error occurred or interrupted by callback
2544 g_propagate_error(err, fail_fast_error);
2545 return FALSE;
2546 }
2547 }
2548
2549 // At this point, after handles of finished transfers were removed
2550 // from the multi_handle, we could add new waiting transfers.
2551 return prepare_next_transfers(dd, err);
2552 }
2553
2554
2555 static gboolean
lr_perform(LrDownload * dd,GError ** err)2556 lr_perform(LrDownload *dd, GError **err)
2557 {
2558 CURLMcode cm_rc; // CurlM_ReturnCode
2559
2560 assert(dd);
2561 assert(!err || *err == NULL);
2562
2563 int extra_sleep_trig = 0;
2564 while (1) {
2565
2566 int still_running = 0;
2567
2568 cm_rc = curl_multi_perform(dd->multi_handle, &still_running);
2569
2570 if (lr_interrupt) {
2571 // Check interrupt after each call of curl_multi_perform
2572 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_INTERRUPTED,
2573 "Interrupted by signal");
2574 return FALSE;
2575 }
2576
2577 if (cm_rc != CURLM_OK) {
2578 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURLM,
2579 "curl_multi_perform() error: %s",
2580 curl_multi_strerror(cm_rc));
2581 return FALSE;
2582 }
2583
2584 // Check if any handle finished and potentially add one or more
2585 // waiting downloads to the multi_handle.
2586 int rc = check_transfer_statuses(dd, err);
2587 if (!rc)
2588 return FALSE;
2589
2590 // Leave if there's nothing to wait for
2591 if (!still_running && !dd->running_transfers)
2592 break;
2593
2594 long curl_timeout = -1;
2595
2596 cm_rc = curl_multi_timeout(dd->multi_handle, &curl_timeout);
2597 if (cm_rc != CURLM_OK) {
2598 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURLM,
2599 "curl_multi_timeout() error: %s",
2600 curl_multi_strerror(cm_rc));
2601 return FALSE;
2602 }
2603
2604 if (curl_timeout <= 0) // No wait
2605 continue;
2606
2607 if (curl_timeout > 500) // Wait no more than 500ms
2608 curl_timeout = 500;
2609
2610 int numfds;
2611 cm_rc = curl_multi_wait(dd->multi_handle, NULL, 0, curl_timeout, &numfds);
2612 if (cm_rc != CURLM_OK) {
2613 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURLM,
2614 "curl_multi_wait() error: %s",
2615 curl_multi_strerror(cm_rc));
2616 return FALSE;
2617 }
2618 // 'numfds' being zero means either a timeout or no file descriptors to wait for.
2619 // If libcurl considers the bandwidth to be exceeded,
2620 // curl_multi_wait() is not interested in these file descriptors (for the moment),
2621 // so there are no file descriptors to wait for and curl_multi_wait() will return immediately.
2622 // Libcurl solved the problem by introducing a new function curl_multi_poll() in version 7.66.0.
2623 // Because librepo must work with older libcurl, new function cannot be used.
2624 // Alternative solution:
2625 // Try timeout on first occurrence, then assume no file descriptors to wait for. In this case,
2626 // sleep to avoid busy-looping.
2627 if (numfds == 0) {
2628 if (extra_sleep_trig == 0) {
2629 extra_sleep_trig = 1; // first occurrence
2630 } else {
2631 long sleep_ms;
2632 // Sleep, to avoid busy-looping
2633 if (curl_multi_timeout(dd->multi_handle, &sleep_ms) == CURLM_OK && sleep_ms > 0) {
2634 if (sleep_ms > curl_timeout)
2635 sleep_ms = curl_timeout;
2636 // ts.tv_nsec must be in the range 0 to 999999999. No problem because curl_timeout <=500.
2637 struct timespec ts = {0, sleep_ms * 1000000L};
2638 nanosleep(&ts, NULL);
2639 }
2640 }
2641 } else {
2642 extra_sleep_trig = 0;
2643 }
2644 }
2645
2646 return TRUE;
2647 }
2648
2649 gboolean
lr_download(GSList * targets,gboolean failfast,GError ** err)2650 lr_download(GSList *targets,
2651 gboolean failfast,
2652 GError **err)
2653 {
2654 gboolean ret = FALSE;
2655 LrDownload dd; // dd stands for Download Data
2656 GError *tmp_err = NULL;
2657
2658 assert(!err || *err == NULL);
2659
2660 if (lr_interrupt) {
2661 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_INTERRUPTED,
2662 "Interrupted by signal");
2663 return FALSE;
2664 }
2665
2666 if (!targets) {
2667 g_debug("%s: No targets", __func__);
2668 return TRUE;
2669 }
2670
2671 // XXX: Downloader configuration (max parallel connections etc.)
2672 // is taken from the handle of the first target.
2673 LrHandle *lr_handle = ((LrDownloadTarget *) targets->data)->handle;
2674
2675 // Prepare download data
2676 dd.failfast = failfast;
2677
2678 if (lr_handle) {
2679 dd.max_parallel_connections = lr_handle->maxparalleldownloads;
2680 dd.max_connection_per_host = lr_handle->maxdownloadspermirror;
2681 dd.max_mirrors_to_try = lr_handle->maxmirrortries;
2682 dd.allowed_mirror_failures = lr_handle->allowed_mirror_failures;
2683 dd.adaptivemirrorsorting = lr_handle->adaptivemirrorsorting;
2684 } else {
2685 // No handle, this is allowed when a complete URL is passed
2686 // via relative_url param.
2687 dd.max_parallel_connections = LRO_MAXPARALLELDOWNLOADS_DEFAULT;
2688 dd.max_connection_per_host = LRO_MAXDOWNLOADSPERMIRROR_DEFAULT;
2689 dd.max_mirrors_to_try = LRO_MAXMIRRORTRIES_DEFAULT;
2690 dd.allowed_mirror_failures = LRO_ALLOWEDMIRRORFAILURES_DEFAULT;
2691 dd.adaptivemirrorsorting = LRO_ADAPTIVEMIRRORSORTING_DEFAULT;
2692 }
2693
2694 dd.multi_handle = curl_multi_init();
2695 if (!dd.multi_handle) {
2696 // Something went wrong
2697 g_set_error(err, LR_DOWNLOADER_ERROR, LRE_CURLM,
2698 "curl_multi_init() call failed");
2699 return FALSE;
2700 }
2701
2702 // Prepare list of LrTargets and LrHandleMirrors
2703 dd.handle_mirrors = NULL;
2704 dd.targets = NULL;
2705 for (GSList *elem = targets; elem; elem = g_slist_next(elem)) {
2706 LrDownloadTarget *dtarget = elem->data;
2707
2708 // Assertions
2709 assert(dtarget);
2710 assert(dtarget->path);
2711 assert((dtarget->fd > 0 && !dtarget->fn) || (dtarget->fd < 0 && dtarget->fn));
2712 g_debug("%s: Target: %s (%s)", __func__,
2713 dtarget->path,
2714 (dtarget->baseurl) ? dtarget->baseurl : "-");
2715
2716 // Cleanup of LrDownloadTarget
2717 lr_downloadtarget_reset(dtarget);
2718
2719 // Create and fill LrTarget
2720 LrTarget *target = lr_malloc0(sizeof(*target));
2721 target->state = LR_DS_WAITING;
2722 target->target = dtarget;
2723 target->original_offset = -1;
2724 target->resume = dtarget->resume;
2725 target->target->rcode = LRE_UNFINISHED;
2726 target->target->err = "Not finished";
2727 target->handle = dtarget->handle;
2728 dd.targets = g_slist_append(dd.targets, target);
2729 // Add list of handle internal mirrors to dd.handle_mirrors
2730 // if doesn't exists yet and set the list reference
2731 // to the target.
2732 dd.handle_mirrors = lr_prepare_lrmirrors(dd.handle_mirrors, target);
2733 }
2734
2735 dd.running_transfers = NULL;
2736
2737 // Prepare the first set of transfers
2738 if (!prepare_next_transfers(&dd, &tmp_err))
2739 goto lr_download_cleanup;
2740
2741 // Perform!
2742 g_debug("%s: Downloading started", __func__);
2743 ret = lr_perform(&dd, &tmp_err);
2744
2745 assert(ret || tmp_err);
2746
2747 lr_download_cleanup:
2748
2749 if (tmp_err) {
2750 // If there was an error, stop all transfers that are in progress.
2751 g_info("Error while downloading: %s", tmp_err->message);
2752
2753 for (GSList *elem = dd.running_transfers; elem; elem = g_slist_next(elem)){
2754 LrTarget *target = elem->data;
2755
2756 curl_multi_remove_handle(dd.multi_handle, target->curl_handle);
2757 curl_easy_cleanup(target->curl_handle);
2758 target->curl_handle = NULL;
2759 fclose(target->f);
2760 target->f = NULL;
2761 g_free(target->headercb_interrupt_reason);
2762 target->headercb_interrupt_reason = NULL;
2763
2764 // Call end callback
2765 LrEndCb end_cb = target->target->endcb;
2766 if (end_cb) {
2767 gchar *msg = g_strdup_printf("Not finished - interrupted by "
2768 "error: %s", tmp_err->message);
2769 end_cb(target->target->cbdata, LR_TRANSFER_ERROR, msg);
2770 // No need to check end_cb return value, because there
2771 // already was an error
2772 g_free(msg);
2773 }
2774
2775 lr_downloadtarget_set_error(target->target, LRE_UNFINISHED,
2776 "Not finished - interrupted by error: %s",
2777 tmp_err->message);
2778 }
2779
2780 g_slist_free(dd.running_transfers);
2781 dd.running_transfers = NULL;
2782
2783 g_propagate_error(err, tmp_err);
2784 }
2785
2786 assert(dd.running_transfers == NULL);
2787
2788 curl_multi_cleanup(dd.multi_handle);
2789
2790 // Clean up dd.handle_mirrors
2791 for (GSList *elem = dd.handle_mirrors; elem; elem = g_slist_next(elem)) {
2792 LrHandleMirrors *handle_mirrors = elem->data;
2793 for (GSList *el = handle_mirrors->lrmirrors; el; el = g_slist_next(el)) {
2794 LrMirror *mirror = el->data;
2795 lr_free(mirror);
2796 }
2797 g_slist_free(handle_mirrors->lrmirrors);
2798 lr_free(handle_mirrors);
2799 }
2800 g_slist_free(dd.handle_mirrors);
2801
2802 // Clean up targets
2803 for (GSList *elem = dd.targets; elem; elem = g_slist_next(elem)) {
2804 LrTarget *target = elem->data;
2805 assert(target->curl_handle == NULL);
2806 assert(target->f == NULL);
2807
2808 // Remove file created for the target if download was
2809 // unsuccessful and the file doesn't exists before or
2810 // its original content was overwritten
2811 if (target->state != LR_DS_FINISHED) {
2812 if (!target->resume || target->original_offset == 0) {
2813 // Remove target file if the file doesn't
2814 // exist before or was empty or was overwritten
2815 if (target->target->fn) {
2816 // We can remove only files that were specified by fn
2817 if (unlink(target->target->fn) != 0) {
2818 g_warning("Error while removing: %s", g_strerror(errno));
2819 }
2820 }
2821 }
2822 }
2823
2824 g_slist_free(target->tried_mirrors);
2825 lr_free(target);
2826 }
2827 g_slist_free(dd.targets);
2828
2829 return ret;
2830 }
2831
2832 gboolean
lr_download_target(LrDownloadTarget * target,GError ** err)2833 lr_download_target(LrDownloadTarget *target,
2834 GError **err)
2835 {
2836 gboolean ret;
2837 GSList *list = NULL;
2838
2839 assert(!err || *err == NULL);
2840
2841 if (!target)
2842 return TRUE;
2843
2844 list = g_slist_prepend(list, target);
2845
2846 ret = lr_download(list, TRUE, err);
2847
2848 g_slist_free(list);
2849
2850 return ret;
2851 }
2852
2853 gboolean
lr_download_url(LrHandle * lr_handle,const char * url,int fd,GError ** err)2854 lr_download_url(LrHandle *lr_handle, const char *url, int fd, GError **err)
2855 {
2856 return lr_yum_download_url(lr_handle, url, fd, FALSE, FALSE, err);
2857 }
2858
2859 int
lr_multi_progress_func(void * ptr,double total_to_download,double now_downloaded)2860 lr_multi_progress_func(void* ptr,
2861 double total_to_download,
2862 double now_downloaded)
2863 {
2864 LrCallbackData *cbdata = ptr;
2865 LrSharedCallbackData *shared_cbdata = cbdata->sharedcbdata;
2866
2867 if (cbdata->downloaded > now_downloaded
2868 || cbdata->total != total_to_download)
2869 {
2870 // Reset counters
2871 // This is not first mirror for the transfer,
2872 // we have already downloaded some data
2873 cbdata->total = total_to_download;
2874
2875 // Call progress cb with zeroized params
2876 // This should tell progress cb, that the total_to_download
2877 // size is changed.
2878 int ret = shared_cbdata->cb(cbdata->userdata, 0.0, 0.0);
2879 if (ret != LR_CB_OK)
2880 return ret;
2881 }
2882
2883 cbdata->downloaded = now_downloaded;
2884
2885 // Prepare values for the user callback
2886 double totalsize = 0.0;
2887 double downloaded = 0.0;
2888
2889 for (GSList *elem = shared_cbdata->singlecbdata; elem; elem = g_slist_next(elem)) {
2890 LrCallbackData *singlecbdata = elem->data;
2891 totalsize += singlecbdata->total;
2892 downloaded += singlecbdata->downloaded;
2893 }
2894
2895 if (downloaded > totalsize)
2896 totalsize = downloaded;
2897
2898 // Call user callback
2899 return shared_cbdata->cb(cbdata->userdata,
2900 totalsize,
2901 downloaded);
2902 }
2903
2904 int
lr_multi_mf_func(void * ptr,const char * msg,const char * url)2905 lr_multi_mf_func(void *ptr, const char *msg, const char *url)
2906 {
2907 LrCallbackData *cbdata = ptr;
2908 LrSharedCallbackData *shared_cbdata = cbdata->sharedcbdata;
2909 return shared_cbdata->mfcb(cbdata->userdata, msg, url);
2910 }
2911
2912 gboolean
lr_download_single_cb(GSList * targets,gboolean failfast,LrProgressCb cb,LrMirrorFailureCb mfcb,GError ** err)2913 lr_download_single_cb(GSList *targets,
2914 gboolean failfast,
2915 LrProgressCb cb,
2916 LrMirrorFailureCb mfcb,
2917 GError **err)
2918 {
2919 gboolean ret;
2920 LrSharedCallbackData shared_cbdata;
2921
2922 assert(!err || *err == NULL);
2923
2924 shared_cbdata.cb = cb;
2925 shared_cbdata.mfcb = mfcb;
2926 shared_cbdata.singlecbdata = NULL;
2927
2928 // "Inject" callbacks and callback data to the targets
2929 for (GSList *elem = targets; elem; elem = g_slist_next(elem)) {
2930 LrDownloadTarget *target = elem->data;
2931
2932 LrCallbackData *lrcbdata = lr_malloc0(sizeof(*lrcbdata));
2933 lrcbdata->downloaded = 0.0;
2934 lrcbdata->total = 0.0;
2935 lrcbdata->userdata = target->cbdata;
2936 lrcbdata->sharedcbdata = &shared_cbdata;
2937
2938 target->progresscb = (cb) ? lr_multi_progress_func : NULL;
2939 target->mirrorfailurecb = (mfcb) ? lr_multi_mf_func : NULL;
2940 target->cbdata = lrcbdata;
2941
2942 shared_cbdata.singlecbdata = g_slist_append(shared_cbdata.singlecbdata,
2943 lrcbdata);
2944 }
2945
2946 ret = lr_download(targets, failfast, err);
2947
2948 // Remove callbacks and callback data
2949 for (GSList *elem = targets; elem; elem = g_slist_next(elem)) {
2950 LrDownloadTarget *target = elem->data;
2951 LrCallbackData *cbdata = target->cbdata;
2952 target->cbdata = cbdata->userdata;
2953 target->progresscb = NULL;
2954 target->mirrorfailurecb = NULL;
2955 lr_free(cbdata);
2956 }
2957 g_slist_free(shared_cbdata.singlecbdata);
2958
2959 return ret;
2960 }
2961