1 /* sched.c
2 * Links internal scheduler
3 * (c) 2002 Mikulas Patocka
4 * This file is a part of the Links program, released under GPL.
5 */
6
7 #include "links.h"
8
9 static tcount connection_count = 0;
10
11 static int active_connections = 0;
12
13 tcount netcfg_stamp = 0;
14
15 struct list_head queue = {&queue, &queue};
16
17 struct h_conn {
18 list_entry_1st
19 unsigned char *host;
20 int conn;
21 list_entry_last
22 };
23
24 static struct list_head h_conns = {&h_conns, &h_conns};
25
26 struct list_head keepalive_connections = {&keepalive_connections, &keepalive_connections};
27
28 /* prototypes */
29 static void send_connection_info(struct connection *c);
30 static void del_keepalive_socket(struct k_conn *kc);
31 static void check_keepalive_connections(void);
32 #ifdef DEBUG
33 static void check_queue_bugs(void);
34 #else
35 #define check_queue_bugs() do { } while (0)
36 #endif
37
connect_info(int type)38 unsigned long connect_info(int type)
39 {
40 int i = 0;
41 struct connection *ce;
42 struct list_head *lce;
43 switch (type) {
44 case CI_FILES:
45 return list_size(&queue);
46 case CI_CONNECTING:
47 foreach(struct connection, ce, lce, queue) i += ce->state > S_WAIT && ce->state < S_TRANS;
48 return i;
49 case CI_TRANSFER:
50 foreach(struct connection, ce, lce, queue) i += ce->state == S_TRANS;
51 return i;
52 case CI_KEEP:
53 #ifndef DOS
54 /* this is very slow on DOS */
55 check_keepalive_connections();
56 #endif
57 return list_size(&keepalive_connections);
58 default:
59 internal_error("connect_info: bad request");
60 }
61 return 0;
62 }
63
getpri(struct connection * c)64 static int getpri(struct connection *c)
65 {
66 int i;
67 for (i = 0; i < N_PRI; i++) if (c->pri[i]) return i;
68 internal_error("connection has no owner");
69 return N_PRI;
70 }
71
connection_disappeared(struct connection * c,tcount count)72 static int connection_disappeared(struct connection *c, tcount count)
73 {
74 struct connection *d;
75 struct list_head *ld;
76 foreach(struct connection, d, ld, queue) if (c == d && count == d->count) return 0;
77 return 1;
78 }
79
is_host_on_list(struct connection * c)80 static struct h_conn *is_host_on_list(struct connection *c)
81 {
82 unsigned char *ho;
83 struct h_conn *h;
84 struct list_head *lh;
85 if (!(ho = get_host_name(c->url))) return NULL;
86 foreach(struct h_conn, h, lh, h_conns) if (!strcmp(cast_const_char h->host, cast_const_char ho)) {
87 mem_free(ho);
88 return h;
89 }
90 mem_free(ho);
91 return NULL;
92 }
93
94 static int st_r = 0;
95
stat_timer(void * c_)96 static void stat_timer(void *c_)
97 {
98 struct connection *c = (struct connection *)c_;
99 struct remaining_info *r = &c->prg;
100 uttime now = get_time();
101 uttime a = now - r->last_time;
102 if (a > SPD_DISP_TIME * 100)
103 a = SPD_DISP_TIME * 100;
104 if (getpri(c) == PRI_CANCEL && (c->est_length > (longlong)memory_cache_size * MAX_CACHED_OBJECT || c->from > (longlong)memory_cache_size * MAX_CACHED_OBJECT)) register_bottom_half(check_queue, NULL);
105 if (c->state > S_WAIT) {
106 r->loaded = c->received;
107 if ((r->size = c->est_length) < (r->pos = c->from) && r->size != -1)
108 r->size = c->from;
109 r->dis_b += a;
110 while (r->dis_b >= SPD_DISP_TIME * CURRENT_SPD_SEC) {
111 r->cur_loaded -= r->data_in_secs[0];
112 memmove(r->data_in_secs, r->data_in_secs + 1, sizeof(off_t) * (CURRENT_SPD_SEC - 1));
113 r->data_in_secs[CURRENT_SPD_SEC - 1] = 0;
114 r->dis_b -= SPD_DISP_TIME;
115 }
116 r->data_in_secs[CURRENT_SPD_SEC - 1] += r->loaded - r->last_loaded;
117 r->cur_loaded += r->loaded - r->last_loaded;
118 r->last_loaded = r->loaded;
119 r->elapsed += a;
120 }
121 r->last_time = now;
122 r->timer = install_timer(SPD_DISP_TIME, stat_timer, c);
123 if (!st_r) send_connection_info(c);
124 }
125
setcstate(struct connection * c,int state)126 void setcstate(struct connection *c, int state)
127 {
128 struct status *stat;
129 struct list_head *lstat;
130 if (c->state < 0 && state >= 0) c->prev_error = c->state;
131 if ((c->state = state) == S_TRANS) {
132 struct remaining_info *r = &c->prg;
133 if (r->timer == NULL) {
134 tcount count = c->count;
135 if (!r->valid) {
136 memset(r, 0, sizeof(struct remaining_info));
137 r->valid = 1;
138 }
139 r->last_time = get_time();
140 r->last_loaded = r->loaded;
141 st_r = 1;
142 stat_timer(c);
143 st_r = 0;
144 if (connection_disappeared(c, count)) return;
145 }
146 } else {
147 struct remaining_info *r = &c->prg;
148 if (r->timer != NULL) kill_timer(r->timer), r->timer = NULL;
149 }
150 foreach(struct status, stat, lstat, c->statuss) {
151 stat->state = state;
152 stat->prev_error = c->prev_error;
153 }
154 if (state >= 0) send_connection_info(c);
155 }
156
is_host_on_keepalive_list(struct connection * c)157 static struct k_conn *is_host_on_keepalive_list(struct connection *c)
158 {
159 unsigned char *ho;
160 int po;
161 void (*ph)(struct connection *);
162 struct k_conn *h;
163 struct list_head *lh;
164 if ((po = get_port(c->url)) == -1) return NULL;
165 if (!(ph = get_protocol_handle(c->url))) return NULL;
166 if (!(ho = get_keepalive_id(c->url))) return NULL;
167 foreach(struct k_conn, h, lh, keepalive_connections)
168 if (h->protocol == ph && h->port == po && !strcmp(cast_const_char h->host, cast_const_char ho)) {
169 mem_free(ho);
170 return h;
171 }
172 mem_free(ho);
173 return NULL;
174 }
175
get_keepalive_socket(struct connection * c,int * protocol_data)176 int get_keepalive_socket(struct connection *c, int *protocol_data)
177 {
178 struct k_conn *k;
179 int cc;
180 if (c->tries > 0 || c->unrestartable) return -1;
181 if (!(k = is_host_on_keepalive_list(c))) return -1;
182 cc = k->conn;
183 if (protocol_data) *protocol_data = k->protocol_data;
184 #ifdef HAVE_SSL
185 freeSSL(c->ssl);
186 c->ssl = k->ssl;
187 #endif
188 memcpy(&c->last_lookup_state, &k->last_lookup_state, sizeof(struct lookup_state));
189 del_from_list(k);
190 mem_free(k->host);
191 mem_free(k);
192 c->sock1 = cc;
193 if (max_tries == 1) c->tries = -1;
194 c->keepalive = 1;
195 return 0;
196 }
197
abort_all_keepalive_connections(void)198 static int abort_all_keepalive_connections(void)
199 {
200 int did_something = 0;
201 while (!list_empty(keepalive_connections)) {
202 del_keepalive_socket(list_struct(keepalive_connections.next, struct k_conn));
203 did_something = 1;
204 }
205 check_keepalive_connections();
206 return did_something;
207 }
208
free_connection_data(struct connection * c)209 static void free_connection_data(struct connection *c)
210 {
211 struct h_conn *h;
212 int rs;
213 if (c->sock1 != -1) set_handlers(c->sock1, NULL, NULL, NULL);
214 close_socket(&c->sock2);
215 if (c->pid) {
216 EINTRLOOP(rs, kill(c->pid, SIGINT));
217 EINTRLOOP(rs, kill(c->pid, SIGTERM));
218 EINTRLOOP(rs, kill(c->pid, SIGKILL));
219 c->pid = 0;
220 }
221 if (!c->running) {
222 internal_error("connection already suspended");
223 }
224 c->running = 0;
225 if (c->dnsquery) kill_dns_request(&c->dnsquery);
226 if (c->buffer) {
227 mem_free(c->buffer);
228 c->buffer = NULL;
229 }
230 if (c->newconn) {
231 mem_free(c->newconn);
232 c->newconn = NULL;
233 }
234 if (c->info) {
235 mem_free(c->info);
236 c->info = NULL;
237 }
238 clear_connection_timeout(c);
239 if (--active_connections < 0) {
240 internal_error("active connections underflow");
241 active_connections = 0;
242 }
243 if (c->state != S_WAIT) {
244 if ((h = is_host_on_list(c))) {
245 if (!--h->conn) {
246 del_from_list(h);
247 mem_free(h->host);
248 mem_free(h);
249 }
250 } else internal_error("suspending connection that is not on the list (state %d)", c->state);
251 }
252 }
253
send_connection_info(struct connection * c)254 static void send_connection_info(struct connection *c)
255 {
256 #if 0
257 int st = c->state;
258 tcount count = c->count;
259 struct status *stat;
260 struct list_head *lstat;
261 /*debug("reporting conn info %s", c->url);*/
262 foreach(struct status, stat, lstat, c->statuss) {
263 /*debug("status list &%p %p %p, status %p %p %p", &c->statuss, c->statuss.next, c->statuss.prev, lstat, lstat->next, lstat->prev);*/
264 stat->ce = c->cache;
265 verify_list_entry(lstat);
266 lstat = lstat->next;
267 if (stat->end) stat->end(stat, stat->data);
268 if (st >= 0 && connection_disappeared(c, count)) return;
269 verify_list_entry(lstat);
270 lstat = lstat->prev;
271 }
272 /*debug("end report %s", c->url);*/
273 #else
274 if (!list_empty(c->statuss)) {
275 int st = c->state;
276 tcount count = c->count;
277 struct list_head *lstat = c->statuss.next;
278 while (1) {
279 int e;
280 struct status *xstat = list_struct(lstat, struct status);
281 xstat->ce = c->cache;
282 lstat = lstat->next;
283 e = lstat == &c->statuss;
284 if (xstat->end) xstat->end(xstat, xstat->data);
285 if (e) return;
286 if (st >= 0 && connection_disappeared(c, count)) return;
287 }
288 }
289 #endif
290 }
291
del_connection(struct connection * c)292 static void del_connection(struct connection *c)
293 {
294 struct cache_entry *ce = c->cache;
295 if (ce) ce->refcount++;
296 del_from_list(c);
297 send_connection_info(c);
298 if (ce) ce->refcount--;
299 if (c->detached) {
300 if (ce && !ce->url[0] && !is_entry_used(ce) && !ce->refcount)
301 delete_cache_entry(ce);
302 } else {
303 if (ce)
304 trim_cache_entry(ce);
305 }
306 mem_free(c->url);
307 if (c->prev_url) mem_free(c->prev_url);
308 #ifdef HAVE_SSL
309 freeSSL(c->ssl);
310 #endif
311 mem_free(c);
312 }
313
add_keepalive_socket(struct connection * c,uttime timeout,int protocol_data)314 void add_keepalive_socket(struct connection *c, uttime timeout, int protocol_data)
315 {
316 struct k_conn *k;
317 int rs;
318 free_connection_data(c);
319 if (c->sock1 == -1) {
320 internal_error("keepalive connection not connected");
321 goto del;
322 }
323 k = mem_alloc(sizeof(struct k_conn));
324 if (c->netcfg_stamp != netcfg_stamp ||
325 #ifdef HAVE_SSL
326 ssl_not_reusable(c->ssl) ||
327 #endif
328 (k->port = get_port(c->url)) == -1 ||
329 !(k->protocol = get_protocol_handle(c->url)) ||
330 !(k->host = get_keepalive_id(c->url))) {
331 mem_free(k);
332 del_connection(c);
333 goto clos;
334 }
335 k->conn = c->sock1;
336 k->timeout = timeout;
337 k->add_time = get_absolute_time();
338 k->protocol_data = protocol_data;
339 #ifdef HAVE_SSL
340 k->ssl = c->ssl;
341 c->ssl = NULL;
342 #endif
343 memcpy(&k->last_lookup_state, &c->last_lookup_state, sizeof(struct lookup_state));
344 add_to_list(keepalive_connections, k);
345 del:
346 del_connection(c);
347 check_queue_bugs();
348 register_bottom_half(check_queue, NULL);
349 return;
350 clos:
351 EINTRLOOP(rs, close(c->sock1));
352 check_queue_bugs();
353 register_bottom_half(check_queue, NULL);
354 }
355
del_keepalive_socket(struct k_conn * kc)356 static void del_keepalive_socket(struct k_conn *kc)
357 {
358 int rs;
359 del_from_list(kc);
360 #ifdef HAVE_SSL
361 freeSSL(kc->ssl);
362 #endif
363 EINTRLOOP(rs, close(kc->conn));
364 mem_free(kc->host);
365 mem_free(kc);
366 }
367
368 static struct timer *keepalive_timeout = NULL;
369
keepalive_timer(void * x)370 static void keepalive_timer(void *x)
371 {
372 keepalive_timeout = NULL;
373 check_keepalive_connections();
374 }
375
check_keepalive_connections(void)376 static void check_keepalive_connections(void)
377 {
378 struct k_conn *kc;
379 struct list_head *lkc;
380 uttime ct = get_absolute_time();
381 int p = 0;
382 if (keepalive_timeout != NULL)
383 kill_timer(keepalive_timeout), keepalive_timeout = NULL;
384 foreach(struct k_conn, kc, lkc, keepalive_connections) {
385 if (can_read(kc->conn) || ct - kc->add_time > kc->timeout) {
386 lkc = lkc->prev;
387 del_keepalive_socket(kc);
388 } else
389 p++;
390 }
391 for (; p > MAX_KEEPALIVE_CONNECTIONS; p--)
392 if (!list_empty(keepalive_connections))
393 del_keepalive_socket(list_struct(keepalive_connections.prev, struct k_conn));
394 else
395 internal_error("keepalive list empty");
396 if (!list_empty(keepalive_connections))
397 keepalive_timeout = install_timer(KEEPALIVE_CHECK_TIME, keepalive_timer, NULL);
398 }
399
add_to_queue(struct connection * c)400 static void add_to_queue(struct connection *c)
401 {
402 struct connection *cc;
403 struct list_head *lcc;
404 int pri = getpri(c);
405 foreach(struct connection, cc, lcc, queue) if (getpri(cc) > pri) break;
406 add_before_list_entry(lcc, &c->list_entry);
407 }
408
sort_queue(void)409 static void sort_queue(void)
410 {
411 struct connection *c, *n;
412 struct list_head *lc;
413 int swp;
414 do {
415 swp = 0;
416 foreach(struct connection, c, lc, queue) if (c->list_entry.next != &queue) {
417 n = list_struct(c->list_entry.next, struct connection);
418 if (getpri(n) < getpri(c)) {
419 del_from_list(c);
420 add_after_pos(n, c);
421 swp = 1;
422 }
423 }
424 } while (swp);
425 }
426
interrupt_connection(struct connection * c)427 static void interrupt_connection(struct connection *c)
428 {
429 #ifdef HAVE_SSL
430 freeSSL(c->ssl);
431 c->ssl = NULL;
432 #endif
433 close_socket(&c->sock1);
434 free_connection_data(c);
435 }
436
suspend_connection(struct connection * c)437 static void suspend_connection(struct connection *c)
438 {
439 interrupt_connection(c);
440 setcstate(c, S_WAIT);
441 }
442
try_to_suspend_connection(struct connection * c,unsigned char * ho)443 static int try_to_suspend_connection(struct connection *c, unsigned char *ho)
444 {
445 int pri = getpri(c);
446 struct connection *d;
447 struct list_head *ld;
448 foreachback(struct connection, d, ld, queue) {
449 if (getpri(d) <= pri) return -1;
450 if (d->state == S_WAIT) continue;
451 if (d->unrestartable == 2 && getpri(d) < PRI_CANCEL) continue;
452 if (ho) {
453 unsigned char *h;
454 if (!(h = get_host_name(d->url))) continue;
455 if (strcmp(cast_const_char h, cast_const_char ho)) {
456 mem_free(h);
457 continue;
458 }
459 mem_free(h);
460 }
461 suspend_connection(d);
462 return 0;
463 }
464 return -1;
465 }
466
is_noproxy_url(unsigned char * url)467 int is_noproxy_url(unsigned char *url)
468 {
469 unsigned char *host = get_host_name(url);
470 if (!proxies.only_proxies) {
471 unsigned char *np = proxies.no_proxy;
472 int host_l = (int)strlen(cast_const_char host);
473 if (*np) while (1) {
474 int l = (int)strcspn(cast_const_char np, ",");
475 if (l > host_l)
476 goto no_match;
477 if (l < host_l && host[host_l - l - 1] != '.')
478 goto no_match;
479 if (casecmp(np, host + (host_l - l), l))
480 goto no_match;
481 mem_free(host);
482 return 1;
483 no_match:
484 if (!np[l])
485 break;
486 np += l + 1;
487 }
488 }
489 mem_free(host);
490 return 0;
491 }
492
run_connection(struct connection * c)493 static void run_connection(struct connection *c)
494 {
495 struct h_conn *hc;
496 void (*func)(struct connection *);
497 if (c->running) {
498 internal_error("connection already running");
499 return;
500 }
501
502 memset(&c->last_lookup_state, 0, sizeof(struct lookup_state));
503
504 if (is_noproxy_url(remove_proxy_prefix(c->url))) {
505 c->socks_proxy[0] = 0;
506 c->dns_append[0] = 0;
507 } else {
508 safe_strncpy(c->socks_proxy, proxies.socks_proxy, sizeof c->socks_proxy);
509 safe_strncpy(c->dns_append, proxies.dns_append, sizeof c->dns_append);
510 }
511
512 if (proxies.only_proxies && !is_proxy_url(c->url) && casecmp(c->url, cast_uchar "data:", 5) && (!*c->socks_proxy || url_bypasses_socks(c->url))) {
513 setcstate(c, S_NO_PROXY);
514 del_connection(c);
515 return;
516 }
517
518 if (!(func = get_protocol_handle(c->url))) {
519 s_bad_url:
520 if (is_proxy_url(c->url)) setcstate(c, S_BAD_PROXY);
521 else setcstate(c, S_BAD_URL);
522 del_connection(c);
523 return;
524 }
525 if (!(hc = is_host_on_list(c))) {
526 hc = mem_alloc(sizeof(struct h_conn));
527 if (!(hc->host = get_host_name(c->url))) {
528 mem_free(hc);
529 goto s_bad_url;
530 }
531 hc->conn = 0;
532 add_to_list(h_conns, hc);
533 }
534 hc->conn++;
535 active_connections++;
536 c->keepalive = 0;
537 c->running = 1;
538 func(c);
539 }
540
is_connection_seekable(struct connection * c)541 static int is_connection_seekable(struct connection *c)
542 {
543 unsigned char *protocol = get_protocol_name(c->url);
544 if (!casestrcmp(protocol, cast_uchar "http") ||
545 !casestrcmp(protocol, cast_uchar "https") ||
546 !casestrcmp(protocol, cast_uchar "proxy")) {
547 unsigned char *d;
548 mem_free(protocol);
549 if (!c->cache || !c->cache->head)
550 return 1;
551 d = parse_http_header(c->cache->head, cast_uchar "Accept-Ranges", NULL);
552 if (d) {
553 mem_free(d);
554 return 1;
555 }
556 return 0;
557 }
558 if (!casestrcmp(protocol, cast_uchar "ftp")) {
559 mem_free(protocol);
560 return 1;
561 }
562 mem_free(protocol);
563 return 0;
564 }
565
is_connection_restartable(struct connection * c)566 int is_connection_restartable(struct connection *c)
567 {
568 return !(c->unrestartable >= 2 || (c->tries + 1 >= (max_tries ? max_tries : 1000)));
569 }
570
is_last_try(struct connection * c)571 int is_last_try(struct connection *c)
572 {
573 int is_restartable;
574 c->tries++;
575 is_restartable = is_connection_restartable(c) && c->tries < 10;
576 c->tries--;
577 return !is_restartable;
578 }
579
retry_connection(struct connection * c)580 void retry_connection(struct connection *c)
581 {
582 interrupt_connection(c);
583 if (!is_connection_restartable(c)) {
584 del_connection(c);
585 check_queue_bugs();
586 register_bottom_half(check_queue, NULL);
587 } else {
588 c->tries++;
589 c->prev_error = c->state;
590 run_connection(c);
591 }
592 }
593
abort_connection(struct connection * c)594 void abort_connection(struct connection *c)
595 {
596 if (c->running) interrupt_connection(c);
597 del_connection(c);
598 check_queue_bugs();
599 register_bottom_half(check_queue, NULL);
600 }
601
try_connection(struct connection * c)602 static int try_connection(struct connection *c)
603 {
604 struct h_conn *hc = NULL;
605 if ((hc = is_host_on_list(c))) {
606 if (hc->conn >= max_connections_to_host) {
607 if (try_to_suspend_connection(c, hc->host)) return 0;
608 else return -1;
609 }
610 }
611 if (active_connections >= max_connections) {
612 if (try_to_suspend_connection(c, NULL)) return 0;
613 else return -1;
614 }
615 run_connection(c);
616 return 1;
617 }
618
619 #ifdef DEBUG
check_queue_bugs(void)620 static void check_queue_bugs(void)
621 {
622 struct connection *d;
623 struct list_head *ld;
624 int p = 0, ps = 0;
625 int cc;
626 again:
627 cc = 0;
628 foreach(struct connection, d, ld, queue) {
629 int q = getpri(d);
630 cc += d->running;
631 if (q < p) {
632 if (!ps) {
633 internal_error("queue is not sorted");
634 sort_queue();
635 ps = 1;
636 goto again;
637 } else {
638 internal_error("queue is not sorted even after sort_queue!");
639 break;
640 }
641 } else p = q;
642 if (d->state < 0) {
643 internal_error("interrupted connection on queue (conn %s, state %d)", d->url, d->state);
644 ld = ld->prev;
645 abort_connection(d);
646 }
647 }
648 if (cc != active_connections) {
649 internal_error("bad number of active connections (counted %d, stored %d)", cc, active_connections);
650 active_connections = cc;
651 }
652 }
653 #endif
654
check_queue(void * dummy)655 void check_queue(void *dummy)
656 {
657 struct connection *c;
658 struct list_head *lc;
659 again:
660 check_queue_bugs();
661 check_keepalive_connections();
662 foreach(struct connection, c, lc, queue) {
663 struct connection *d;
664 struct list_head *ld;
665 int cp = getpri(c);
666 foreachfrom(struct connection, d, ld, queue, &c->list_entry) {
667 if (getpri(d) != cp)
668 break;
669 if (!d->state && is_host_on_keepalive_list(d))
670 if (try_connection(d))
671 goto again;
672 }
673 foreachfrom(struct connection, d, ld, queue, &c->list_entry) {
674 if (getpri(d) != cp)
675 break;
676 if (!d->state)
677 if (try_connection(d))
678 goto again;
679 }
680 lc = ld->prev;
681 }
682 again2:
683 foreachback(struct connection, c, lc, queue) {
684 if (getpri(c) < PRI_CANCEL) break;
685 if (c->state == S_WAIT) {
686 setcstate(c, S_INTERRUPTED);
687 del_connection(c);
688 goto again2;
689 } else if (c->est_length > (longlong)memory_cache_size * MAX_CACHED_OBJECT || c->from > (longlong)memory_cache_size * MAX_CACHED_OBJECT) {
690 setcstate(c, S_INTERRUPTED);
691 abort_connection(c);
692 goto again2;
693 }
694 }
695 check_queue_bugs();
696 }
697
get_proxy_string(unsigned char * url)698 unsigned char *get_proxy_string(unsigned char *url)
699 {
700 if (is_noproxy_url(url))
701 return NULL;
702 if (*proxies.http_proxy && !casecmp(url, cast_uchar "http://", 7)) return proxies.http_proxy;
703 if (*proxies.ftp_proxy && !casecmp(url, cast_uchar "ftp://", 6)) return proxies.ftp_proxy;
704 #ifdef HAVE_SSL
705 if (*proxies.https_proxy && !casecmp(url, cast_uchar "https://", 8)) return proxies.https_proxy;
706 #endif
707 return NULL;
708 }
709
get_proxy(unsigned char * url)710 unsigned char *get_proxy(unsigned char *url)
711 {
712 unsigned char *proxy = get_proxy_string(url);
713 unsigned char *u;
714 u = stracpy(cast_uchar "");
715 if (proxy) {
716 add_to_strn(&u, cast_uchar "proxy://");
717 add_to_strn(&u, proxy);
718 add_to_strn(&u, cast_uchar "/");
719 }
720 add_to_strn(&u, url);
721 return u;
722 }
723
is_proxy_url(unsigned char * url)724 int is_proxy_url(unsigned char *url)
725 {
726 return !casecmp(url, cast_uchar "proxy://", 8);
727 }
728
remove_proxy_prefix(unsigned char * url)729 unsigned char *remove_proxy_prefix(unsigned char *url)
730 {
731 unsigned char *x = NULL;
732 if (is_proxy_url(url))
733 x = get_url_data(url);
734 if (!x)
735 x = url;
736 return x;
737 }
738
get_allow_flags(unsigned char * url)739 int get_allow_flags(unsigned char *url)
740 {
741 return !casecmp(url, cast_uchar "smb://", 6) ? ALLOW_SMB :
742 !casecmp(url, cast_uchar "file://", 7) ? ALLOW_FILE : 0;
743 }
744
disallow_url(unsigned char * url,int allow_flags)745 int disallow_url(unsigned char *url, int allow_flags)
746 {
747 if (!casecmp(url, cast_uchar "smb://", 6) && !(allow_flags & ALLOW_SMB) && !smb_options.allow_hyperlinks_to_smb) {
748 return S_SMB_NOT_ALLOWED;
749 }
750 if (!casecmp(url, cast_uchar "file://", 7) && !(allow_flags & ALLOW_FILE)) {
751 return S_FILE_NOT_ALLOWED;
752 }
753 return 0;
754 }
755
756 /* prev_url is a pointer to previous url or NULL */
757 /* prev_url will NOT be deallocated */
load_url(unsigned char * url,unsigned char * prev_url,struct status * stat,int pri,int no_cache,int no_compress,int allow_flags,off_t position)758 void load_url(unsigned char *url, unsigned char *prev_url, struct status *stat, int pri, int no_cache, int no_compress, int allow_flags, off_t position)
759 {
760 struct cache_entry *e = NULL;
761 struct connection *c;
762 struct list_head *lc;
763 unsigned char *u;
764 int must_detach = 0;
765 int err;
766
767 if (stat) {
768 stat->c = NULL;
769 stat->ce = NULL;
770 stat->state = S_OUT_OF_MEM;
771 stat->prev_error = 0;
772 stat->pri = pri;
773 }
774 #ifdef DEBUG
775 foreach(struct connection, c, lc, queue) {
776 struct status *st;
777 struct list_head *lst;
778 foreach(struct status, st, lst, c->statuss) {
779 if (st == stat) {
780 internal_error("status already assigned to '%s'", c->url);
781 stat->state = S_INTERNAL;
782 if (stat->end) stat->end(stat, stat->data);
783 goto ret;
784 }
785 }
786 }
787 #endif
788 if (is_url_blocked(url)) {
789 if (stat) {
790 stat->state = S_BLOCKED_URL;
791 if (stat->end) stat->end(stat, stat->data);
792 }
793 goto ret;
794 }
795 err = disallow_url(url, allow_flags);
796 if (err) {
797 if (stat) {
798 stat->state = err;
799 if (stat->end) stat->end(stat, stat->data);
800 }
801 goto ret;
802 }
803 if (no_cache <= NC_CACHE && !find_in_cache(url, &e)) {
804 if (e->incomplete) {
805 e->refcount--;
806 goto skip_cache;
807 }
808 if (!aggressive_cache && no_cache > NC_ALWAYS_CACHE) {
809 if (e->expire_time && e->expire_time < get_absolute_seconds()) {
810 if (no_cache < NC_IF_MOD) no_cache = NC_IF_MOD;
811 e->refcount--;
812 goto skip_cache;
813 }
814 }
815 if (no_compress) {
816 unsigned char *enc;
817 enc = parse_http_header(e->head, cast_uchar "Content-Encoding", NULL);
818 if (enc) {
819 mem_free(enc);
820 e->refcount--;
821 must_detach = 1;
822 goto skip_cache;
823 }
824 }
825 if (stat) {
826 stat->ce = e;
827 stat->state = S__OK;
828 if (stat->end) stat->end(stat, stat->data);
829 }
830 e->refcount--;
831 goto ret;
832 }
833 skip_cache:
834 if (is_proxy_url(url)) {
835 if (stat) {
836 stat->state = S_BAD_URL;
837 if (stat->end) stat->end(stat, stat->data);
838 }
839 goto ret;
840 }
841 u = get_proxy(url);
842 foreach(struct connection, c, lc, queue) if (!c->detached && !strcmp(cast_const_char c->url, cast_const_char u)) {
843 if (c->from < position) continue;
844 if (no_compress && !c->no_compress) {
845 unsigned char *enc;
846 if ((c->state >= S_WAIT && c->state < S_TRANS) || !c->cache) {
847 must_detach = 1;
848 break;
849 }
850 enc = parse_http_header(c->cache->head, cast_uchar "Content-Encoding", NULL);
851 if (enc) {
852 mem_free(enc);
853 must_detach = 1;
854 break;
855 }
856 }
857 mem_free(u);
858 if (getpri(c) > pri) {
859 del_from_list(c);
860 c->pri[pri]++;
861 add_to_queue(c);
862 register_bottom_half(check_queue, NULL);
863 } else c->pri[pri]++;
864 if (stat) {
865 stat->prg = &c->prg;
866 stat->c = c;
867 stat->ce = c->cache;
868 add_to_list(c->statuss, stat);
869 setcstate(c, c->state);
870 }
871 check_queue_bugs();
872 goto ret;
873 }
874 c = mem_calloc(sizeof(struct connection));
875 c->count = connection_count++;
876 c->url = u;
877 c->prev_url = stracpy(prev_url);
878 c->running = 0;
879 c->prev_error = 0;
880 if (position || must_detach) {
881 c->from = position;
882 } else if (no_cache >= NC_IF_MOD || !e) {
883 c->from = 0;
884 } else {
885 struct fragment *frag;
886 struct list_head *lfrag;
887 c->from = 0;
888 foreach(struct fragment, frag, lfrag, e->frag) {
889 if (frag->offset != c->from)
890 break;
891 c->from += frag->length;
892 }
893 }
894 memset(c->pri, 0, sizeof c->pri);
895 c->pri[pri] = 1;
896 c->no_cache = no_cache;
897 c->sock1 = c->sock2 = -1;
898 c->dnsquery = NULL;
899 c->tries = 0;
900 c->netcfg_stamp = netcfg_stamp;
901 init_list(c->statuss);
902 c->info = NULL;
903 c->buffer = NULL;
904 c->newconn = NULL;
905 c->cache = NULL;
906 c->est_length = -1;
907 c->unrestartable = 0;
908 #ifdef HAVE_ANY_COMPRESSION
909 c->no_compress = http_options.no_compression || no_compress || dmp == D_SOURCE;
910 #else
911 c->no_compress = 1;
912 #endif
913 c->prg.timer = NULL;
914 c->timer = NULL;
915 if (position || must_detach) {
916 if (new_cache_entry(cast_uchar "", &c->cache)) {
917 mem_free(c->url);
918 if (c->prev_url) mem_free(c->prev_url);
919 mem_free(c);
920 if (stat) {
921 stat->state = S_OUT_OF_MEM;
922 if (stat->end) stat->end(stat, stat->data);
923 }
924 goto ret;
925 }
926 c->cache->refcount--;
927 detach_cache_entry(c->cache);
928 c->detached = 2;
929 }
930 if (stat) {
931 stat->prg = &c->prg;
932 stat->c = c;
933 stat->ce = NULL;
934 add_to_list(c->statuss, stat);
935 }
936 add_to_queue(c);
937 setcstate(c, S_WAIT);
938 check_queue_bugs();
939 register_bottom_half(check_queue, NULL);
940
941 ret:
942 return;
943 }
944
change_connection(struct status * oldstat,struct status * newstat,int newpri)945 void change_connection(struct status *oldstat, struct status *newstat, int newpri)
946 {
947 struct connection *c;
948 int oldpri;
949 oldpri = oldstat->pri;
950 if (oldstat->state < 0) {
951 if (newstat) {
952 struct cache_entry *ce = oldstat->ce;
953 if (ce) ce->refcount++;
954 newstat->ce = oldstat->ce;
955 newstat->state = oldstat->state;
956 newstat->prev_error = oldstat->prev_error;
957 if (newstat->end) newstat->end(newstat, newstat->data);
958 if (ce) ce->refcount--;
959 }
960 return;
961 }
962 check_queue_bugs();
963 c = oldstat->c;
964 if (--c->pri[oldpri] < 0) {
965 internal_error("priority counter underflow");
966 c->pri[oldpri] = 0;
967 }
968 c->pri[newpri]++;
969 del_from_list(oldstat);
970 oldstat->state = S_INTERRUPTED;
971 if (newstat) {
972 newstat->prg = &c->prg;
973 add_to_list(c->statuss, newstat);
974 newstat->state = c->state;
975 newstat->prev_error = c->prev_error;
976 newstat->pri = newpri;
977 newstat->c = c;
978 newstat->ce = c->cache;
979 }
980 if (c->detached && !newstat) {
981 setcstate(c, S_INTERRUPTED);
982 abort_connection(c);
983 }
984 sort_queue();
985 check_queue_bugs();
986 register_bottom_half(check_queue, NULL);
987 }
988
detach_connection(struct status * stat,off_t pos,int want_to_restart,int dont_check_refcount)989 void detach_connection(struct status *stat, off_t pos, int want_to_restart, int dont_check_refcount)
990 {
991 struct connection *c;
992 int i, n_users;
993 off_t l;
994 if (stat->state < 0) return;
995 c = stat->c;
996 if (c->no_compress && want_to_restart) return;
997 if (!c->cache || (!dont_check_refcount && c->cache->refcount)) return;
998 want_to_restart |= pos > c->from && is_connection_seekable(c);
999 if (c->detached) goto detach_done;
1000 if (c->est_length == -1) l = c->from;
1001 else l = c->est_length;
1002 if (!dont_check_refcount && l < (longlong)memory_cache_size * MAX_CACHED_OBJECT && !want_to_restart) return;
1003 n_users = 0;
1004 for (i = 0; i < PRI_CANCEL; i++) n_users += c->pri[i];
1005 if (!n_users) internal_error("detaching free connection");
1006 if (n_users != 1) return;
1007 shrink_memory(SH_CHECK_QUOTA, 0);
1008 detach_cache_entry(c->cache);
1009 c->detached = 1;
1010 detach_done:
1011 free_entry_to(c->cache, pos);
1012
1013 if (c->detached < 2 && want_to_restart) {
1014 int running = c->running;
1015 c->no_compress = 1;
1016 if (running) interrupt_connection(c);
1017 c->from = pos;
1018 if (running) run_connection(c);
1019 c->detached = 2;
1020 }
1021 }
1022
get_timeout_value(struct connection * c)1023 static uttime get_timeout_value(struct connection *c)
1024 {
1025 uttime t;
1026 if (c->state == S_CONN || c->state == S_CONN_ANOTHER)
1027 t = timeout_multiple_addresses * (c->tries < 1 ? 1 : c->tries + 1);
1028 else if (c->unrestartable)
1029 t = unrestartable_receive_timeout;
1030 else
1031 t = receive_timeout;
1032 t *= 500;
1033 return t;
1034 }
1035
connection_timeout(void * c_)1036 static void connection_timeout(void *c_)
1037 {
1038 struct connection *c = (struct connection *)c_;
1039 c->timer = NULL;
1040 if (c->state == S_CONN || c->state == S_CONN_ANOTHER) {
1041 retry_connect(c, get_error_from_errno(ETIMEDOUT), 0);
1042 return;
1043 }
1044 setcstate(c, S_TIMEOUT);
1045 retry_connection(c);
1046 }
1047
connection_timeout_1(void * c_)1048 static void connection_timeout_1(void *c_)
1049 {
1050 struct connection *c = (struct connection *)c_;
1051 c->timer = install_timer(get_timeout_value(c), connection_timeout, c);
1052 }
1053
clear_connection_timeout(struct connection * c)1054 void clear_connection_timeout(struct connection *c)
1055 {
1056 if (c->timer != NULL) kill_timer(c->timer), c->timer = NULL;
1057 }
1058
set_connection_timeout(struct connection * c)1059 void set_connection_timeout(struct connection *c)
1060 {
1061 clear_connection_timeout(c);
1062 c->timer = install_timer(get_timeout_value(c), connection_timeout_1, c);
1063 }
1064
set_connection_timeout_keepal(struct connection * c)1065 void set_connection_timeout_keepal(struct connection *c)
1066 {
1067 if (c->keepalive && !c->unrestartable) {
1068 clear_connection_timeout(c);
1069 c->timer = install_timer(timeout_multiple_addresses * 1000, connection_timeout, c);
1070 return;
1071 }
1072 set_connection_timeout(c);
1073 }
1074
abort_all_connections(void)1075 void abort_all_connections(void)
1076 {
1077 while (!list_empty(queue)) {
1078 struct connection *c = list_struct(queue.next, struct connection);
1079 setcstate(c, S_INTERRUPTED);
1080 abort_connection(c);
1081 }
1082 abort_all_keepalive_connections();
1083 }
1084
abort_background_connections(void)1085 int abort_background_connections(void)
1086 {
1087 int did_something = 0;
1088 struct connection *c;
1089 struct list_head *lc;
1090 again:
1091 foreach(struct connection, c, lc, queue) {
1092 if (getpri(c) >= PRI_CANCEL) {
1093 setcstate(c, S_INTERRUPTED);
1094 abort_connection(c);
1095 did_something = 1;
1096 goto again;
1097 }
1098 }
1099 return did_something | abort_all_keepalive_connections();
1100 }
1101
is_entry_used(struct cache_entry * e)1102 int is_entry_used(struct cache_entry *e)
1103 {
1104 struct connection *c;
1105 struct list_head *lc;
1106 foreach(struct connection, c, lc, queue) if (c->cache == e) return 1;
1107 return 0;
1108 }
1109
1110 struct blacklist_entry {
1111 list_entry_1st
1112 int flags;
1113 list_entry_last
1114 unsigned char host[1];
1115 };
1116
1117 static struct list_head blacklist = { &blacklist, &blacklist };
1118
add_blacklist_entry(unsigned char * host,int flags)1119 void add_blacklist_entry(unsigned char *host, int flags)
1120 {
1121 struct blacklist_entry *b;
1122 struct list_head *lb;
1123 size_t sl;
1124 foreach(struct blacklist_entry, b, lb, blacklist) if (!casestrcmp(host, b->host)) {
1125 b->flags |= flags;
1126 return;
1127 }
1128 sl = strlen(cast_const_char host);
1129 if (sl > MAXINT - sizeof(struct blacklist_entry)) overalloc();
1130 b = mem_alloc(sizeof(struct blacklist_entry) + sl);
1131 b->flags = flags;
1132 strcpy(cast_char b->host, cast_const_char host);
1133 add_to_list(blacklist, b);
1134 }
1135
del_blacklist_entry(unsigned char * host,int flags)1136 void del_blacklist_entry(unsigned char *host, int flags)
1137 {
1138 struct blacklist_entry *b;
1139 struct list_head *lb;
1140 foreach(struct blacklist_entry, b, lb, blacklist) if (!casestrcmp(host, b->host)) {
1141 b->flags &= ~flags;
1142 if (!b->flags) {
1143 del_from_list(b);
1144 mem_free(b);
1145 }
1146 return;
1147 }
1148 }
1149
get_blacklist_flags(unsigned char * host)1150 int get_blacklist_flags(unsigned char *host)
1151 {
1152 struct blacklist_entry *b;
1153 struct list_head *lb;
1154 foreach(struct blacklist_entry, b, lb, blacklist) if (!casestrcmp(host, b->host)) return b->flags;
1155 return 0;
1156 }
1157
free_blacklist(void)1158 void free_blacklist(void)
1159 {
1160 free_list(struct blacklist_entry, blacklist);
1161 }
1162