1 /*
2 * lftp - file transfer program
3 *
4 * Copyright (c) 1996-2016 by Alexander V. Lukyanov (lav@yars.free.net)
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include <config.h>
21 #include "trio.h"
22 #include <unistd.h>
23 #include <sys/types.h>
24 #include <sys/socket.h>
25 #include <fcntl.h>
26 #include <errno.h>
27 #include <stdarg.h>
28 #include <time.h>
29 #include <fnmatch.h>
30 #include <locale.h>
31 #include <assert.h>
32 #include "Http.h"
33 #include "ResMgr.h"
34 #include "log.h"
35 #include "url.h"
36 #include "HttpAuth.h"
37 #include "HttpDir.h"
38 #include "misc.h"
39 #include "buffer_ssl.h"
40 #include "buffer_zlib.h"
41
42 #include "ascii_ctype.h"
43
44 #if !HAVE_DECL_STRPTIME
45 CDECL char *strptime(const char *buf, const char *format, struct tm *tm);
46 #endif
47
48 #define super NetAccess
49
50 #define max_buf 0x10000
51
52 #define HTTP_DEFAULT_PORT "80"
53 #define HTTP_DEFAULT_PROXY_PORT "3128"
54 #define HTTPS_DEFAULT_PORT "443"
55
56 enum {
57 H_Continue=100,
58 H_Switching_Protocols=101,
59 H_Processing=102,
60 H_Ok=200,
61 H_Created=201,
62 H_Accepted=202,
63 H_Non_Authoritative_Information=203,
64 H_No_Content=204,
65 H_Reset_Content=205,
66 H_Partial_Content=206,
67 H_Multi_Status=207,
68 H_Moved_Permanently=301,
69 H_Found=302,
70 H_See_Other=303,
71 H_Not_Modified=304,
72 H_Use_Proxy=305,
73 H_Temporary_Redirect=307,
74 H_Permanent_Redirect=308,
75 H_Bad_Request=400,
76 H_Unauthorized=401,
77 H_Payment_Required=402,
78 H_Forbidden=403,
79 H_Not_Found=404,
80 H_Method_Not_Allowed=405,
81 H_Not_Acceptable=406,
82 H_Proxy_Authentication_Required=407,
83 H_Request_Timeout=408,
84 H_Conflict=409,
85 H_Gone=410,
86 H_Length_Required=411,
87 H_Precondition_Failed=412,
88 H_Request_Entity_Too_Large=413,
89 H_Request_URI_Too_Long=414,
90 H_Unsupported_Media_Type=415,
91 H_Requested_Range_Not_Satisfiable=416,
92 H_Expectation_Failed=417,
93 H_Unprocessable_Entity=422,
94 H_Locked=423,
95 H_Failed_Dependency=424,
96 H_Too_Many_Requests=429,
97 H_Internal_Server_Error=500,
98 H_Not_Implemented=501,
99 H_Bad_Gateway=502,
100 H_Service_Unavailable=503,
101 H_Gateway_Timeout=504,
102 H_HTTP_Version_Not_Supported=505,
103 H_Insufficient_Storage=507,
104 };
105
106 /* Some status code validation macros: */
107 #define H_2XX(x) (((x) >= 200) && ((x) <= 299))
108 #define H_5XX(x) (((x) >= 500) && ((x) <= 599))
109 #define H_PARTIAL(x) ((x) == H_Partial_Content)
110 #define H_REDIRECTED(x) (((x) == H_Moved_Permanently) || ((x) == H_Found) || ((x) == H_See_Other) || ((x) == H_Temporary_Redirect) || ((x) == H_Permanent_Redirect))
111 #define H_EMPTY(x) (((x) == H_No_Content) || ((x) == H_Reset_Content))
112 #define H_CONTINUE(x) ((x) == H_Continue || (x) == H_Processing)
113 #define H_REQUESTED_RANGE_NOT_SATISFIABLE(x) ((x) == H_Requested_Range_Not_Satisfiable)
114 #define H_TRANSIENT(x) ((x)==H_Request_Timeout || (x)==H_Bad_Gateway || (x)==H_Service_Unavailable || (x)==H_Gateway_Timeout)
115 #define H_UNSUPPORTED(x) ((x)==H_Bad_Request || (x)==H_Not_Implemented)
116 #define H_AUTH_REQ(x) ((x)==H_Unauthorized || (x)==H_Proxy_Authentication_Required)
117
118 #ifndef EINPROGRESS
119 #define EINPROGRESS -1
120 #endif
121
122 enum { CHUNK_SIZE_UNKNOWN=-1 };
123
Connection(int s,const char * c)124 Http::Connection::Connection(int s,const char *c)
125 : closure(c), sock(s)
126 {
127 }
~Connection()128 Http::Connection::~Connection()
129 {
130 close(sock);
131 /* make sure we free buffers before ssl */
132 recv_buf=0;
133 send_buf=0;
134 }
MakeBuffers()135 void Http::Connection::MakeBuffers()
136 {
137 send_buf=new IOBufferFDStream(
138 new FDStream(sock,"<output-socket>"),IOBuffer::PUT);
139 recv_buf=new IOBufferFDStream(
140 new FDStream(sock,"<input-socket>"),IOBuffer::GET);
141 }
142
Init()143 void Http::Init()
144 {
145 state=DISCONNECTED;
146 tunnel_state=NO_TUNNEL;
147 body_size=-1;
148 bytes_received=0;
149 status_code=0;
150 status_consumed=0;
151 proto_version=0x10;
152 sent_eot=false;
153 last_method=0;
154
155 default_cwd="/";
156
157 keep_alive=false;
158 keep_alive_max=-1;
159
160 array_send=0;
161
162 chunked=false;
163 chunked_trailer=false;
164 chunk_size=CHUNK_SIZE_UNKNOWN;
165 chunk_pos=0;
166
167 request_pos=0;
168
169 no_ranges=false;
170 seen_ranges_bytes=false;
171 entity_date_set=false;
172 sending_proppatch=false;
173
174 no_cache_this=false;
175 no_cache=false;
176
177 auth_sent[0]=auth_sent[1]=0;
178 auth_scheme[0]=auth_scheme[1]=HttpAuth::NONE;
179
180 use_propfind_now=true;
181
182 retry_after=0;
183
184 hftp=false;
185 https=false;
186 use_head=true;
187
188 user_agent=0;
189 special=HTTP_NONE;
190 }
191
Http()192 Http::Http() : super()
193 {
194 Init();
195 Reconfig();
196 }
Http(const Http * f)197 Http::Http(const Http *f) : super(f)
198 {
199 Init();
200 Reconfig();
201 }
202
~Http()203 Http::~Http()
204 {
205 Close();
206 Disconnect();
207 }
208
MoveConnectionHere(Http * o)209 void Http::MoveConnectionHere(Http *o)
210 {
211 conn=o->conn.borrow();
212 conn->ResumeInternal();
213 rate_limit=o->rate_limit.borrow();
214 last_method=o->last_method; o->last_method=0;
215 last_uri.move_here(o->last_uri);
216 last_url.move_here(o->last_url);
217 timeout_timer.Reset(o->timeout_timer);
218 state=CONNECTED;
219 tunnel_state=o->tunnel_state;
220 o->Disconnect();
221 ResumeInternal();
222 }
223
DisconnectLL()224 void Http::DisconnectLL()
225 {
226 Enter(this);
227 rate_limit=0;
228 if(conn)
229 {
230 LogNote(7,_("Closing HTTP connection"));
231 conn=0;
232 }
233
234 if(!Error() && !H_AUTH_REQ(status_code))
235 auth_sent[0]=auth_sent[1]=0;
236
237 if(state!=DONE && (real_pos>0 || special==HTTP_POST)
238 && !Error() && !H_AUTH_REQ(status_code)) {
239 if(last_method && !strcmp(last_method,"POST"))
240 SetError(FATAL,_("POST method failed"));
241 else if(ModeIs(STORE))
242 SetError(STORE_FAILED,0);
243 else if(fragile)
244 SetError(FRAGILE_FAILED,0);
245 }
246 if(ModeIs(STORE) && H_AUTH_REQ(status_code))
247 pos=real_pos=request_pos; // resend all the data again
248
249 last_method=0;
250 last_uri.unset();
251 last_url.unset();
252 ResetRequestData();
253 state=DISCONNECTED;
254 Leave(this);
255 }
256
ResetRequestData()257 void Http::ResetRequestData()
258 {
259 body_size=-1;
260 bytes_received=0;
261 real_pos=no_ranges?0:-1;
262 status.set(0);
263 status_consumed=0;
264 line.set(0);
265 sent_eot=false;
266 keep_alive=false;
267 keep_alive_max=-1;
268 array_send=fileset_for_info?fileset_for_info->curr_index():0;
269 chunked=false;
270 chunked_trailer=false;
271 chunk_size=CHUNK_SIZE_UNKNOWN;
272 chunk_pos=0;
273 request_pos=0;
274 propfind=0;
275 inflate=0;
276 seen_ranges_bytes=false;
277 entity_date_set=false;
278 }
279
Close()280 void Http::Close()
281 {
282 if(mode==CLOSED)
283 return;
284 if(conn && conn->recv_buf)
285 conn->recv_buf->Roll(); // try to read any remaining data
286 if(conn && keep_alive && (keep_alive_max>0 || keep_alive_max==-1)
287 && !ModeIs(STORE) && !conn->recv_buf->Eof() && (state==RECEIVING_BODY || state==DONE))
288 {
289 conn->recv_buf->Resume();
290 conn->recv_buf->Roll();
291 if(xstrcmp(last_method,"HEAD"))
292 {
293 // check if all data are in buffer
294 if(!chunked) // chunked is a bit complex, so don't handle it
295 {
296 bytes_received+=conn->recv_buf->Size();
297 conn->recv_buf->Skip(conn->recv_buf->Size());
298 }
299 if(!(body_size>=0 && bytes_received==body_size))
300 goto disconnect;
301 }
302 // can reuse the connection.
303 state=CONNECTED;
304 ResetRequestData();
305 rate_limit=0;
306 }
307 else
308 {
309 disconnect:
310 Disconnect();
311 DontSleep();
312 }
313 array_send=0;
314 no_cache_this=false;
315 auth_sent[0]=auth_sent[1]=0;
316 auth_scheme[0]=auth_scheme[1]=HttpAuth::NONE;
317 no_ranges=!QueryBool("use-range",hostname);
318 use_propfind_now=QueryBool("use-propfind",hostname);
319 special=HTTP_NONE;
320 special_data.set(0);
321 sending_proppatch=false;
322 super::Close();
323 }
324
Send(const xstring & str)325 void Http::Send(const xstring& str)
326 {
327 if(str.length()==0)
328 return;
329 LogSend(5,str);
330 conn->send_buf->Put(str);
331 }
332
Send(const char * format,...)333 void Http::Send(const char *format,...)
334 {
335 va_list va;
336 va_start(va,format);
337 xstring& str=xstring::vformat(format,va);
338 va_end(va);
339 Send(str);
340 }
341
Send(const HttpHeader * hdr)342 void Http::Send(const HttpHeader *hdr)
343 {
344 Send("%s: %s\r\n",hdr->GetName(),hdr->GetValue());
345 }
346
AppendHostEncoded(xstring & buf,const char * host)347 void Http::AppendHostEncoded(xstring& buf,const char *host)
348 {
349 if(is_ipv6_address(host))
350 buf.append('[').append(host).append(']');
351 else
352 buf.append_url_encoded(host,URL_HOST_UNSAFE);
353 }
354
SendMethod(const char * method,const char * efile)355 void Http::SendMethod(const char *method,const char *efile)
356 {
357 xstring& stripped_hostname=xstring::get_tmp(hostname);
358 stripped_hostname.truncate_at('%');
359 xstring ehost;
360 AppendHostEncoded(ehost,xidna_to_ascii(stripped_hostname));
361 if(portname) {
362 ehost.append(':');
363 ehost.append(url::encode(portname,URL_PORT_UNSAFE));
364 }
365 if(!use_head && !strcmp(method,"HEAD"))
366 method="GET";
367 last_method=method;
368 if(file_url)
369 {
370 efile=file_url;
371 if(!proxy)
372 efile+=url::path_index(efile);
373 else if(!strncmp(efile,"hftp://",7))
374 efile++;
375 }
376
377 if(hftp && mode!=LONG_LIST && mode!=CHANGE_DIR && mode!=MAKE_DIR
378 && mode!=REMOVE && mode!=REMOVE_DIR
379 && (strlen(efile)<7 || strncmp(efile+strlen(efile)-7,";type=",6))
380 && QueryBool("use-type",hostname))
381 {
382 efile=xstring::format("%s;type=%c",efile,ascii?'a':'i');
383 }
384
385 /*
386 Handle the case when the user has not given us
387 get http://foobar.org (note the absense of the trailing /
388
389 It fixes segfault with a certain webserver which I've
390 seen ... (Geoffrey Lee <glee@gnupilgrims.org>).
391 */
392 if(*efile=='\0')
393 efile="/";
394
395 last_uri.set(efile+(proxy?url::path_index(efile):0));
396 if(last_uri.length()==0)
397 last_uri.set("/");
398 if(proxy)
399 last_url.set(efile);
400
401 Send("%s %s HTTP/1.1\r\n",method,efile);
402 Send("Host: %s\r\n",ehost.get());
403 if(user_agent && user_agent[0])
404 Send("User-Agent: %s\r\n",user_agent);
405 if(!hftp)
406 {
407 const char *content_type=0;
408 if(!strcmp(method,"PUT"))
409 content_type=Query("put-content-type",hostname);
410 else if(!strcmp(method,"POST"))
411 content_type=Query("post-content-type",hostname);
412 if(content_type && content_type[0])
413 Send("Content-Type: %s\r\n",content_type);
414
415 const char *accept=Query("accept",hostname);
416 if(accept && accept[0])
417 Send("Accept: %s\r\n",accept);
418 accept=Query("accept-language",hostname);
419 if(accept && accept[0])
420 Send("Accept-Language: %s\r\n",accept);
421 accept=Query("accept-charset",hostname);
422 if(accept && accept[0])
423 Send("Accept-Charset: %s\r\n",accept);
424 accept=Query("accept-encoding",hostname);
425 if(accept && accept[0])
426 Send("Accept-Encoding: %s\r\n",accept);
427
428 const char *referer=Query("referer",hostname);
429 const char *slash="";
430 if(!xstrcmp(referer,"."))
431 {
432 referer=GetConnectURL(NO_USER+NO_PASSWORD);
433 if(last_char(referer)!='/' && !cwd.is_file)
434 slash="/";
435 }
436 if(referer && referer[0])
437 Send("Referer: %s%s\r\n",referer,slash);
438
439 xstring cookie;
440 MakeCookie(cookie,hostname,efile+(proxy?url::path_index(efile):0));
441 if(cookie.length()>0)
442 Send("Cookie: %s\r\n",cookie.get());
443 }
444 }
445
SendBasicAuth(const char * tag,const char * auth)446 void Http::SendBasicAuth(const char *tag,const char *auth)
447 {
448 if(!auth || !*auth)
449 return;
450 int auth_len=strlen(auth);
451 char *buf64=string_alloca(base64_length(auth_len)+1);
452 base64_encode(auth,buf64,auth_len);
453 Send("%s: Basic %s\r\n",tag,buf64);
454 }
SendBasicAuth(const char * tag,const char * user,const char * pass)455 void Http::SendBasicAuth(const char *tag,const char *user,const char *pass)
456 {
457 /* Basic scheme */
458 SendBasicAuth(tag,xstring::cat(user,":",pass,NULL));
459 }
460
SendAuth(HttpAuth::target_t target,const char * user,const char * uri)461 void Http::SendAuth(HttpAuth::target_t target,const char *user,const char *uri)
462 {
463 auth_scheme[target]=HttpAuth::NONE;
464 if(!user)
465 return;
466 HttpAuth *auth=HttpAuth::Get(target,GetFileURL(file,NO_USER),user);
467 if(auth && auth->Update(last_method,uri)) {
468 auth_sent[target]++;
469 Send(auth->GetHeader());
470 }
471 }
472
SendProxyAuth()473 void Http::SendProxyAuth()
474 {
475 SendAuth(HttpAuth::PROXY,proxy_user,last_url);
476 }
477
SendAuth()478 void Http::SendAuth()
479 {
480 if(hftp && !auth_scheme[HttpAuth::WWW] && user && pass && QueryBool("use-authorization",proxy)) {
481 SendBasicAuth("Authorization",user,pass);
482 return;
483 }
484 SendAuth(HttpAuth::WWW,user?user:auth_user,last_uri);
485 }
SendCacheControl()486 void Http::SendCacheControl()
487 {
488 const char *cc_setting=Query("cache-control",hostname);
489 const char *cc_no_cache=(no_cache || no_cache_this)?"no-cache":0;
490 if(!*cc_setting)
491 cc_setting=0;
492 if(!cc_setting && !cc_no_cache)
493 return;
494 int cc_no_cache_len=xstrlen(cc_no_cache);
495 if(cc_no_cache && cc_setting)
496 {
497 const char *pos=strstr(cc_setting,cc_no_cache);
498 if(pos && (pos==cc_setting || pos[-1]==' ')
499 && (pos[cc_no_cache_len]==0 || pos[cc_no_cache_len]==' '))
500 cc_no_cache=0, cc_no_cache_len=0;
501 }
502 xstring& cc=xstring::join(",",2,cc_no_cache,cc_setting);
503 if(*cc)
504 Send("Cache-Control: %s\r\n",cc.get());
505 }
506
ModeSupported()507 bool Http::ModeSupported()
508 {
509 switch((open_mode)mode)
510 {
511 case CLOSED:
512 case QUOTE_CMD:
513 case LIST:
514 case CHANGE_MODE:
515 case LINK:
516 case SYMLINK:
517 return false;
518 case CONNECT_VERIFY:
519 case RETRIEVE:
520 case STORE:
521 case MAKE_DIR:
522 case CHANGE_DIR:
523 case ARRAY_INFO:
524 case REMOVE_DIR:
525 case REMOVE:
526 case LONG_LIST:
527 case RENAME:
528 return true;
529 case MP_LIST:
530 #if USE_EXPAT
531 return QueryBool("use-propfind",hostname);
532 #else
533 // without XML parser it is meaningless to retrieve XML file info.
534 return false;
535 #endif
536 }
537 abort(); // should not happen
538 }
539
DirFile(xstring & path,const xstring & ecwd,const xstring & efile) const540 void Http::DirFile(xstring& path,const xstring& ecwd,const xstring& efile) const
541 {
542 const int base=path.length();
543
544 if(efile[0]=='/') {
545 path.append(efile);
546 } else if(efile[0]=='~' || ecwd.length()==0 || (ecwd.eq("~") && !hftp)) {
547 path.append('/');
548 path.append(efile);
549 } else {
550 size_t min_len=path.length()+1;
551 if(ecwd[0]!='/')
552 path.append('/');
553 path.append(ecwd);
554 if(ecwd.last_char()!='/' && efile.length()>0)
555 path.append('/');
556
557 // reduce . and .. at beginning of efile:
558 // * get the minimum path length (so that we don't remove ~user)
559 // * skip .; handle .. using basename_ptr to chomp the path.
560 if(path[min_len]=='~') {
561 while(path[min_len] && path[min_len]!='/')
562 ++min_len;
563 if(path[min_len]=='/')
564 ++min_len;
565 }
566 const char *e=efile;
567 while(e[0]=='.') {
568 if(e[1]=='/' || e[1]==0)
569 ++e;
570 else if(e[1]=='.' && (e[2]=='/' || e[2]==0)) {
571 if(path.length()<=min_len)
572 break;
573 const char *bn=basename_ptr(path+min_len);
574 path.truncate(bn-path);
575 e+=2;
576 } else
577 break;
578 if(*e=='/')
579 ++e;
580 }
581 path.append(e);
582 }
583
584 // remove "/~" or "/~/"
585 if(path[base+1]=='~' && path[base+2]==0)
586 path.truncate(base+1);
587 else if(path[base+1]=='~' && path[base+2]=='/')
588 path.set_substr(base,2,"");
589 }
590
SendPropfind(const xstring & efile,int depth)591 void Http::SendPropfind(const xstring& efile,int depth)
592 {
593 SendMethod("PROPFIND",efile);
594 Send("Depth: %d\r\n",depth);
595 if(allprop.length()>0)
596 {
597 Send("Content-Type: text/xml\r\n");
598 Send("Content-Length: %d\r\n",int(allprop.length()));
599 }
600 }
SendPropfindBody()601 void Http::SendPropfindBody()
602 {
603 Send(allprop);
604 }
605
FormatLastModified(time_t lm)606 const xstring& Http::FormatLastModified(time_t lm)
607 {
608 static const char weekday_names[][4]={
609 "Sun","Mon","Tue","Wed","Thu","Fri","Sat"
610 };
611 const struct tm *t=gmtime(&lm);
612 return xstring::format("%s, %2d %s %04d %02d:%02d:%02d GMT",
613 weekday_names[t->tm_wday],t->tm_mday,month_names[t->tm_mon],
614 t->tm_year+1900,t->tm_hour,t->tm_min,t->tm_sec);
615 }
616
SendProppatch(const xstring & efile)617 void Http::SendProppatch(const xstring& efile)
618 {
619 SendMethod("PROPPATCH",efile);
620 xstring prop(
621 "<?xml version=\"1.0\" ?>"
622 "<propertyupdate xmlns=\"DAV:\">"
623 "<set>"
624 "<prop>"
625 "<getlastmodified>");
626 prop.append(FormatLastModified(entity_date)).append(
627 "</getlastmodified>"
628 "</prop>"
629 "</set>"
630 "</propertyupdate>");
631 Send("Content-Type: text/xml\r\n");
632 Send("Content-Length: %d\r\n",int(prop.length()));
633 Send("\r\n");
634 Send(prop);
635 }
636
SendRequest(const char * connection,const char * f)637 void Http::SendRequest(const char *connection,const char *f)
638 {
639 xstring efile;
640 xstring ecwd;
641 bool add_slash=true;
642
643 if(mode==CHANGE_DIR && new_cwd && new_cwd->url)
644 {
645 const char *efile_c=new_cwd->url+url::path_index(new_cwd->url);
646 if(!*efile_c)
647 efile_c="/";
648 efile.set(efile_c);
649 add_slash=false;
650 }
651 else
652 efile.set(url::encode(f,URL_PATH_UNSAFE));
653
654 if(cwd.url)
655 ecwd.set(cwd.url+url::path_index(cwd.url));
656 else
657 {
658 ecwd.set(url::encode(cwd,URL_PATH_UNSAFE));
659 if(hftp && ecwd[0]=='/' && ecwd[1]!='~')
660 {
661 // root directory in ftp urls needs special encoding. (/%2Fpath)
662 ecwd.set_substr(1,0,"%2F");
663 }
664 }
665
666 if(cwd.is_file)
667 {
668 if(efile[0])
669 ecwd.truncate(basename_ptr(ecwd+(!strncmp(ecwd,"/~",2)))-ecwd);
670 add_slash=false;
671 }
672 if(mode==CHANGE_DIR && new_cwd && !new_cwd->url)
673 add_slash=!new_cwd->is_file;
674
675 xstring pfile;
676 if(proxy && !https)
677 {
678 const char *proto="http";
679 if(hftp)
680 proto="ftp";
681 pfile.vset(proto,"://",NULL);
682 if(hftp && user && pass)
683 {
684 pfile.append(url::encode(user,URL_USER_UNSAFE));
685 if(!QueryBool("use-authorization",proxy))
686 {
687 pfile.append(':');
688 pfile.append(url::encode(pass,URL_PASS_UNSAFE));
689 }
690 pfile.append('@');
691 }
692 AppendHostEncoded(pfile,hostname);
693 if(portname)
694 {
695 pfile.append(':');
696 pfile.append(url::encode(portname,URL_PORT_UNSAFE));
697 }
698 }
699 else
700 {
701 pfile.set("");
702 }
703
704 DirFile(pfile,ecwd,efile);
705 efile.set(pfile);
706
707 if(pos==0)
708 real_pos=0;
709 if(ModeIs(STORE)) // can't seek before writing
710 real_pos=pos;
711
712 #ifdef DEBUG_MP_LIST
713 if(mode==RETRIEVE && file[0]==0)
714 mode=MP_LIST;
715 #endif
716
717 switch((open_mode)mode)
718 {
719 case CLOSED:
720 case CONNECT_VERIFY:
721 abort(); // cannot happen
722
723 case QUOTE_CMD:
724 switch(special)
725 {
726 case HTTP_POST:
727 entity_size=special_data.length();
728 goto send_post;
729 case HTTP_MOVE:
730 case HTTP_COPY:
731 SendMethod(special==HTTP_MOVE?"MOVE":"COPY",efile);
732 Send("Destination: %s\r\n",special_data.get());
733 break;
734 case HTTP_PROPFIND:
735 SendMethod("PROPFIND",efile);
736 Send("Depth: 1\r\n"); // directory listing required
737 break;
738 case HTTP_NONE:
739 abort(); // cannot happen
740 }
741 break;
742
743 case LIST:
744 case CHANGE_MODE:
745 case LINK:
746 case SYMLINK:
747 abort(); // unsupported
748
749 case RETRIEVE:
750 retrieve:
751 SendMethod("GET",efile);
752 if(pos>0 && !no_ranges)
753 {
754 if(limit==FILE_END)
755 Send("Range: bytes=%lld-\r\n",(long long)pos);
756 else
757 Send("Range: bytes=%lld-%lld\r\n",(long long)pos,(long long)limit-1);
758 }
759 break;
760
761 case STORE:
762 if(sending_proppatch) {
763 SendProppatch(efile);
764 break;
765 }
766 if(hftp || strcasecmp(Query("put-method",hostname),"POST"))
767 SendMethod("PUT",efile);
768 else
769 {
770 send_post:
771 SendMethod("POST",efile);
772 pos=0;
773 }
774 if(entity_size>=0)
775 Send("Content-length: %lld\r\n",(long long)(entity_size-pos));
776 if(pos>0 && entity_size<0)
777 {
778 request_pos=pos;
779 if(limit==FILE_END)
780 Send("Range: bytes=%lld-\r\n",(long long)pos);
781 else
782 Send("Range: bytes=%lld-%lld\r\n",(long long)pos,(long long)limit-1);
783 }
784 else if(pos>0)
785 {
786 request_pos=pos;
787 Send("Range: bytes=%lld-%lld/%lld\r\n",(long long)pos,
788 (long long)((limit==FILE_END || limit>entity_size ? entity_size : limit)-1),
789 (long long)entity_size);
790 }
791 if(entity_date!=NO_DATE)
792 {
793 Send("Last-Modified: %s\r\n",FormatLastModified(entity_date).get());
794 Send("X-OC-MTime: %ld\r\n",(long)entity_date); // for OwnCloud
795 }
796 break;
797
798 case CHANGE_DIR:
799 case LONG_LIST:
800 case MP_LIST:
801 case MAKE_DIR:
802 if(last_char(efile)!='/' && add_slash)
803 efile.append('/');
804 if(mode==CHANGE_DIR)
805 {
806 if(use_propfind_now)
807 SendPropfind(efile,0);
808 else
809 SendMethod("HEAD",efile);
810 }
811 else if(mode==LONG_LIST)
812 goto retrieve;
813 else if(mode==MAKE_DIR)
814 {
815 if(QueryBool("use-mkcol"))
816 SendMethod("MKCOL",efile);
817 else
818 {
819 SendMethod("PUT",efile);
820 Send("Content-Length: 0\r\n");
821 }
822 pos=entity_size=0;
823 }
824 else if(mode==MP_LIST)
825 {
826 SendPropfind(efile,1);
827 pos=0;
828 }
829 break;
830
831 case(REMOVE):
832 SendMethod("DELETE",efile);
833 Send("Depth: 0\r\n"); // deny directory removal
834 break;
835
836 case(REMOVE_DIR):
837 if(efile.last_char()!='/')
838 efile.append('/');
839 SendMethod("DELETE",efile);
840 break;
841
842 case ARRAY_INFO:
843 if(use_propfind_now)
844 SendPropfind(efile,0);
845 else
846 SendMethod("HEAD",efile);
847 break;
848
849 case RENAME:
850 {
851 SendMethod("MOVE",efile);
852 Send("Destination: %s\r\n",GetFileURL(file1).get());
853 }
854 }
855 if(proxy && !https)
856 SendProxyAuth();
857 SendAuth();
858 if(no_cache || no_cache_this)
859 Send("Pragma: no-cache\r\n"); // for HTTP/1.0 compatibility
860 SendCacheControl();
861 if(mode==ARRAY_INFO && !use_head)
862 connection="close";
863 else if(!ModeIs(STORE))
864 connection="keep-alive";
865 if(mode!=ARRAY_INFO || connection)
866 Send("Connection: %s\r\n",connection?connection:"close");
867 Send("\r\n");
868 if(special==HTTP_POST)
869 {
870 if(special_data)
871 Send("%s",special_data.get());
872 entity_size=NO_SIZE;
873 }
874 else if(!xstrcmp(last_method,"PROPFIND"))
875 SendPropfindBody();
876
877 keep_alive=false;
878 chunked=false;
879 chunked_trailer=false;
880 chunk_size=CHUNK_SIZE_UNKNOWN;
881 chunk_pos=0;
882 request_pos=0;
883 inflate=0;
884 no_ranges=!QueryBool("use-range",hostname);
885
886 conn->send_buf->SetPos(0);
887 }
888
SendArrayInfoRequest()889 int Http::SendArrayInfoRequest()
890 {
891 // skip to next needed file
892 for(FileInfo *fi=fileset_for_info->curr(); fi; fi=fileset_for_info->next())
893 if(fi->need)
894 break;
895 if(array_send<fileset_for_info->curr_index())
896 array_send=fileset_for_info->curr_index();
897
898 if(state!=CONNECTED)
899 return 0;
900
901 int m=1;
902 if(keep_alive && use_head)
903 {
904 m=keep_alive_max;
905 if(m==-1)
906 m=100;
907 }
908 int req_count=0;
909 while(array_send-fileset_for_info->curr_index()<m
910 && array_send<fileset_for_info->count())
911 {
912 FileInfo *fi=(*fileset_for_info)[array_send++];
913 if(fi->need==0)
914 continue;
915 xstring *name=&fi->name;
916 if(fi->filetype==fi->DIRECTORY && name->last_char()!='/') {
917 name=&xstring::get_tmp(*name);
918 name->append('/');
919 }
920 if(fi->uri)
921 file_url.set(dir_file(GetConnectURL(),fi->uri));
922 else
923 file_url.unset();
924 SendRequest(array_send==fileset_for_info->count()-1 ? 0 : "keep-alive", *name);
925 req_count++;
926 }
927 return req_count;
928 }
929
ProceedArrayInfo()930 void Http::ProceedArrayInfo()
931 {
932 for(;;)
933 {
934 // skip to next needed file
935 FileInfo *fi=fileset_for_info->next();
936 if(!fi || fi->need)
937 break;
938 }
939 if(!fileset_for_info->curr())
940 {
941 LogNote(10,"that was the last file info");
942 // received all requested info.
943 state=DONE;
944 return;
945 }
946 // we can avoid reconnection if server supports it.
947 if(keep_alive && (keep_alive_max>1 || keep_alive_max==-1)
948 && (use_head || use_propfind_now))
949 {
950 // we'll have to receive next header, unset the status
951 status.set(0);
952 status_code=0;
953
954 state=CONNECTED;
955 SendArrayInfoRequest();
956 state=RECEIVING_HEADER;
957 }
958 else
959 {
960 Disconnect();
961 DontSleep();
962 }
963 }
964
NewAuth(const char * hdr,HttpAuth::target_t target,const char * user,const char * pass)965 void Http::NewAuth(const char *hdr,HttpAuth::target_t target,const char *user,const char *pass)
966 {
967 if(!user || !pass)
968 return;
969
970 // FIXME: keep a request queue, get the URI from the queue.
971 const char *uri=GetFileURL(file,NO_USER);
972
973 Ref<HttpAuth::Challenge> chal(new HttpAuth::Challenge(hdr));
974 bool stale=chal->GetParam("stale").eq_nc("true");
975
976 if(auth_sent[target]>(stale?1:0))
977 return;
978
979 HttpAuth::scheme_t new_scheme=chal->GetSchemeCode();
980 if(new_scheme<=auth_scheme[target])
981 return;
982 if(HttpAuth::New(target,uri,chal.borrow(),user,pass))
983 auth_scheme[target]=new_scheme;
984 }
985
HandleHeaderLine(const char * name,const char * value)986 void Http::HandleHeaderLine(const char *name,const char *value)
987 {
988 // use a perfect hash
989 #define hh(L,C) ((L)+(C)*3)
990 #define hhc(S,C) hh(sizeof((S))-1,(C))
991 #define case_hh(S,C) case hhc((S),(C)): if(strcasecmp(name,(S))) break;
992 switch(hh(strlen(name),c_toupper(name[0]))) {
993 case_hh("Content-Length",'C') {
994 long long bs=0;
995 if(1!=sscanf(value,"%lld",&bs))
996 return;
997 if(bs<0) // try to workaround broken servers
998 bs+=0x100000000LL;
999 body_size=bs;
1000 if(mode==ARRAY_INFO && H_2XX(status_code)
1001 && xstrcmp(last_method,"PROPFIND"))
1002 {
1003 FileInfo *fi=fileset_for_info->curr();
1004 fi->SetSize(body_size);
1005 TrySuccess();
1006 }
1007 return;
1008 }
1009 case_hh("Content-Range",'C') {
1010 long long first,last,fsize;
1011 if(H_REQUESTED_RANGE_NOT_SATISFIABLE(status_code))
1012 {
1013 if(sscanf(value,"%*[^/]/%lld",&fsize)!=1)
1014 return;
1015 if(opt_size)
1016 *opt_size=fsize;
1017 return;
1018 }
1019 if(sscanf(value,"%*s %lld-%lld/%lld",&first,&last,&fsize)!=3)
1020 return;
1021 real_pos=first;
1022 if(last==-1)
1023 last=fsize-first-1;
1024 if(body_size<0)
1025 body_size=last-first+1;
1026 if(!ModeIs(STORE) && !ModeIs(MAKE_DIR))
1027 entity_size=fsize;
1028 if(opt_size && H_2XX(status_code))
1029 *opt_size=fsize;
1030 return;
1031 }
1032 case_hh("Last-Modified",'L') {
1033 if(!H_2XX(status_code))
1034 return;
1035
1036 time_t t=Http::atotm(value);
1037 if(t==ATOTM_ERROR)
1038 return;
1039
1040 if(opt_date)
1041 *opt_date=t;
1042
1043 if(mode==ARRAY_INFO && !propfind)
1044 {
1045 FileInfo *fi=fileset_for_info->curr();
1046 fi->SetDate(t,0);
1047 TrySuccess();
1048 }
1049 return;
1050 }
1051 case_hh("Location",'L')
1052 if(value[0]=='/' && value[1]=='/')
1053 location.vset(GetProto(),":",value,NULL);
1054 else if(value[0]=='/')
1055 location.vset(GetConnectURL(NO_PATH).get(),value,NULL);
1056 else
1057 location.set(value);
1058
1059 location_permanent = (status_code==H_Moved_Permanently
1060 || status_code==H_Permanent_Redirect);
1061 location_mode=OpenMode();
1062 if(status_code==H_See_Other)
1063 location_mode=RETRIEVE;
1064
1065 if(location_mode!=QUOTE_CMD) {
1066 ParsedURL url(location,true);
1067 location_file.set(url.path);
1068 } else {
1069 xstring &tmp=xstring::get_tmp(file,file.instr(' ')+1);
1070 tmp.append(location+url::path_index(location));
1071 if(special_data)
1072 tmp.append(' ').append(special_data);
1073 location_file.set(tmp);
1074 }
1075 return;
1076
1077 case_hh("Retry-After",'R')
1078 retry_after=0;
1079 sscanf(value,"%ld",&retry_after);
1080 return;
1081
1082 case_hh("Keep-Alive",'K') {
1083 keep_alive=true;
1084 const char *m=strstr(value,"max=");
1085 if(m) {
1086 if(sscanf(m+4,"%d",&keep_alive_max)!=1)
1087 keep_alive=false;
1088 } else
1089 keep_alive_max=100;
1090 return;
1091 }
1092 case_hh("Proxy-Connection",'P')
1093 goto case_Connection;
1094 case_hh("Connection",'C')
1095 case_Connection:
1096 if(!strcasecmp(value,"keep-alive"))
1097 keep_alive=true;
1098 else if(!strcasecmp(value,"close"))
1099 keep_alive=false;
1100 return;
1101
1102 case_hh("Transfer-Encoding",'T')
1103 if(!strcasecmp(value,"identity"))
1104 return;
1105 if(!strcasecmp(value,"chunked"))
1106 {
1107 chunked=true;
1108 chunked_trailer=false;
1109 chunk_size=CHUNK_SIZE_UNKNOWN; // expecting first chunk
1110 chunk_pos=0;
1111 }
1112 return;
1113
1114 case_hh("Content-Encoding",'C')
1115 content_encoding.set(value);
1116 return;
1117
1118 case_hh("Accept-Ranges",'A')
1119 if(!strcasecmp(value,"none"))
1120 no_ranges=true;
1121 if(strstr(value,"bytes"))
1122 seen_ranges_bytes=true;
1123 return;
1124
1125 case_hh("Set-Cookie",'S')
1126 if(!hftp && QueryBool("set-cookies",hostname))
1127 SetCookie(value);
1128 return;
1129
1130 case_hh("Content-Disposition",'C') {
1131 const char *filename=strstr(value,"filename=");
1132 if(!filename)
1133 return;
1134 filename=HttpHeader::extract_quoted_value(filename+9);
1135 SetSuggestedFileName(filename);
1136 return;
1137 }
1138 case_hh("Content-Type",'C') {
1139 entity_content_type.set(value);
1140 const char *cs=strstr(value,"charset=");
1141 if(cs)
1142 {
1143 cs=HttpHeader::extract_quoted_value(cs+8);
1144 entity_charset.set(cs);
1145 }
1146 return;
1147 }
1148 case_hh("WWW-Authenticate",'W') {
1149 if(status_code!=H_Unauthorized)
1150 return;
1151 if(user && pass)
1152 NewAuth(value,HttpAuth::WWW,user,pass);
1153 else
1154 NewAuth(value,HttpAuth::WWW,auth_user,auth_pass);
1155 return;
1156 }
1157 case_hh("Proxy-Authenticate",'P') {
1158 if(status_code!=H_Proxy_Authentication_Required)
1159 return;
1160 NewAuth(value,HttpAuth::PROXY,proxy_user,proxy_pass);
1161 return;
1162 }
1163 case_hh("X-OC-MTime",'X') {
1164 if(!strcasecmp(value,"accepted"))
1165 entity_date_set=true;
1166 }
1167 default:
1168 break;
1169 }
1170 LogNote(10,"unhandled header line `%s'",name);
1171 }
1172
1173 static
find_eol(const char * p,int len,int * eol_size)1174 const char *find_eol(const char *p,int len,int *eol_size)
1175 {
1176 *eol_size=1;
1177 for(int i=0; i<len; i++,p++)
1178 {
1179 if(p[0]=='\n')
1180 return p;
1181 if(i+1<len && p[1]=='\n' && p[0]=='\r')
1182 {
1183 *eol_size=2;
1184 return p;
1185 }
1186 }
1187 *eol_size=0;
1188 return 0;
1189 }
1190
GetBetterConnection(int level)1191 void Http::GetBetterConnection(int level)
1192 {
1193 if(level==0)
1194 return;
1195 for(FA *fo=FirstSameSite(); fo!=0; fo=NextSameSite(fo))
1196 {
1197 Http *o=(Http*)fo; // we are sure it is Http.
1198
1199 if(!o->conn || o->state==CONNECTING)
1200 continue;
1201
1202 if(o->tunnel_state==TUNNEL_WAITING)
1203 continue;
1204
1205 if(o->state!=CONNECTED || o->mode!=CLOSED)
1206 {
1207 if(level<2)
1208 continue;
1209 if(!connection_takeover || (o->priority>=priority && !o->IsSuspended()))
1210 continue;
1211 o->Disconnect();
1212 return;
1213 }
1214
1215 // so borrow the connection
1216 MoveConnectionHere(o);
1217 return;
1218 }
1219 }
1220
Do()1221 int Http::Do()
1222 {
1223 int m=STALL;
1224 int res;
1225 const char *error;
1226 const char *buf;
1227 int len;
1228
1229 // check if idle time exceeded
1230 if(mode==CLOSED && conn && idle_timer.Stopped())
1231 {
1232 LogNote(1,_("Closing idle connection"));
1233 Disconnect();
1234 return m;
1235 }
1236
1237 if(home.path==0)
1238 set_home(default_cwd);
1239
1240 if(Error())
1241 return m;
1242
1243 if(propfind)
1244 {
1245 if(propfind->Error())
1246 {
1247 propfind=0;
1248 if(mode==CHANGE_DIR)
1249 {
1250 SetError(NO_FILE,propfind->ErrorText());
1251 return MOVED;
1252 }
1253 if(propfind->ErrorFatal())
1254 fileset_for_info->next();
1255 Disconnect();
1256 return MOVED;
1257 }
1258 if(propfind->Eof())
1259 {
1260 LogNote(9,"got EOF on PROPFIND reply");
1261 const char *b;
1262 int len;
1263 propfind->Get(&b,&len);
1264 Ref<FileSet> fs(HttpListInfo::ParseProps(b,len,GetCwd()));
1265 propfind=0;
1266 if(fs)
1267 {
1268 if(mode==CHANGE_DIR)
1269 {
1270 fs->rewind();
1271 FileInfo *fi=fs->curr();
1272 if(fi && fi->Has(fi->TYPE))
1273 {
1274 LogNote(9,"new-cwd: %s",fi->GetLongName());
1275 new_cwd->is_file=(fi->filetype!=fi->DIRECTORY);
1276 if(new_cwd->url.last_char()=='/' && new_cwd->is_file)
1277 new_cwd->url.rtrim('/');
1278 else if(new_cwd->url.last_char()!='/' && !new_cwd->is_file)
1279 new_cwd->url.append('/');
1280 }
1281 }
1282 else if(mode==ARRAY_INFO)
1283 fileset_for_info->Merge(fs);
1284 }
1285 m=MOVED;
1286 state=DONE;
1287 if(mode==CHANGE_DIR)
1288 {
1289 cwd.Set(new_cwd);
1290 cache->SetDirectory(this, "", !cwd.is_file);
1291 return m;
1292 }
1293 else if(mode==ARRAY_INFO)
1294 ProceedArrayInfo();
1295 }
1296 }
1297
1298 switch(state)
1299 {
1300 case DISCONNECTED:
1301 if(mode==CLOSED || !hostname)
1302 return m;
1303 if(ModeIs(STORE) && pos>0 && entity_size>=0 && pos>=entity_size)
1304 {
1305 state=DONE;
1306 return MOVED;
1307 }
1308 if(mode==ARRAY_INFO)
1309 {
1310 // check if we have anything to request
1311 SendArrayInfoRequest();
1312 if(!fileset_for_info->curr())
1313 {
1314 state=DONE;
1315 return MOVED;
1316 }
1317 }
1318 if(!hftp && mode==QUOTE_CMD && !special)
1319 {
1320 handle_quote_cmd:
1321 if(file && !strncasecmp(file,"Set-Cookie ",11))
1322 SetCookie(file+11);
1323 else if(file && !strncasecmp(file,"POST ",5))
1324 special=HTTP_POST;
1325 else if(file && !strncasecmp(file,"COPY ",5))
1326 special=HTTP_COPY;
1327 else if(file && !strncasecmp(file,"MOVE ",5))
1328 special=HTTP_MOVE;
1329 else if(file && !strncasecmp(file,"PROPFIND ",9))
1330 special=HTTP_PROPFIND;
1331 else
1332 {
1333 SetError(NOT_SUPP,0);
1334 return MOVED;
1335 }
1336 if(special)
1337 {
1338 // METHOD encoded_path data
1339 const char *scan=file;
1340 while(*scan && *scan!=' ')
1341 scan++;
1342 while(*scan==' ')
1343 scan++;
1344 file_url.set(https?"https://":"http://");
1345 AppendHostEncoded(file_url,hostname);
1346 if(portname)
1347 {
1348 file_url.append(':');
1349 file_url.append_url_encoded(portname,URL_PORT_UNSAFE);
1350 }
1351 if(*scan!='/' && cwd)
1352 {
1353 if(cwd[0]!='/')
1354 file_url.append('/');
1355 file_url.append_url_encoded(cwd,URL_PATH_UNSAFE);
1356 }
1357 if(*scan!='/' && file_url.last_char()!='/')
1358 file_url.append('/');
1359 file_url.append(scan);
1360 file_url.truncate_at(' ');
1361
1362 scan=strchr(scan,' ');
1363 while(scan && *scan==' ')
1364 scan++;
1365 special_data.set(scan);
1366 return MOVED;
1367 }
1368 state=DONE;
1369 return MOVED;
1370 }
1371 if(!special && !ModeSupported())
1372 {
1373 SetError(NOT_SUPP);
1374 return MOVED;
1375 }
1376 if(hftp)
1377 {
1378 if(!proxy)
1379 {
1380 // problem here: hftp cannot work without proxy
1381 SetError(FATAL,_("ftp over http cannot work without proxy, set hftp:proxy."));
1382 return MOVED;
1383 }
1384 }
1385
1386 // walk through Http classes and try to find identical idle session
1387 // first try "easy" cases of session take-over.
1388 for(int i=0; i<3; i++)
1389 {
1390 if(i>=2 && (connection_limit==0 || connection_limit>CountConnections()))
1391 break;
1392 GetBetterConnection(i);
1393 if(state!=DISCONNECTED)
1394 return MOVED;
1395 }
1396
1397 if(!resolver && mode!=CONNECT_VERIFY && !ReconnectAllowed())
1398 return m;
1399
1400 if(https)
1401 m|=Resolve(HTTPS_DEFAULT_PORT,"https","tcp");
1402 else
1403 m|=Resolve(HTTP_DEFAULT_PORT,"http","tcp");
1404 if(!peer)
1405 return m;
1406
1407 if(mode==CONNECT_VERIFY)
1408 return m;
1409
1410 if(!ReconnectAllowed())
1411 return m;
1412
1413 if(!NextTry())
1414 return MOVED;
1415
1416 retry_after=0;
1417
1418 res=SocketCreateTCP(peer[peer_curr].sa.sa_family);
1419 if(res==-1)
1420 {
1421 saved_errno=errno;
1422 if(peer_curr+1<peer.count())
1423 {
1424 peer_curr++;
1425 retries--;
1426 return MOVED;
1427 }
1428 if(NonFatalError(saved_errno))
1429 return m;
1430 SetError(SEE_ERRNO,xstring::format(
1431 _("cannot create socket of address family %d"),
1432 peer[peer_curr].sa.sa_family));
1433 return MOVED;
1434 }
1435 conn=new Connection(res,hostname);
1436
1437 SayConnectingTo();
1438 res=SocketConnect(conn->sock,&peer[peer_curr]);
1439 if(res==-1 && errno!=EINPROGRESS)
1440 {
1441 saved_errno=errno;
1442 NextPeer();
1443 LogError(0,"connect: %s\n",strerror(saved_errno));
1444 Disconnect();
1445 if(NotSerious(saved_errno))
1446 return MOVED;
1447 goto system_error;
1448 }
1449 state=CONNECTING;
1450 m=MOVED;
1451 timeout_timer.Reset();
1452
1453 case CONNECTING:
1454 res=Poll(conn->sock,POLLOUT,&error);
1455 if(res==-1)
1456 {
1457 LogError(0,_("Socket error (%s) - reconnecting"),error);
1458 Disconnect(error);
1459 NextPeer();
1460 return MOVED;
1461 }
1462 if(!(res&POLLOUT))
1463 {
1464 if(CheckTimeout())
1465 {
1466 NextPeer();
1467 return MOVED;
1468 }
1469 Block(conn->sock,POLLOUT);
1470 return m;
1471 }
1472
1473 m=MOVED;
1474 state=CONNECTED;
1475 #if USE_SSL
1476 if(proxy?!strncmp(proxy,"https://",8):https)
1477 {
1478 conn->MakeSSLBuffers();
1479 }
1480 else
1481 #endif
1482 {
1483 conn->MakeBuffers();
1484 #if USE_SSL
1485 if(proxy && https)
1486 {
1487 // have to setup a tunnel.
1488 xstring ehost;
1489 AppendHostEncoded(ehost,hostname);
1490 const char *port_to_use=portname?portname.get():HTTPS_DEFAULT_PORT;
1491 const char *eport=url::encode(port_to_use,URL_PORT_UNSAFE);
1492 Send("CONNECT %s:%s HTTP/1.1\r\n",ehost.get(),eport);
1493 SendProxyAuth();
1494 Send("\r\n");
1495 tunnel_state=TUNNEL_WAITING;
1496 state=RECEIVING_HEADER;
1497 return MOVED;
1498 }
1499 #endif // USE_SSL
1500 }
1501 /*fallthrough*/
1502 case CONNECTED:
1503 if(mode==CONNECT_VERIFY)
1504 return MOVED;
1505
1506 if(mode==QUOTE_CMD && !special)
1507 goto handle_quote_cmd;
1508 if(conn->recv_buf->Eof())
1509 {
1510 LogError(0,_("Peer closed connection"));
1511 Disconnect();
1512 return MOVED;
1513 }
1514 if(mode==CLOSED)
1515 return m;
1516 if(!special && !ModeSupported())
1517 {
1518 SetError(NOT_SUPP);
1519 return MOVED;
1520 }
1521 ExpandTildeInCWD();
1522 if(ModeIs(STORE) && pos>0 && entity_size>=0 && pos>=entity_size)
1523 {
1524 state=DONE;
1525 return MOVED;
1526 }
1527 if(mode==ARRAY_INFO)
1528 {
1529 if(SendArrayInfoRequest()==0) {
1530 // nothing to do
1531 state=DONE;
1532 return MOVED;
1533 }
1534 }
1535 else
1536 {
1537 LogNote(9,_("Sending request..."));
1538 SendRequest();
1539 }
1540
1541 state=RECEIVING_HEADER;
1542 m=MOVED;
1543 if(ModeIs(STORE))
1544 rate_limit=new RateLimit(hostname);
1545
1546 case RECEIVING_HEADER:
1547 if(conn->send_buf->Error() || conn->recv_buf->Error())
1548 {
1549 if((ModeIs(STORE) || special) && status_code && !H_2XX(status_code))
1550 goto pre_RECEIVING_BODY; // assume error.
1551 handle_buf_error:
1552 if(conn->send_buf->Error())
1553 {
1554 LogError(0,"send: %s",conn->send_buf->ErrorText());
1555 if(conn->send_buf->ErrorFatal())
1556 SetError(FATAL,conn->send_buf->ErrorText());
1557 }
1558 if(conn->recv_buf->Error())
1559 {
1560 LogError(0,"recv: %s",conn->recv_buf->ErrorText());
1561 if(conn->recv_buf->ErrorFatal())
1562 SetError(FATAL,conn->recv_buf->ErrorText());
1563 }
1564 Disconnect();
1565 return MOVED;
1566 }
1567 timeout_timer.Reset(conn->send_buf->EventTime());
1568 timeout_timer.Reset(conn->recv_buf->EventTime());
1569 if(CheckTimeout())
1570 return MOVED;
1571 conn->recv_buf->Get(&buf,&len);
1572 if(!buf)
1573 {
1574 // eof
1575 LogError(0,_("Hit EOF while fetching headers"));
1576 // workaround some broken servers
1577 if(H_REDIRECTED(status_code) && location)
1578 goto pre_RECEIVING_BODY;
1579 Disconnect();
1580 return MOVED;
1581 }
1582 if(len>0)
1583 {
1584 int eol_size;
1585 const char *eol=find_eol(buf,len,&eol_size);
1586 if(eol)
1587 {
1588 // empty line indicates end of headers.
1589 if(eol==buf && status)
1590 {
1591 LogRecv(4,"");
1592 conn->recv_buf->Skip(eol_size);
1593 if(tunnel_state==TUNNEL_WAITING)
1594 {
1595 if(H_2XX(status_code))
1596 {
1597 #if USE_SSL
1598 if(https)
1599 conn->MakeSSLBuffers();
1600 #endif
1601 tunnel_state=TUNNEL_ESTABLISHED;
1602 ResetRequestData();
1603 state=CONNECTED;
1604 return MOVED;
1605 }
1606 }
1607 if(chunked_trailer)
1608 {
1609 chunked_trailer=false;
1610 chunked=false;
1611 if(propfind) {
1612 // we avoid the DONE state since we have yet to handle propfind data
1613 propfind->PutEOF();
1614 state=CONNECTED;
1615 } else
1616 state=DONE;
1617 return MOVED;
1618 }
1619 if(H_CONTINUE(status_code))
1620 {
1621 status.set(0);
1622 status_code=0;
1623 return MOVED;
1624 }
1625 if(mode==ARRAY_INFO)
1626 {
1627 if(!xstrcmp(last_method,"PROPFIND"))
1628 {
1629 if(H_UNSUPPORTED(status_code))
1630 {
1631 ResMgr::Set("http:use-propfind",hostname,"no");
1632 use_propfind_now=false;
1633 Disconnect();
1634 DontSleep();
1635 return MOVED;
1636 }
1637 goto pre_RECEIVING_BODY;
1638 }
1639 FileInfo *fi=fileset_for_info->curr();
1640 if(H_REDIRECTED(status_code)) {
1641 HandleRedirection();
1642 if(location)
1643 fi->SetRedirect(location);
1644 } else if(H_2XX(status_code) && !fi->Has(fi->TYPE)) {
1645 fi->SetType(last_uri.last_char()=='/'?fi->DIRECTORY:fi->NORMAL);
1646 }
1647 ProceedArrayInfo();
1648 return MOVED;
1649 }
1650 else if(ModeIs(STORE) || ModeIs(MAKE_DIR) || sending_proppatch)
1651 {
1652 if((sent_eot || pos==entity_size || sending_proppatch) && H_2XX(status_code))
1653 {
1654 state=DONE;
1655 Disconnect();
1656 state=DONE;
1657 if(ModeIs(STORE) && entity_date!=NO_DATE && !entity_date_set
1658 && use_propfind_now) {
1659 // send PROPPATCH in a separate request.
1660 sending_proppatch=true;
1661 state=DISCONNECTED;
1662 }
1663 return MOVED;
1664 }
1665 if(H_2XX(status_code))
1666 {
1667 // should never happen
1668 LogError(0,"Unexpected success, the server did not accept full request body");
1669 Disconnect();
1670 return MOVED;
1671 }
1672 // going to pre_RECEIVING_BODY to catch error
1673 }
1674 goto pre_RECEIVING_BODY;
1675 }
1676 len=eol-buf;
1677 line.nset(buf,len);
1678
1679 conn->recv_buf->Skip(len+eol_size);
1680
1681 LogRecv(4,line);
1682 m=MOVED;
1683
1684 if(status==0)
1685 {
1686 // it's status line
1687 status.set(line);
1688 int ver_major,ver_minor;
1689 if(3!=sscanf(status,"HTTP/%d.%d %n%d",&ver_major,&ver_minor,
1690 &status_consumed,&status_code))
1691 {
1692 // simple 0.9 ?
1693 proto_version=0x09;
1694 status_code=H_Ok;
1695 LogError(0,_("Could not parse HTTP status line"));
1696 if(ModeIs(STORE))
1697 {
1698 state=DONE;
1699 Disconnect();
1700 state=DONE;
1701 return MOVED;
1702 }
1703 conn->recv_buf->UnSkip(len+eol_size);
1704 goto pre_RECEIVING_BODY;
1705 }
1706 proto_version=(ver_major<<4)+ver_minor;
1707
1708 // HTTP/1.1 does keep-alive by default
1709 if(proto_version>=0x11)
1710 keep_alive=true;
1711
1712 if(!H_2XX(status_code))
1713 {
1714 if(H_CONTINUE(status_code))
1715 return MOVED;
1716
1717 if(H_5XX(status_code)) // server failed, try another
1718 NextPeer();
1719 if(status_code==H_Gateway_Timeout)
1720 {
1721 const char *cc=Query("cache-control");
1722 if(cc && strstr(cc,"only-if-cached"))
1723 {
1724 if(mode!=ARRAY_INFO)
1725 {
1726 SetError(NO_FILE,_("Object is not cached and http:cache-control has only-if-cached"));
1727 return MOVED;
1728 }
1729 status_code=H_Not_Acceptable; // so that no retry will be attempted
1730 }
1731 }
1732 // check for retriable codes
1733 if(H_TRANSIENT(status_code))
1734 {
1735 Disconnect();
1736 return MOVED;
1737 }
1738 if(status_code==H_Too_Many_Requests)
1739 {
1740 Disconnect();
1741 if(retry_after)
1742 reconnect_timer.StopDelayed(retry_after);
1743 return MOVED;
1744 }
1745
1746 if(mode==ARRAY_INFO)
1747 TrySuccess();
1748
1749 return MOVED;
1750 }
1751 }
1752 else
1753 {
1754 // header line.
1755 char *colon=strchr(line.get_non_const(),':');
1756 if(colon)
1757 {
1758 *colon=0; // terminate the header tag
1759 const char *value=colon+1;
1760 while(*value==' ')
1761 value++;
1762 HandleHeaderLine(line,value);
1763 }
1764 }
1765 }
1766 }
1767
1768 if(ModeIs(STORE) && (!status || H_CONTINUE(status_code)) && !sent_eot)
1769 Block(conn->sock,POLLOUT);
1770
1771 return m;
1772
1773 pre_RECEIVING_BODY:
1774
1775 // 204 No Content
1776 if(H_EMPTY(status_code) && body_size<0)
1777 body_size=0;
1778
1779 if(H_REDIRECTED(status_code))
1780 {
1781 // check if it is redirection to the same server
1782 // or to directory instead of file.
1783 // FIXME.
1784 }
1785
1786 if(H_REQUESTED_RANGE_NOT_SATISFIABLE(status_code))
1787 {
1788 // file is smaller than requested
1789 state=DONE;
1790 return MOVED;
1791 }
1792
1793 if((status_code==H_Unauthorized && auth_scheme[HttpAuth::WWW])
1794 || (status_code==H_Proxy_Authentication_Required && auth_scheme[HttpAuth::PROXY])) {
1795 // retry with authentication
1796 retries--;
1797 state=RECEIVING_BODY;
1798 LogErrorText();
1799 Disconnect();
1800 DontSleep();
1801 return MOVED;
1802 }
1803
1804 if(!H_2XX(status_code))
1805 {
1806 xstring err;
1807 int code=NO_FILE;
1808
1809 if(H_REDIRECTED(status_code))
1810 {
1811 HandleRedirection();
1812 err.setf("%s (%s -> %s)",status+status_consumed,file.get(),
1813 location?location.get():"nowhere");
1814 code=FILE_MOVED;
1815 }
1816 else
1817 {
1818 const char *closure=file;
1819 if(H_UNSUPPORTED(status_code) || status_code==H_Method_Not_Allowed)
1820 {
1821 if(H_UNSUPPORTED(status_code))
1822 {
1823 if(!xstrcmp(last_method,"PROPFIND"))
1824 ResMgr::Set("http:use-propfind",hostname,"no");
1825 if(!xstrcmp(last_method,"MKCOL"))
1826 ResMgr::Set("http:use-mkcol",hostname,"no");
1827 }
1828 if(mode==CHANGE_DIR && !xstrcmp(last_method,"PROPFIND"))
1829 {
1830 use_propfind_now=false;
1831 Disconnect();
1832 DontSleep();
1833 return MOVED;
1834 }
1835 code=NOT_SUPP;
1836 closure=last_method;
1837 }
1838 if(closure && closure[0])
1839 err.setf("%s (%s)",status+status_consumed,closure);
1840 else
1841 err.setf("%s (%s%s)",status+status_consumed,cwd.path.get(),
1842 (last_char(cwd)=='/')?"":"/");
1843 }
1844 state=RECEIVING_BODY;
1845 LogErrorText();
1846 if(mode==ARRAY_INFO)
1847 {
1848 if(!H_TRANSIENT(status_code))
1849 fileset_for_info->next();
1850 Disconnect();
1851 DontSleep();
1852 }
1853 else
1854 SetError(code,err);
1855 return MOVED;
1856 }
1857 if(!xstrcmp(last_method,"PROPFIND")
1858 && (mode==ARRAY_INFO || mode==CHANGE_DIR)) {
1859 LogNote(9,"accepting XML for PROPFIND...");
1860 propfind=new IOBufferFileAccess(this);
1861 }
1862
1863 if(mode==CHANGE_DIR && !propfind)
1864 {
1865 cwd.Set(new_cwd);
1866 cache->SetDirectory(this, "", !cwd.is_file);
1867 state=DONE;
1868 return MOVED;
1869 }
1870
1871 // Many servers send application/x-gzip with x-gzip encoding,
1872 // don't decode in such a case.
1873 if(CompressedContentEncoding() && !CompressedContentType()
1874 && QueryBool("decode",hostname)) {
1875 // inflated size is unknown beforehand
1876 entity_size=NO_SIZE;
1877 if(opt_size)
1878 *opt_size=NO_SIZE;
1879 // start the inflation
1880 inflate=new DirectedBuffer(DirectedBuffer::GET);
1881 inflate->SetTranslator(new DataInflator());
1882 }
1883 // sometimes it's possible to derive entity size from body size.
1884 if(entity_size==NO_SIZE && body_size!=NO_SIZE
1885 && pos==0 && !ModeIs(STORE) && !ModeIs(MAKE_DIR) && !inflate) {
1886 entity_size=body_size;
1887 if(opt_size && H_2XX(status_code))
1888 *opt_size=body_size;
1889 }
1890
1891 LogNote(9,_("Receiving body..."));
1892 rate_limit=new RateLimit(hostname);
1893 if(real_pos<0) // assume Range: did not work
1894 {
1895 if(!ModeIs(STORE) && !ModeIs(MAKE_DIR) && body_size>=0)
1896 {
1897 entity_size=body_size;
1898 if(opt_size && H_2XX(status_code))
1899 *opt_size=entity_size;
1900 }
1901 real_pos=0;
1902 }
1903 state=RECEIVING_BODY;
1904 m=MOVED;
1905 /*passthrough*/
1906 case RECEIVING_BODY:
1907 if(conn->recv_buf->Error() || conn->send_buf->Error())
1908 goto handle_buf_error;
1909 if(conn->recv_buf->Size()>=rate_limit->BytesAllowedToGet())
1910 {
1911 conn->recv_buf->Suspend();
1912 TimeoutS(1);
1913 }
1914 else if(conn->recv_buf->Size()>=max_buf)
1915 {
1916 conn->recv_buf->Suspend();
1917 m=MOVED;
1918 }
1919 else
1920 {
1921 if(conn->recv_buf->IsSuspended())
1922 {
1923 conn->recv_buf->Resume();
1924 if(conn->recv_buf->Size()>0 || (conn->recv_buf->Size()==0 && conn->recv_buf->Eof()))
1925 m=MOVED;
1926 }
1927 timeout_timer.Reset(conn->send_buf->EventTime());
1928 timeout_timer.Reset(conn->recv_buf->EventTime());
1929 if(conn->recv_buf->Size()==0)
1930 {
1931 // check if ranges were emulated by squid
1932 bool no_ranges_if_timeout=(bytes_received==0 && !seen_ranges_bytes);
1933 if(CheckTimeout())
1934 {
1935 if(no_ranges_if_timeout)
1936 {
1937 no_ranges=true;
1938 real_pos=0; // so that pget would know immediately.
1939 }
1940 return MOVED;
1941 }
1942 }
1943 }
1944 return m;
1945
1946 case DONE:
1947 return m;
1948 }
1949 return m;
1950
1951 system_error:
1952 assert(saved_errno!=0);
1953 if(NonFatalError(saved_errno))
1954 return m;
1955 SetError(SEE_ERRNO,0);
1956 Disconnect();
1957 return MOVED;
1958 }
1959
HandleRedirection()1960 void Http::HandleRedirection()
1961 {
1962 bool is_url=(location && url::is_url(location));
1963 if(location && !is_url
1964 && mode==QUOTE_CMD && !strncasecmp(file,"POST ",5)
1965 && tunnel_state!=TUNNEL_WAITING)
1966 {
1967 const char *the_file=file;
1968
1969 const char *scan=file+5;
1970 while(*scan==' ')
1971 scan++;
1972 char *the_post_file=alloca_strdup(scan);
1973 char *space=strchr(the_post_file,' ');
1974 if(space)
1975 *space=0;
1976 the_file=the_post_file;
1977
1978 char *new_location=alloca_strdup2(GetConnectURL(),
1979 strlen(the_file)+strlen(location));
1980 int p_ind=url::path_index(new_location);
1981 if(location[0]=='/')
1982 strcpy(new_location+p_ind,location);
1983 else
1984 {
1985 if(the_file[0]=='/')
1986 strcpy(new_location+p_ind,the_file);
1987 else
1988 {
1989 char *slash=strrchr(new_location,'/');
1990 strcpy(slash+1,the_file);
1991 }
1992 char *slash=strrchr(new_location,'/');
1993 strcpy(slash+1,location);
1994 }
1995 location.set(new_location);
1996 } else if(is_url && !hftp) {
1997 ParsedURL url(location);
1998 if(url.proto.eq(GetProto()) && !xstrcasecmp(url.host,hostname)
1999 && user && !url.user) {
2000 // use the same user name after redirect to the same site.
2001 url.user.set(user);
2002 location.truncate();
2003 url.CombineTo(location);
2004 }
2005 }
2006 }
2007
New()2008 FileAccess *Http::New() { return new Http(); }
New()2009 FileAccess *HFtp::New() { return new HFtp(); }
2010
ClassInit()2011 void Http::ClassInit()
2012 {
2013 // register the class
2014 Register("http",Http::New);
2015 Register("hftp",HFtp::New);
2016 #if USE_SSL
2017 Register("https",Https::New);
2018 #endif
2019 }
2020
SuspendInternal()2021 void Http::SuspendInternal()
2022 {
2023 super::SuspendInternal();
2024 if(conn)
2025 conn->SuspendInternal();
2026 }
ResumeInternal()2027 void Http::ResumeInternal()
2028 {
2029 if(conn)
2030 conn->ResumeInternal();
2031 super::ResumeInternal();
2032 }
2033
Read(Buffer * buf,int size)2034 int Http::Read(Buffer *buf,int size)
2035 {
2036 if(Error())
2037 return error_code;
2038 if(mode==CLOSED)
2039 return 0;
2040 if(state==DONE)
2041 return 0; // eof
2042 int res=DO_AGAIN;
2043 if(state==RECEIVING_BODY && real_pos>=0)
2044 {
2045 Enter(this);
2046 res=_Read(buf,size);
2047 if(res>0)
2048 {
2049 pos+=res;
2050 if(rate_limit)
2051 rate_limit->BytesGot(res);
2052 TrySuccess();
2053 }
2054 Leave(this);
2055 }
2056 return res;
2057 }
_Skip(int to_skip)2058 void Http::_Skip(int to_skip)
2059 {
2060 if(inflate)
2061 inflate->Skip(to_skip);
2062 else
2063 conn->recv_buf->Skip(to_skip);
2064 _UpdatePos(to_skip);
2065 }
_UpdatePos(int to_skip)2066 void Http::_UpdatePos(int to_skip)
2067 {
2068 if(!inflate) {
2069 if(chunked)
2070 chunk_pos+=to_skip;
2071 bytes_received+=to_skip;
2072 }
2073 real_pos+=to_skip;
2074 }
_Read(Buffer * buf,int size)2075 int Http::_Read(Buffer *buf,int size)
2076 {
2077 const char *buf1;
2078 int size1;
2079 Buffer *src_buf=conn->recv_buf.get_non_const();
2080 get_again:
2081 if(conn->recv_buf->Size()==0 && conn->recv_buf->Error())
2082 {
2083 LogError(0,"recv: %s",conn->recv_buf->ErrorText());
2084 if(conn->recv_buf->ErrorFatal())
2085 SetError(FATAL,conn->recv_buf->ErrorText());
2086 Disconnect();
2087 return DO_AGAIN;
2088 }
2089 conn->recv_buf->Get(&buf1,&size1);
2090 if(buf1==0) // eof
2091 {
2092 LogNote(9,_("Hit EOF"));
2093 if(bytes_received<body_size || chunked)
2094 {
2095 LogError(0,_("Received not enough data, retrying"));
2096 Disconnect();
2097 return DO_AGAIN;
2098 }
2099 return 0;
2100 }
2101 if(!chunked)
2102 {
2103 if(body_size>=0 && bytes_received>=body_size
2104 && (!inflate || inflate->Size()==0))
2105 {
2106 LogNote(9,_("Received all"));
2107 return 0; // all received
2108 }
2109 if(entity_size>=0 && pos>=entity_size)
2110 {
2111 LogNote(9,_("Received all (total)"));
2112 return 0;
2113 }
2114 }
2115 if(size1==0 && (!inflate || inflate->Size()==0))
2116 return DO_AGAIN;
2117 if(chunked && size1>0)
2118 {
2119 if(chunked_trailer && state==RECEIVING_HEADER)
2120 return DO_AGAIN;
2121 const char *nl;
2122 if(chunk_size==CHUNK_SIZE_UNKNOWN) // expecting first/next chunk
2123 {
2124 nl=(const char*)memchr(buf1,'\n',size1);
2125 if(nl==0) // not yet
2126 {
2127 not_yet:
2128 if(conn->recv_buf->Eof())
2129 Disconnect(); // connection closed too early
2130 return DO_AGAIN;
2131 }
2132 if(!is_ascii_xdigit(*buf1)
2133 || sscanf(buf1,"%lx",&chunk_size)!=1)
2134 {
2135 Fatal(_("chunked format violated"));
2136 return FATAL;
2137 }
2138 conn->recv_buf->Skip(nl-buf1+1);
2139 chunk_pos=0;
2140 LogNote(9,"next chunk size: %ld",chunk_size);
2141 goto get_again;
2142 }
2143 if(chunk_size==0) // eof
2144 {
2145 LogNote(9,_("Received last chunk"));
2146 // headers may follow
2147 chunked_trailer=true;
2148 state=RECEIVING_HEADER;
2149 body_size=bytes_received;
2150 Timeout(0);
2151 return DO_AGAIN;
2152 }
2153 if(chunk_pos==chunk_size)
2154 {
2155 if(size1<2)
2156 goto not_yet;
2157 if(buf1[0]!='\r' || buf1[1]!='\n')
2158 {
2159 Fatal(_("chunked format violated"));
2160 return FATAL;
2161 }
2162 conn->recv_buf->Skip(2);
2163 chunk_size=CHUNK_SIZE_UNKNOWN;
2164 goto get_again;
2165 }
2166 // ok, now we may get portion of data
2167 if(size1>chunk_size-chunk_pos)
2168 size1=chunk_size-chunk_pos;
2169 }
2170
2171 if(!chunked)
2172 {
2173 // limit by body_size.
2174 if(body_size>=0 && size1+bytes_received>=body_size)
2175 size1=body_size-bytes_received;
2176 }
2177
2178 int bytes_allowed=0x10000000;
2179 if(rate_limit)
2180 bytes_allowed=rate_limit->BytesAllowedToGet();
2181
2182 if(inflate) {
2183 // do the inflation, if there are not enough inflated data
2184 if(size1>bytes_allowed)
2185 size1=bytes_allowed;
2186 if(inflate->Size()<size && size1>0) {
2187 inflate->PutTranslated(buf1,size1);
2188 conn->recv_buf->Skip(size1);
2189 if(chunked)
2190 chunk_pos+=size1;
2191 bytes_received+=size1;
2192 if(inflate->Error())
2193 SetError(FATAL,inflate->ErrorText());
2194 }
2195 inflate->Get(&buf1,&size1);
2196 src_buf=inflate.get_non_const();
2197 } else {
2198 if(size1>bytes_allowed)
2199 size1=bytes_allowed;
2200 }
2201 if(size1==0)
2202 return DO_AGAIN;
2203 if(norest_manual && real_pos==0 && pos>0)
2204 return DO_AGAIN;
2205 if(real_pos<pos)
2206 {
2207 off_t to_skip=pos-real_pos;
2208 if(to_skip>size1)
2209 to_skip=size1;
2210 _Skip(to_skip);
2211 goto get_again;
2212 }
2213 if(size>size1)
2214 size=size1;
2215 size=buf->MoveDataHere(src_buf,size);
2216 _UpdatePos(size);
2217 return size;
2218 }
2219
Done()2220 int Http::Done()
2221 {
2222 if(mode==CLOSED)
2223 return OK;
2224 if(Error())
2225 return error_code;
2226 if(state==DONE)
2227 return OK;
2228 if(mode==CONNECT_VERIFY && (peer || conn))
2229 return OK;
2230 if((mode==REMOVE || mode==REMOVE_DIR || mode==RENAME)
2231 && state==RECEIVING_BODY)
2232 return OK;
2233 return IN_PROGRESS;
2234 }
2235
Buffered()2236 int Http::Buffered()
2237 {
2238 if(!ModeIs(STORE) || !conn || !conn->send_buf)
2239 return 0;
2240 return conn->send_buf->Size()+SocketBuffered(conn->sock);
2241 }
2242
Write(const void * buf,int size)2243 int Http::Write(const void *buf,int size)
2244 {
2245 if(!ModeIs(STORE))
2246 return(0);
2247
2248 Resume();
2249 Do();
2250 if(Error())
2251 return(error_code);
2252
2253 if(state!=RECEIVING_HEADER || status!=0 || conn->send_buf->Size()!=0)
2254 return DO_AGAIN;
2255
2256 {
2257 int allowed=rate_limit->BytesAllowedToPut();
2258 if(allowed==0)
2259 return DO_AGAIN;
2260 if(size>allowed)
2261 size=allowed;
2262 }
2263 if(size+conn->send_buf->Size()>=max_buf)
2264 size=max_buf-conn->send_buf->Size();
2265 if(entity_size!=NO_SIZE && pos+size>entity_size)
2266 {
2267 size=entity_size-pos;
2268 // tried to write more than originally requested. Make it retry with Open:
2269 if(size==0)
2270 return STORE_FAILED;
2271 }
2272 if(size<=0)
2273 return 0;
2274
2275 conn->send_buf->Put((const char*)buf,size);
2276
2277 if(retries>0 && conn->send_buf->GetPos()-conn->send_buf->Size()>Buffered()+0x1000)
2278 TrySuccess();
2279 rate_limit->BytesPut(size);
2280 pos+=size;
2281 real_pos+=size;
2282 return(size);
2283 }
2284
SendEOT()2285 int Http::SendEOT()
2286 {
2287 if(sent_eot)
2288 return OK;
2289 if(Error())
2290 return(error_code);
2291 if(ModeIs(STORE))
2292 {
2293 if(state==RECEIVING_HEADER && conn->send_buf->Size()==0)
2294 {
2295 if(entity_size==NO_SIZE || pos<entity_size)
2296 {
2297 shutdown(conn->sock,1);
2298 keep_alive=false;
2299 }
2300 sent_eot=true;
2301 return(OK);
2302 }
2303 return(DO_AGAIN);
2304 }
2305 return(OK);
2306 }
2307
StoreStatus()2308 int Http::StoreStatus()
2309 {
2310 if(!sent_eot && state==RECEIVING_HEADER)
2311 SendEOT();
2312 return Done();
2313 }
2314
CurrentStatus()2315 const char *Http::CurrentStatus()
2316 {
2317 switch(state)
2318 {
2319 case DISCONNECTED:
2320 if(hostname)
2321 {
2322 if(resolver)
2323 return(_("Resolving host address..."));
2324 if(!ReconnectAllowed())
2325 return DelayingMessage();
2326 }
2327 return "";
2328 case CONNECTING:
2329 return(_("Connecting..."));
2330 case CONNECTED:
2331 return(_("Connection idle"));
2332 case RECEIVING_HEADER:
2333 if(ModeIs(STORE) && !sent_eot && !status)
2334 return(_("Sending data"));
2335 if(tunnel_state==TUNNEL_WAITING)
2336 return(_("Connecting..."));
2337 if(!status)
2338 return(_("Waiting for response..."));
2339 return(_("Fetching headers..."));
2340 case RECEIVING_BODY:
2341 return(_("Receiving data"));
2342 case DONE:
2343 return "";
2344 }
2345 abort();
2346 }
2347
Reconfig(const char * name)2348 void Http::Reconfig(const char *name)
2349 {
2350 const char *c=hostname;
2351
2352 super::Reconfig(name);
2353
2354 no_cache = !QueryBool("cache",c);
2355 if(!hftp && NoProxy(hostname))
2356 SetProxy(0);
2357 else
2358 {
2359 const char *p=0;
2360 if(hftp && vproto && !strcmp(vproto,"ftp"))
2361 {
2362 p=ResMgr::Query("ftp:proxy",c);
2363 if(p && strncmp(p,"http://",7) && strncmp(p,"https://",8))
2364 p=0;
2365 }
2366 if(!p)
2367 {
2368 if(https)
2369 p=ResMgr::Query("https:proxy",c);
2370 else
2371 p=Query("proxy",c);
2372 // if no hftp:proxy is specified, try http:proxy.
2373 if(hftp && !p)
2374 p=ResMgr::Query("http:proxy",c);
2375 }
2376 SetProxy(p);
2377 }
2378
2379 if(conn)
2380 SetSocketBuffer(conn->sock);
2381 if(proxy && proxy_port==0)
2382 proxy_port.set(HTTP_DEFAULT_PROXY_PORT);
2383
2384 user_agent=ResMgr::Query("http:user-agent",c);
2385 use_propfind_now=(use_propfind_now && QueryBool("use-propfind",c));
2386 no_ranges=(no_ranges || !QueryBool("use-range",hostname));
2387
2388 if(QueryBool("use-allprop",c)) {
2389 allprop.set( // PROPFIND request
2390 "<?xml version=\"1.0\" ?>"
2391 "<propfind xmlns=\"DAV:\">"
2392 "<allprop/>"
2393 "</propfind>\r\n");
2394 } else {
2395 allprop.unset();
2396 }
2397
2398 if(!user || !pass) {
2399 // get auth info from http:authorization setting
2400 const char *auth_c=Query("authorization",hostname);
2401 if(auth_c && *auth_c) {
2402 char *auth=alloca_strdup(auth_c);
2403 char *colon=strchr(auth,':');
2404 if(colon) {
2405 *colon=0;
2406 auth_user.set(auth);
2407 auth_pass.set(colon+1);
2408 }
2409 }
2410 }
2411 }
2412
SameSiteAs(const FileAccess * fa) const2413 bool Http::SameSiteAs(const FileAccess *fa) const
2414 {
2415 if(!SameProtoAs(fa))
2416 return false;
2417 Http *o=(Http*)fa;
2418 return(!xstrcasecmp(hostname,o->hostname) && !xstrcmp(portname,o->portname)
2419 && !xstrcmp(user,o->user) && !xstrcmp(pass,o->pass));
2420 }
2421
SameLocationAs(const FileAccess * fa) const2422 bool Http::SameLocationAs(const FileAccess *fa) const
2423 {
2424 if(!SameSiteAs(fa))
2425 return false;
2426 Http *o=(Http*)fa;
2427 if(cwd!=o->cwd)
2428 return false;
2429 return true;
2430 }
2431
ResetLocationData()2432 void Http::ResetLocationData()
2433 {
2434 super::ResetLocationData();
2435 Reconfig();
2436 state=DISCONNECTED;
2437 use_propfind_now=QueryBool("use-propfind",hostname);
2438 no_ranges=!QueryBool("use-range",hostname);
2439 }
2440
MakeDirList(ArgV * args)2441 DirList *Http::MakeDirList(ArgV *args)
2442 {
2443 return new HttpDirList(this,args);
2444 }
2445 #include "FileGlob.h"
MakeGlob(const char * pattern)2446 Glob *Http::MakeGlob(const char *pattern)
2447 {
2448 return new GenericGlob(this,pattern);
2449 }
MakeListInfo(const char * path)2450 ListInfo *Http::MakeListInfo(const char *path)
2451 {
2452 return new HttpListInfo(this,path);
2453 }
2454
CookieClosureMatch(const char * closure_c,const char * hostname,const char * efile)2455 bool Http::CookieClosureMatch(const char *closure_c,
2456 const char *hostname,const char *efile)
2457 {
2458 if(!closure_c)
2459 return true;
2460 char *closure=alloca_strdup2(closure_c,1);
2461 char *path=0;
2462
2463 char *scan=closure;
2464 for(;;)
2465 {
2466 char *slash=strchr(scan,';');
2467 if(!slash)
2468 break;
2469 *slash++=0;
2470 while(*slash && *slash==' ')
2471 slash++;
2472 if(!strncmp(slash,"path=",5))
2473 path=slash+5;
2474 else if(!strncmp(slash,"secure",6) && (slash[6]==';' || slash[6]==0))
2475 {
2476 if(!https)
2477 return false;
2478 }
2479 }
2480 if(closure[0] && 0!=fnmatch(closure,hostname,FNM_PATHNAME))
2481 return false;
2482 if(!path)
2483 return true;
2484 int path_len=strlen(path);
2485 if(path_len>0 && path[path_len-1]=='/')
2486 path_len--;
2487 if(!strncmp(efile,path,path_len)
2488 && (efile[path_len]==0 || efile[path_len]=='/'))
2489 return true;
2490 return false;
2491 }
2492
CookieMerge(xstring & all,const char * cookie_c)2493 void Http::CookieMerge(xstring &all,const char *cookie_c)
2494 {
2495 char *value=alloca_strdup(cookie_c);
2496
2497 for(char *entry=strtok(value,";"); entry; entry=strtok(0,";"))
2498 {
2499 if(*entry==' ')
2500 entry++;
2501 if(*entry==0)
2502 break;
2503 if(!strncasecmp(entry,"path=",5)
2504 || !strncasecmp(entry,"expires=",8)
2505 || !strncasecmp(entry,"domain=",7)
2506 || (!strncasecmp(entry,"secure",6)
2507 && (entry[6]==' ' || entry[6]==0 || entry[6]==';')))
2508 continue; // filter out path= expires= domain= secure
2509
2510 char *c_name=entry;
2511 char *c_value=strchr(entry,'=');
2512 if(c_value)
2513 *c_value++=0;
2514 else
2515 c_value=c_name, c_name=0;
2516 int c_name_len=xstrlen(c_name);
2517
2518 for(unsigned i=all.skip_all(0,' '); i<all.length(); i=all.skip_all(i+1,' '))
2519 {
2520 const char *scan=all+i;
2521 const char *semicolon=strchr(scan,';');
2522 const char *eq=strchr(scan,'=');
2523 if(semicolon && eq>semicolon)
2524 eq=0;
2525 if((eq==0 && c_name==0)
2526 || (eq-scan==c_name_len && !strncmp(scan,c_name,c_name_len)))
2527 {
2528 // remove old cookie.
2529 if(!semicolon)
2530 all.truncate(i);
2531 else
2532 all.set_substr(i,all.skip_all(semicolon+1-all,' ')-i,"",0);
2533 break;
2534 }
2535 if(!semicolon)
2536 break;
2537 i=semicolon+1-all;
2538 }
2539
2540 // append cookie.
2541 all.rtrim(' ');
2542 all.rtrim(';');
2543 int c_len=all.length();
2544 if(c_len>0 && all[c_len-1]!=';')
2545 all.append("; ");
2546 if(c_name)
2547 all.vappend(c_name,"=",c_value,NULL);
2548 else
2549 all.append(c_value);
2550 }
2551 }
2552
MakeCookie(xstring & all_cookies,const char * hostname,const char * efile)2553 void Http::MakeCookie(xstring &all_cookies,const char *hostname,const char *efile)
2554 {
2555 Resource *scan=0;
2556 const char *closure;
2557 for(;;)
2558 {
2559 const char *cookie=ResMgr::QueryNext("http:cookie",&closure,&scan);
2560 if(cookie==0)
2561 break;
2562 if(!CookieClosureMatch(closure,hostname,efile))
2563 continue;
2564 CookieMerge(all_cookies,cookie);
2565 }
2566 }
2567
SetCookie(const char * value_const)2568 void Http::SetCookie(const char *value_const)
2569 {
2570 char *value=alloca_strdup(value_const);
2571 const char *domain=hostname;
2572 const char *path=0;
2573 bool secure=false;
2574
2575 for(char *entry=strtok(value,";"); entry; entry=strtok(0,";"))
2576 {
2577 while(*entry==' ') // skip spaces.
2578 entry++;
2579 if(*entry==0)
2580 break;
2581
2582 if(!strncasecmp(entry,"expires=",8))
2583 continue; // not used yet (FIXME)
2584
2585 if(!strncasecmp(entry,"secure",6)
2586 && (entry[6]==' ' || entry[6]==0 || entry[6]==';'))
2587 {
2588 secure=true;
2589 continue;
2590 }
2591
2592 if(!strncasecmp(entry,"path=",5))
2593 {
2594 path=alloca_strdup(entry+5);
2595 continue;
2596 }
2597
2598 if(!strncasecmp(entry,"domain=",7))
2599 {
2600 char *new_domain=alloca_strdup(entry+6);
2601 if(new_domain[1]=='.')
2602 new_domain[0]='*';
2603 else
2604 new_domain++;
2605 char *end=strchr(new_domain,';');
2606 if(end)
2607 *end=0;
2608 domain=new_domain;
2609 continue;
2610 }
2611 }
2612
2613 xstring closure(domain);
2614 if(path && path[0] && strcmp(path,"/"))
2615 closure.append(";path=").append(path);
2616 if(secure)
2617 closure.append(";secure");
2618
2619 xstring c(Query("cookie",closure));
2620 CookieMerge(c,value_const);
2621 ResMgr::Set("http:cookie",closure,c);
2622 }
2623
2624 #if USE_SSL
2625 #undef super
2626 #define super Http
Https()2627 Https::Https()
2628 {
2629 https=true;
2630 res_prefix="http";
2631 }
~Https()2632 Https::~Https()
2633 {
2634 }
Https(const Https * o)2635 Https::Https(const Https *o) : super(o)
2636 {
2637 https=true;
2638 res_prefix="http";
2639 Reconfig(0);
2640 }
New()2641 FileAccess *Https::New(){ return new Https();}
2642
MakeSSLBuffers()2643 void Http::Connection::MakeSSLBuffers()
2644 {
2645 ssl=new lftp_ssl(sock,lftp_ssl::CLIENT,closure);
2646 ssl->load_keys();
2647 IOBufferSSL *send_buf_ssl=new IOBufferSSL(ssl,IOBuffer::PUT);
2648 IOBufferSSL *recv_buf_ssl=new IOBufferSSL(ssl,IOBuffer::GET);
2649 send_buf=send_buf_ssl;
2650 recv_buf=recv_buf_ssl;
2651 }
2652 #endif
2653
2654 #undef super
2655 #define super Http
HFtp()2656 HFtp::HFtp()
2657 {
2658 hftp=true;
2659 default_cwd="~";
2660 Reconfig(0);
2661 }
~HFtp()2662 HFtp::~HFtp()
2663 {
2664 }
HFtp(const HFtp * o)2665 HFtp::HFtp(const HFtp *o) : super(o)
2666 {
2667 hftp=true;
2668 Reconfig(0);
2669 }
Login(const char * u,const char * p)2670 void HFtp::Login(const char *u,const char *p)
2671 {
2672 super::Login(u,p);
2673 if(u)
2674 {
2675 home.Set("~");
2676 cwd.Set(home,false,0);
2677 }
2678 }
Reconfig(const char * name)2679 void HFtp::Reconfig(const char *name)
2680 {
2681 super::Reconfig(name);
2682 use_head=QueryBool("use-head");
2683 }
2684
LogErrorText()2685 void Http::LogErrorText()
2686 {
2687 if(!conn || !conn->recv_buf)
2688 return;
2689 conn->recv_buf->Roll();
2690 int size=conn->recv_buf->Size();
2691 if(size==0)
2692 return;
2693 Buffer tmpbuf;
2694 size=_Read(&tmpbuf,size);
2695 if(size<=0)
2696 return;
2697 tmpbuf.SpaceAdd(size);
2698 const char *buf0=tmpbuf.Get();
2699 char *buf=alloca_strdup(buf0);
2700 remove_tags(buf);
2701 for(char *line=strtok(buf,"\n"); line; line=strtok(0,"\n")) {
2702 rtrim(line);
2703 if(*line)
2704 Log::global->Format(4,"<--* %s\n",line);
2705 }
2706 }
2707
2708
2709 /* The functions http_atotm and check_end are taken from wget */
2710 #define ISSPACE(c) is_ascii_space((c))
2711 #define ISDIGIT(c) is_ascii_digit((c))
2712
2713 /* Check whether the result of strptime() indicates success.
2714 strptime() returns the pointer to how far it got to in the string.
2715 The processing has been successful if the string is at `GMT' or
2716 `+X', or at the end of the string.
2717
2718 In extended regexp parlance, the function returns 1 if P matches
2719 "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of
2720 strptime()) is considered a failure and 0 is returned. */
2721 static int
check_end(const char * p)2722 check_end (const char *p)
2723 {
2724 if (!p)
2725 return 0;
2726 while (ISSPACE (*p))
2727 ++p;
2728 if (!*p
2729 || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T')
2730 || (p[0] == 'U' && p[1] == 'T' && p[2] == 'C')
2731 || ((p[0] == '+' || p[1] == '-') && ISDIGIT (p[1])))
2732 return 1;
2733 else
2734 return 0;
2735 }
2736
2737 /* Convert TIME_STRING time to time_t. TIME_STRING can be in any of
2738 the three formats RFC2068 allows the HTTP servers to emit --
2739 RFC1123-date, RFC850-date or asctime-date. Timezones are ignored,
2740 and should be GMT.
2741
2742 We use strptime() to recognize various dates, which makes it a
2743 little bit slacker than the RFC1123/RFC850/asctime (e.g. it always
2744 allows shortened dates and months, one-digit days, etc.). It also
2745 allows more than one space anywhere where the specs require one SP.
2746 The routine should probably be even more forgiving (as recommended
2747 by RFC2068), but I do not have the time to write one.
2748
2749 Return the computed time_t representation, or ATOTM_ERROR if all the
2750 schemes fail.
2751
2752 Needless to say, what we *really* need here is something like
2753 Marcus Hennecke's atotm(), which is forgiving, fast, to-the-point,
2754 and does not use strptime(). atotm() is to be found in the sources
2755 of `phttpd', a little-known HTTP server written by Peter Erikson. */
2756 time_t
atotm(const char * time_string)2757 Http::atotm (const char *time_string)
2758 {
2759 struct tm t;
2760
2761 /* Roger Beeman says: "This function dynamically allocates struct tm
2762 t, but does no initialization. The only field that actually
2763 needs initialization is tm_isdst, since the others will be set by
2764 strptime. Since strptime does not set tm_isdst, it will return
2765 the data structure with whatever data was in tm_isdst to begin
2766 with. For those of us in timezones where DST can occur, there
2767 can be a one hour shift depending on the previous contents of the
2768 data area where the data structure is allocated." */
2769 t.tm_isdst = -1;
2770
2771 /* Note that under foreign locales Solaris strptime() fails to
2772 recognize English dates, which renders this function useless. I
2773 assume that other non-GNU strptime's are plagued by the same
2774 disease. We solve this by setting only LC_MESSAGES in
2775 i18n_initialize(), instead of LC_ALL.
2776
2777 Another solution could be to temporarily set locale to C, invoke
2778 strptime(), and restore it back. This is slow and dirty,
2779 however, and locale support other than LC_MESSAGES can mess other
2780 things, so I rather chose to stick with just setting LC_MESSAGES.
2781
2782 Also note that none of this is necessary under GNU strptime(),
2783 because it recognizes both international and local dates. */
2784
2785 /* NOTE: We don't use `%n' for white space, as OSF's strptime uses
2786 it to eat all white space up to (and including) a newline, and
2787 the function fails if there is no newline (!).
2788
2789 Let's hope all strptime() implementations use ` ' to skip *all*
2790 whitespace instead of just one (it works that way on all the
2791 systems I've tested it on). */
2792
2793 time_t ut=ATOTM_ERROR;
2794
2795 setlocale(LC_TIME,"C"); // we need english month and week day names
2796
2797 /* RFC1123: Thu, 29 Jan 1998 22:12:57 */
2798 if (check_end (strptime (time_string, "%a, %d %b %Y %T", &t)))
2799 ut=mktime_from_utc (&t);
2800 /* RFC850: Thu, 29-Jan-98 22:12:57 */
2801 else if (check_end (strptime (time_string, "%a, %d-%b-%y %T", &t)))
2802 ut=mktime_from_utc (&t);
2803 /* asctime: Thu Jan 29 22:12:57 1998 */
2804 else if (check_end (strptime (time_string, "%a %b %d %T %Y", &t)))
2805 ut=mktime_from_utc (&t);
2806
2807 setlocale(LC_TIME,""); // restore locale
2808
2809 return ut;
2810 }
2811
IsCompressed(const char * s)2812 bool Http::IsCompressed(const char *s)
2813 {
2814 static const char *const values[] = {
2815 "x-gzip", "gzip", "deflate", "compress", "x-compress", NULL
2816 };
2817 for(const char *const *v=values; *v; v++)
2818 if(!strcmp(s,*v))
2819 return true;
2820 return false;
2821 }
2822
CompressedContentEncoding() const2823 bool Http::CompressedContentEncoding() const
2824 {
2825 return content_encoding && IsCompressed(content_encoding);
2826 }
CompressedContentType() const2827 bool Http::CompressedContentType() const
2828 {
2829 if(file.ends_with(".gz") || file.ends_with(".Z") || file.ends_with(".tgz"))
2830 return true;
2831 static const char app[]="application/";
2832 return entity_content_type && entity_content_type.begins_with(app)
2833 && IsCompressed(entity_content_type+sizeof(app)-1);
2834 }
2835
2836 #include "modconfig.h"
2837 #ifdef MODULE_PROTO_HTTP
module_init()2838 void module_init()
2839 {
2840 Http::ClassInit();
2841 }
2842 #endif
2843