1 /*
2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9 /* DEBUG: section 85 Client-side Request Routines */
10
11 /*
12 * General logic of request processing:
13 *
14 * We run a series of tests to determine if access will be permitted, and to do
15 * any redirection. Then we call into the result clientStream to retrieve data.
16 * From that point on it's up to reply management.
17 */
18
19 #include "squid.h"
20 #include "acl/FilledChecklist.h"
21 #include "acl/Gadgets.h"
22 #include "anyp/PortCfg.h"
23 #include "base/AsyncJobCalls.h"
24 #include "client_side.h"
25 #include "client_side_reply.h"
26 #include "client_side_request.h"
27 #include "ClientRequestContext.h"
28 #include "clientStream.h"
29 #include "comm/Connection.h"
30 #include "comm/Write.h"
31 #include "err_detail_type.h"
32 #include "errorpage.h"
33 #include "fd.h"
34 #include "fde.h"
35 #include "format/Token.h"
36 #include "gopher.h"
37 #include "helper.h"
38 #include "helper/Reply.h"
39 #include "http.h"
40 #include "http/Stream.h"
41 #include "HttpHdrCc.h"
42 #include "HttpReply.h"
43 #include "HttpRequest.h"
44 #include "ip/QosConfig.h"
45 #include "ipcache.h"
46 #include "log/access_log.h"
47 #include "MemObject.h"
48 #include "Parsing.h"
49 #include "profiler/Profiler.h"
50 #include "redirect.h"
51 #include "rfc1738.h"
52 #include "SquidConfig.h"
53 #include "SquidTime.h"
54 #include "Store.h"
55 #include "StrList.h"
56 #include "tools.h"
57 #include "wordlist.h"
58 #if USE_AUTH
59 #include "auth/UserRequest.h"
60 #endif
61 #if USE_ADAPTATION
62 #include "adaptation/AccessCheck.h"
63 #include "adaptation/Answer.h"
64 #include "adaptation/Iterator.h"
65 #include "adaptation/Service.h"
66 #if ICAP_CLIENT
67 #include "adaptation/icap/History.h"
68 #endif
69 #endif
70 #if USE_OPENSSL
71 #include "ssl/ServerBump.h"
72 #include "ssl/support.h"
73 #endif
74
75 #if LINGERING_CLOSE
76 #define comm_close comm_lingering_close
77 #endif
78
79 static const char *const crlf = "\r\n";
80
81 #if FOLLOW_X_FORWARDED_FOR
82 static void clientFollowXForwardedForCheck(allow_t answer, void *data);
83 #endif /* FOLLOW_X_FORWARDED_FOR */
84
85 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
86
87 CBDATA_CLASS_INIT(ClientRequestContext);
88
89 /* Local functions */
90 /* other */
91 static void clientAccessCheckDoneWrapper(allow_t, void *);
92 #if USE_OPENSSL
93 static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
94 #endif
95 static int clientHierarchical(ClientHttpRequest * http);
96 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
97 static HLPCB clientRedirectDoneWrapper;
98 static HLPCB clientStoreIdDoneWrapper;
99 static void checkNoCacheDoneWrapper(allow_t, void *);
100 SQUIDCEXTERN CSR clientGetMoreData;
101 SQUIDCEXTERN CSS clientReplyStatus;
102 SQUIDCEXTERN CSD clientReplyDetach;
103 static void checkFailureRatio(err_type, hier_code);
104
~ClientRequestContext()105 ClientRequestContext::~ClientRequestContext()
106 {
107 /*
108 * Release our "lock" on our parent, ClientHttpRequest, if we
109 * still have one
110 */
111
112 if (http)
113 cbdataReferenceDone(http);
114
115 delete error;
116 debugs(85,3, "ClientRequestContext destructed, this=" << this);
117 }
118
ClientRequestContext(ClientHttpRequest * anHttp)119 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) :
120 http(cbdataReference(anHttp)),
121 acl_checklist(NULL),
122 redirect_state(REDIRECT_NONE),
123 store_id_state(REDIRECT_NONE),
124 host_header_verify_done(false),
125 http_access_done(false),
126 adapted_http_access_done(false),
127 #if USE_ADAPTATION
128 adaptation_acl_check_done(false),
129 #endif
130 redirect_done(false),
131 store_id_done(false),
132 no_cache_done(false),
133 interpreted_req_hdrs(false),
134 tosToClientDone(false),
135 nfmarkToClientDone(false),
136 #if USE_OPENSSL
137 sslBumpCheckDone(false),
138 #endif
139 error(NULL),
140 readNextRequest(false)
141 {
142 debugs(85, 3, "ClientRequestContext constructed, this=" << this);
143 }
144
145 CBDATA_CLASS_INIT(ClientHttpRequest);
146
ClientHttpRequest(ConnStateData * aConn)147 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
148 #if USE_ADAPTATION
149 AsyncJob("ClientHttpRequest"),
150 #endif
151 request(NULL),
152 uri(NULL),
153 log_uri(NULL),
154 req_sz(0),
155 logType(LOG_TAG_NONE),
156 calloutContext(NULL),
157 maxReplyBodySize_(0),
158 entry_(NULL),
159 loggingEntry_(NULL),
160 conn_(NULL)
161 #if USE_OPENSSL
162 , sslBumpNeed_(Ssl::bumpEnd)
163 #endif
164 #if USE_ADAPTATION
165 , request_satisfaction_mode(false)
166 , request_satisfaction_offset(0)
167 #endif
168 {
169 setConn(aConn);
170 al = new AccessLogEntry;
171 al->cache.start_time = current_time;
172 if (aConn) {
173 al->tcpClient = clientConnection = aConn->clientConnection;
174 al->cache.port = aConn->port;
175 al->cache.caddr = aConn->log_addr;
176
177 #if USE_OPENSSL
178 if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
179 if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get())
180 al->cache.sslClientCert.resetWithoutLocking(SSL_get_peer_certificate(ssl));
181 }
182 #endif
183 }
184 dlinkAdd(this, &active, &ClientActiveRequests);
185 }
186
187 /*
188 * returns true if client specified that the object must come from the cache
189 * without contacting origin server
190 */
191 bool
onlyIfCached() const192 ClientHttpRequest::onlyIfCached()const
193 {
194 assert(request);
195 return request->cache_control &&
196 request->cache_control->hasOnlyIfCached();
197 }
198
199 /**
200 * This function is designed to serve a fairly specific purpose.
201 * Occasionally our vBNS-connected caches can talk to each other, but not
202 * the rest of the world. Here we try to detect frequent failures which
203 * make the cache unusable (e.g. DNS lookup and connect() failures). If
204 * the failure:success ratio goes above 1.0 then we go into "hit only"
205 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
206 * will only fetch HITs from us if they are using the ICP protocol. We
207 * stay in this mode for 5 minutes.
208 *
209 * Duane W., Sept 16, 1996
210 */
211 static void
checkFailureRatio(err_type etype,hier_code hcode)212 checkFailureRatio(err_type etype, hier_code hcode)
213 {
214 // Can be set at compile time with -D compiler flag
215 #ifndef FAILURE_MODE_TIME
216 #define FAILURE_MODE_TIME 300
217 #endif
218
219 if (hcode == HIER_NONE)
220 return;
221
222 // don't bother when ICP is disabled.
223 if (Config.Port.icp <= 0)
224 return;
225
226 static double magic_factor = 100.0;
227 double n_good;
228 double n_bad;
229
230 n_good = magic_factor / (1.0 + request_failure_ratio);
231
232 n_bad = magic_factor - n_good;
233
234 switch (etype) {
235
236 case ERR_DNS_FAIL:
237
238 case ERR_CONNECT_FAIL:
239 case ERR_SECURE_CONNECT_FAIL:
240
241 case ERR_READ_ERROR:
242 ++n_bad;
243 break;
244
245 default:
246 ++n_good;
247 }
248
249 request_failure_ratio = n_bad / n_good;
250
251 if (hit_only_mode_until > squid_curtime)
252 return;
253
254 if (request_failure_ratio < 1.0)
255 return;
256
257 debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
258 std::setprecision(3) << request_failure_ratio);
259
260 debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
261 FAILURE_MODE_TIME / 60 << " minutes...");
262
263 hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
264
265 request_failure_ratio = 0.8; /* reset to something less than 1.0 */
266 }
267
~ClientHttpRequest()268 ClientHttpRequest::~ClientHttpRequest()
269 {
270 debugs(33, 3, "httpRequestFree: " << uri);
271 PROF_start(httpRequestFree);
272
273 // Even though freeResources() below may destroy the request,
274 // we no longer set request->body_pipe to NULL here
275 // because we did not initiate that pipe (ConnStateData did)
276
277 /* the ICP check here was erroneous
278 * - StoreEntry::releaseRequest was always called if entry was valid
279 */
280
281 logRequest();
282
283 loggingEntry(NULL);
284
285 if (request)
286 checkFailureRatio(request->errType, al->hier.code);
287
288 freeResources();
289
290 #if USE_ADAPTATION
291 announceInitiatorAbort(virginHeadSource);
292
293 if (adaptedBodySource != NULL)
294 stopConsumingFrom(adaptedBodySource);
295 #endif
296
297 if (calloutContext)
298 delete calloutContext;
299
300 clientConnection = NULL;
301
302 if (conn_)
303 cbdataReferenceDone(conn_);
304
305 /* moving to the next connection is handled by the context free */
306 dlinkDelete(&active, &ClientActiveRequests);
307
308 PROF_stop(httpRequestFree);
309 }
310
311 /**
312 * Create a request and kick it off
313 *
314 * \retval 0 success
315 * \retval -1 failure
316 *
317 * TODO: Pass in the buffers to be used in the inital Read request, as they are
318 * determined by the user
319 */
320 int
clientBeginRequest(const HttpRequestMethod & method,char const * url,CSCB * streamcallback,CSD * streamdetach,ClientStreamData streamdata,HttpHeader const * header,char * tailbuf,size_t taillen,const MasterXaction::Pointer & mx)321 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
322 CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
323 char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
324 {
325 size_t url_sz;
326 ClientHttpRequest *http = new ClientHttpRequest(NULL);
327 HttpRequest *request;
328 StoreIOBuffer tempBuffer;
329 if (http->al != NULL)
330 http->al->cache.start_time = current_time;
331 /* this is only used to adjust the connection offset in client_side.c */
332 http->req_sz = 0;
333 tempBuffer.length = taillen;
334 tempBuffer.data = tailbuf;
335 /* client stream setup */
336 clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
337 clientReplyStatus, new clientReplyContext(http), streamcallback,
338 streamdetach, streamdata, tempBuffer);
339 /* make it visible in the 'current acctive requests list' */
340 /* Set flags */
341 /* internal requests only makes sense in an
342 * accelerator today. TODO: accept flags ? */
343 http->flags.accel = true;
344 /* allow size for url rewriting */
345 url_sz = strlen(url) + Config.appendDomainLen + 5;
346 http->uri = (char *)xcalloc(url_sz, 1);
347 strcpy(http->uri, url); // XXX: polluting http->uri before parser validation
348
349 request = HttpRequest::FromUrlXXX(http->uri, mx, method);
350 if (!request) {
351 debugs(85, 5, "Invalid URL: " << http->uri);
352 return -1;
353 }
354
355 /*
356 * now update the headers in request with our supplied headers.
357 * HttpRequest::FromUrl() should return a blank header set, but
358 * we use Update to be sure of correctness.
359 */
360 if (header)
361 request->header.update(header);
362
363 /* http struct now ready */
364
365 /*
366 * build new header list *? TODO
367 */
368 request->flags.accelerated = http->flags.accel;
369
370 /* this is an internally created
371 * request, not subject to acceleration
372 * target overrides */
373 /*
374 * FIXME? Do we want to detect and handle internal requests of internal
375 * objects ?
376 */
377
378 /* Internally created requests cannot have bodies today */
379 request->content_length = 0;
380
381 request->client_addr.setNoAddr();
382
383 #if FOLLOW_X_FORWARDED_FOR
384 request->indirect_client_addr.setNoAddr();
385 #endif /* FOLLOW_X_FORWARDED_FOR */
386
387 request->my_addr.setNoAddr(); /* undefined for internal requests */
388
389 request->my_addr.port(0);
390
391 request->http_ver = Http::ProtocolVersion();
392
393 http->initRequest(request);
394
395 /* optional - skip the access check ? */
396 http->calloutContext = new ClientRequestContext(http);
397
398 http->calloutContext->http_access_done = false;
399
400 http->calloutContext->redirect_done = true;
401
402 http->calloutContext->no_cache_done = true;
403
404 http->doCallouts();
405
406 return 0;
407 }
408
409 bool
httpStateIsValid()410 ClientRequestContext::httpStateIsValid()
411 {
412 ClientHttpRequest *http_ = http;
413
414 if (cbdataReferenceValid(http_))
415 return true;
416
417 http = NULL;
418
419 cbdataReferenceDone(http_);
420
421 return false;
422 }
423
424 #if FOLLOW_X_FORWARDED_FOR
425 /**
426 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
427 * against the followXFF ACL, or cleans up and passes control to
428 * clientAccessCheck().
429 *
430 * The trust model here is a little ambiguous. So to clarify the logic:
431 * - we may always use the direct client address as the client IP.
432 * - these trust tests merey tell whether we trust given IP enough to believe the
433 * IP string which it appended to the X-Forwarded-For: header.
434 * - if at any point we don't trust what an IP adds we stop looking.
435 * - at that point the current contents of indirect_client_addr are the value set
436 * by the last previously trusted IP.
437 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
438 */
439 static void
clientFollowXForwardedForCheck(allow_t answer,void * data)440 clientFollowXForwardedForCheck(allow_t answer, void *data)
441 {
442 ClientRequestContext *calloutContext = (ClientRequestContext *) data;
443
444 if (!calloutContext->httpStateIsValid())
445 return;
446
447 ClientHttpRequest *http = calloutContext->http;
448 HttpRequest *request = http->request;
449
450 if (answer.allowed() && request->x_forwarded_for_iterator.size() != 0) {
451
452 /*
453 * Remove the last comma-delimited element from the
454 * x_forwarded_for_iterator and use it to repeat the cycle.
455 */
456 const char *p;
457 const char *asciiaddr;
458 int l;
459 Ip::Address addr;
460 p = request->x_forwarded_for_iterator.termedBuf();
461 l = request->x_forwarded_for_iterator.size();
462
463 /*
464 * XXX x_forwarded_for_iterator should really be a list of
465 * IP addresses, but it's a String instead. We have to
466 * walk backwards through the String, biting off the last
467 * comma-delimited part each time. As long as the data is in
468 * a String, we should probably implement and use a variant of
469 * strListGetItem() that walks backwards instead of forwards
470 * through a comma-separated list. But we don't even do that;
471 * we just do the work in-line here.
472 */
473 /* skip trailing space and commas */
474 while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
475 --l;
476 request->x_forwarded_for_iterator.cut(l);
477 /* look for start of last item in list */
478 while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
479 --l;
480 asciiaddr = p+l;
481 if ((addr = asciiaddr)) {
482 request->indirect_client_addr = addr;
483 request->x_forwarded_for_iterator.cut(l);
484 calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
485 if (!Config.onoff.acl_uses_indirect_client) {
486 /* override the default src_addr tested if we have to go deeper than one level into XFF */
487 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
488 }
489 calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
490 return;
491 }
492 }
493
494 /* clean up, and pass control to clientAccessCheck */
495 if (Config.onoff.log_uses_indirect_client) {
496 /*
497 * Ensure that the access log shows the indirect client
498 * instead of the direct client.
499 */
500 http->al->cache.caddr = request->indirect_client_addr;
501 if (ConnStateData *conn = http->getConn())
502 conn->log_addr = request->indirect_client_addr;
503 }
504 request->x_forwarded_for_iterator.clean();
505 request->flags.done_follow_x_forwarded_for = true;
506
507 if (!answer.someRuleMatched()) {
508 debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
509 }
510
511 /* process actual access ACL as normal. */
512 calloutContext->clientAccessCheck();
513 }
514 #endif /* FOLLOW_X_FORWARDED_FOR */
515
516 static void
hostHeaderIpVerifyWrapper(const ipcache_addrs * ia,const Dns::LookupDetails & dns,void * data)517 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const Dns::LookupDetails &dns, void *data)
518 {
519 ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
520 c->hostHeaderIpVerify(ia, dns);
521 }
522
523 void
hostHeaderIpVerify(const ipcache_addrs * ia,const Dns::LookupDetails & dns)524 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const Dns::LookupDetails &dns)
525 {
526 Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
527
528 // note the DNS details for the transaction stats.
529 http->request->recordLookup(dns);
530
531 if (ia != NULL && ia->count > 0) {
532 // Is the NAT destination IP in DNS?
533 for (int i = 0; i < ia->count; ++i) {
534 if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
535 debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
536 http->request->flags.hostVerified = true;
537 http->doCallouts();
538 return;
539 }
540 debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
541 }
542 }
543 debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
544 hostHeaderVerifyFailed("local IP", "any domain IP");
545 }
546
547 void
hostHeaderVerifyFailed(const char * A,const char * B)548 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
549 {
550 // IP address validation for Host: failed. Admin wants to ignore them.
551 // NP: we do not yet handle CONNECT tunnels well, so ignore for them
552 if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
553 debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
554 " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
555
556 // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
557 // The problems here are about WRITE for new cache content, which means flags.cachable
558 http->request->flags.cachable = false; // MUST NOT cache (for now)
559 // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
560 http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
561 // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
562 http->doCallouts();
563 return;
564 }
565
566 debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
567 http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
568 if (const char *ua = http->request->header.getStr(Http::HdrType::USER_AGENT))
569 debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << ua);
570 debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
571
572 // IP address validation for Host: failed. reject the connection.
573 clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
574 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
575 assert (repContext);
576 repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
577 http->request->method, NULL,
578 http->getConn()->clientConnection->remote,
579 http->request,
580 NULL,
581 #if USE_AUTH
582 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
583 http->getConn()->getAuth() : http->request->auth_user_request);
584 #else
585 NULL);
586 #endif
587 node = (clientStreamNode *)http->client_stream.tail->data;
588 clientStreamRead(node, http, node->readBuffer);
589 }
590
591 void
hostHeaderVerify()592 ClientRequestContext::hostHeaderVerify()
593 {
594 // Require a Host: header.
595 const char *host = http->request->header.getStr(Http::HdrType::HOST);
596
597 if (!host) {
598 // TODO: dump out the HTTP/1.1 error about missing host header.
599 // otherwise this is fine, can't forge a header value when its not even set.
600 debugs(85, 3, HERE << "validate skipped with no Host: header present.");
601 http->doCallouts();
602 return;
603 }
604
605 if (http->request->flags.internal) {
606 // TODO: kill this when URL handling allows partial URLs out of accel mode
607 // and we no longer screw with the URL just to add our internal host there
608 debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
609 http->doCallouts();
610 return;
611 }
612
613 // Locate if there is a port attached, strip ready for IP lookup
614 char *portStr = NULL;
615 char *hostB = xstrdup(host);
616 host = hostB;
617 if (host[0] == '[') {
618 // IPv6 literal.
619 portStr = strchr(hostB, ']');
620 if (portStr && *(++portStr) != ':') {
621 portStr = NULL;
622 }
623 } else {
624 // Domain or IPv4 literal with port
625 portStr = strrchr(hostB, ':');
626 }
627
628 uint16_t port = 0;
629 if (portStr) {
630 *portStr = '\0'; // strip the ':'
631 if (*(++portStr) != '\0') {
632 char *end = NULL;
633 int64_t ret = strtoll(portStr, &end, 10);
634 if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
635 // invalid port details. Replace the ':'
636 *(--portStr) = ':';
637 portStr = NULL;
638 } else
639 port = (ret & 0xFFFF);
640 }
641 }
642
643 debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
644 if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
645 // verify the Host: port (if any) matches the apparent destination
646 if (portStr && port != http->getConn()->clientConnection->local.port()) {
647 debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
648 " matches Host: port " << port << " (" << portStr << ")");
649 hostHeaderVerifyFailed("intercepted port", portStr);
650 } else {
651 // XXX: match the scheme default port against the apparent destination
652
653 // verify the destination DNS is one of the Host: headers IPs
654 ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
655 }
656 } else if (!Config.onoff.hostStrictVerify) {
657 debugs(85, 3, "validate skipped.");
658 http->doCallouts();
659 } else if (strlen(host) != strlen(http->request->url.host())) {
660 // Verify forward-proxy requested URL domain matches the Host: header
661 debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
662 hostHeaderVerifyFailed(host, http->request->url.host());
663 } else if (matchDomainName(host, http->request->url.host()) != 0) {
664 // Verify forward-proxy requested URL domain matches the Host: header
665 debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
666 hostHeaderVerifyFailed(host, http->request->url.host());
667 } else if (portStr && port != http->request->url.port()) {
668 // Verify forward-proxy requested URL domain matches the Host: header
669 debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
670 hostHeaderVerifyFailed("URL port", portStr);
671 } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
672 // Verify forward-proxy requested URL domain matches the Host: header
673 // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
674 debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
675 hostHeaderVerifyFailed("URL port", "default port");
676 } else {
677 // Okay no problem.
678 debugs(85, 3, "validate passed.");
679 http->request->flags.hostVerified = true;
680 http->doCallouts();
681 }
682 safe_free(hostB);
683 }
684
685 /* This is the entry point for external users of the client_side routines */
686 void
clientAccessCheck()687 ClientRequestContext::clientAccessCheck()
688 {
689 #if FOLLOW_X_FORWARDED_FOR
690 if (!http->request->flags.doneFollowXff() &&
691 Config.accessList.followXFF &&
692 http->request->header.has(Http::HdrType::X_FORWARDED_FOR)) {
693
694 /* we always trust the direct client address for actual use */
695 http->request->indirect_client_addr = http->request->client_addr;
696 http->request->indirect_client_addr.port(0);
697
698 /* setup the XFF iterator for processing */
699 http->request->x_forwarded_for_iterator = http->request->header.getList(Http::HdrType::X_FORWARDED_FOR);
700
701 /* begin by checking to see if we trust direct client enough to walk XFF */
702 acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
703 acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
704 return;
705 }
706 #endif
707
708 if (Config.accessList.http) {
709 acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
710 acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
711 } else {
712 debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
713 clientAccessCheckDone(ACCESS_DENIED);
714 }
715 }
716
717 /**
718 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
719 * The default here is to allow all. Since the earlier http_access should do a default deny all.
720 * This check is just for a last-minute denial based on adapted request headers.
721 */
722 void
clientAccessCheck2()723 ClientRequestContext::clientAccessCheck2()
724 {
725 if (Config.accessList.adapted_http) {
726 acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
727 acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
728 } else {
729 debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
730 clientAccessCheckDone(ACCESS_ALLOWED);
731 }
732 }
733
734 void
clientAccessCheckDoneWrapper(allow_t answer,void * data)735 clientAccessCheckDoneWrapper(allow_t answer, void *data)
736 {
737 ClientRequestContext *calloutContext = (ClientRequestContext *) data;
738
739 if (!calloutContext->httpStateIsValid())
740 return;
741
742 calloutContext->clientAccessCheckDone(answer);
743 }
744
745 void
clientAccessCheckDone(const allow_t & answer)746 ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
747 {
748 acl_checklist = NULL;
749 err_type page_id;
750 Http::StatusCode status;
751 debugs(85, 2, "The request " << http->request->method << ' ' <<
752 http->uri << " is " << answer <<
753 "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
754
755 #if USE_AUTH
756 char const *proxy_auth_msg = "<null>";
757 if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
758 proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
759 else if (http->request->auth_user_request != NULL)
760 proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
761 #endif
762
763 if (!answer.allowed()) {
764 // auth has a grace period where credentials can be expired but okay not to challenge.
765
766 /* Send an auth challenge or error */
767 // XXX: do we still need aclIsProxyAuth() ?
768 bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
769 debugs(85, 5, "Access Denied: " << http->uri);
770 debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
771 #if USE_AUTH
772 if (auth_challenge)
773 debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
774 #endif
775
776 /*
777 * NOTE: get page_id here, based on AclMatchedName because if
778 * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
779 * the clientCreateStoreEntry() call just below. Pedro Ribeiro
780 * <pribeiro@isel.pt>
781 */
782 page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
783
784 http->logType = LOG_TCP_DENIED;
785
786 if (auth_challenge) {
787 #if USE_AUTH
788 if (http->request->flags.sslBumped) {
789 /*SSL Bumped request, authentication is not possible*/
790 status = Http::scForbidden;
791 } else if (!http->flags.accel) {
792 /* Proxy authorisation needed */
793 status = Http::scProxyAuthenticationRequired;
794 } else {
795 /* WWW authorisation needed */
796 status = Http::scUnauthorized;
797 }
798 #else
799 // need auth, but not possible to do.
800 status = Http::scForbidden;
801 #endif
802 if (page_id == ERR_NONE)
803 page_id = ERR_CACHE_ACCESS_DENIED;
804 } else {
805 status = Http::scForbidden;
806
807 if (page_id == ERR_NONE)
808 page_id = ERR_ACCESS_DENIED;
809 }
810
811 Ip::Address tmpnoaddr;
812 tmpnoaddr.setNoAddr();
813 error = clientBuildError(page_id, status,
814 NULL,
815 http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
816 http->request
817 );
818
819 #if USE_AUTH
820 error->auth_user_request =
821 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
822 http->getConn()->getAuth() : http->request->auth_user_request;
823 #endif
824
825 readNextRequest = true;
826 }
827
828 /* ACCESS_ALLOWED continues here ... */
829 xfree(http->uri);
830 http->uri = SBufToCstring(http->request->effectiveRequestUri());
831 http->doCallouts();
832 }
833
834 #if USE_ADAPTATION
835 void
noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)836 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
837 {
838 debugs(93,3,HERE << this << " adaptationAclCheckDone called");
839
840 #if ICAP_CLIENT
841 Adaptation::Icap::History::Pointer ih = request->icapHistory();
842 if (ih != NULL) {
843 if (getConn() != NULL && getConn()->clientConnection != NULL) {
844 ih->rfc931 = getConn()->clientConnection->rfc931;
845 #if USE_OPENSSL
846 if (getConn()->clientConnection->isOpen()) {
847 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl.get());
848 }
849 #endif
850 }
851 ih->log_uri = log_uri;
852 ih->req_sz = req_sz;
853 }
854 #endif
855
856 if (!g) {
857 debugs(85,3, HERE << "no adaptation needed");
858 doCallouts();
859 return;
860 }
861
862 startAdaptation(g);
863 }
864
865 #endif
866
867 static void
clientRedirectAccessCheckDone(allow_t answer,void * data)868 clientRedirectAccessCheckDone(allow_t answer, void *data)
869 {
870 ClientRequestContext *context = (ClientRequestContext *)data;
871 ClientHttpRequest *http = context->http;
872 context->acl_checklist = NULL;
873
874 if (answer.allowed())
875 redirectStart(http, clientRedirectDoneWrapper, context);
876 else {
877 Helper::Reply const nilReply(Helper::Error);
878 context->clientRedirectDone(nilReply);
879 }
880 }
881
882 void
clientRedirectStart()883 ClientRequestContext::clientRedirectStart()
884 {
885 debugs(33, 5, HERE << "'" << http->uri << "'");
886 (void)SyncNotes(*http->al, *http->request);
887 if (Config.accessList.redirector) {
888 acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
889 acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
890 } else
891 redirectStart(http, clientRedirectDoneWrapper, this);
892 }
893
894 /**
895 * This methods handles Access checks result of StoreId access list.
896 * Will handle as "ERR" (no change) in a case Access is not allowed.
897 */
898 static void
clientStoreIdAccessCheckDone(allow_t answer,void * data)899 clientStoreIdAccessCheckDone(allow_t answer, void *data)
900 {
901 ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
902 ClientHttpRequest *http = context->http;
903 context->acl_checklist = NULL;
904
905 if (answer.allowed())
906 storeIdStart(http, clientStoreIdDoneWrapper, context);
907 else {
908 debugs(85, 3, "access denied expected ERR reply handling: " << answer);
909 Helper::Reply const nilReply(Helper::Error);
910 context->clientStoreIdDone(nilReply);
911 }
912 }
913
914 /**
915 * Start locating an alternative storeage ID string (if any) from admin
916 * configured helper program. This is an asynchronous operation terminating in
917 * ClientRequestContext::clientStoreIdDone() when completed.
918 */
919 void
clientStoreIdStart()920 ClientRequestContext::clientStoreIdStart()
921 {
922 debugs(33, 5,"'" << http->uri << "'");
923
924 if (Config.accessList.store_id) {
925 acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
926 acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
927 } else
928 storeIdStart(http, clientStoreIdDoneWrapper, this);
929 }
930
931 static int
clientHierarchical(ClientHttpRequest * http)932 clientHierarchical(ClientHttpRequest * http)
933 {
934 HttpRequest *request = http->request;
935 HttpRequestMethod method = request->method;
936
937 // intercepted requests MUST NOT (yet) be sent to peers unless verified
938 if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
939 return 0;
940
941 /*
942 * IMS needs a private key, so we can use the hierarchy for IMS only if our
943 * neighbors support private keys
944 */
945
946 if (request->flags.ims && !neighbors_do_private_keys)
947 return 0;
948
949 /*
950 * This is incorrect: authenticating requests can be sent via a hierarchy
951 * (they can even be cached if the correct headers are set on the reply)
952 */
953 if (request->flags.auth)
954 return 0;
955
956 if (method == Http::METHOD_TRACE)
957 return 1;
958
959 if (method != Http::METHOD_GET)
960 return 0;
961
962 if (request->flags.loopDetected)
963 return 0;
964
965 if (request->url.getScheme() == AnyP::PROTO_HTTP)
966 return method.respMaybeCacheable();
967
968 if (request->url.getScheme() == AnyP::PROTO_GOPHER)
969 return gopherCachable(request);
970
971 if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
972 return 0;
973
974 return 1;
975 }
976
977 static void
clientCheckPinning(ClientHttpRequest * http)978 clientCheckPinning(ClientHttpRequest * http)
979 {
980 HttpRequest *request = http->request;
981 HttpHeader *req_hdr = &request->header;
982 ConnStateData *http_conn = http->getConn();
983
984 /* Internal requests such as those from ESI includes may be without
985 * a client connection
986 */
987 if (!http_conn)
988 return;
989
990 request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
991 if (!request->flags.connectionAuthDisabled) {
992 if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
993 if (http_conn->pinning.auth) {
994 request->flags.connectionAuth = true;
995 request->flags.auth = true;
996 } else {
997 request->flags.connectionProxyAuth = true;
998 }
999 // These should already be linked correctly.
1000 assert(request->clientConnectionManager == http_conn);
1001 }
1002 }
1003
1004 /* check if connection auth is used, and flag as candidate for pinning
1005 * in such case.
1006 * Note: we may need to set flags.connectionAuth even if the connection
1007 * is already pinned if it was pinned earlier due to proxy auth
1008 */
1009 if (!request->flags.connectionAuth) {
1010 if (req_hdr->has(Http::HdrType::AUTHORIZATION) || req_hdr->has(Http::HdrType::PROXY_AUTHORIZATION)) {
1011 HttpHeaderPos pos = HttpHeaderInitPos;
1012 HttpHeaderEntry *e;
1013 int may_pin = 0;
1014 while ((e = req_hdr->getEntry(&pos))) {
1015 if (e->id == Http::HdrType::AUTHORIZATION || e->id == Http::HdrType::PROXY_AUTHORIZATION) {
1016 const char *value = e->value.rawBuf();
1017 if (strncasecmp(value, "NTLM ", 5) == 0
1018 ||
1019 strncasecmp(value, "Negotiate ", 10) == 0
1020 ||
1021 strncasecmp(value, "Kerberos ", 9) == 0) {
1022 if (e->id == Http::HdrType::AUTHORIZATION) {
1023 request->flags.connectionAuth = true;
1024 may_pin = 1;
1025 } else {
1026 request->flags.connectionProxyAuth = true;
1027 may_pin = 1;
1028 }
1029 }
1030 }
1031 }
1032 if (may_pin && !request->pinnedConnection()) {
1033 // These should already be linked correctly. Just need the ServerConnection to pinn.
1034 assert(request->clientConnectionManager == http_conn);
1035 }
1036 }
1037 }
1038 }
1039
1040 static void
clientInterpretRequestHeaders(ClientHttpRequest * http)1041 clientInterpretRequestHeaders(ClientHttpRequest * http)
1042 {
1043 HttpRequest *request = http->request;
1044 HttpHeader *req_hdr = &request->header;
1045 bool no_cache = false;
1046
1047 request->imslen = -1;
1048 request->ims = req_hdr->getTime(Http::HdrType::IF_MODIFIED_SINCE);
1049
1050 if (request->ims > 0)
1051 request->flags.ims = true;
1052
1053 if (!request->flags.ignoreCc) {
1054 if (request->cache_control) {
1055 if (request->cache_control->hasNoCache())
1056 no_cache=true;
1057
1058 // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1059 } else if (req_hdr->has(Http::HdrType::PRAGMA))
1060 no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1061 }
1062
1063 if (request->method == Http::METHOD_OTHER) {
1064 no_cache=true;
1065 }
1066
1067 if (no_cache) {
1068 #if USE_HTTP_VIOLATIONS
1069
1070 if (Config.onoff.reload_into_ims)
1071 request->flags.nocacheHack = true;
1072 else if (refresh_nocache_hack)
1073 request->flags.nocacheHack = true;
1074 else
1075 #endif
1076
1077 request->flags.noCache = true;
1078 }
1079
1080 /* ignore range header in non-GETs or non-HEADs */
1081 if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1082 // XXX: initialize if we got here without HttpRequest::parseHeader()
1083 if (!request->range)
1084 request->range = req_hdr->getRange();
1085
1086 if (request->range) {
1087 request->flags.isRanged = true;
1088 clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1089 /* XXX: This is suboptimal. We should give the stream the range set,
1090 * and thereby let the top of the stream set the offset when the
1091 * size becomes known. As it is, we will end up requesting from 0
1092 * for evey -X range specification.
1093 * RBC - this may be somewhat wrong. We should probably set the range
1094 * iter up at this point.
1095 */
1096 node->readBuffer.offset = request->range->lowestOffset(0);
1097 }
1098 }
1099
1100 /* Only HEAD and GET requests permit a Range or Request-Range header.
1101 * If these headers appear on any other type of request, delete them now.
1102 */
1103 else {
1104 req_hdr->delById(Http::HdrType::RANGE);
1105 req_hdr->delById(Http::HdrType::REQUEST_RANGE);
1106 request->ignoreRange("neither HEAD nor GET");
1107 }
1108
1109 if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1110 request->flags.auth = true;
1111
1112 clientCheckPinning(http);
1113
1114 if (!request->url.userInfo().isEmpty())
1115 request->flags.auth = true;
1116
1117 if (req_hdr->has(Http::HdrType::VIA)) {
1118 String s = req_hdr->getList(Http::HdrType::VIA);
1119 /*
1120 * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1121 * Note ThisCache2 has a space prepended to the hostname so we don't
1122 * accidentally match super-domains.
1123 */
1124
1125 if (strListIsSubstr(&s, ThisCache2, ',')) {
1126 debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1127 request, (ObjPackMethod) & httpRequestPack);
1128 request->flags.loopDetected = true;
1129 }
1130
1131 #if USE_FORW_VIA_DB
1132 fvdbCountVia(s.termedBuf());
1133
1134 #endif
1135
1136 s.clean();
1137 }
1138
1139 #if USE_FORW_VIA_DB
1140
1141 if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1142 String s = req_hdr->getList(Http::HdrType::X_FORWARDED_FOR);
1143 fvdbCountForw(s.termedBuf());
1144 s.clean();
1145 }
1146
1147 #endif
1148
1149 request->flags.cachable = http->request->maybeCacheable();
1150
1151 if (clientHierarchical(http))
1152 request->flags.hierarchical = true;
1153
1154 debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1155 (request->flags.noCache ? "SET" : "NOT SET"));
1156 debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1157 (request->flags.cachable ? "SET" : "NOT SET"));
1158 debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1159 (request->flags.hierarchical ? "SET" : "NOT SET"));
1160
1161 }
1162
1163 void
clientRedirectDoneWrapper(void * data,const Helper::Reply & result)1164 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1165 {
1166 ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1167
1168 if (!calloutContext->httpStateIsValid())
1169 return;
1170
1171 calloutContext->clientRedirectDone(result);
1172 }
1173
1174 void
clientStoreIdDoneWrapper(void * data,const Helper::Reply & result)1175 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1176 {
1177 ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1178
1179 if (!calloutContext->httpStateIsValid())
1180 return;
1181
1182 calloutContext->clientStoreIdDone(result);
1183 }
1184
1185 void
clientRedirectDone(const Helper::Reply & reply)1186 ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1187 {
1188 HttpRequest *old_request = http->request;
1189 debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1190 assert(redirect_state == REDIRECT_PENDING);
1191 redirect_state = REDIRECT_DONE;
1192
1193 // Put helper response Notes into the transaction state record (ALE) eventually
1194 // do it early to ensure that no matter what the outcome the notes are present.
1195 if (http->al != NULL)
1196 (void)SyncNotes(*http->al, *old_request);
1197
1198 UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1199
1200 switch (reply.result) {
1201 case Helper::TimedOut:
1202 if (Config.onUrlRewriteTimeout.action != toutActBypass) {
1203 http->calloutsError(ERR_GATEWAY_FAILURE, ERR_DETAIL_REDIRECTOR_TIMEDOUT);
1204 debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1205 }
1206 break;
1207
1208 case Helper::Unknown:
1209 case Helper::TT:
1210 // Handler in redirect.cc should have already mapped Unknown
1211 // IF it contained valid entry for the old URL-rewrite helper protocol
1212 debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1213 break;
1214
1215 case Helper::BrokenHelper:
1216 debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1217 break;
1218
1219 case Helper::Error:
1220 // no change to be done.
1221 break;
1222
1223 case Helper::Okay: {
1224 // #1: redirect with a specific status code OK status=NNN url="..."
1225 // #2: redirect with a default status code OK url="..."
1226 // #3: re-write the URL OK rewrite-url="..."
1227
1228 const char *statusNote = reply.notes.findFirst("status");
1229 const char *urlNote = reply.notes.findFirst("url");
1230
1231 if (urlNote != NULL) {
1232 // HTTP protocol redirect to be done.
1233
1234 // TODO: change default redirect status for appropriate requests
1235 // Squid defaults to 302 status for now for better compatibility with old clients.
1236 // HTTP/1.0 client should get 302 (Http::scFound)
1237 // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1238 // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1239 Http::StatusCode status = Http::scFound;
1240 if (statusNote != NULL) {
1241 const char * result = statusNote;
1242 status = static_cast<Http::StatusCode>(atoi(result));
1243 }
1244
1245 if (status == Http::scMovedPermanently
1246 || status == Http::scFound
1247 || status == Http::scSeeOther
1248 || status == Http::scPermanentRedirect
1249 || status == Http::scTemporaryRedirect) {
1250 http->redirect.status = status;
1251 http->redirect.location = xstrdup(urlNote);
1252 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1253 } else {
1254 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1255 }
1256 } else {
1257 // URL-rewrite wanted. Ew.
1258 urlNote = reply.notes.findFirst("rewrite-url");
1259
1260 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1261 if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1262 AnyP::Uri tmpUrl;
1263 if (tmpUrl.parse(old_request->method, SBuf(urlNote))) {
1264 HttpRequest *new_request = old_request->clone();
1265 new_request->url = tmpUrl;
1266 debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1267
1268 // update the new request to flag the re-writing was done on it
1269 new_request->flags.redirected = true;
1270
1271 // unlink bodypipe from the old request. Not needed there any longer.
1272 if (old_request->body_pipe != NULL) {
1273 old_request->body_pipe = NULL;
1274 debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1275 " from request " << old_request << " to " << new_request);
1276 }
1277
1278 http->resetRequest(new_request);
1279 old_request = nullptr;
1280 } else {
1281 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1282 old_request->method << " " << urlNote << " " << old_request->http_ver);
1283 }
1284 }
1285 }
1286 }
1287 break;
1288 }
1289
1290 /* FIXME PIPELINE: This is innacurate during pipelining */
1291
1292 if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1293 fd_note(http->getConn()->clientConnection->fd, http->uri);
1294
1295 assert(http->uri);
1296
1297 http->doCallouts();
1298 }
1299
1300 /**
1301 * This method handles the different replies from StoreID helper.
1302 */
1303 void
clientStoreIdDone(const Helper::Reply & reply)1304 ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1305 {
1306 HttpRequest *old_request = http->request;
1307 debugs(85, 5, "'" << http->uri << "' result=" << reply);
1308 assert(store_id_state == REDIRECT_PENDING);
1309 store_id_state = REDIRECT_DONE;
1310
1311 // Put helper response Notes into the transaction state record (ALE) eventually
1312 // do it early to ensure that no matter what the outcome the notes are present.
1313 if (http->al != NULL)
1314 (void)SyncNotes(*http->al, *old_request);
1315
1316 UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1317
1318 switch (reply.result) {
1319 case Helper::Unknown:
1320 case Helper::TT:
1321 // Handler in redirect.cc should have already mapped Unknown
1322 // IF it contained valid entry for the old helper protocol
1323 debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1324 break;
1325
1326 case Helper::TimedOut:
1327 // Timeouts for storeID are not implemented
1328 case Helper::BrokenHelper:
1329 debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1330 break;
1331
1332 case Helper::Error:
1333 // no change to be done.
1334 break;
1335
1336 case Helper::Okay: {
1337 const char *urlNote = reply.notes.findFirst("store-id");
1338
1339 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1340 if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1341 // Debug section required for some very specific cases.
1342 debugs(85, 9, "Setting storeID with: " << urlNote );
1343 http->request->store_id = urlNote;
1344 http->store_id = urlNote;
1345 }
1346 }
1347 break;
1348 }
1349
1350 http->doCallouts();
1351 }
1352
1353 /** Test cache allow/deny configuration
1354 * Sets flags.cachable=1 if caching is not denied.
1355 */
1356 void
checkNoCache()1357 ClientRequestContext::checkNoCache()
1358 {
1359 if (Config.accessList.noCache) {
1360 acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1361 acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1362 } else {
1363 /* unless otherwise specified, we try to cache. */
1364 checkNoCacheDone(ACCESS_ALLOWED);
1365 }
1366 }
1367
1368 static void
checkNoCacheDoneWrapper(allow_t answer,void * data)1369 checkNoCacheDoneWrapper(allow_t answer, void *data)
1370 {
1371 ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1372
1373 if (!calloutContext->httpStateIsValid())
1374 return;
1375
1376 calloutContext->checkNoCacheDone(answer);
1377 }
1378
1379 void
checkNoCacheDone(const allow_t & answer)1380 ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1381 {
1382 acl_checklist = NULL;
1383 if (answer.denied()) {
1384 http->request->flags.noCache = true; // do not read reply from cache
1385 http->request->flags.cachable = false; // do not store reply into cache
1386 }
1387 http->doCallouts();
1388 }
1389
1390 #if USE_OPENSSL
1391 bool
sslBumpAccessCheck()1392 ClientRequestContext::sslBumpAccessCheck()
1393 {
1394 if (!http->getConn()) {
1395 http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1396 return false;
1397 }
1398
1399 const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1400 if (http->request->flags.forceTunnel) {
1401 debugs(85, 5, "not needed; already decided to tunnel " << http->getConn());
1402 if (bumpMode != Ssl::bumpEnd)
1403 http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1404 return false;
1405 }
1406
1407 // If SSL connection tunneling or bumping decision has been made, obey it.
1408 if (bumpMode != Ssl::bumpEnd) {
1409 debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1410 "), " << "ignoring ssl_bump for " << http->getConn());
1411
1412 // We need the following "if" for transparently bumped TLS connection,
1413 // because in this case we are running ssl_bump access list before
1414 // the doCallouts runs. It can be removed after the bug #4340 fixed.
1415 // We do not want to proceed to bumping steps:
1416 // - if the TLS connection with the client is already established
1417 // because we are accepting normal HTTP requests on TLS port,
1418 // or because of the client-first bumping mode
1419 // - When the bumping is already started
1420 if (!http->getConn()->switchedToHttps() &&
1421 !http->getConn()->serverBump())
1422 http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1423 http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1424 return false;
1425 }
1426
1427 // If we have not decided yet, decide whether to bump now.
1428
1429 // Bumping here can only start with a CONNECT request on a bumping port
1430 // (bumping of intercepted SSL conns is decided before we get 1st request).
1431 // We also do not bump redirected CONNECT requests.
1432 if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1433 !Config.accessList.ssl_bump ||
1434 !http->getConn()->port->flags.tunnelSslBumping) {
1435 http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1436 debugs(85, 5, HERE << "cannot SslBump this request");
1437 return false;
1438 }
1439
1440 // Do not bump during authentication: clients would not proxy-authenticate
1441 // if we delay a 407 response and respond with 200 OK to CONNECT.
1442 if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1443 http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1444 debugs(85, 5, HERE << "no SslBump during proxy authentication");
1445 return false;
1446 }
1447
1448 if (error) {
1449 debugs(85, 5, "SslBump applies. Force bump action on error " << errorTypeName(error->type));
1450 http->sslBumpNeed(Ssl::bumpBump);
1451 http->al->ssl.bumpMode = Ssl::bumpBump;
1452 return false;
1453 }
1454
1455 debugs(85, 5, HERE << "SslBump possible, checking ACL");
1456
1457 ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1458 aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1459 return true;
1460 }
1461
1462 /**
1463 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1464 * as ACLFilledChecklist callback
1465 */
1466 static void
sslBumpAccessCheckDoneWrapper(allow_t answer,void * data)1467 sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1468 {
1469 ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1470
1471 if (!calloutContext->httpStateIsValid())
1472 return;
1473 calloutContext->sslBumpAccessCheckDone(answer);
1474 }
1475
1476 void
sslBumpAccessCheckDone(const allow_t & answer)1477 ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1478 {
1479 if (!httpStateIsValid())
1480 return;
1481
1482 const Ssl::BumpMode bumpMode = answer.allowed() ?
1483 static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpSplice;
1484 http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1485 http->al->ssl.bumpMode = bumpMode; // for logging
1486
1487 if (bumpMode == Ssl::bumpTerminate) {
1488 const Comm::ConnectionPointer clientConn = http->getConn() ? http->getConn()->clientConnection : nullptr;
1489 if (Comm::IsConnOpen(clientConn)) {
1490 debugs(85, 3, "closing after Ssl::bumpTerminate ");
1491 clientConn->close();
1492 }
1493 return;
1494 }
1495
1496 http->doCallouts();
1497 }
1498 #endif
1499
1500 /*
1501 * Identify requests that do not go through the store and client side stream
1502 * and forward them to the appropriate location. All other requests, request
1503 * them.
1504 */
1505 void
processRequest()1506 ClientHttpRequest::processRequest()
1507 {
1508 debugs(85, 4, request->method << ' ' << uri);
1509
1510 const bool untouchedConnect = request->method == Http::METHOD_CONNECT && !redirect.status;
1511
1512 #if USE_OPENSSL
1513 if (untouchedConnect && sslBumpNeeded()) {
1514 assert(!request->flags.forceTunnel);
1515 sslBumpStart();
1516 return;
1517 }
1518 #endif
1519
1520 if (untouchedConnect || request->flags.forceTunnel) {
1521 getConn()->stopReading(); // tunnels read for themselves
1522 tunnelStart(this);
1523 return;
1524 }
1525
1526 httpStart();
1527 }
1528
1529 void
httpStart()1530 ClientHttpRequest::httpStart()
1531 {
1532 PROF_start(httpStart);
1533 logType = LOG_TAG_NONE;
1534 debugs(85, 4, logType.c_str() << " for '" << uri << "'");
1535
1536 /* no one should have touched this */
1537 assert(out.offset == 0);
1538 /* Use the Stream Luke */
1539 clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1540 clientStreamRead(node, this, node->readBuffer);
1541 PROF_stop(httpStart);
1542 }
1543
1544 #if USE_OPENSSL
1545
1546 void
sslBumpNeed(Ssl::BumpMode mode)1547 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1548 {
1549 debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1550 sslBumpNeed_ = mode;
1551 }
1552
1553 // called when comm_write has completed
1554 static void
SslBumpEstablish(const Comm::ConnectionPointer &,char *,size_t,Comm::Flag errflag,int,void * data)1555 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1556 {
1557 ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1558 debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1559
1560 assert(r && cbdataReferenceValid(r));
1561 r->sslBumpEstablish(errflag);
1562 }
1563
1564 void
sslBumpEstablish(Comm::Flag errflag)1565 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1566 {
1567 // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1568 if (errflag == Comm::ERR_CLOSING)
1569 return;
1570
1571 if (errflag) {
1572 debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1573 getConn()->clientConnection->close();
1574 return;
1575 }
1576
1577 // We lack HttpReply which logRequest() uses to log the status code.
1578 // TODO: Use HttpReply instead of the "200 Connection established" string.
1579 al->http.code = 200;
1580
1581 #if USE_AUTH
1582 // Preserve authentication info for the ssl-bumped request
1583 if (request->auth_user_request != NULL)
1584 getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1585 #endif
1586
1587 assert(sslBumpNeeded());
1588 getConn()->switchToHttps(request, sslBumpNeed_);
1589 }
1590
1591 void
sslBumpStart()1592 ClientHttpRequest::sslBumpStart()
1593 {
1594 debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1595 "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1596 getConn()->sslBumpMode = sslBumpNeed_;
1597
1598 AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1599 CommIoCbPtrFun(&SslBumpEstablish, this));
1600
1601 if (request->flags.interceptTproxy || request->flags.intercepted) {
1602 CommIoCbParams ¶ms = GetCommParams<CommIoCbParams>(bumpCall);
1603 params.flag = Comm::OK;
1604 params.conn = getConn()->clientConnection;
1605 ScheduleCallHere(bumpCall);
1606 return;
1607 }
1608
1609 // send an HTTP 200 response to kick client SSL negotiation
1610 // TODO: Unify with tunnel.cc and add a Server(?) header
1611 static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1612 Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1613 }
1614
1615 #endif
1616
1617 bool
gotEnough() const1618 ClientHttpRequest::gotEnough() const
1619 {
1620 /** TODO: should be querying the stream. */
1621 int64_t contentLength =
1622 memObject()->getReply()->bodySize(request->method);
1623 assert(contentLength >= 0);
1624
1625 if (out.offset < contentLength)
1626 return false;
1627
1628 return true;
1629 }
1630
1631 void
storeEntry(StoreEntry * newEntry)1632 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1633 {
1634 entry_ = newEntry;
1635 }
1636
1637 void
loggingEntry(StoreEntry * newEntry)1638 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1639 {
1640 if (loggingEntry_)
1641 loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1642
1643 loggingEntry_ = newEntry;
1644
1645 if (loggingEntry_)
1646 loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1647 }
1648
1649 void
initRequest(HttpRequest * aRequest)1650 ClientHttpRequest::initRequest(HttpRequest *aRequest)
1651 {
1652 assignRequest(aRequest);
1653 if (const auto csd = getConn()) {
1654 if (!csd->connectionTag().isEmpty()) {
1655 if (!request->notes)
1656 request->notes = new NotePairs;
1657 // TODO: assert if "clt_conn_tag" already added?
1658 request->notes->add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1659 }
1660 }
1661 // al is created in the constructor
1662 assert(al);
1663 if (!al->request) {
1664 al->request = request;
1665 HTTPMSGLOCK(al->request);
1666 (void)SyncNotes(*al, *request);
1667 }
1668 }
1669
1670 void
resetRequest(HttpRequest * newRequest)1671 ClientHttpRequest::resetRequest(HttpRequest *newRequest)
1672 {
1673 assert(request != newRequest);
1674 clearRequest();
1675 assignRequest(newRequest);
1676 xfree(uri);
1677 uri = SBufToCstring(request->effectiveRequestUri());
1678 }
1679
1680 void
assignRequest(HttpRequest * newRequest)1681 ClientHttpRequest::assignRequest(HttpRequest *newRequest)
1682 {
1683 assert(newRequest);
1684 assert(!request);
1685 const_cast<HttpRequest *&>(request) = newRequest;
1686 HTTPMSGLOCK(request);
1687 setLogUriToRequestUri();
1688 }
1689
1690 void
clearRequest()1691 ClientHttpRequest::clearRequest()
1692 {
1693 HttpRequest *oldRequest = request;
1694 HTTPMSGUNLOCK(oldRequest);
1695 const_cast<HttpRequest *&>(request) = nullptr;
1696 absorbLogUri(nullptr);
1697 }
1698
1699 /*
1700 * doCallouts() - This function controls the order of "callout"
1701 * executions, including non-blocking access control checks, the
1702 * redirector, and ICAP. Previously, these callouts were chained
1703 * together such that "clientAccessCheckDone()" would call
1704 * "clientRedirectStart()" and so on.
1705 *
1706 * The ClientRequestContext (aka calloutContext) class holds certain
1707 * state data for the callout/callback operations. Previously
1708 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1709 * for a short time. ClientRequestContext would then delete itself
1710 * and pass control back to ClientHttpRequest when all callouts
1711 * were finished.
1712 *
1713 * This caused some problems for ICAP because we want to make the
1714 * ICAP callout after checking ACLs, but before checking the no_cache
1715 * list. We can't stuff the ICAP state into the ClientRequestContext
1716 * class because we still need the ICAP state after ClientRequestContext
1717 * goes away.
1718 *
1719 * Note that ClientRequestContext is created before the first call
1720 * to doCallouts().
1721 *
1722 * If one of the callouts notices that ClientHttpRequest is no
1723 * longer valid, it should call cbdataReferenceDone() so that
1724 * ClientHttpRequest's reference count goes to zero and it will get
1725 * deleted. ClientHttpRequest will then delete ClientRequestContext.
1726 *
1727 * Note that we set the _done flags here before actually starting
1728 * the callout. This is strictly for convenience.
1729 */
1730
1731 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1732 nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1733
1734 void
doCallouts()1735 ClientHttpRequest::doCallouts()
1736 {
1737 assert(calloutContext);
1738
1739 if (!calloutContext->error) {
1740 // CVE-2009-0801: verify the Host: header is consistent with other known details.
1741 if (!calloutContext->host_header_verify_done) {
1742 debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1743 calloutContext->host_header_verify_done = true;
1744 calloutContext->hostHeaderVerify();
1745 return;
1746 }
1747
1748 if (!calloutContext->http_access_done) {
1749 debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1750 calloutContext->http_access_done = true;
1751 calloutContext->clientAccessCheck();
1752 return;
1753 }
1754
1755 #if USE_ADAPTATION
1756 if (!calloutContext->adaptation_acl_check_done) {
1757 calloutContext->adaptation_acl_check_done = true;
1758 if (Adaptation::AccessCheck::Start(
1759 Adaptation::methodReqmod, Adaptation::pointPreCache,
1760 request, NULL, calloutContext->http->al, this))
1761 return; // will call callback
1762 }
1763 #endif
1764
1765 if (!calloutContext->redirect_done) {
1766 calloutContext->redirect_done = true;
1767
1768 if (Config.Program.redirect) {
1769 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1770 calloutContext->redirect_state = REDIRECT_PENDING;
1771 calloutContext->clientRedirectStart();
1772 return;
1773 }
1774 }
1775
1776 if (!calloutContext->adapted_http_access_done) {
1777 debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1778 calloutContext->adapted_http_access_done = true;
1779 calloutContext->clientAccessCheck2();
1780 return;
1781 }
1782
1783 if (!calloutContext->store_id_done) {
1784 calloutContext->store_id_done = true;
1785
1786 if (Config.Program.store_id) {
1787 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1788 calloutContext->store_id_state = REDIRECT_PENDING;
1789 calloutContext->clientStoreIdStart();
1790 return;
1791 }
1792 }
1793
1794 if (!calloutContext->interpreted_req_hdrs) {
1795 debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1796 calloutContext->interpreted_req_hdrs = 1;
1797 clientInterpretRequestHeaders(this);
1798 }
1799
1800 if (!calloutContext->no_cache_done) {
1801 calloutContext->no_cache_done = true;
1802
1803 if (Config.accessList.noCache && request->flags.cachable) {
1804 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1805 calloutContext->checkNoCache();
1806 return;
1807 }
1808 }
1809 } // if !calloutContext->error
1810
1811 if (!calloutContext->tosToClientDone) {
1812 calloutContext->tosToClientDone = true;
1813 if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1814 ACLFilledChecklist ch(NULL, request, NULL);
1815 ch.al = calloutContext->http->al;
1816 ch.src_addr = request->client_addr;
1817 ch.my_addr = request->my_addr;
1818 ch.syncAle(request, log_uri);
1819 tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1820 if (tos)
1821 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1822 }
1823 }
1824
1825 if (!calloutContext->nfmarkToClientDone) {
1826 calloutContext->nfmarkToClientDone = true;
1827 if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1828 ACLFilledChecklist ch(NULL, request, NULL);
1829 ch.al = calloutContext->http->al;
1830 ch.src_addr = request->client_addr;
1831 ch.my_addr = request->my_addr;
1832 ch.syncAle(request, log_uri);
1833 nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1834 if (mark)
1835 Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1836 }
1837 }
1838
1839 #if USE_OPENSSL
1840 // Even with calloutContext->error, we call sslBumpAccessCheck() to decide
1841 // whether SslBump applies to this transaction. If it applies, we will
1842 // attempt to bump the client to serve the error.
1843 if (!calloutContext->sslBumpCheckDone) {
1844 calloutContext->sslBumpCheckDone = true;
1845 if (calloutContext->sslBumpAccessCheck())
1846 return;
1847 /* else no ssl bump required*/
1848 }
1849 #endif
1850
1851 if (calloutContext->error) {
1852 // XXX: prformance regression. c_str() reallocates
1853 SBuf storeUriBuf(request->storeId());
1854 const char *storeUri = storeUriBuf.c_str();
1855 StoreEntry *e = storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1856 #if USE_OPENSSL
1857 if (sslBumpNeeded()) {
1858 // We have to serve an error, so bump the client first.
1859 sslBumpNeed(Ssl::bumpClientFirst);
1860 // set final error but delay sending until we bump
1861 Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e, Ssl::bumpClientFirst);
1862 errorAppendEntry(e, calloutContext->error);
1863 calloutContext->error = NULL;
1864 getConn()->setServerBump(srvBump);
1865 e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1866 } else
1867 #endif
1868 {
1869 // send the error to the client now
1870 clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1871 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1872 assert (repContext);
1873 repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1874 errorAppendEntry(e, calloutContext->error);
1875 calloutContext->error = NULL;
1876 if (calloutContext->readNextRequest && getConn())
1877 getConn()->flags.readMore = true; // resume any pipeline reads.
1878 node = (clientStreamNode *)client_stream.tail->data;
1879 clientStreamRead(node, this, node->readBuffer);
1880 e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1881 return;
1882 }
1883 }
1884
1885 cbdataReferenceDone(calloutContext->http);
1886 delete calloutContext;
1887 calloutContext = NULL;
1888 #if HEADERS_LOG
1889
1890 headersLog(0, 1, request->method, request);
1891 #endif
1892
1893 debugs(83, 3, HERE << "calling processRequest()");
1894 processRequest();
1895
1896 #if ICAP_CLIENT
1897 Adaptation::Icap::History::Pointer ih = request->icapHistory();
1898 if (ih != NULL)
1899 ih->logType = logType;
1900 #endif
1901 }
1902
1903 void
setLogUriToRequestUri()1904 ClientHttpRequest::setLogUriToRequestUri()
1905 {
1906 assert(request);
1907 const auto canonicalUri = request->canonicalCleanUrl();
1908 absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1909 }
1910
1911 void
setLogUriToRawUri(const char * rawUri,const HttpRequestMethod & method)1912 ClientHttpRequest::setLogUriToRawUri(const char *rawUri, const HttpRequestMethod &method)
1913 {
1914 assert(rawUri);
1915 // Should(!request);
1916
1917 // TODO: SBuf() performance regression, fix by converting rawUri to SBuf
1918 char *canonicalUri = urlCanonicalCleanWithoutRequest(SBuf(rawUri), method, AnyP::UriScheme());
1919
1920 absorbLogUri(AnyP::Uri::cleanup(canonicalUri));
1921
1922 char *cleanedRawUri = AnyP::Uri::cleanup(rawUri);
1923 al->setVirginUrlForMissingRequest(SBuf(cleanedRawUri));
1924 xfree(cleanedRawUri);
1925 }
1926
1927 void
absorbLogUri(char * aUri)1928 ClientHttpRequest::absorbLogUri(char *aUri)
1929 {
1930 xfree(log_uri);
1931 const_cast<char *&>(log_uri) = aUri;
1932 }
1933
1934 void
setErrorUri(const char * aUri)1935 ClientHttpRequest::setErrorUri(const char *aUri)
1936 {
1937 assert(!uri);
1938 assert(aUri);
1939 // Should(!request);
1940
1941 uri = xstrdup(aUri);
1942 // TODO: SBuf() performance regression, fix by converting setErrorUri() parameter to SBuf
1943 const SBuf errorUri(aUri);
1944 const auto canonicalUri = urlCanonicalCleanWithoutRequest(errorUri, HttpRequestMethod(), AnyP::UriScheme());
1945 absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1946
1947 al->setVirginUrlForMissingRequest(errorUri);
1948 }
1949
1950 #if !_USE_INLINE_
1951 #include "client_side_request.cci"
1952 #endif
1953
1954 // XXX: This should not be a _request_ method. Move range_iter elsewhere.
1955 int64_t
prepPartialResponseGeneration()1956 ClientHttpRequest::prepPartialResponseGeneration()
1957 {
1958 assert(request);
1959 assert(request->range);
1960
1961 range_iter.pos = request->range->begin();
1962 range_iter.end = request->range->end();
1963 range_iter.debt_size = 0;
1964 const auto multipart = request->range->specs.size() > 1;
1965 if (multipart)
1966 range_iter.boundary = rangeBoundaryStr();
1967 range_iter.valid = true; // TODO: Remove.
1968 range_iter.updateSpec(); // TODO: Refactor to initialize rather than update.
1969
1970 assert(range_iter.pos != range_iter.end);
1971 const auto &firstRange = *range_iter.pos;
1972 assert(firstRange);
1973 out.offset = firstRange->offset;
1974
1975 return multipart ? mRangeCLen() : firstRange->length;
1976 }
1977
1978 #if USE_ADAPTATION
1979 /// Initiate an asynchronous adaptation transaction which will call us back.
1980 void
startAdaptation(const Adaptation::ServiceGroupPointer & g)1981 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1982 {
1983 debugs(85, 3, HERE << "adaptation needed for " << this);
1984 assert(!virginHeadSource);
1985 assert(!adaptedBodySource);
1986 virginHeadSource = initiateAdaptation(
1987 new Adaptation::Iterator(request, NULL, al, g));
1988
1989 // we could try to guess whether we can bypass this adaptation
1990 // initiation failure, but it should not really happen
1991 Must(initiated(virginHeadSource));
1992 }
1993
1994 void
noteAdaptationAnswer(const Adaptation::Answer & answer)1995 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1996 {
1997 assert(cbdataReferenceValid(this)); // indicates bug
1998 clearAdaptation(virginHeadSource);
1999 assert(!adaptedBodySource);
2000
2001 switch (answer.kind) {
2002 case Adaptation::Answer::akForward:
2003 handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
2004 break;
2005
2006 case Adaptation::Answer::akBlock:
2007 handleAdaptationBlock(answer);
2008 break;
2009
2010 case Adaptation::Answer::akError:
2011 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
2012 break;
2013 }
2014 }
2015
2016 void
handleAdaptedHeader(HttpMsg * msg)2017 ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
2018 {
2019 assert(msg);
2020
2021 if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
2022
2023 // update the new message to flag whether URL re-writing was done on it
2024 if (request->effectiveRequestUri() != new_req->effectiveRequestUri())
2025 new_req->flags.redirected = true;
2026 resetRequest(new_req);
2027 assert(request->method.id());
2028 } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
2029 debugs(85,3,HERE << "REQMOD reply is HTTP reply");
2030
2031 // subscribe to receive reply body
2032 if (new_rep->body_pipe != NULL) {
2033 adaptedBodySource = new_rep->body_pipe;
2034 int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
2035 assert(consumer_ok);
2036 }
2037
2038 clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2039 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2040 assert(repContext);
2041 repContext->createStoreEntry(request->method, request->flags);
2042
2043 request_satisfaction_mode = true;
2044 request_satisfaction_offset = 0;
2045 storeEntry()->replaceHttpReply(new_rep);
2046 storeEntry()->timestampsSet();
2047
2048 if (!adaptedBodySource) // no body
2049 storeEntry()->complete();
2050 clientGetMoreData(node, this);
2051 }
2052
2053 // we are done with getting headers (but may be receiving body)
2054 clearAdaptation(virginHeadSource);
2055
2056 if (!request_satisfaction_mode)
2057 doCallouts();
2058 }
2059
2060 void
handleAdaptationBlock(const Adaptation::Answer & answer)2061 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
2062 {
2063 request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
2064 AclMatchedName = answer.ruleId.termedBuf();
2065 assert(calloutContext);
2066 calloutContext->clientAccessCheckDone(ACCESS_DENIED);
2067 AclMatchedName = NULL;
2068 }
2069
2070 void
resumeBodyStorage()2071 ClientHttpRequest::resumeBodyStorage()
2072 {
2073 if (!adaptedBodySource)
2074 return;
2075
2076 noteMoreBodyDataAvailable(adaptedBodySource);
2077 }
2078
2079 void
noteMoreBodyDataAvailable(BodyPipe::Pointer)2080 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
2081 {
2082 assert(request_satisfaction_mode);
2083 assert(adaptedBodySource != NULL);
2084
2085 if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
2086 const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
2087
2088 if (spaceAvailable < contentSize ) {
2089 // No or partial body data consuming
2090 typedef NullaryMemFunT<ClientHttpRequest> Dialer;
2091 AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
2092 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
2093 storeEntry()->deferProducer(call);
2094 }
2095
2096 if (!spaceAvailable)
2097 return;
2098
2099 if (spaceAvailable < contentSize )
2100 contentSize = spaceAvailable;
2101
2102 BodyPipeCheckout bpc(*adaptedBodySource);
2103 const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
2104 storeEntry()->write(ioBuf);
2105 // assume StoreEntry::write() writes the entire ioBuf
2106 request_satisfaction_offset += ioBuf.length;
2107 bpc.buf.consume(contentSize);
2108 bpc.checkIn();
2109 }
2110
2111 if (adaptedBodySource->exhausted())
2112 endRequestSatisfaction();
2113 // else wait for more body data
2114 }
2115
2116 void
noteBodyProductionEnded(BodyPipe::Pointer)2117 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
2118 {
2119 assert(!virginHeadSource);
2120 // should we end request satisfaction now?
2121 if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
2122 endRequestSatisfaction();
2123 }
2124
2125 void
endRequestSatisfaction()2126 ClientHttpRequest::endRequestSatisfaction()
2127 {
2128 debugs(85,4, HERE << this << " ends request satisfaction");
2129 assert(request_satisfaction_mode);
2130 stopConsumingFrom(adaptedBodySource);
2131
2132 // TODO: anything else needed to end store entry formation correctly?
2133 storeEntry()->complete();
2134 }
2135
2136 void
noteBodyProducerAborted(BodyPipe::Pointer)2137 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2138 {
2139 assert(!virginHeadSource);
2140 stopConsumingFrom(adaptedBodySource);
2141
2142 debugs(85,3, HERE << "REQMOD body production failed");
2143 if (request_satisfaction_mode) { // too late to recover or serve an error
2144 request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2145 const Comm::ConnectionPointer c = getConn()->clientConnection;
2146 Must(Comm::IsConnOpen(c));
2147 c->close(); // drastic, but we may be writing a response already
2148 } else {
2149 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2150 }
2151 }
2152
2153 void
handleAdaptationFailure(int errDetail,bool bypassable)2154 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2155 {
2156 debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2157
2158 const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2159 const bool usedPipe = request->body_pipe != NULL &&
2160 request->body_pipe->consumedSize() > 0;
2161
2162 if (bypassable && !usedStore && !usedPipe) {
2163 debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2164 if (calloutContext)
2165 doCallouts();
2166 return;
2167 }
2168
2169 debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2170
2171 clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2172 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2173 assert(repContext);
2174
2175 calloutsError(ERR_ICAP_FAILURE, errDetail);
2176
2177 if (calloutContext)
2178 doCallouts();
2179 }
2180
2181 void
callException(const std::exception & ex)2182 ClientHttpRequest::callException(const std::exception &ex)
2183 {
2184 if (const auto clientConn = getConn() ? getConn()->clientConnection : nullptr) {
2185 if (Comm::IsConnOpen(clientConn)) {
2186 debugs(85, 3, "closing after exception: " << ex.what());
2187 clientConn->close(); // initiate orderly top-to-bottom cleanup
2188 return;
2189 }
2190 }
2191 debugs(85, DBG_IMPORTANT, "ClientHttpRequest exception without connection. Ignoring " << ex.what());
2192 // XXX: Normally, we mustStop() but we cannot do that here because it is
2193 // likely to leave Http::Stream and ConnStateData with a dangling http
2194 // pointer. See r13480 or XXX in Http::Stream class description.
2195 }
2196 #endif
2197
2198 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2199 void
calloutsError(const err_type error,const int errDetail)2200 ClientHttpRequest::calloutsError(const err_type error, const int errDetail)
2201 {
2202 // The original author of the code also wanted to pass an errno to
2203 // setReplyToError, but it seems unlikely that the errno reflects the
2204 // true cause of the error at this point, so I did not pass it.
2205 if (calloutContext) {
2206 Ip::Address noAddr;
2207 noAddr.setNoAddr();
2208 ConnStateData * c = getConn();
2209 calloutContext->error = clientBuildError(error, Http::scInternalServerError,
2210 NULL,
2211 c != NULL ? c->clientConnection->remote : noAddr,
2212 request
2213 );
2214 #if USE_AUTH
2215 calloutContext->error->auth_user_request =
2216 c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2217 #endif
2218 calloutContext->error->detailError(errDetail);
2219 calloutContext->readNextRequest = true;
2220 if (c != NULL)
2221 c->expectNoForwarding();
2222 }
2223 //else if(calloutContext == NULL) is it possible?
2224 }
2225
2226