1 /*
2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9 /* DEBUG: section 73 HTTP Request */
10
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "acl/AclSizeLimit.h"
14 #include "acl/FilledChecklist.h"
15 #include "client_side.h"
16 #include "client_side_request.h"
17 #include "dns/LookupDetails.h"
18 #include "Downloader.h"
19 #include "err_detail_type.h"
20 #include "globals.h"
21 #include "gopher.h"
22 #include "http.h"
23 #include "http/one/RequestParser.h"
24 #include "http/Stream.h"
25 #include "HttpHdrCc.h"
26 #include "HttpHeaderRange.h"
27 #include "HttpRequest.h"
28 #include "log/Config.h"
29 #include "MemBuf.h"
30 #include "sbuf/StringConvert.h"
31 #include "SquidConfig.h"
32 #include "Store.h"
33
34 #if USE_AUTH
35 #include "auth/UserRequest.h"
36 #endif
37 #if ICAP_CLIENT
38 #include "adaptation/icap/icap_log.h"
39 #endif
40
HttpRequest(const MasterXaction::Pointer & mx)41 HttpRequest::HttpRequest(const MasterXaction::Pointer &mx) :
42 HttpMsg(hoRequest),
43 masterXaction(mx)
44 {
45 assert(mx);
46 init();
47 }
48
HttpRequest(const HttpRequestMethod & aMethod,AnyP::ProtocolType aProtocol,const char * aSchemeImg,const char * aUrlpath,const MasterXaction::Pointer & mx)49 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath, const MasterXaction::Pointer &mx) :
50 HttpMsg(hoRequest),
51 masterXaction(mx)
52 {
53 assert(mx);
54 static unsigned int id = 1;
55 debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
56 init();
57 initHTTP(aMethod, aProtocol, aSchemeImg, aUrlpath);
58 }
59
~HttpRequest()60 HttpRequest::~HttpRequest()
61 {
62 clean();
63 debugs(93,7, HERE << "destructed, this=" << this);
64 }
65
66 void
initHTTP(const HttpRequestMethod & aMethod,AnyP::ProtocolType aProtocol,const char * aSchemeImg,const char * aUrlpath)67 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath)
68 {
69 method = aMethod;
70 url.setScheme(aProtocol, aSchemeImg);
71 url.path(aUrlpath);
72 }
73
74 void
init()75 HttpRequest::init()
76 {
77 method = Http::METHOD_NONE;
78 url.clear();
79 #if USE_AUTH
80 auth_user_request = NULL;
81 #endif
82 flags = RequestFlags();
83 range = NULL;
84 ims = -1;
85 imslen = 0;
86 lastmod = -1;
87 client_addr.setEmpty();
88 my_addr.setEmpty();
89 body_pipe = NULL;
90 // hier
91 dnsWait = -1;
92 errType = ERR_NONE;
93 errDetail = ERR_DETAIL_NONE;
94 peer_login = NULL; // not allocated/deallocated by this class
95 peer_domain = NULL; // not allocated/deallocated by this class
96 peer_host = NULL;
97 vary_headers = SBuf();
98 myportname = null_string;
99 tag = null_string;
100 #if USE_AUTH
101 extacl_user = null_string;
102 extacl_passwd = null_string;
103 #endif
104 extacl_log = null_string;
105 extacl_message = null_string;
106 pstate = psReadyToParseStartLine;
107 #if FOLLOW_X_FORWARDED_FOR
108 indirect_client_addr.setEmpty();
109 #endif /* FOLLOW_X_FORWARDED_FOR */
110 #if USE_ADAPTATION
111 adaptHistory_ = NULL;
112 #endif
113 #if ICAP_CLIENT
114 icapHistory_ = NULL;
115 #endif
116 rangeOffsetLimit = -2; //a value of -2 means not checked yet
117 forcedBodyContinuation = false;
118 }
119
120 void
clean()121 HttpRequest::clean()
122 {
123 // we used to assert that the pipe is NULL, but now the request only
124 // points to a pipe that is owned and initiated by another object.
125 body_pipe = NULL;
126 #if USE_AUTH
127 auth_user_request = NULL;
128 #endif
129 vary_headers.clear();
130 url.clear();
131
132 header.clean();
133
134 if (cache_control) {
135 delete cache_control;
136 cache_control = NULL;
137 }
138
139 if (range) {
140 delete range;
141 range = NULL;
142 }
143
144 myportname.clean();
145
146 notes = NULL;
147
148 tag.clean();
149 #if USE_AUTH
150 extacl_user.clean();
151 extacl_passwd.clean();
152 #endif
153 extacl_log.clean();
154
155 extacl_message.clean();
156
157 etag.clean();
158
159 #if USE_ADAPTATION
160 adaptHistory_ = NULL;
161 #endif
162 #if ICAP_CLIENT
163 icapHistory_ = NULL;
164 #endif
165 }
166
167 void
reset()168 HttpRequest::reset()
169 {
170 clean();
171 init();
172 }
173
174 HttpRequest *
clone() const175 HttpRequest::clone() const
176 {
177 HttpRequest *copy = new HttpRequest(masterXaction);
178 copy->method = method;
179 // TODO: move common cloning clone to Msg::copyTo() or copy ctor
180 copy->header.append(&header);
181 copy->hdrCacheInit();
182 copy->hdr_sz = hdr_sz;
183 copy->http_ver = http_ver;
184 copy->pstate = pstate; // TODO: should we assert a specific state here?
185 copy->body_pipe = body_pipe;
186
187 copy->url = url;
188
189 // range handled in hdrCacheInit()
190 copy->ims = ims;
191 copy->imslen = imslen;
192 copy->hier = hier; // Is it safe to copy? Should we?
193
194 copy->errType = errType;
195
196 // XXX: what to do with copy->peer_login?
197
198 copy->lastmod = lastmod;
199 copy->etag = etag;
200 copy->vary_headers = vary_headers;
201 // XXX: what to do with copy->peer_domain?
202
203 copy->tag = tag;
204 copy->extacl_log = extacl_log;
205 copy->extacl_message = extacl_message;
206
207 const bool inheritWorked = copy->inheritProperties(this);
208 assert(inheritWorked);
209
210 return copy;
211 }
212
213 bool
inheritProperties(const HttpMsg * aMsg)214 HttpRequest::inheritProperties(const HttpMsg *aMsg)
215 {
216 const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
217 if (!aReq)
218 return false;
219
220 client_addr = aReq->client_addr;
221 #if FOLLOW_X_FORWARDED_FOR
222 indirect_client_addr = aReq->indirect_client_addr;
223 #endif
224 my_addr = aReq->my_addr;
225
226 dnsWait = aReq->dnsWait;
227
228 #if USE_ADAPTATION
229 adaptHistory_ = aReq->adaptHistory();
230 #endif
231 #if ICAP_CLIENT
232 icapHistory_ = aReq->icapHistory();
233 #endif
234
235 // This may be too conservative for the 204 No Content case
236 // may eventually need cloneNullAdaptationImmune() for that.
237 flags = aReq->flags.cloneAdaptationImmune();
238
239 errType = aReq->errType;
240 errDetail = aReq->errDetail;
241 #if USE_AUTH
242 auth_user_request = aReq->auth_user_request;
243 extacl_user = aReq->extacl_user;
244 extacl_passwd = aReq->extacl_passwd;
245 #endif
246
247 myportname = aReq->myportname;
248
249 forcedBodyContinuation = aReq->forcedBodyContinuation;
250
251 // main property is which connection the request was received on (if any)
252 clientConnectionManager = aReq->clientConnectionManager;
253
254 downloader = aReq->downloader;
255
256 notes = aReq->notes;
257
258 sources = aReq->sources;
259 return true;
260 }
261
262 /**
263 * Checks the first line of an HTTP request is valid
264 * currently just checks the request method is present.
265 *
266 * NP: Other errors are left for detection later in the parse.
267 */
268 bool
sanityCheckStartLine(const char * buf,const size_t hdr_len,Http::StatusCode * error)269 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
270 {
271 // content is long enough to possibly hold a reply
272 // 2 being magic size of a 1-byte request method plus space delimiter
273 if (hdr_len < 2) {
274 // this is ony a real error if the headers apparently complete.
275 if (hdr_len > 0) {
276 debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
277 *error = Http::scInvalidHeader;
278 }
279 return false;
280 }
281
282 /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
283 HttpRequestMethod m;
284 m.HttpRequestMethodXXX(buf);
285 if (m == Http::METHOD_NONE) {
286 debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
287 *error = Http::scInvalidHeader;
288 return false;
289 }
290
291 return true;
292 }
293
294 bool
parseFirstLine(const char * start,const char * end)295 HttpRequest::parseFirstLine(const char *start, const char *end)
296 {
297 method.HttpRequestMethodXXX(start);
298
299 if (method == Http::METHOD_NONE)
300 return false;
301
302 // XXX: performance regression, strcspn() over the method bytes a second time.
303 // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
304 const char *t = start + strcspn(start, w_space);
305
306 start = t + strspn(t, w_space); // skip w_space after method
307
308 const char *ver = findTrailingHTTPVersion(start, end);
309
310 if (ver) {
311 end = ver - 1;
312
313 while (xisspace(*end)) // find prev non-space
314 --end;
315
316 ++end; // back to space
317
318 if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
319 debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
320 return false;
321 }
322 } else {
323 http_ver.major = 0;
324 http_ver.minor = 9;
325 }
326
327 if (end < start) // missing URI
328 return false;
329
330 return url.parse(method, SBuf(start, size_t(end-start)));
331 }
332
333 /* swaps out request using httpRequestPack */
334 void
swapOut(StoreEntry * e)335 HttpRequest::swapOut(StoreEntry * e)
336 {
337 assert(e);
338 e->buffer();
339 pack(e);
340 e->flush();
341 }
342
343 /* packs request-line and headers, appends <crlf> terminator */
344 void
pack(Packable * p) const345 HttpRequest::pack(Packable * p) const
346 {
347 assert(p);
348 /* pack request-line */
349 p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
350 SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
351 http_ver.major, http_ver.minor);
352 /* headers */
353 header.packInto(p);
354 /* trailer */
355 p->append("\r\n", 2);
356 }
357
358 /*
359 * A wrapper for debugObj()
360 */
361 void
httpRequestPack(void * obj,Packable * p)362 httpRequestPack(void *obj, Packable *p)
363 {
364 HttpRequest *request = static_cast<HttpRequest*>(obj);
365 request->pack(p);
366 }
367
368 /* returns the length of request line + headers + crlf */
369 int
prefixLen() const370 HttpRequest::prefixLen() const
371 {
372 return method.image().length() + 1 +
373 url.path().length() + 1 +
374 4 + 1 + 3 + 2 +
375 header.len + 2;
376 }
377
378 /* sync this routine when you update HttpRequest struct */
379 void
hdrCacheInit()380 HttpRequest::hdrCacheInit()
381 {
382 HttpMsg::hdrCacheInit();
383
384 assert(!range);
385 range = header.getRange();
386 }
387
388 #if ICAP_CLIENT
389 Adaptation::Icap::History::Pointer
icapHistory() const390 HttpRequest::icapHistory() const
391 {
392 if (!icapHistory_) {
393 if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
394 icapHistory_ = new Adaptation::Icap::History();
395 debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
396 }
397 }
398
399 return icapHistory_;
400 }
401 #endif
402
403 #if USE_ADAPTATION
404 Adaptation::History::Pointer
adaptHistory(bool createIfNone) const405 HttpRequest::adaptHistory(bool createIfNone) const
406 {
407 if (!adaptHistory_ && createIfNone) {
408 adaptHistory_ = new Adaptation::History();
409 debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
410 }
411
412 return adaptHistory_;
413 }
414
415 Adaptation::History::Pointer
adaptLogHistory() const416 HttpRequest::adaptLogHistory() const
417 {
418 return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
419 }
420
421 void
adaptHistoryImport(const HttpRequest & them)422 HttpRequest::adaptHistoryImport(const HttpRequest &them)
423 {
424 if (!adaptHistory_) {
425 adaptHistory_ = them.adaptHistory_; // may be nil
426 } else {
427 // check that histories did not diverge
428 Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
429 }
430 }
431
432 #endif
433
434 bool
multipartRangeRequest() const435 HttpRequest::multipartRangeRequest() const
436 {
437 return (range && range->specs.size() > 1);
438 }
439
440 bool
bodyNibbled() const441 HttpRequest::bodyNibbled() const
442 {
443 return body_pipe != NULL && body_pipe->consumedSize() > 0;
444 }
445
446 void
detailError(err_type aType,int aDetail)447 HttpRequest::detailError(err_type aType, int aDetail)
448 {
449 if (errType || errDetail)
450 debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
451 debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
452 // checking type and detail separately may cause inconsistency, but
453 // may result in more details available if they only become available later
454 if (!errType)
455 errType = aType;
456 if (!errDetail)
457 errDetail = aDetail;
458 }
459
460 void
clearError()461 HttpRequest::clearError()
462 {
463 debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
464 errType = ERR_NONE;
465 errDetail = ERR_DETAIL_NONE;
466 }
467
468 void
packFirstLineInto(Packable * p,bool full_uri) const469 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
470 {
471 const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
472
473 // form HTTP request-line
474 p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
475 SQUIDSBUFPRINT(method.image()),
476 SQUIDSBUFPRINT(tmp),
477 http_ver.major, http_ver.minor);
478 }
479
480 /*
481 * Indicate whether or not we would expect an entity-body
482 * along with this request
483 */
484 bool
expectingBody(const HttpRequestMethod &,int64_t & theSize) const485 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
486 {
487 bool expectBody = false;
488
489 /*
490 * Note: Checks for message validity is in clientIsContentLengthValid().
491 * this just checks if a entity-body is expected based on HTTP message syntax
492 */
493 if (header.chunked()) {
494 expectBody = true;
495 theSize = -1;
496 } else if (content_length >= 0) {
497 expectBody = true;
498 theSize = content_length;
499 } else {
500 expectBody = false;
501 // theSize undefined
502 }
503
504 return expectBody;
505 }
506
507 /*
508 * Create a Request from a URL and METHOD.
509 *
510 * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
511 * If the request cannot be created cleanly, NULL is returned
512 */
513 HttpRequest *
FromUrl(const SBuf & url,const MasterXaction::Pointer & mx,const HttpRequestMethod & method)514 HttpRequest::FromUrl(const SBuf &url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method)
515 {
516 std::unique_ptr<HttpRequest> req(new HttpRequest(mx));
517 if (req->url.parse(method, url)) {
518 req->method = method;
519 return req.release();
520 }
521 return nullptr;
522 }
523
524 HttpRequest *
FromUrlXXX(const char * url,const MasterXaction::Pointer & mx,const HttpRequestMethod & method)525 HttpRequest::FromUrlXXX(const char * url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method)
526 {
527 return FromUrl(SBuf(url), mx, method);
528 }
529
530 /**
531 * Are responses to this request possible cacheable ?
532 * If false then no matter what the response must not be cached.
533 */
534 bool
maybeCacheable()535 HttpRequest::maybeCacheable()
536 {
537 // Intercepted request with Host: header which cannot be trusted.
538 // Because it failed verification, or someone bypassed the security tests
539 // we cannot cache the reponse for sharing between clients.
540 // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
541 if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
542 return false;
543
544 switch (url.getScheme()) {
545 case AnyP::PROTO_HTTP:
546 case AnyP::PROTO_HTTPS:
547 if (!method.respMaybeCacheable())
548 return false;
549
550 // RFC 7234 section 5.2.1.5:
551 // "cache MUST NOT store any part of either this request or any response to it"
552 //
553 // NP: refresh_pattern ignore-no-store only applies to response messages
554 // this test is handling request message CC header.
555 if (!flags.ignoreCc && cache_control && cache_control->hasNoStore())
556 return false;
557 break;
558
559 case AnyP::PROTO_GOPHER:
560 if (!gopherCachable(this))
561 return false;
562 break;
563
564 case AnyP::PROTO_CACHE_OBJECT:
565 return false;
566
567 //case AnyP::PROTO_FTP:
568 default:
569 break;
570 }
571
572 return true;
573 }
574
575 bool
conditional() const576 HttpRequest::conditional() const
577 {
578 return flags.ims ||
579 header.has(Http::HdrType::IF_MATCH) ||
580 header.has(Http::HdrType::IF_NONE_MATCH);
581 }
582
583 void
recordLookup(const Dns::LookupDetails & dns)584 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
585 {
586 if (dns.wait >= 0) { // known delay
587 if (dnsWait >= 0) // have recorded DNS wait before
588 dnsWait += dns.wait;
589 else
590 dnsWait = dns.wait;
591 }
592 }
593
594 int64_t
getRangeOffsetLimit()595 HttpRequest::getRangeOffsetLimit()
596 {
597 /* -2 is the starting value of rangeOffsetLimit.
598 * If it is -2, that means we haven't checked it yet.
599 * Otherwise, return the current value */
600 if (rangeOffsetLimit != -2)
601 return rangeOffsetLimit;
602
603 rangeOffsetLimit = 0; // default value for rangeOffsetLimit
604
605 ACLFilledChecklist ch(NULL, this, NULL);
606 ch.src_addr = client_addr;
607 ch.my_addr = my_addr;
608
609 for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
610 /* if there is no ACL list or if the ACLs listed match use this limit value */
611 if (!l->aclList || ch.fastCheck(l->aclList).allowed()) {
612 rangeOffsetLimit = l->size; // may be -1
613 debugs(58, 4, rangeOffsetLimit);
614 break;
615 }
616 }
617
618 return rangeOffsetLimit;
619 }
620
621 void
ignoreRange(const char * reason)622 HttpRequest::ignoreRange(const char *reason)
623 {
624 if (range) {
625 debugs(73, 3, static_cast<void*>(range) << " for " << reason);
626 delete range;
627 range = NULL;
628 }
629 // Some callers also reset isRanged but it may not be safe for all callers:
630 // isRanged is used to determine whether a weak ETag comparison is allowed,
631 // and that check should not ignore the Range header if it was present.
632 // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
633 }
634
635 bool
canHandle1xx() const636 HttpRequest::canHandle1xx() const
637 {
638 // old clients do not support 1xx unless they sent Expect: 100-continue
639 // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
640 if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
641 return false;
642
643 // others must support 1xx control messages
644 return true;
645 }
646
647 ConnStateData *
pinnedConnection()648 HttpRequest::pinnedConnection()
649 {
650 if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
651 return clientConnectionManager.get();
652 return NULL;
653 }
654
655 const SBuf
storeId()656 HttpRequest::storeId()
657 {
658 if (store_id.size() != 0) {
659 debugs(73, 3, "sent back store_id: " << store_id);
660 return StringToSBuf(store_id);
661 }
662 debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
663 return effectiveRequestUri();
664 }
665
666 const SBuf &
effectiveRequestUri() const667 HttpRequest::effectiveRequestUri() const
668 {
669 if (method.id() == Http::METHOD_CONNECT || url.getScheme() == AnyP::PROTO_AUTHORITY_FORM)
670 return url.authority(true); // host:port
671 return url.absolute();
672 }
673
674 char *
canonicalCleanUrl() const675 HttpRequest::canonicalCleanUrl() const
676 {
677 return urlCanonicalCleanWithoutRequest(effectiveRequestUri(), method, url.getScheme());
678 }
679
680 void
manager(const CbcPointer<ConnStateData> & aMgr,const AccessLogEntryPointer & al)681 HttpRequest::manager(const CbcPointer<ConnStateData> &aMgr, const AccessLogEntryPointer &al)
682 {
683 clientConnectionManager = aMgr;
684
685 if (!clientConnectionManager.valid())
686 return;
687
688 AnyP::PortCfgPointer port = clientConnectionManager->port;
689 if (port) {
690 myportname = port->name;
691 flags.ignoreCc = port->ignore_cc;
692 }
693
694 if (auto clientConnection = clientConnectionManager->clientConnection) {
695 client_addr = clientConnection->remote; // XXX: remove request->client_addr member.
696 #if FOLLOW_X_FORWARDED_FOR
697 // indirect client gets stored here because it is an HTTP header result (from X-Forwarded-For:)
698 // not details about the TCP connection itself
699 indirect_client_addr = clientConnection->remote;
700 #endif /* FOLLOW_X_FORWARDED_FOR */
701 my_addr = clientConnection->local;
702
703 flags.intercepted = ((clientConnection->flags & COMM_INTERCEPTION) != 0);
704 flags.interceptTproxy = ((clientConnection->flags & COMM_TRANSPARENT) != 0 ) ;
705 const bool proxyProtocolPort = port ? port->flags.proxySurrogate : false;
706 if (flags.interceptTproxy && !proxyProtocolPort) {
707 if (Config.accessList.spoof_client_ip) {
708 ACLFilledChecklist *checklist = new ACLFilledChecklist(Config.accessList.spoof_client_ip, this, clientConnection->rfc931);
709 checklist->al = al;
710 checklist->syncAle(this, nullptr);
711 flags.spoofClientIp = checklist->fastCheck().allowed();
712 delete checklist;
713 } else
714 flags.spoofClientIp = true;
715 } else
716 flags.spoofClientIp = false;
717 }
718 }
719
720