1 /*
2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9 /* DEBUG: section 88 Client-side Reply Routines */
10
11 #include "squid.h"
12 #include "acl/FilledChecklist.h"
13 #include "acl/Gadgets.h"
14 #include "anyp/PortCfg.h"
15 #include "client_side_reply.h"
16 #include "errorpage.h"
17 #include "ETag.h"
18 #include "fd.h"
19 #include "fde.h"
20 #include "format/Token.h"
21 #include "FwdState.h"
22 #include "globals.h"
23 #include "http/Stream.h"
24 #include "HttpHeaderTools.h"
25 #include "HttpReply.h"
26 #include "HttpRequest.h"
27 #include "ip/QosConfig.h"
28 #include "ipcache.h"
29 #include "log/access_log.h"
30 #include "MemObject.h"
31 #include "mime_header.h"
32 #include "neighbors.h"
33 #include "refresh.h"
34 #include "RequestFlags.h"
35 #include "SquidConfig.h"
36 #include "SquidTime.h"
37 #include "Store.h"
38 #include "StrList.h"
39 #include "tools.h"
40 #if USE_AUTH
41 #include "auth/UserRequest.h"
42 #endif
43 #if USE_DELAY_POOLS
44 #include "DelayPools.h"
45 #endif
46 #if USE_SQUID_ESI
47 #include "esi/Esi.h"
48 #endif
49
50 #include <memory>
51
52 CBDATA_CLASS_INIT(clientReplyContext);
53
54 /* Local functions */
55 extern "C" CSS clientReplyStatus;
56 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
57
58 /* privates */
59
~clientReplyContext()60 clientReplyContext::~clientReplyContext()
61 {
62 deleting = true;
63 /* This may trigger a callback back into SendMoreData as the cbdata
64 * is still valid
65 */
66 removeClientStoreReference(&sc, http);
67 /* old_entry might still be set if we didn't yet get the reply
68 * code in HandleIMSReply() */
69 removeStoreReference(&old_sc, &old_entry);
70 safe_free(tempBuffer.data);
71 cbdataReferenceDone(http);
72 HTTPMSGUNLOCK(reply);
73 }
74
clientReplyContext(ClientHttpRequest * clientContext)75 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
76 purgeStatus(Http::scNone),
77 lookingforstore(0),
78 http(cbdataReference(clientContext)),
79 headers_sz(0),
80 sc(NULL),
81 old_reqsize(0),
82 reqsize(0),
83 reqofs(0),
84 #if USE_CACHE_DIGESTS
85 lookup_type(NULL),
86 #endif
87 ourNode(NULL),
88 reply(NULL),
89 old_entry(NULL),
90 old_sc(NULL),
91 old_lastmod(-1),
92 deleting(false),
93 collapsedRevalidation(crNone)
94 {
95 *tempbuf = 0;
96 }
97
98 /** Create an error in the store awaiting the client side to read it.
99 *
100 * This may be better placed in the clientStream logic, but it has not been
101 * relocated there yet
102 */
103 void
setReplyToError(err_type err,Http::StatusCode status,const HttpRequestMethod & method,char const * uri,Ip::Address & addr,HttpRequest * failedrequest,const char * unparsedrequest,Auth::UserRequest::Pointer auth_user_request)104 clientReplyContext::setReplyToError(
105 err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
106 Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
107 #if USE_AUTH
108 Auth::UserRequest::Pointer auth_user_request
109 #else
110 void*
111 #endif
112 )
113 {
114 ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
115
116 if (unparsedrequest)
117 errstate->request_hdrs = xstrdup(unparsedrequest);
118
119 #if USE_AUTH
120 errstate->auth_user_request = auth_user_request;
121 #endif
122 setReplyToError(method, errstate);
123 }
124
setReplyToError(const HttpRequestMethod & method,ErrorState * errstate)125 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
126 {
127 if (errstate->httpStatus == Http::scNotImplemented && http->request)
128 /* prevent confusion over whether we default to persistent or not */
129 http->request->flags.proxyKeepalive = false;
130
131 http->al->http.code = errstate->httpStatus;
132
133 if (http->request)
134 http->request->ignoreRange("responding with a Squid-generated error");
135
136 createStoreEntry(method, RequestFlags());
137 assert(errstate->callback_data == NULL);
138 errorAppendEntry(http->storeEntry(), errstate);
139 /* Now the caller reads to get this */
140 }
141
142 void
setReplyToReply(HttpReply * futureReply)143 clientReplyContext::setReplyToReply(HttpReply *futureReply)
144 {
145 Must(futureReply);
146 http->al->http.code = futureReply->sline.status();
147
148 HttpRequestMethod method;
149 if (http->request) { // nil on responses to unparsable requests
150 http->request->ignoreRange("responding with a Squid-generated reply");
151 method = http->request->method;
152 }
153
154 createStoreEntry(method, RequestFlags());
155
156 http->storeEntry()->storeErrorResponse(futureReply);
157 /* Now the caller reads to get futureReply */
158 }
159
160 // Assumes that the entry contains an error response without Content-Range.
161 // To use with regular entries, make HTTP Range header removal conditional.
setReplyToStoreEntry(StoreEntry * entry,const char * reason)162 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
163 {
164 entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
165 sc = storeClientListAdd(entry, this);
166 #if USE_DELAY_POOLS
167 sc->setDelayId(DelayId::DelayClient(http));
168 #endif
169 reqofs = 0;
170 reqsize = 0;
171 if (http->request)
172 http->request->ignoreRange(reason);
173 flags.storelogiccomplete = 1;
174 http->storeEntry(entry);
175 }
176
177 void
removeStoreReference(store_client ** scp,StoreEntry ** ep)178 clientReplyContext::removeStoreReference(store_client ** scp,
179 StoreEntry ** ep)
180 {
181 StoreEntry *e;
182 store_client *sc_tmp = *scp;
183
184 if ((e = *ep) != NULL) {
185 *ep = NULL;
186 storeUnregister(sc_tmp, e, this);
187 *scp = NULL;
188 e->unlock("clientReplyContext::removeStoreReference");
189 }
190 }
191
192 void
removeClientStoreReference(store_client ** scp,ClientHttpRequest * aHttpRequest)193 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
194 {
195 StoreEntry *reference = aHttpRequest->storeEntry();
196 removeStoreReference(scp, &reference);
197 aHttpRequest->storeEntry(reference);
198 }
199
200 void
saveState()201 clientReplyContext::saveState()
202 {
203 assert(old_sc == NULL);
204 debugs(88, 3, "clientReplyContext::saveState: saving store context");
205 old_entry = http->storeEntry();
206 old_sc = sc;
207 old_lastmod = http->request->lastmod;
208 old_etag = http->request->etag;
209 old_reqsize = reqsize;
210 tempBuffer.offset = reqofs;
211 /* Prevent accessing the now saved entries */
212 http->storeEntry(NULL);
213 sc = NULL;
214 reqsize = 0;
215 reqofs = 0;
216 }
217
218 void
restoreState()219 clientReplyContext::restoreState()
220 {
221 assert(old_sc != NULL);
222 debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
223 removeClientStoreReference(&sc, http);
224 http->storeEntry(old_entry);
225 sc = old_sc;
226 reqsize = old_reqsize;
227 reqofs = tempBuffer.offset;
228 http->request->lastmod = old_lastmod;
229 http->request->etag = old_etag;
230 /* Prevent accessed the old saved entries */
231 old_entry = NULL;
232 old_sc = NULL;
233 old_lastmod = -1;
234 old_etag.clean();
235 old_reqsize = 0;
236 tempBuffer.offset = 0;
237 }
238
239 void
startError(ErrorState * err)240 clientReplyContext::startError(ErrorState * err)
241 {
242 createStoreEntry(http->request->method, RequestFlags());
243 triggerInitialStoreRead();
244 errorAppendEntry(http->storeEntry(), err);
245 }
246
247 clientStreamNode *
getNextNode() const248 clientReplyContext::getNextNode() const
249 {
250 return (clientStreamNode *)ourNode->node.next->data;
251 }
252
253 /* This function is wrong - the client parameters don't include the
254 * header offset
255 */
256 void
triggerInitialStoreRead()257 clientReplyContext::triggerInitialStoreRead()
258 {
259 /* when confident, 0 becomes reqofs, and then this factors into
260 * startSendProcess
261 */
262 assert(reqofs == 0);
263 StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
264 storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
265 }
266
267 /* there is an expired entry in the store.
268 * setup a temporary buffer area and perform an IMS to the origin
269 */
270 void
processExpired()271 clientReplyContext::processExpired()
272 {
273 const char *url = storeId();
274 debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
275 const time_t lastmod = http->storeEntry()->lastModified();
276 assert(lastmod >= 0);
277 /*
278 * check if we are allowed to contact other servers
279 * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
280 * a stale entry *if* it matches client requirements
281 */
282
283 if (http->onlyIfCached()) {
284 processOnlyIfCachedMiss();
285 return;
286 }
287
288 http->logType = LOG_TCP_REFRESH;
289 http->request->flags.refresh = true;
290 #if STORE_CLIENT_LIST_DEBUG
291 /* Prevent a race with the store client memory free routines
292 */
293 assert(storeClientIsThisAClient(sc, this));
294 #endif
295 /* Prepare to make a new temporary request */
296 saveState();
297
298 // TODO: support collapsed revalidation for Vary-controlled entries
299 const bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
300 !Store::Root().smpAware() &&
301 http->request->vary_headers.isEmpty();
302
303 StoreEntry *entry = nullptr;
304 if (collapsingAllowed) {
305 if ((entry = storeGetPublicByRequest(http->request, ksRevalidation)))
306 entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
307 }
308
309 if (entry) {
310 entry->ensureMemObject(url, http->log_uri, http->request->method);
311 debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
312 collapsedRevalidation = crSlave;
313 } else {
314 entry = storeCreateEntry(url,
315 http->log_uri, http->request->flags, http->request->method);
316 /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
317
318 if (collapsingAllowed && Store::Root().allowCollapsing(entry, http->request->flags, http->request->method)) {
319 debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
320 collapsedRevalidation = crInitiator;
321 } else {
322 collapsedRevalidation = crNone;
323 }
324 }
325
326 sc = storeClientListAdd(entry, this);
327 #if USE_DELAY_POOLS
328 /* delay_id is already set on original store client */
329 sc->setDelayId(DelayId::DelayClient(http));
330 #endif
331
332 http->request->lastmod = lastmod;
333
334 if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
335 ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
336 if (old_entry->hasEtag(etag) && !etag.weak)
337 http->request->etag = etag.str;
338 }
339
340 debugs(88, 5, "lastmod " << entry->lastModified());
341 http->storeEntry(entry);
342 assert(http->out.offset == 0);
343 assert(http->request->clientConnectionManager == http->getConn());
344
345 if (collapsedRevalidation != crSlave) {
346 /*
347 * A refcounted pointer so that FwdState stays around as long as
348 * this clientReplyContext does
349 */
350 Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
351 FwdState::Start(conn, http->storeEntry(), http->request, http->al);
352 }
353 /* Register with storage manager to receive updates when data comes in. */
354
355 if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
356 debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
357
358 {
359 /* start counting the length from 0 */
360 StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
361 storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
362 }
363 }
364
365 void
sendClientUpstreamResponse()366 clientReplyContext::sendClientUpstreamResponse()
367 {
368 StoreIOBuffer tempresult;
369 removeStoreReference(&old_sc, &old_entry);
370
371 if (collapsedRevalidation)
372 http->storeEntry()->clearPublicKeyScope();
373
374 /* here the data to send is the data we just received */
375 tempBuffer.offset = 0;
376 old_reqsize = 0;
377 /* sendMoreData tracks the offset as well.
378 * Force it back to zero */
379 reqofs = 0;
380 assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
381 /* TODO: provide sendMoreData with the ready parsed reply */
382 tempresult.length = reqsize;
383 tempresult.data = tempbuf;
384 sendMoreData(tempresult);
385 }
386
387 void
HandleIMSReply(void * data,StoreIOBuffer result)388 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
389 {
390 clientReplyContext *context = (clientReplyContext *)data;
391 context->handleIMSReply(result);
392 }
393
394 void
sendClientOldEntry()395 clientReplyContext::sendClientOldEntry()
396 {
397 /* Get the old request back */
398 restoreState();
399 /* here the data to send is in the next nodes buffers already */
400 assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
401 /* sendMoreData tracks the offset as well.
402 * Force it back to zero */
403 reqofs = 0;
404 StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
405 sendMoreData(tempresult);
406 }
407
408 /* This is the workhorse of the HandleIMSReply callback.
409 *
410 * It is called when we've got data back from the origin following our
411 * IMS request to revalidate a stale entry.
412 */
413 void
handleIMSReply(StoreIOBuffer result)414 clientReplyContext::handleIMSReply(StoreIOBuffer result)
415 {
416 if (deleting)
417 return;
418
419 debugs(88, 3, http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes");
420
421 if (http->storeEntry() == NULL)
422 return;
423
424 if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
425 return;
426
427 if (collapsedRevalidation == crSlave && !http->storeEntry()->mayStartHitting()) {
428 debugs(88, 3, "CF slave hit private non-shareable " << *http->storeEntry() << ". MISS");
429 // restore context to meet processMiss() expectations
430 restoreState();
431 http->logType = LOG_TCP_MISS;
432 processMiss();
433 return;
434 }
435
436 /* update size of the request */
437 reqsize = result.length + reqofs;
438
439 const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
440
441 // request to origin was aborted
442 if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
443 debugs(88, 3, "request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client");
444 http->logType = LOG_TCP_REFRESH_FAIL_OLD;
445 sendClientOldEntry();
446 }
447
448 const HttpReply *old_rep = old_entry->getReply();
449
450 // origin replied 304
451 if (status == Http::scNotModified) {
452 http->logType = LOG_TCP_REFRESH_UNMODIFIED;
453 http->request->flags.staleIfHit = false; // old_entry is no longer stale
454
455 // update headers on existing entry
456 Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
457
458 // if client sent IMS
459
460 if (http->request->flags.ims && !old_entry->modifiedSince(http->request->ims, http->request->imslen)) {
461 // forward the 304 from origin
462 debugs(88, 3, "origin replied 304, revalidating existing entry and forwarding 304 to client");
463 sendClientUpstreamResponse();
464 } else {
465 // send existing entry, it's still valid
466 debugs(88, 3, "origin replied 304, revalidating existing entry and sending " <<
467 old_rep->sline.status() << " to client");
468 sendClientOldEntry();
469 }
470 }
471
472 // origin replied with a non-error code
473 else if (status > Http::scNone && status < Http::scInternalServerError) {
474 const HttpReply *new_rep = http->storeEntry()->getReply();
475 // RFC 7234 section 4: a cache MUST use the most recent response
476 // (as determined by the Date header field)
477 if (new_rep->olderThan(old_rep)) {
478 http->logType.err.ignored = true;
479 debugs(88, 3, "origin replied " << status <<
480 " but with an older date header, sending old entry (" <<
481 old_rep->sline.status() << ") to client");
482 sendClientOldEntry();
483 } else {
484 http->logType = LOG_TCP_REFRESH_MODIFIED;
485 debugs(88, 3, "origin replied " << status <<
486 ", replacing existing entry and forwarding to client");
487 sendClientUpstreamResponse();
488 }
489 }
490
491 // origin replied with an error
492 else if (http->request->flags.failOnValidationError) {
493 http->logType = LOG_TCP_REFRESH_FAIL_ERR;
494 debugs(88, 3, "origin replied with error " << status <<
495 ", forwarding to client due to fail_on_validation_err");
496 sendClientUpstreamResponse();
497 } else {
498 // ignore and let client have old entry
499 http->logType = LOG_TCP_REFRESH_FAIL_OLD;
500 debugs(88, 3, "origin replied with error " <<
501 status << ", sending old entry (" << old_rep->sline.status() << ") to client");
502 sendClientOldEntry();
503 }
504 }
505
506 SQUIDCEXTERN CSR clientGetMoreData;
507 SQUIDCEXTERN CSD clientReplyDetach;
508
509 /**
510 * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
511 * have been parsed. Normally this should be a single call, but
512 * it might take more than one. As soon as we have the headers,
513 * we hand off to clientSendMoreData, processExpired, or
514 * processMiss.
515 */
516 void
CacheHit(void * data,StoreIOBuffer result)517 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
518 {
519 clientReplyContext *context = (clientReplyContext *)data;
520 context->cacheHit(result);
521 }
522
523 /**
524 * Process a possible cache HIT.
525 */
526 void
cacheHit(StoreIOBuffer result)527 clientReplyContext::cacheHit(StoreIOBuffer result)
528 {
529 /** Ignore if the HIT object is being deleted. */
530 if (deleting) {
531 debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
532 return;
533 }
534
535 StoreEntry *e = http->storeEntry();
536
537 HttpRequest *r = http->request;
538
539 debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
540
541 if (http->storeEntry() == NULL) {
542 debugs(88, 3, "clientCacheHit: request aborted");
543 return;
544 } else if (result.flags.error) {
545 /* swap in failure */
546 debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
547 http->logType = LOG_TCP_SWAPFAIL_MISS;
548 removeClientStoreReference(&sc, http);
549 processMiss();
550 return;
551 }
552
553 // The previously identified hit suddenly became unsharable!
554 // This is common for collapsed forwarding slaves but might also
555 // happen to regular hits because we are called asynchronously.
556 if (!e->mayStartHitting()) {
557 debugs(88, 3, "unsharable " << *e << ". MISS");
558 http->logType = LOG_TCP_MISS;
559 processMiss();
560 return;
561 }
562
563 if (result.length == 0) {
564 debugs(88, 5, "store IO buffer has no content. MISS");
565 /* the store couldn't get enough data from the file for us to id the
566 * object
567 */
568 /* treat as a miss */
569 http->logType = LOG_TCP_MISS;
570 processMiss();
571 return;
572 }
573
574 assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
575 /* update size of the request */
576 reqsize = result.length + reqofs;
577
578 /*
579 * Got the headers, now grok them
580 */
581 assert(http->logType.oldType == LOG_TCP_HIT);
582
583 if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
584 debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
585 http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
586 processMiss();
587 return;
588 }
589
590 switch (varyEvaluateMatch(e, r)) {
591
592 case VARY_NONE:
593 /* No variance detected. Continue as normal */
594 break;
595
596 case VARY_MATCH:
597 /* This is the correct entity for this request. Continue */
598 debugs(88, 2, "clientProcessHit: Vary MATCH!");
599 break;
600
601 case VARY_OTHER:
602 /* This is not the correct entity for this request. We need
603 * to requery the cache.
604 */
605 removeClientStoreReference(&sc, http);
606 e = NULL;
607 /* Note: varyEvalyateMatch updates the request with vary information
608 * so we only get here once. (it also takes care of cancelling loops)
609 */
610 debugs(88, 2, "clientProcessHit: Vary detected!");
611 clientGetMoreData(ourNode, http);
612 return;
613
614 case VARY_CANCEL:
615 /* varyEvaluateMatch found a object loop. Process as miss */
616 debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
617 http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
618 processMiss();
619 return;
620 }
621
622 if (r->method == Http::METHOD_PURGE) {
623 debugs(88, 5, "PURGE gets a HIT");
624 removeClientStoreReference(&sc, http);
625 e = NULL;
626 purgeRequest();
627 return;
628 }
629
630 if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
631 debugs(88, 5, "negative-HIT");
632 http->logType = LOG_TCP_NEGATIVE_HIT;
633 sendMoreData(result);
634 return;
635 } else if (blockedHit()) {
636 debugs(88, 5, "send_hit forces a MISS");
637 http->logType = LOG_TCP_MISS;
638 processMiss();
639 return;
640 } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
641 debugs(88, 5, "clientCacheHit: in refreshCheck() block");
642 /*
643 * We hold a stale copy; it needs to be validated
644 */
645 /*
646 * The 'needValidation' flag is used to prevent forwarding
647 * loops between siblings. If our copy of the object is stale,
648 * then we should probably only use parents for the validation
649 * request. Otherwise two siblings could generate a loop if
650 * both have a stale version of the object.
651 */
652 r->flags.needValidation = true;
653
654 if (e->lastModified() < 0) {
655 debugs(88, 3, "validate HIT object? NO. Can't calculate entry modification time. Do MISS.");
656 /*
657 * We cannot revalidate entries without knowing their
658 * modification time.
659 * XXX: BUG 1890 objects without Date do not get one added.
660 */
661 http->logType = LOG_TCP_MISS;
662 processMiss();
663 } else if (r->flags.noCache) {
664 debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
665 /*
666 * This did not match a refresh pattern that overrides no-cache
667 * we should honour the client no-cache header.
668 */
669 http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
670 processMiss();
671 } else if (r->url.getScheme() == AnyP::PROTO_HTTP || r->url.getScheme() == AnyP::PROTO_HTTPS) {
672 debugs(88, 3, "validate HIT object? YES.");
673 /*
674 * Object needs to be revalidated
675 * XXX This could apply to FTP as well, if Last-Modified is known.
676 */
677 processExpired();
678 } else {
679 debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
680 /*
681 * We don't know how to re-validate other protocols. Handle
682 * them as if the object has expired.
683 */
684 http->logType = LOG_TCP_MISS;
685 processMiss();
686 }
687 return;
688 } else if (r->conditional()) {
689 debugs(88, 5, "conditional HIT");
690 if (processConditional(result))
691 return;
692 }
693
694 /*
695 * plain ol' cache hit
696 */
697 debugs(88, 5, "plain old HIT");
698
699 #if USE_DELAY_POOLS
700 if (e->store_status != STORE_OK)
701 http->logType = LOG_TCP_MISS;
702 else
703 #endif
704 if (e->mem_status == IN_MEMORY)
705 http->logType = LOG_TCP_MEM_HIT;
706 else if (Config.onoff.offline)
707 http->logType = LOG_TCP_OFFLINE_HIT;
708
709 sendMoreData(result);
710 }
711
712 /**
713 * Prepare to fetch the object as it's a cache miss of some kind.
714 */
715 void
processMiss()716 clientReplyContext::processMiss()
717 {
718 char *url = http->uri;
719 HttpRequest *r = http->request;
720 ErrorState *err = NULL;
721 debugs(88, 4, r->method << ' ' << url);
722
723 /**
724 * We might have a left-over StoreEntry from a failed cache hit
725 * or IMS request.
726 */
727 if (http->storeEntry()) {
728 if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
729 debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
730 debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
731 http->storeEntry()->dump(1);
732 }
733
734 removeClientStoreReference(&sc, http);
735 }
736
737 /** Check if its a PURGE request to be actioned. */
738 if (r->method == Http::METHOD_PURGE) {
739 purgeRequest();
740 return;
741 }
742
743 /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
744 if (r->method == Http::METHOD_OTHER) {
745 purgeAllCached();
746 }
747
748 /** Check if 'only-if-cached' flag is set. Action if so. */
749 if (http->onlyIfCached()) {
750 processOnlyIfCachedMiss();
751 return;
752 }
753
754 Comm::ConnectionPointer conn = http->getConn() != nullptr ? http->getConn()->clientConnection : nullptr;
755 /// Deny loops
756 if (r->flags.loopDetected) {
757 http->al->http.code = Http::scForbidden;
758 Ip::Address tmp_noaddr;
759 tmp_noaddr.setNoAddr();
760 err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr, conn ? conn->remote : tmp_noaddr, http->request);
761 createStoreEntry(r->method, RequestFlags());
762 errorAppendEntry(http->storeEntry(), err);
763 triggerInitialStoreRead();
764 return;
765 } else {
766 assert(http->out.offset == 0);
767 createStoreEntry(r->method, r->flags);
768 triggerInitialStoreRead();
769
770 if (http->redirect.status) {
771 HttpReply *rep = new HttpReply;
772 http->logType = LOG_TCP_REDIRECT;
773 http->storeEntry()->releaseRequest();
774 rep->redirect(http->redirect.status, http->redirect.location);
775 http->storeEntry()->replaceHttpReply(rep);
776 http->storeEntry()->complete();
777 return;
778 }
779
780 assert(r->clientConnectionManager == http->getConn());
781
782 /** Start forwarding to get the new object from network */
783 FwdState::Start(conn, http->storeEntry(), r, http->al);
784 }
785 }
786
787 /**
788 * client issued a request with an only-if-cached cache-control directive;
789 * we did not find a cached object that can be returned without
790 * contacting other servers;
791 * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
792 */
793 void
processOnlyIfCachedMiss()794 clientReplyContext::processOnlyIfCachedMiss()
795 {
796 debugs(88, 4, http->request->method << ' ' << http->uri);
797 http->al->http.code = Http::scGatewayTimeout;
798 Ip::Address tmp_noaddr;
799 tmp_noaddr.setNoAddr();
800 ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
801 http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
802 http->request);
803 removeClientStoreReference(&sc, http);
804 startError(err);
805 }
806
807 /// process conditional request from client
808 bool
processConditional(StoreIOBuffer & result)809 clientReplyContext::processConditional(StoreIOBuffer &result)
810 {
811 StoreEntry *const e = http->storeEntry();
812
813 if (e->getReply()->sline.status() != Http::scOkay) {
814 debugs(88, 4, "Reply code " << e->getReply()->sline.status() << " != 200");
815 http->logType = LOG_TCP_MISS;
816 processMiss();
817 return true;
818 }
819
820 HttpRequest &r = *http->request;
821
822 if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
823 // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
824 sendPreconditionFailedError();
825 return true;
826 }
827
828 if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
829 // RFC 7232: If-None-Match recipient MUST ignore IMS
830 r.flags.ims = false;
831 r.ims = -1;
832 r.imslen = 0;
833 r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
834
835 if (e->hasIfNoneMatchEtag(r)) {
836 sendNotModifiedOrPreconditionFailedError();
837 return true;
838 }
839
840 // None-Match is true (no ETag matched); treat as an unconditional hit
841 return false;
842 }
843
844 if (r.flags.ims) {
845 // handle If-Modified-Since requests from the client
846 if (e->modifiedSince(r.ims, r.imslen)) {
847 // Modified-Since is true; treat as an unconditional hit
848 return false;
849
850 } else {
851 // otherwise reply with 304 Not Modified
852 sendNotModified();
853 }
854 return true;
855 }
856
857 return false;
858 }
859
860 /// whether squid.conf send_hit prevents us from serving this hit
861 bool
blockedHit() const862 clientReplyContext::blockedHit() const
863 {
864 if (!Config.accessList.sendHit)
865 return false; // hits are not blocked by default
866
867 if (http->flags.internal)
868 return false; // internal content "hits" cannot be blocked
869
870 if (const HttpReply *rep = http->storeEntry()->getReply()) {
871 std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
872 chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
873 HTTPMSGLOCK(chl->reply);
874 return !chl->fastCheck().allowed(); // when in doubt, block
875 }
876
877 // This does not happen, I hope, because we are called from CacheHit, which
878 // is called via a storeClientCopy() callback, and store should initialize
879 // the reply before calling that callback.
880 debugs(88, 3, "Missing reply!");
881 return false;
882 }
883
884 void
purgeRequestFindObjectToPurge()885 clientReplyContext::purgeRequestFindObjectToPurge()
886 {
887 /* Try to find a base entry */
888 http->flags.purging = true;
889 lookingforstore = 1;
890
891 // TODO: can we use purgeAllCached() here instead of doing the
892 // getPublicByRequestMethod() dance?
893 StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
894 }
895
896 // Purges all entries with a given url
897 // TODO: move to SideAgent parent, when we have one
898 /*
899 * We probably cannot purge Vary-affected responses because their MD5
900 * keys depend on vary headers.
901 */
902 void
purgeEntriesByUrl(HttpRequest * req,const char * url)903 purgeEntriesByUrl(HttpRequest * req, const char *url)
904 {
905 for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
906 if (m.respMaybeCacheable()) {
907 const cache_key *key = storeKeyPublic(url, m);
908 debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key));
909 #if USE_HTCP
910 neighborsHtcpClear(nullptr, req, m, HTCP_CLR_INVALIDATION);
911 #endif
912 Store::Root().evictIfFound(key);
913 }
914 }
915 }
916
917 void
purgeAllCached()918 clientReplyContext::purgeAllCached()
919 {
920 // XXX: performance regression, c_str() reallocates
921 SBuf url(http->request->effectiveRequestUri());
922 purgeEntriesByUrl(http->request, url.c_str());
923 }
924
925 void
created(StoreEntry * newEntry)926 clientReplyContext::created(StoreEntry *newEntry)
927 {
928 if (lookingforstore == 1)
929 purgeFoundGet(newEntry);
930 else if (lookingforstore == 2)
931 purgeFoundHead(newEntry);
932 else if (lookingforstore == 3)
933 purgeDoPurgeGet(newEntry);
934 else if (lookingforstore == 4)
935 purgeDoPurgeHead(newEntry);
936 else if (lookingforstore == 5)
937 identifyFoundObject(newEntry);
938 }
939
940 void
purgeFoundGet(StoreEntry * newEntry)941 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
942 {
943 if (newEntry->isNull()) {
944 lookingforstore = 2;
945 StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
946 } else
947 purgeFoundObject (newEntry);
948 }
949
950 void
purgeFoundHead(StoreEntry * newEntry)951 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
952 {
953 if (newEntry->isNull())
954 purgeDoMissPurge();
955 else
956 purgeFoundObject (newEntry);
957 }
958
959 void
purgeFoundObject(StoreEntry * entry)960 clientReplyContext::purgeFoundObject(StoreEntry *entry)
961 {
962 assert (entry && !entry->isNull());
963
964 if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
965 http->logType = LOG_TCP_DENIED;
966 Ip::Address tmp_noaddr;
967 tmp_noaddr.setNoAddr(); // TODO: make a global const
968 ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
969 http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
970 http->request);
971 startError(err);
972 return; // XXX: leaking unused entry if some store does not keep it
973 }
974
975 StoreIOBuffer localTempBuffer;
976 /* Swap in the metadata */
977 http->storeEntry(entry);
978
979 http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
980 http->storeEntry()->ensureMemObject(storeId(), http->log_uri,
981 http->request->method);
982
983 sc = storeClientListAdd(http->storeEntry(), this);
984
985 http->logType = LOG_TCP_HIT;
986
987 reqofs = 0;
988
989 localTempBuffer.offset = http->out.offset;
990
991 localTempBuffer.length = next()->readBuffer.length;
992
993 localTempBuffer.data = next()->readBuffer.data;
994
995 storeClientCopy(sc, http->storeEntry(),
996 localTempBuffer, CacheHit, this);
997 }
998
999 void
purgeRequest()1000 clientReplyContext::purgeRequest()
1001 {
1002 debugs(88, 3, "Config2.onoff.enable_purge = " <<
1003 Config2.onoff.enable_purge);
1004
1005 if (!Config2.onoff.enable_purge) {
1006 http->logType = LOG_TCP_DENIED;
1007 Ip::Address tmp_noaddr;
1008 tmp_noaddr.setNoAddr();
1009 ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
1010 http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request);
1011 startError(err);
1012 return;
1013 }
1014
1015 /* Release both IP cache */
1016 ipcacheInvalidate(http->request->url.host());
1017
1018 if (!http->flags.purging)
1019 purgeRequestFindObjectToPurge();
1020 else
1021 purgeDoMissPurge();
1022 }
1023
1024 void
purgeDoMissPurge()1025 clientReplyContext::purgeDoMissPurge()
1026 {
1027 http->logType = LOG_TCP_MISS;
1028 lookingforstore = 3;
1029 StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
1030 }
1031
1032 void
purgeDoPurgeGet(StoreEntry * newEntry)1033 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
1034 {
1035 assert (newEntry);
1036 /* Move to new() when that is created */
1037 purgeStatus = Http::scNotFound;
1038
1039 if (!newEntry->isNull()) {
1040 /* Release the cached URI */
1041 debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
1042 #if USE_HTCP
1043 neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1044 #endif
1045 newEntry->release(true);
1046 purgeStatus = Http::scOkay;
1047 }
1048
1049 lookingforstore = 4;
1050 StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
1051 }
1052
1053 void
purgeDoPurgeHead(StoreEntry * newEntry)1054 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
1055 {
1056 if (newEntry && !newEntry->isNull()) {
1057 debugs(88, 4, "HEAD " << newEntry->url());
1058 #if USE_HTCP
1059 neighborsHtcpClear(newEntry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1060 #endif
1061 newEntry->release(true);
1062 purgeStatus = Http::scOkay;
1063 }
1064
1065 /* And for Vary, release the base URI if none of the headers was included in the request */
1066 if (!http->request->vary_headers.isEmpty()
1067 && http->request->vary_headers.find('=') != SBuf::npos) {
1068 // XXX: performance regression, c_str() reallocates
1069 SBuf tmp(http->request->effectiveRequestUri());
1070 StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1071
1072 if (entry) {
1073 debugs(88, 4, "Vary GET " << entry->url());
1074 #if USE_HTCP
1075 neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1076 #endif
1077 entry->release(true);
1078 purgeStatus = Http::scOkay;
1079 }
1080
1081 entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1082
1083 if (entry) {
1084 debugs(88, 4, "Vary HEAD " << entry->url());
1085 #if USE_HTCP
1086 neighborsHtcpClear(entry, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1087 #endif
1088 entry->release(true);
1089 purgeStatus = Http::scOkay;
1090 }
1091 }
1092
1093 if (purgeStatus == Http::scNone)
1094 purgeStatus = Http::scNotFound;
1095
1096 /*
1097 * Make a new entry to hold the reply to be written
1098 * to the client.
1099 */
1100 /* FIXME: This doesn't need to go through the store. Simply
1101 * push down the client chain
1102 */
1103 createStoreEntry(http->request->method, RequestFlags());
1104
1105 triggerInitialStoreRead();
1106
1107 HttpReply *rep = new HttpReply;
1108 rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1109 http->storeEntry()->replaceHttpReply(rep);
1110 http->storeEntry()->complete();
1111 }
1112
1113 void
traceReply(clientStreamNode * node)1114 clientReplyContext::traceReply(clientStreamNode * node)
1115 {
1116 clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1117 StoreIOBuffer localTempBuffer;
1118 createStoreEntry(http->request->method, RequestFlags());
1119 localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1120 localTempBuffer.length = nextNode->readBuffer.length;
1121 localTempBuffer.data = nextNode->readBuffer.data;
1122 storeClientCopy(sc, http->storeEntry(),
1123 localTempBuffer, SendMoreData, this);
1124 http->storeEntry()->releaseRequest();
1125 http->storeEntry()->buffer();
1126 HttpReply *rep = new HttpReply;
1127 rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1128 http->storeEntry()->replaceHttpReply(rep);
1129 http->request->swapOut(http->storeEntry());
1130 http->storeEntry()->complete();
1131 }
1132
1133 #define SENDING_BODY 0
1134 #define SENDING_HDRSONLY 1
1135 int
checkTransferDone()1136 clientReplyContext::checkTransferDone()
1137 {
1138 StoreEntry *entry = http->storeEntry();
1139
1140 if (entry == NULL)
1141 return 0;
1142
1143 /*
1144 * For now, 'done_copying' is used for special cases like
1145 * Range and HEAD requests.
1146 */
1147 if (http->flags.done_copying)
1148 return 1;
1149
1150 if (http->request->flags.chunkedReply && !flags.complete) {
1151 // last-chunk was not sent
1152 return 0;
1153 }
1154
1155 /*
1156 * Handle STORE_OK objects.
1157 * objectLen(entry) will be set proprely.
1158 * RC: Does objectLen(entry) include the Headers?
1159 * RC: Yes.
1160 */
1161 if (entry->store_status == STORE_OK) {
1162 return storeOKTransferDone();
1163 } else {
1164 return storeNotOKTransferDone();
1165 }
1166 }
1167
1168 int
storeOKTransferDone() const1169 clientReplyContext::storeOKTransferDone() const
1170 {
1171 assert(http->storeEntry()->objectLen() >= 0);
1172 assert(http->storeEntry()->objectLen() >= headers_sz);
1173 if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1174 debugs(88,3,HERE << "storeOKTransferDone " <<
1175 " out.offset=" << http->out.offset <<
1176 " objectLen()=" << http->storeEntry()->objectLen() <<
1177 " headers_sz=" << headers_sz);
1178 return 1;
1179 }
1180
1181 return 0;
1182 }
1183
1184 int
storeNotOKTransferDone() const1185 clientReplyContext::storeNotOKTransferDone() const
1186 {
1187 /*
1188 * Now, handle STORE_PENDING objects
1189 */
1190 MemObject *mem = http->storeEntry()->mem_obj;
1191 assert(mem != NULL);
1192 assert(http->request != NULL);
1193 /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1194 HttpReply const *curReply = mem->getReply();
1195
1196 if (headers_sz == 0)
1197 /* haven't found end of headers yet */
1198 return 0;
1199
1200 /*
1201 * Figure out how much data we are supposed to send.
1202 * If we are sending a body and we don't have a content-length,
1203 * then we must wait for the object to become STORE_OK.
1204 */
1205 if (curReply->content_length < 0)
1206 return 0;
1207
1208 uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1209
1210 if (http->out.size < expectedLength)
1211 return 0;
1212 else {
1213 debugs(88,3,HERE << "storeNotOKTransferDone " <<
1214 " out.size=" << http->out.size <<
1215 " expectedLength=" << expectedLength);
1216 return 1;
1217 }
1218 }
1219
1220 /* A write has completed, what is the next status based on the
1221 * canonical request data?
1222 * 1 something is wrong
1223 * 0 nothing is wrong.
1224 *
1225 */
1226 int
clientHttpRequestStatus(int fd,ClientHttpRequest const * http)1227 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1228 {
1229 #if SIZEOF_INT64_T == 4
1230 if (http->out.size > 0x7FFF0000) {
1231 debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1232 if (http->getConn())
1233 debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1234 debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1235 debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1236 return 1;
1237 }
1238
1239 if (http->out.offset > 0x7FFF0000) {
1240 debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1241 if (http->getConn())
1242 debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1243 debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1244 debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1245 return 1;
1246 }
1247
1248 #endif
1249 return 0;
1250 }
1251
1252 /* Preconditions:
1253 * *http is a valid structure.
1254 * fd is either -1, or an open fd.
1255 *
1256 * TODO: enumify this
1257 *
1258 * This function is used by any http request sink, to determine the status
1259 * of the object.
1260 */
1261 clientStream_status_t
clientReplyStatus(clientStreamNode * aNode,ClientHttpRequest * http)1262 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1263 {
1264 clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1265 assert (context);
1266 assert (context->http == http);
1267 return context->replyStatus();
1268 }
1269
1270 clientStream_status_t
replyStatus()1271 clientReplyContext::replyStatus()
1272 {
1273 int done;
1274 /* Here because lower nodes don't need it */
1275
1276 if (http->storeEntry() == NULL) {
1277 debugs(88, 5, "clientReplyStatus: no storeEntry");
1278 return STREAM_FAILED; /* yuck, but what can we do? */
1279 }
1280
1281 if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1282 /* TODO: Could upstream read errors (result.flags.error) be
1283 * lost, and result in undersize requests being considered
1284 * complete. Should we tcp reset such connections ?
1285 */
1286 debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1287 return STREAM_FAILED;
1288 }
1289
1290 if ((done = checkTransferDone()) != 0 || flags.complete) {
1291 debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1292 /* Ok we're finished, but how? */
1293
1294 if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1295 debugs(88, 5, "clientReplyStatus: truncated response body");
1296 return STREAM_UNPLANNED_COMPLETE;
1297 }
1298
1299 if (!done) {
1300 debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1301 return STREAM_FAILED;
1302 }
1303
1304 const int64_t expectedBodySize =
1305 http->storeEntry()->getReply()->bodySize(http->request->method);
1306 if (expectedBodySize >= 0 && !http->gotEnough()) {
1307 debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1308 return STREAM_UNPLANNED_COMPLETE;
1309 }
1310
1311 debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1312 http->request->flags.proxyKeepalive);
1313 return STREAM_COMPLETE;
1314 }
1315
1316 // XXX: Should this be checked earlier? We could return above w/o checking.
1317 if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1318 /* 4096 is a margin for the HTTP headers included in out.offset */
1319 debugs(88, 5, "clientReplyStatus: client reply body is too large");
1320 return STREAM_FAILED;
1321 }
1322
1323 return STREAM_NONE;
1324 }
1325
1326 /* Responses with no body will not have a content-type header,
1327 * which breaks the rep_mime_type acl, which
1328 * coincidentally, is the most common acl for reply access lists.
1329 * A better long term fix for this is to allow acl matchs on the various
1330 * status codes, and then supply a default ruleset that puts these
1331 * codes before any user defines access entries. That way the user
1332 * can choose to block these responses where appropriate, but won't get
1333 * mysterious breakages.
1334 */
1335 bool
alwaysAllowResponse(Http::StatusCode sline) const1336 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1337 {
1338 bool result;
1339
1340 switch (sline) {
1341
1342 case Http::scContinue:
1343
1344 case Http::scSwitchingProtocols:
1345
1346 case Http::scProcessing:
1347
1348 case Http::scNoContent:
1349
1350 case Http::scNotModified:
1351 result = true;
1352 break;
1353
1354 default:
1355 result = false;
1356 }
1357
1358 return result;
1359 }
1360
1361 /**
1362 * Generate the reply headers sent to client.
1363 *
1364 * Filters out unwanted entries and hop-by-hop from original reply header
1365 * then adds extra entries if we have more info than origin server
1366 * then adds Squid specific entries
1367 */
1368 void
buildReplyHeader()1369 clientReplyContext::buildReplyHeader()
1370 {
1371 HttpHeader *hdr = &reply->header;
1372 const bool is_hit = http->logType.isTcpHit();
1373 HttpRequest *request = http->request;
1374 #if DONT_FILTER_THESE
1375 /* but you might want to if you run Squid as an HTTP accelerator */
1376 /* hdr->delById(HDR_ACCEPT_RANGES); */
1377 hdr->delById(HDR_ETAG);
1378 #endif
1379
1380 if (is_hit || collapsedRevalidation == crSlave)
1381 hdr->delById(Http::HdrType::SET_COOKIE);
1382 // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1383
1384 // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1385 // remove the Proxy-Authenticate header
1386 if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0)) {
1387 #if USE_ADAPTATION
1388 // but allow adaptation services to authenticate clients
1389 // via request satisfaction
1390 if (!http->requestSatisfactionMode())
1391 #endif
1392 reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1393 }
1394
1395 reply->header.removeHopByHopEntries();
1396
1397 // if (request->range)
1398 // clientBuildRangeHeader(http, reply);
1399
1400 /*
1401 * Add a estimated Age header on cache hits.
1402 */
1403 if (is_hit) {
1404 /*
1405 * Remove any existing Age header sent by upstream caches
1406 * (note that the existing header is passed along unmodified
1407 * on cache misses)
1408 */
1409 hdr->delById(Http::HdrType::AGE);
1410 /*
1411 * This adds the calculated object age. Note that the details of the
1412 * age calculation is performed by adjusting the timestamp in
1413 * StoreEntry::timestampsSet(), not here.
1414 */
1415 if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1416 hdr->delById(Http::HdrType::DATE);
1417 hdr->putTime(Http::HdrType::DATE, squid_curtime);
1418 } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1419 // Swap the Date: header to current time if we are simulating an origin
1420 HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1421 if (h)
1422 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1423 hdr->delById(Http::HdrType::DATE);
1424 hdr->putTime(Http::HdrType::DATE, squid_curtime);
1425 h = hdr->findEntry(Http::HdrType::EXPIRES);
1426 if (h && http->storeEntry()->expires >= 0) {
1427 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1428 hdr->delById(Http::HdrType::EXPIRES);
1429 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1430 }
1431 if (http->storeEntry()->timestamp <= squid_curtime) {
1432 // put X-Cache-Age: instead of Age:
1433 char age[64];
1434 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1435 hdr->putExt("X-Cache-Age", age);
1436 }
1437 } else if (http->storeEntry()->timestamp <= squid_curtime) {
1438 hdr->putInt(Http::HdrType::AGE,
1439 squid_curtime - http->storeEntry()->timestamp);
1440 /* Signal old objects. NB: rfc 2616 is not clear,
1441 * by implication, on whether we should do this to all
1442 * responses, or only cache hits.
1443 * 14.46 states it ONLY applys for heuristically caclulated
1444 * freshness values, 13.2.4 doesn't specify the same limitation.
1445 * We interpret RFC 2616 under the combination.
1446 */
1447 /* TODO: if maxage or s-maxage is present, don't do this */
1448
1449 if (squid_curtime - http->storeEntry()->timestamp >= 86400)
1450 hdr->putWarning(113, "This cache hit is still fresh and more than 1 day old");
1451 }
1452 }
1453
1454 /* RFC 2616: Section 14.18
1455 *
1456 * Add a Date: header if missing.
1457 * We have access to a clock therefore are required to amend any shortcoming in servers.
1458 *
1459 * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1460 */
1461 if ( !hdr->has(Http::HdrType::DATE) ) {
1462 if (!http->storeEntry())
1463 hdr->putTime(Http::HdrType::DATE, squid_curtime);
1464 else if (http->storeEntry()->timestamp > 0)
1465 hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1466 else {
1467 debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1468 /* dump something useful about the problem */
1469 http->storeEntry()->dump(DBG_IMPORTANT);
1470 }
1471 }
1472
1473 // add Warnings required by RFC 2616 if serving a stale hit
1474 if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1475 hdr->putWarning(110, "Response is stale");
1476 if (http->request->flags.needValidation)
1477 hdr->putWarning(111, "Revalidation failed");
1478 }
1479
1480 /* Filter unproxyable authentication types */
1481 if (http->logType.oldType != LOG_TCP_DENIED &&
1482 hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1483 HttpHeaderPos pos = HttpHeaderInitPos;
1484 HttpHeaderEntry *e;
1485
1486 int connection_auth_blocked = 0;
1487 while ((e = hdr->getEntry(&pos))) {
1488 if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1489 const char *value = e->value.rawBuf();
1490
1491 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1492 (value[4] == '\0' || value[4] == ' '))
1493 ||
1494 (strncasecmp(value, "Negotiate", 9) == 0 &&
1495 (value[9] == '\0' || value[9] == ' '))
1496 ||
1497 (strncasecmp(value, "Kerberos", 8) == 0 &&
1498 (value[8] == '\0' || value[8] == ' '))) {
1499 if (request->flags.connectionAuthDisabled) {
1500 hdr->delAt(pos, connection_auth_blocked);
1501 continue;
1502 }
1503 request->flags.mustKeepalive = true;
1504 if (!request->flags.accelerated && !request->flags.intercepted) {
1505 httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1506 /*
1507 We send "Connection: Proxy-Support" header to mark
1508 Proxy-Support as a hop-by-hop header for intermediaries that do not
1509 understand the semantics of this header. The RFC should have included
1510 this recommendation.
1511 */
1512 httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1513 }
1514 break;
1515 }
1516 }
1517 }
1518
1519 if (connection_auth_blocked)
1520 hdr->refreshMask();
1521 }
1522
1523 #if USE_AUTH
1524 /* Handle authentication headers */
1525 if (http->logType.oldType == LOG_TCP_DENIED &&
1526 ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1527 reply->sline.status() == Http::scUnauthorized)
1528 ) {
1529 /* Add authentication header */
1530 /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1531 * depends on authenticate behaviour: all schemes to date send no extra
1532 * data on 407/401 responses, and do not check the accel state on 401/407
1533 * responses
1534 */
1535 authenticateFixHeader(reply, request->auth_user_request, request, 0, 1);
1536 } else if (request->auth_user_request != NULL)
1537 authenticateFixHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1538 #endif
1539
1540 /* Append X-Cache */
1541 httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1542 is_hit ? "HIT" : "MISS", getMyHostname());
1543
1544 #if USE_CACHE_DIGESTS
1545 /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1546 httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1547 lookup_type ? lookup_type : "NONE",
1548 getMyHostname(), getMyPort());
1549
1550 #endif
1551
1552 const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1553 reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1554 (request->http_ver >= Http::ProtocolVersion(1,1));
1555
1556 /* Check whether we should send keep-alive */
1557 if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1558 debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1559 request->flags.proxyKeepalive = false;
1560 } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1561 debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1562 request->flags.proxyKeepalive = false;
1563 } else if (request->flags.proxyKeepalive && shutting_down) {
1564 debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1565 request->flags.proxyKeepalive = false;
1566 } else if (request->flags.connectionAuth && !reply->keep_alive) {
1567 debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1568 request->flags.proxyKeepalive = false;
1569 } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1570 debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1571 request->flags.proxyKeepalive = false;
1572 } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1573 debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1574 request->flags.proxyKeepalive = false;
1575 } else if (request->flags.sslBumped && !reply->persistent()) {
1576 // We do not really have to close, but we pretend we are a tunnel.
1577 debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1578 request->flags.proxyKeepalive = false;
1579 } else if (request->pinnedConnection() && !reply->persistent()) {
1580 // The peer wants to close the pinned connection
1581 debugs(88, 3, "pinned reply forces close");
1582 request->flags.proxyKeepalive = false;
1583 } else if (http->getConn()) {
1584 ConnStateData * conn = http->getConn();
1585 if (!Comm::IsConnOpen(conn->port->listenConn)) {
1586 // The listening port closed because of a reconfigure
1587 debugs(88, 3, "listening port closed");
1588 request->flags.proxyKeepalive = false;
1589 }
1590 }
1591
1592 // Decide if we send chunked reply
1593 if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1594 debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1595 request->flags.chunkedReply = true;
1596 hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1597 }
1598
1599 hdr->addVia(reply->sline.version);
1600
1601 /* Signal keep-alive or close explicitly */
1602 hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1603
1604 #if ADD_X_REQUEST_URI
1605 /*
1606 * Knowing the URI of the request is useful when debugging persistent
1607 * connections in a client; we cannot guarantee the order of http headers,
1608 * but X-Request-URI is likely to be the very last header to ease use from a
1609 * debugger [hdr->entries.count-1].
1610 */
1611 hdr->putStr(Http::HdrType::X_REQUEST_URI,
1612 http->memOjbect()->url ? http->memObject()->url : http->uri);
1613
1614 #endif
1615
1616 /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1617 if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1618 if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1619 hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1620 }
1621 /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1622 }
1623
1624 httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1625 }
1626
1627 void
cloneReply()1628 clientReplyContext::cloneReply()
1629 {
1630 assert(reply == NULL);
1631
1632 reply = http->storeEntry()->getReply()->clone();
1633 HTTPMSGLOCK(reply);
1634
1635 if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1636 /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1637 reply->sline.version = Http::ProtocolVersion();
1638 }
1639
1640 /* do header conversions */
1641 buildReplyHeader();
1642 }
1643
1644 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1645 /// We cannot just ignore the entry because it may be locking or otherwise
1646 /// holding an associated cache resource of some sort.
1647 void
forgetHit()1648 clientReplyContext::forgetHit()
1649 {
1650 StoreEntry *e = http->storeEntry();
1651 assert(e); // or we are not dealing with a hit
1652 // We probably have not locked the entry earlier, unfortunately. We lock it
1653 // now so that we can unlock two lines later (and trigger cleanup).
1654 // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1655 // used so inconsistently that simply adding locking there leads to bugs.
1656 e->lock("clientReplyContext::forgetHit");
1657 http->storeEntry(NULL);
1658 e->unlock("clientReplyContext::forgetHit"); // may delete e
1659 }
1660
1661 void
identifyStoreObject()1662 clientReplyContext::identifyStoreObject()
1663 {
1664 HttpRequest *r = http->request;
1665
1666 // client sent CC:no-cache or some other condition has been
1667 // encountered which prevents delivering a public/cached object.
1668 if (!r->flags.noCache || r->flags.internal) {
1669 lookingforstore = 5;
1670 StoreEntry::getPublicByRequest (this, r);
1671 } else {
1672 identifyFoundObject (NullStoreEntry::getInstance());
1673 }
1674 }
1675
1676 /**
1677 * Check state of the current StoreEntry object.
1678 * to see if we can determine the final status of the request.
1679 */
1680 void
identifyFoundObject(StoreEntry * newEntry)1681 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1682 {
1683 StoreEntry *e = newEntry;
1684 HttpRequest *r = http->request;
1685
1686 /** \li If the entry received isNull() then we ignore it. */
1687 if (e->isNull()) {
1688 http->storeEntry(NULL);
1689 } else {
1690 http->storeEntry(e);
1691 }
1692
1693 e = http->storeEntry();
1694
1695 /* Release IP-cache entries on reload */
1696 /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1697 * 'invalidate' the cached IP entries for this request ???
1698 */
1699 if (r->flags.noCache || r->flags.noCacheHack())
1700 ipcacheInvalidateNegative(r->url.host());
1701
1702 #if USE_CACHE_DIGESTS
1703 lookup_type = http->storeEntry() ? "HIT" : "MISS";
1704 #endif
1705
1706 if (NULL == http->storeEntry()) {
1707 /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1708 debugs(85, 3, "StoreEntry is NULL - MISS");
1709 http->logType = LOG_TCP_MISS;
1710 doGetMoreData();
1711 return;
1712 }
1713
1714 if (Config.onoff.offline) {
1715 /** \li If we are running in offline mode set to HIT */
1716 debugs(85, 3, "offline HIT " << *e);
1717 http->logType = LOG_TCP_HIT;
1718 doGetMoreData();
1719 return;
1720 }
1721
1722 if (http->redirect.status) {
1723 /** \li If redirection status is True force this to be a MISS */
1724 debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1725 forgetHit();
1726 http->logType = LOG_TCP_REDIRECT;
1727 doGetMoreData();
1728 return;
1729 }
1730
1731 if (!e->validToSend()) {
1732 debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1733 forgetHit();
1734 http->logType = LOG_TCP_MISS;
1735 doGetMoreData();
1736 return;
1737 }
1738
1739 if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1740 /* \li Special entries are always hits, no matter what the client says */
1741 debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1742 http->logType = LOG_TCP_HIT;
1743 doGetMoreData();
1744 return;
1745 }
1746
1747 if (r->flags.noCache) {
1748 debugs(85, 3, "no-cache REFRESH MISS " << *e);
1749 forgetHit();
1750 http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1751 doGetMoreData();
1752 return;
1753 }
1754
1755 debugs(85, 3, "default HIT " << *e);
1756 http->logType = LOG_TCP_HIT;
1757 doGetMoreData();
1758 }
1759
1760 /**
1761 * Request more data from the store for the client Stream
1762 * This is *the* entry point to this module.
1763 *
1764 * Preconditions:
1765 * - This is the head of the list.
1766 * - There is at least one more node.
1767 * - Data context is not null
1768 */
1769 void
clientGetMoreData(clientStreamNode * aNode,ClientHttpRequest * http)1770 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1771 {
1772 /* Test preconditions */
1773 assert(aNode != NULL);
1774 assert(cbdataReferenceValid(aNode));
1775 assert(aNode->node.prev == NULL);
1776 assert(aNode->node.next != NULL);
1777 clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1778 assert (context);
1779 assert(context->http == http);
1780
1781 clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1782
1783 if (!context->ourNode)
1784 context->ourNode = aNode;
1785
1786 /* no cbdatareference, this is only used once, and safely */
1787 if (context->flags.storelogiccomplete) {
1788 StoreIOBuffer tempBuffer;
1789 tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1790 tempBuffer.length = next->readBuffer.length;
1791 tempBuffer.data = next->readBuffer.data;
1792
1793 storeClientCopy(context->sc, http->storeEntry(),
1794 tempBuffer, clientReplyContext::SendMoreData, context);
1795 return;
1796 }
1797
1798 if (context->http->request->method == Http::METHOD_PURGE) {
1799 context->purgeRequest();
1800 return;
1801 }
1802
1803 // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1804
1805 if (context->http->request->method == Http::METHOD_TRACE) {
1806 if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1807 context->traceReply(aNode);
1808 return;
1809 }
1810
1811 /* continue forwarding, not finished yet. */
1812 http->logType = LOG_TCP_MISS;
1813
1814 context->doGetMoreData();
1815 } else
1816 context->identifyStoreObject();
1817 }
1818
1819 void
doGetMoreData()1820 clientReplyContext::doGetMoreData()
1821 {
1822 /* We still have to do store logic processing - vary, cache hit etc */
1823 if (http->storeEntry() != NULL) {
1824 /* someone found the object in the cache for us */
1825 StoreIOBuffer localTempBuffer;
1826
1827 http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1828
1829 http->storeEntry()->ensureMemObject(storeId(), http->log_uri, http->request->method);
1830
1831 sc = storeClientListAdd(http->storeEntry(), this);
1832 #if USE_DELAY_POOLS
1833 sc->setDelayId(DelayId::DelayClient(http));
1834 #endif
1835
1836 assert(http->logType.oldType == LOG_TCP_HIT);
1837 reqofs = 0;
1838 /* guarantee nothing has been sent yet! */
1839 assert(http->out.size == 0);
1840 assert(http->out.offset == 0);
1841
1842 if (ConnStateData *conn = http->getConn()) {
1843 if (Ip::Qos::TheConfig.isHitTosActive()) {
1844 Ip::Qos::doTosLocalHit(conn->clientConnection);
1845 }
1846
1847 if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1848 Ip::Qos::doNfmarkLocalHit(conn->clientConnection);
1849 }
1850 }
1851
1852 localTempBuffer.offset = reqofs;
1853 localTempBuffer.length = getNextNode()->readBuffer.length;
1854 localTempBuffer.data = getNextNode()->readBuffer.data;
1855 storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1856 } else {
1857 /* MISS CASE, http->logType is already set! */
1858 processMiss();
1859 }
1860 }
1861
1862 /** The next node has removed itself from the stream. */
1863 void
clientReplyDetach(clientStreamNode * node,ClientHttpRequest * http)1864 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1865 {
1866 /** detach from the stream */
1867 clientStreamDetach(node, http);
1868 }
1869
1870 /**
1871 * Accepts chunk of a http message in buf, parses prefix, filters headers and
1872 * such, writes processed message to the message recipient
1873 */
1874 void
SendMoreData(void * data,StoreIOBuffer result)1875 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1876 {
1877 clientReplyContext *context = static_cast<clientReplyContext *>(data);
1878 context->sendMoreData (result);
1879 }
1880
1881 void
makeThisHead()1882 clientReplyContext::makeThisHead()
1883 {
1884 /* At least, I think thats what this does */
1885 dlinkDelete(&http->active, &ClientActiveRequests);
1886 dlinkAdd(http, &http->active, &ClientActiveRequests);
1887 }
1888
1889 bool
errorInStream(StoreIOBuffer const & result,size_t const & sizeToProcess) const1890 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1891 {
1892 return /* aborted request */
1893 (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1894 /* Upstream read error */ (result.flags.error) ||
1895 /* Upstream EOF */ (sizeToProcess == 0);
1896 }
1897
1898 void
sendStreamError(StoreIOBuffer const & result)1899 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1900 {
1901 /** call clientWriteComplete so the client socket gets closed
1902 *
1903 * We call into the stream, because we don't know that there is a
1904 * client socket!
1905 */
1906 debugs(88, 5, "A stream error has occurred, marking as complete and sending no data.");
1907 StoreIOBuffer localTempBuffer;
1908 flags.complete = 1;
1909 http->request->flags.streamError = true;
1910 localTempBuffer.flags.error = result.flags.error;
1911 clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1912 localTempBuffer);
1913 }
1914
1915 void
pushStreamData(StoreIOBuffer const & result,char * source)1916 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1917 {
1918 StoreIOBuffer localTempBuffer;
1919
1920 if (result.length == 0) {
1921 debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1922 flags.complete = 1;
1923 }
1924
1925 assert(result.offset - headers_sz == next()->readBuffer.offset);
1926 localTempBuffer.offset = result.offset - headers_sz;
1927 localTempBuffer.length = result.length;
1928
1929 if (localTempBuffer.length)
1930 localTempBuffer.data = source;
1931
1932 clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1933 localTempBuffer);
1934 }
1935
1936 clientStreamNode *
next() const1937 clientReplyContext::next() const
1938 {
1939 assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1940 return getNextNode();
1941 }
1942
1943 void
sendBodyTooLargeError()1944 clientReplyContext::sendBodyTooLargeError()
1945 {
1946 Ip::Address tmp_noaddr;
1947 tmp_noaddr.setNoAddr(); // TODO: make a global const
1948 http->logType = LOG_TCP_DENIED_REPLY;
1949 ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1950 http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1951 http->request);
1952 removeClientStoreReference(&(sc), http);
1953 HTTPMSGUNLOCK(reply);
1954 startError(err);
1955
1956 }
1957
1958 /// send 412 (Precondition Failed) to client
1959 void
sendPreconditionFailedError()1960 clientReplyContext::sendPreconditionFailedError()
1961 {
1962 http->logType = LOG_TCP_HIT;
1963 Ip::Address tmp_noaddr;
1964 tmp_noaddr.setNoAddr();
1965 ErrorState *const err =
1966 clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1967 NULL, http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request);
1968 removeClientStoreReference(&sc, http);
1969 HTTPMSGUNLOCK(reply);
1970 startError(err);
1971 }
1972
1973 /// send 304 (Not Modified) to client
1974 void
sendNotModified()1975 clientReplyContext::sendNotModified()
1976 {
1977 StoreEntry *e = http->storeEntry();
1978 const time_t timestamp = e->timestamp;
1979 HttpReply *const temprep = e->getReply()->make304();
1980 // log as TCP_INM_HIT if code 304 generated for
1981 // If-None-Match request
1982 if (!http->request->flags.ims)
1983 http->logType = LOG_TCP_INM_HIT;
1984 else
1985 http->logType = LOG_TCP_IMS_HIT;
1986 removeClientStoreReference(&sc, http);
1987 createStoreEntry(http->request->method, RequestFlags());
1988 e = http->storeEntry();
1989 // Copy timestamp from the original entry so the 304
1990 // reply has a meaningful Age: header.
1991 e->timestampsSet();
1992 e->timestamp = timestamp;
1993 e->replaceHttpReply(temprep);
1994 e->complete();
1995 /*
1996 * TODO: why put this in the store and then serialise it and
1997 * then parse it again. Simply mark the request complete in
1998 * our context and write the reply struct to the client side.
1999 */
2000 triggerInitialStoreRead();
2001 }
2002
2003 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
2004 /// depending on request method
2005 void
sendNotModifiedOrPreconditionFailedError()2006 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
2007 {
2008 if (http->request->method == Http::METHOD_GET ||
2009 http->request->method == Http::METHOD_HEAD)
2010 sendNotModified();
2011 else
2012 sendPreconditionFailedError();
2013 }
2014
2015 void
processReplyAccess()2016 clientReplyContext::processReplyAccess ()
2017 {
2018 /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
2019 assert(reply);
2020
2021 /** Don't block our own responses or HTTP status messages */
2022 if (http->logType.oldType == LOG_TCP_DENIED ||
2023 http->logType.oldType == LOG_TCP_DENIED_REPLY ||
2024 alwaysAllowResponse(reply->sline.status())) {
2025 headers_sz = reply->hdr_sz;
2026 processReplyAccessResult(ACCESS_ALLOWED);
2027 return;
2028 }
2029
2030 /** Check for reply to big error */
2031 if (reply->expectedBodyTooLarge(*http->request)) {
2032 sendBodyTooLargeError();
2033 return;
2034 }
2035
2036 headers_sz = reply->hdr_sz;
2037
2038 /** check for absent access controls (permit by default) */
2039 if (!Config.accessList.reply) {
2040 processReplyAccessResult(ACCESS_ALLOWED);
2041 return;
2042 }
2043
2044 /** Process http_reply_access lists */
2045 ACLFilledChecklist *replyChecklist =
2046 clientAclChecklistCreate(Config.accessList.reply, http);
2047 replyChecklist->reply = reply;
2048 HTTPMSGLOCK(replyChecklist->reply);
2049 replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2050 }
2051
2052 void
ProcessReplyAccessResult(allow_t rv,void * voidMe)2053 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2054 {
2055 clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2056 me->processReplyAccessResult(rv);
2057 }
2058
2059 void
processReplyAccessResult(const allow_t & accessAllowed)2060 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2061 {
2062 debugs(88, 2, "The reply for " << http->request->method
2063 << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2064 << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2065
2066 if (!accessAllowed.allowed()) {
2067 ErrorState *err;
2068 err_type page_id;
2069 page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2070
2071 http->logType = LOG_TCP_DENIED_REPLY;
2072
2073 if (page_id == ERR_NONE)
2074 page_id = ERR_ACCESS_DENIED;
2075
2076 Ip::Address tmp_noaddr;
2077 tmp_noaddr.setNoAddr();
2078 err = clientBuildError(page_id, Http::scForbidden, NULL,
2079 http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2080 http->request);
2081
2082 removeClientStoreReference(&sc, http);
2083
2084 HTTPMSGUNLOCK(reply);
2085
2086 startError(err);
2087
2088 return;
2089 }
2090
2091 /* Ok, the reply is allowed, */
2092 http->loggingEntry(http->storeEntry());
2093
2094 ssize_t body_size = reqofs - reply->hdr_sz;
2095 if (body_size < 0) {
2096 reqofs = reply->hdr_sz;
2097 body_size = 0;
2098 }
2099
2100 debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2101 (int) body_size << " bytes after " << reply->hdr_sz <<
2102 " bytes of headers");
2103
2104 #if USE_SQUID_ESI
2105
2106 if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2107 !alwaysAllowResponse(reply->sline.status()) &&
2108 esiEnableProcessing(reply)) {
2109 debugs(88, 2, "Enabling ESI processing for " << http->uri);
2110 clientStreamInsertHead(&http->client_stream, esiStreamRead,
2111 esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2112 }
2113
2114 #endif
2115
2116 if (http->request->method == Http::METHOD_HEAD) {
2117 /* do not forward body for HEAD replies */
2118 body_size = 0;
2119 http->flags.done_copying = true;
2120 flags.complete = 1;
2121 }
2122
2123 assert (!flags.headersSent);
2124 flags.headersSent = true;
2125
2126 StoreIOBuffer localTempBuffer;
2127 char *buf = next()->readBuffer.data;
2128 char *body_buf = buf + reply->hdr_sz;
2129
2130 //Server side may disable ranges under some circumstances.
2131
2132 if ((!http->request->range))
2133 next()->readBuffer.offset = 0;
2134
2135 body_buf -= next()->readBuffer.offset;
2136
2137 if (next()->readBuffer.offset != 0) {
2138 if (next()->readBuffer.offset > body_size) {
2139 /* Can't use any of the body we received. send nothing */
2140 localTempBuffer.length = 0;
2141 localTempBuffer.data = NULL;
2142 } else {
2143 localTempBuffer.length = body_size - next()->readBuffer.offset;
2144 localTempBuffer.data = body_buf + next()->readBuffer.offset;
2145 }
2146 } else {
2147 localTempBuffer.length = body_size;
2148 localTempBuffer.data = body_buf;
2149 }
2150
2151 /* TODO??: move the data in the buffer back by the request header size */
2152 clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2153 http, reply, localTempBuffer);
2154
2155 return;
2156 }
2157
2158 void
sendMoreData(StoreIOBuffer result)2159 clientReplyContext::sendMoreData (StoreIOBuffer result)
2160 {
2161 if (deleting)
2162 return;
2163
2164 StoreEntry *entry = http->storeEntry();
2165
2166 if (ConnStateData * conn = http->getConn()) {
2167 if (!conn->isOpen()) {
2168 debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2169 return;
2170 }
2171 if (conn->pinning.zeroReply) {
2172 debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2173 return;
2174 }
2175
2176 if (reqofs==0 && !http->logType.isTcpHit()) {
2177 if (Ip::Qos::TheConfig.isHitTosActive()) {
2178 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2179 }
2180 if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2181 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2182 }
2183 }
2184
2185 debugs(88, 5, conn->clientConnection <<
2186 " '" << entry->url() << "'" <<
2187 " out.offset=" << http->out.offset);
2188 }
2189
2190 char *buf = next()->readBuffer.data;
2191
2192 if (buf != result.data) {
2193 /* we've got to copy some data */
2194 assert(result.length <= next()->readBuffer.length);
2195 memcpy(buf, result.data, result.length);
2196 }
2197
2198 /* We've got the final data to start pushing... */
2199 flags.storelogiccomplete = 1;
2200
2201 reqofs += result.length;
2202
2203 assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2204
2205 assert(http->request != NULL);
2206
2207 /* ESI TODO: remove this assert once everything is stable */
2208 assert(http->client_stream.head->data
2209 && cbdataReferenceValid(http->client_stream.head->data));
2210
2211 makeThisHead();
2212
2213 debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2214 reqofs << " bytes (" << result.length <<
2215 " new bytes)");
2216
2217 /* update size of the request */
2218 reqsize = reqofs;
2219
2220 if (errorInStream(result, reqofs)) {
2221 sendStreamError(result);
2222 return;
2223 }
2224
2225 if (flags.headersSent) {
2226 pushStreamData (result, buf);
2227 return;
2228 }
2229
2230 cloneReply();
2231
2232 #if USE_DELAY_POOLS
2233 if (sc)
2234 sc->setDelayId(DelayId::DelayClient(http,reply));
2235 #endif
2236
2237 /* handle headers */
2238
2239 if (Config.onoff.log_mime_hdrs) {
2240 size_t k;
2241
2242 if ((k = headersEnd(buf, reqofs))) {
2243 safe_free(http->al->headers.reply);
2244 http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2245 xstrncpy(http->al->headers.reply, buf, k);
2246 }
2247 }
2248
2249 holdingBuffer = result;
2250 processReplyAccess();
2251 return;
2252 }
2253
2254 /* Using this breaks the client layering just a little!
2255 */
2256 void
createStoreEntry(const HttpRequestMethod & m,RequestFlags reqFlags)2257 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2258 {
2259 assert(http != NULL);
2260 /*
2261 * For erroneous requests, we might not have a h->request,
2262 * so make a fake one.
2263 */
2264
2265 if (http->request == NULL) {
2266 const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient);
2267 // XXX: These fake URI parameters shadow the real (or error:...) URI.
2268 // TODO: Either always set the request earlier and assert here OR use
2269 // http->uri (converted to Anyp::Uri) to create this catch-all request.
2270 const_cast<HttpRequest *&>(http->request) = new HttpRequest(m, AnyP::PROTO_NONE, "http", null_string, mx);
2271 HTTPMSGLOCK(http->request);
2272 }
2273
2274 StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2275
2276 // Make entry collapsable ASAP, to increase collapsing chances for others,
2277 // TODO: every must-revalidate and similar request MUST reach the origin,
2278 // but do we have to prohibit others from collapsing on that request?
2279 if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2280 !reqFlags.needValidation &&
2281 (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2282 // make the entry available for future requests now
2283 (void)Store::Root().allowCollapsing(e, reqFlags, m);
2284 }
2285
2286 sc = storeClientListAdd(e, this);
2287
2288 #if USE_DELAY_POOLS
2289 sc->setDelayId(DelayId::DelayClient(http));
2290 #endif
2291
2292 reqofs = 0;
2293
2294 reqsize = 0;
2295
2296 /* I don't think this is actually needed! -- adrian */
2297 /* http->reqbuf = http->norm_reqbuf; */
2298 // assert(http->reqbuf == http->norm_reqbuf);
2299 /* The next line is illegal because we don't know if the client stream
2300 * buffers have been set up
2301 */
2302 // storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2303 // SendMoreData, this);
2304 /* So, we mark the store logic as complete */
2305 flags.storelogiccomplete = 1;
2306
2307 /* and get the caller to request a read, from whereever they are */
2308 /* NOTE: after ANY data flows down the pipe, even one step,
2309 * this function CAN NOT be used to manage errors
2310 */
2311 http->storeEntry(e);
2312 }
2313
2314 ErrorState *
clientBuildError(err_type page_id,Http::StatusCode status,char const * url,Ip::Address & src_addr,HttpRequest * request)2315 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2316 Ip::Address &src_addr, HttpRequest * request)
2317 {
2318 ErrorState *err = new ErrorState(page_id, status, request);
2319 err->src_addr = src_addr;
2320
2321 if (url)
2322 err->url = xstrdup(url);
2323
2324 return err;
2325 }
2326
2327