1 /* 2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors 3 * 4 * Squid software is distributed under GPLv2+ license and includes 5 * contributions from numerous individuals and organizations. 6 * Please see the COPYING and CONTRIBUTORS files for details. 7 */ 8 9 #ifndef SQUID_STORE_H 10 #define SQUID_STORE_H 11 12 #include "base/Packable.h" 13 #include "base/RefCount.h" 14 #include "comm/forward.h" 15 #include "CommRead.h" 16 #include "hash.h" 17 #include "http/forward.h" 18 #include "http/RequestMethod.h" 19 #include "HttpReply.h" 20 #include "MemObject.h" 21 #include "Range.h" 22 #include "RemovalPolicy.h" 23 #include "store/Controller.h" 24 #include "store/forward.h" 25 #include "store_key_md5.h" 26 #include "StoreIOBuffer.h" 27 #include "StoreStats.h" 28 29 #if USE_SQUID_ESI 30 #include "esi/Element.h" 31 #endif 32 33 #include <ostream> 34 35 class AsyncCall; 36 class HttpRequest; 37 class RequestFlags; 38 39 extern StoreIoStats store_io_stats; 40 41 class StoreEntry : public hash_link, public Packable 42 { 43 44 public: 45 static DeferredRead::DeferrableRead DeferReader; 46 bool checkDeferRead(int fd) const; 47 48 virtual const char *getMD5Text() const; 49 StoreEntry(); 50 virtual ~StoreEntry(); 51 52 virtual HttpReply const *getReply() const; 53 virtual void write (StoreIOBuffer); 54 55 /** Check if the Store entry is emtpty 56 * \retval true Store contains 0 bytes of data. 57 * \retval false Store contains 1 or more bytes of data. 58 * \retval false Store contains negative content !!!!!! 59 */ isEmpty()60 virtual bool isEmpty() const { 61 assert (mem_obj); 62 return mem_obj->endOffset() == 0; 63 } 64 virtual bool isAccepting() const; 65 virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const; 66 /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it 67 void lengthWentBad(const char *reason); 68 virtual void complete(); 69 virtual store_client_t storeClientType() const; 70 virtual char const *getSerialisedMetaData(); 71 /// Store a prepared error response. MemObject locks the reply object. 72 void storeErrorResponse(HttpReply *reply); 73 void replaceHttpReply(HttpReply *, bool andStartWriting = true); 74 void startWriting(); ///< pack and write reply headers and, maybe, body 75 /// whether we may start writing to disk (now or in the future) 76 virtual bool mayStartSwapOut(); 77 virtual void trimMemory(const bool preserveSwappable); 78 79 // called when a decision to cache in memory has been made 80 void memOutDecision(const bool willCacheInRam); 81 // called when a decision to cache on disk has been made 82 void swapOutDecision(const MemObject::SwapOut::Decision &decision); 83 84 void abort(); 85 bool makePublic(const KeyScope keyScope = ksDefault); 86 void makePrivate(const bool shareable); 87 /// A low-level method just resetting "private key" flags. 88 /// To avoid key inconsistency please use forcePublicKey() 89 /// or similar instead. 90 void clearPrivate(); 91 bool setPublicKey(const KeyScope keyScope = ksDefault); 92 /// Resets existing public key to a public key with default scope, 93 /// releasing the old default-scope entry (if any). 94 /// Does nothing if the existing public key already has default scope. 95 void clearPublicKeyScope(); 96 97 /// \returns public key (if the entry has it) or nil (otherwise) publicKey()98 const cache_key *publicKey() const { 99 return (!EBIT_TEST(flags, KEY_PRIVATE)) ? 100 reinterpret_cast<const cache_key*>(key): // may be nil 101 nullptr; 102 } 103 104 /// Either fills this entry with private key or changes the existing key 105 /// from public to private. 106 /// \param permanent whether this entry should be private forever. 107 void setPrivateKey(const bool shareable, const bool permanent); 108 109 void expireNow(); 110 /// Makes the StoreEntry private and marks the corresponding entry 111 /// for eventual removal from the Store. 112 void releaseRequest(const bool shareable = false); 113 void negativeCache(); 114 bool cacheNegatively(); /** \todo argh, why both? */ 115 void invokeHandlers(); 116 void cacheInMemory(); ///< start or continue storing in memory cache 117 void swapOut(); 118 /// whether we are in the process of writing this entry to disk swappingOut()119 bool swappingOut() const { return swap_status == SWAPOUT_WRITING; } 120 /// whether the entire entry is now on disk (possibly marked for deletion) swappedOut()121 bool swappedOut() const { return swap_status == SWAPOUT_DONE; } 122 /// whether we failed to write this entry to disk swapoutFailed()123 bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; } 124 void swapOutFileClose(int how); 125 const char *url() const; 126 /// Satisfies cachability requirements shared among disk and RAM caches. 127 /// Encapsulates common checks of mayStartSwapOut() and memoryCachable(). 128 /// TODO: Rename and make private so only those two methods can call this. 129 bool checkCachable(); 130 int checkNegativeHit() const; 131 int locked() const; 132 int validToSend() const; 133 bool memoryCachable(); ///< checkCachable() and can be cached in memory 134 135 /// initialize mem_obj; assert if mem_obj already exists 136 /// avoid this method in favor of createMemObject(trio)! 137 void createMemObject(); 138 139 /// initialize mem_obj with URIs/method; assert if mem_obj already exists 140 void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod); 141 142 /// initialize mem_obj (if needed) and set URIs/method (if missing) 143 void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod); 144 145 void dump(int debug_lvl) const; 146 void hashDelete(); 147 void hashInsert(const cache_key *); 148 void registerAbort(STABH * cb, void *); 149 void reset(); 150 void setMemStatus(mem_status_t); 151 bool timestampsSet(); 152 void unregisterAbort(); 153 void destroyMemObject(); 154 int checkTooSmall(); 155 156 void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback); 157 158 void setNoDelay (bool const); lastModified(const time_t when)159 void lastModified(const time_t when) { lastModified_ = when; } 160 /// \returns entry's 'effective' modification time lastModified()161 time_t lastModified() const { 162 // may still return -1 if timestamp is not set 163 return lastModified_ < 0 ? timestamp : lastModified_; 164 } 165 /// \returns a formatted string with entry's timestamps 166 const char *describeTimestamps() const; 167 // TODO: consider removing currently unsupported imslen parameter 168 bool modifiedSince(const time_t ims, const int imslen = -1) const; 169 /// has ETag matching at least one of the If-Match etags 170 bool hasIfMatchEtag(const HttpRequest &request) const; 171 /// has ETag matching at least one of the If-None-Match etags 172 bool hasIfNoneMatchEtag(const HttpRequest &request) const; 173 /// whether this entry has an ETag; if yes, puts ETag value into parameter 174 bool hasEtag(ETag &etag) const; 175 176 /// the disk this entry is [being] cached on; asserts for entries w/o a disk 177 Store::Disk &disk() const; 178 /// whether one of this StoreEntry owners has locked the corresponding 179 /// disk entry (at the specified disk entry coordinates, if any) 180 bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const; 181 /// Makes hasDisk(dirn, filn) true. The caller should have locked 182 /// the corresponding disk store entry for reading or writing. 183 void attachToDisk(const sdirno, const sfileno, const swap_status_t); 184 /// Makes hasDisk() false. The caller should have unlocked 185 /// the corresponding disk store entry. 186 void detachFromDisk(); 187 188 /// whether there is a corresponding locked transients table entry hasTransients()189 bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; } 190 /// whether there is a corresponding locked shared memory table entry hasMemStore()191 bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; } 192 193 MemObject *mem_obj; 194 RemovalPolicyNode repl; 195 /* START OF ON-DISK STORE_META_STD TLV field */ 196 time_t timestamp; 197 time_t lastref; 198 time_t expires; 199 private: 200 time_t lastModified_; ///< received Last-Modified value or -1; use lastModified() 201 public: 202 uint64_t swap_file_sz; 203 uint16_t refcount; 204 uint16_t flags; 205 /* END OF ON-DISK STORE_META_STD */ 206 207 /// unique ID inside a cache_dir for swapped out entries; -1 for others 208 sfileno swap_filen:25; // keep in sync with SwapFilenMax 209 210 sdirno swap_dirn:7; 211 212 mem_status_t mem_status:3; 213 214 ping_status_t ping_status:3; 215 216 store_status_t store_status:3; 217 218 swap_status_t swap_status:3; 219 220 public: 221 static size_t inUseCount(); 222 static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method); 223 static void getPublicByRequest(StoreClient * aClient, HttpRequest * request); 224 static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method); 225 isNull()226 virtual bool isNull() { 227 return false; 228 }; 229 230 void *operator new(size_t byteCount); 231 void operator delete(void *address); 232 #if USE_SQUID_ESI 233 234 ESIElement::Pointer cachedESITree; 235 #endif 236 virtual int64_t objectLen() const; 237 virtual int64_t contentLen() const; 238 239 /// claim shared ownership of this entry (for use in a given context) 240 /// matching lock() and unlock() contexts eases leak triage but is optional 241 void lock(const char *context); 242 243 /// disclaim shared ownership; may remove entry from store and delete it 244 /// returns remaning lock level (zero for unlocked and possibly gone entry) 245 int unlock(const char *context); 246 247 /// returns a local concurrent use counter, for debugging locks()248 int locks() const { return static_cast<int>(lock_count); } 249 250 /// update last reference timestamp and related Store metadata 251 void touch(); 252 253 /// One of the three methods to get rid of an unlocked StoreEntry object. 254 /// Removes all unlocked (and marks for eventual removal all locked) Store 255 /// entries, including attached and unattached entries that have our key. 256 /// Also destroys us if we are unlocked or makes us private otherwise. 257 /// TODO: remove virtual. 258 virtual void release(const bool shareable = false); 259 260 /// One of the three methods to get rid of an unlocked StoreEntry object. 261 /// May destroy this object if it is unlocked; does nothing otherwise. 262 /// Unlike release(), may not trigger eviction of underlying store entries, 263 /// but, unlike destroyStoreEntry(), does honor an earlier release request. abandon(const char * context)264 void abandon(const char *context) { if (!locked()) doAbandon(context); } 265 266 /// May the caller commit to treating this [previously locked] 267 /// entry as a cache hit? mayStartHitting()268 bool mayStartHitting() const { 269 return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate; 270 } 271 272 #if USE_ADAPTATION 273 /// call back producer when more buffer space is available 274 void deferProducer(const AsyncCall::Pointer &producer); 275 /// calls back producer registered with deferProducer 276 void kickProducer(); 277 #endif 278 279 /* Packable API */ 280 virtual void append(char const *, int); 281 virtual void vappendf(const char *, va_list); 282 virtual void buffer(); 283 virtual void flush(); 284 285 protected: 286 typedef Store::EntryGuard EntryGuard; 287 288 void transientsAbandonmentCheck(); 289 /// does nothing except throwing if disk-associated data members are inconsistent 290 void checkDisk() const; 291 292 private: 293 void doAbandon(const char *context); 294 bool checkTooBig() const; 295 void forcePublicKey(const cache_key *newkey); 296 StoreEntry *adjustVary(); 297 const cache_key *calcPublicKey(const KeyScope keyScope); 298 299 static MemAllocator *pool; 300 301 unsigned short lock_count; /* Assume < 65536! */ 302 303 /// Nobody can find/lock KEY_PRIVATE entries, but some transactions 304 /// (e.g., collapsed requests) find/lock a public entry before it becomes 305 /// private. May such transactions start using the now-private entry 306 /// they previously locked? This member should not affect transactions 307 /// that already started reading from the entry. 308 bool shareableWhenPrivate; 309 310 #if USE_ADAPTATION 311 /// producer callback registered with deferProducer 312 AsyncCall::Pointer deferredProducer; 313 #endif 314 315 bool validLength() const; 316 bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const; 317 318 friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e); 319 }; 320 321 std::ostream &operator <<(std::ostream &os, const StoreEntry &e); 322 323 /// \ingroup StoreAPI 324 class NullStoreEntry:public StoreEntry 325 { 326 327 public: 328 static NullStoreEntry *getInstance(); isNull()329 bool isNull() { 330 return true; 331 } 332 333 const char *getMD5Text() const; getReply()334 HttpReply const *getReply() const { return NULL; } write(StoreIOBuffer)335 void write (StoreIOBuffer) {} 336 isEmpty()337 bool isEmpty () const {return true;} 338 bytesWanted(Range<size_t> const aRange,bool)339 virtual size_t bytesWanted(Range<size_t> const aRange, bool) const { return aRange.end; } 340 341 void operator delete(void *address); complete()342 void complete() {} 343 344 private: storeClientType()345 store_client_t storeClientType() const {return STORE_MEM_CLIENT;} 346 347 char const *getSerialisedMetaData(); mayStartSwapOut()348 virtual bool mayStartSwapOut() { return false; } 349 trimMemory(const bool)350 void trimMemory(const bool) {} 351 352 static NullStoreEntry _instance; 353 }; 354 355 /// \ingroup StoreAPI 356 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata); 357 358 namespace Store { 359 360 /// a smart pointer similar to std::unique_ptr<> that automatically 361 /// release()s and unlock()s the guarded Entry on stack-unwinding failures 362 class EntryGuard { 363 public: 364 /// \param entry either nil or a locked Entry to manage 365 /// \param context default unlock() message EntryGuard(Entry * entry,const char * context)366 EntryGuard(Entry *entry, const char *context): 367 entry_(entry), context_(context) { 368 assert(!entry_ || entry_->locked()); 369 } 370 ~EntryGuard()371 ~EntryGuard() { 372 if (entry_) { 373 // something went wrong -- the caller did not unlockAndReset() us 374 onException(); 375 } 376 } 377 378 EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now) 379 380 /// like std::unique_ptr::get() 381 /// \returns nil or the guarded (locked) entry get()382 Entry *get() { 383 return entry_; 384 } 385 386 /// like std::unique_ptr::reset() 387 /// stops guarding the entry 388 /// unlocks the entry (which may destroy it) 389 void unlockAndReset(const char *resetContext = nullptr) { 390 if (entry_) { 391 entry_->unlock(resetContext ? resetContext : context_); 392 entry_ = nullptr; 393 } 394 } 395 396 private: 397 void onException() noexcept; 398 399 Entry *entry_; ///< the guarded Entry or nil 400 const char *context_; ///< default unlock() message 401 }; 402 403 void Stats(StoreEntry *output); 404 void Maintain(void *unused); 405 }; // namespace Store 406 407 /// \ingroup StoreAPI 408 size_t storeEntryInUse(); 409 410 /// \ingroup StoreAPI 411 const char *storeEntryFlags(const StoreEntry *); 412 413 /// \ingroup StoreAPI 414 void storeEntryReplaceObject(StoreEntry *, HttpReply *); 415 416 /// \ingroup StoreAPI 417 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method); 418 419 /// \ingroup StoreAPI 420 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault); 421 422 /// \ingroup StoreAPI 423 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault); 424 425 /// \ingroup StoreAPI 426 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key. 427 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&); 428 429 /// \ingroup StoreAPI 430 /// Creates a new StoreEntry with mem_obj and sets initial flags/states. 431 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&); 432 433 /// \ingroup StoreAPI 434 void storeInit(void); 435 436 /// \ingroup StoreAPI 437 void storeConfigure(void); 438 439 /// \ingroup StoreAPI 440 void storeFreeMemory(void); 441 442 /// \ingroup StoreAPI 443 int expiresMoreThan(time_t, time_t); 444 445 /// \ingroup StoreAPI 446 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2; 447 448 /// \ingroup StoreAPI 449 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap); 450 451 /// \ingroup StoreAPI 452 int storeTooManyDiskFilesOpen(void); 453 454 /// \ingroup StoreAPI 455 void storeHeapPositionUpdate(StoreEntry *, SwapDir *); 456 457 /// \ingroup StoreAPI 458 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn); 459 460 /// \ingroup StoreAPI 461 void storeFsInit(void); 462 463 /// \ingroup StoreAPI 464 void storeFsDone(void); 465 466 /// \ingroup StoreAPI 467 void storeReplAdd(const char *, REMOVALPOLICYCREATE *); 468 469 /// One of the three methods to get rid of an unlocked StoreEntry object. 470 /// This low-level method ignores lock()ing and release() promises. It never 471 /// leaves the entry in the local store_table. 472 /// TODO: Hide by moving its functionality into the StoreEntry destructor. 473 extern FREE destroyStoreEntry; 474 475 /// \ingroup StoreAPI 476 void storeGetMemSpace(int size); 477 478 #endif /* SQUID_STORE_H */ 479 480