1 // This file is part of Desktop App Toolkit, 2 // a set of libraries for developing nice desktop applications. 3 // 4 // For license and copyright information please follow this link: 5 // https://github.com/desktop-app/legal/blob/master/LEGAL 6 // 7 #pragma once 8 9 #include "storage/cache/storage_cache_database.h" 10 #include "storage/storage_encrypted_file.h" 11 #include "base/binary_guard.h" 12 #include "base/concurrent_timer.h" 13 #include "base/bytes.h" 14 #include "base/flat_set.h" 15 #include <set> 16 #include <rpl/event_stream.h> 17 18 namespace Storage { 19 namespace Cache { 20 namespace details { 21 22 class Cleaner; 23 class Compactor; 24 25 class DatabaseObject { 26 public: 27 using Settings = Cache::Database::Settings; 28 DatabaseObject( 29 crl::weak_on_queue<DatabaseObject> weak, 30 const QString &path, 31 const Settings &settings); 32 void reconfigure(const Settings &settings); 33 void updateSettings(const SettingsUpdate &update); 34 35 void open(EncryptionKey &&key, FnMut<void(Error)> &&done); 36 void close(FnMut<void()> &&done); 37 38 void put( 39 const Key &key, 40 TaggedValue &&value, 41 FnMut<void(Error)> &&done); 42 void get(const Key &key, FnMut<void(TaggedValue&&)> &&done); 43 void remove(const Key &key, FnMut<void(Error)> &&done); 44 45 void putIfEmpty( 46 const Key &key, 47 TaggedValue &&value, 48 FnMut<void(Error)> &&done); 49 void copyIfEmpty( 50 const Key &from, 51 const Key &to, 52 FnMut<void(Error)> &&done); 53 void moveIfEmpty( 54 const Key &from, 55 const Key &to, 56 FnMut<void(Error)> &&done); 57 58 void getWithSizes( 59 const Key &key, 60 std::vector<Key> &&keys, 61 FnMut<void(QByteArray&&, std::vector<int>&&)> &&done); 62 63 rpl::producer<Stats> stats() const; 64 65 void clear(FnMut<void(Error)> &&done); 66 void clearByTag(uint8 tag, FnMut<void(Error)> &&done); 67 void waitForCleaner(FnMut<void()> &&done); 68 69 static QString BinlogFilename(); 70 static QString CompactReadyFilename(); 71 72 void compactorDone(const QString &path, int64 originalReadTill); 73 void compactorFail(); 74 75 struct Entry { 76 Entry() = default; 77 Entry( 78 PlaceId place, 79 uint8 tag, 80 uint32 checksum, 81 size_type size, 82 uint64 useTime); 83 84 uint64 useTime = 0; 85 size_type size = 0; 86 uint32 checksum = 0; 87 PlaceId place = { { 0 } }; 88 uint8 tag = 0; 89 }; 90 using Raw = std::pair<Key, Entry>; 91 std::vector<Raw> getManyRaw(const std::vector<Key> &keys) const; 92 93 ~DatabaseObject(); 94 95 private: 96 struct CleanerWrap { 97 std::unique_ptr<Cleaner> object; 98 base::binary_guard guard; 99 FnMut<void()> done; 100 }; 101 struct CompactorWrap { 102 std::unique_ptr<Compactor> object; 103 int64 excessLength = 0; 104 crl::time nextAttempt = 0; 105 crl::time delayAfterFailure = 10 * crl::time(1000); 106 base::binary_guard guard; 107 }; 108 struct KeyPlaceChange { 109 QString wasPath; 110 QString nowPath; 111 PlaceId nowPlace; 112 }; 113 using Map = std::unordered_map<Key, Entry>; 114 115 template <typename Callback, typename ...Args> 116 void invokeCallback(Callback &&callback, Args &&...args) const; 117 118 Error ioError(const QString &path) const; 119 120 void checkSettings(); 121 QString computePath(Version version) const; 122 QString binlogPath(Version version) const; 123 QString binlogPath() const; 124 QString compactReadyPath(Version version) const; 125 QString compactReadyPath() const; 126 Error openSomeBinlog(EncryptionKey &&key); 127 Error openNewBinlog(EncryptionKey &key); 128 File::Result openBinlog( 129 Version version, 130 File::Mode mode, 131 EncryptionKey &key); 132 bool readHeader(); 133 bool writeHeader(); 134 135 void readBinlog(); 136 template <typename Reader, typename ...Handlers> 137 void readBinlogHelper(Reader &reader, Handlers &&...handlers); 138 template <typename Record, typename Postprocess> 139 bool processRecordStoreGeneric( 140 const Record *record, 141 Postprocess &&postprocess); 142 bool processRecordStore(const Store *record, std::is_class<Store>); 143 bool processRecordStore( 144 const StoreWithTime *record, 145 std::is_class<StoreWithTime>); 146 template <typename Record, typename GetElement> 147 bool processRecordMultiStore( 148 const Record &header, 149 const GetElement &element); 150 template <typename GetElement> 151 bool processRecordMultiRemove( 152 const MultiRemove &header, 153 const GetElement &element); 154 template <typename GetElement> 155 bool processRecordMultiAccess( 156 const MultiAccess &header, 157 const GetElement &element); 158 159 void optimize(); 160 void checkCompactor(); 161 void adjustRelativeTime(); 162 bool startDelayedPruning(); 163 uint64 countRelativeTime() const; 164 EstimatedTimePoint countTimePoint() const; 165 void applyTimePoint(EstimatedTimePoint time); 166 167 uint64 pruneBeforeTime() const; 168 void prune(); 169 void collectTimeStale( 170 base::flat_set<Key> &stale, 171 int64 &staleTotalSize); 172 void collectSizeStale( 173 base::flat_set<Key> &stale, 174 int64 &staleTotalSize); 175 void startStaleClear(); 176 void clearStaleNow(const base::flat_set<Key> &stale); 177 void clearStaleChunkDelayed(); 178 void clearStaleChunk(); 179 180 void updateStats(const Entry &was, const Entry &now); 181 Stats collectStats() const; 182 void pushStatsDelayed(); 183 void pushStats(); 184 185 void setMapEntry(const Key &key, Entry &&entry); 186 void eraseMapEntry(const Map::const_iterator &i); 187 void recordEntryAccess(const Key &key); 188 QByteArray readValueData(PlaceId place, size_type size) const; 189 190 Version findAvailableVersion() const; 191 QString versionPath() const; 192 bool writeVersion(Version version); 193 Version readVersion() const; 194 195 QString placePath(PlaceId place) const; 196 bool isFreePlace(PlaceId place) const; 197 198 KeyPlaceChange chooseKeyPlace( 199 const Key &key, 200 const TaggedValue &value, 201 uint32 checksum); 202 Error writeNewEntry( 203 const QString &path, 204 QByteArray &&content); 205 template <typename StoreRecord> 206 Error writeKeyPlaceGeneric( 207 StoreRecord &&record, 208 const Key &key, 209 const PlaceId &place, 210 const TaggedValue &value, 211 uint32 checksum); 212 Error writeKeyPlace( 213 const Key &key, 214 const PlaceId &place, 215 const TaggedValue &value, 216 uint32 checksum); 217 template <typename StoreRecord> 218 Error writeExistingPlaceGeneric( 219 StoreRecord &&record, 220 const Key &key, 221 const Entry &entry); 222 Error writeExistingPlace( 223 const Key &key, 224 const Entry &entry); 225 void writeMultiRemoveLazy(); 226 Error writeMultiRemove(); 227 void writeMultiAccessLazy(); 228 Error writeMultiAccess(); 229 Error writeMultiAccessBlock(); 230 void writeBundlesLazy(); 231 void writeBundles(); 232 233 void createCleaner(); 234 void cleanerDone(Error error); 235 void clearState(); 236 237 crl::weak_on_queue<DatabaseObject> _weak; 238 QString _base, _path; 239 Settings _settings; 240 EncryptionKey _key; 241 File _binlog; 242 Map _map; 243 std::set<Key> _removing; 244 std::set<Key> _accessed; 245 std::vector<Key> _stale; 246 247 EstimatedTimePoint _time; 248 249 int64 _binlogExcessLength = 0; 250 int64 _totalSize = 0; 251 uint64 _minimalEntryTime = 0; 252 size_type _entriesWithMinimalTimeCount = 0; 253 254 base::flat_map<uint8, TaggedSummary> _taggedStats; 255 rpl::event_stream<Stats> _stats; 256 bool _pushingStats = false; 257 bool _clearingStale = false; 258 259 base::ConcurrentTimer _writeBundlesTimer; 260 base::ConcurrentTimer _pruneTimer; 261 262 CleanerWrap _cleaner; 263 CompactorWrap _compactor; 264 265 }; 266 267 } // namespace details 268 } // namespace Cache 269 } // namespace Storage 270