1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifndef HashStore_h__ 6 #define HashStore_h__ 7 8 #include "Entries.h" 9 #include "ChunkSet.h" 10 11 #include "nsString.h" 12 #include "nsTArray.h" 13 #include "nsIFile.h" 14 #include "nsISupports.h" 15 #include "nsCOMPtr.h" 16 #include "nsClassHashtable.h" 17 #include <string> 18 19 namespace mozilla { 20 namespace safebrowsing { 21 22 // The abstract class of TableUpdateV2 and TableUpdateV4. This 23 // is convenient for passing the TableUpdate* around associated 24 // with v2 and v4 instance. 25 class TableUpdate { 26 public: TableUpdate(const nsACString & aTable)27 TableUpdate(const nsACString& aTable) : mTable(aTable) {} 28 29 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TableUpdate); 30 31 // To be overriden. 32 virtual bool Empty() const = 0; 33 34 // Common interfaces. TableName()35 const nsCString& TableName() const { return mTable; } 36 37 template <typename T> Cast(TableUpdate * aThat)38 static T* Cast(TableUpdate* aThat) { 39 return (T::TAG == aThat->Tag() ? reinterpret_cast<T*>(aThat) : nullptr); 40 } 41 template <typename T> Cast(const TableUpdate * aThat)42 static const T* Cast(const TableUpdate* aThat) { 43 return (T::TAG == aThat->Tag() ? reinterpret_cast<const T*>(aThat) 44 : nullptr); 45 } 46 47 protected: 48 virtual ~TableUpdate() = default; 49 50 private: 51 virtual int Tag() const = 0; 52 53 const nsCString mTable; 54 }; 55 56 typedef nsTArray<RefPtr<TableUpdate>> TableUpdateArray; 57 typedef nsTArray<RefPtr<const TableUpdate>> ConstTableUpdateArray; 58 59 // A table update is built from a single update chunk from the server. As the 60 // protocol parser processes each chunk, it constructs a table update with the 61 // new hashes. 62 class TableUpdateV2 : public TableUpdate { 63 public: TableUpdateV2(const nsACString & aTable)64 explicit TableUpdateV2(const nsACString& aTable) : TableUpdate(aTable) {} 65 Empty()66 bool Empty() const override { 67 return mAddChunks.Length() == 0 && mSubChunks.Length() == 0 && 68 mAddExpirations.Length() == 0 && mSubExpirations.Length() == 0 && 69 mAddPrefixes.Length() == 0 && mSubPrefixes.Length() == 0 && 70 mAddCompletes.Length() == 0 && mSubCompletes.Length() == 0 && 71 mMissPrefixes.Length() == 0; 72 } 73 74 // Throughout, uint32_t aChunk refers only to the chunk number. Chunk data is 75 // stored in the Prefix structures. NewAddChunk(uint32_t aChunk)76 [[nodiscard]] nsresult NewAddChunk(uint32_t aChunk) { 77 return mAddChunks.Set(aChunk); 78 }; NewSubChunk(uint32_t aChunk)79 [[nodiscard]] nsresult NewSubChunk(uint32_t aChunk) { 80 return mSubChunks.Set(aChunk); 81 }; NewAddExpiration(uint32_t aChunk)82 [[nodiscard]] nsresult NewAddExpiration(uint32_t aChunk) { 83 return mAddExpirations.Set(aChunk); 84 }; NewSubExpiration(uint32_t aChunk)85 [[nodiscard]] nsresult NewSubExpiration(uint32_t aChunk) { 86 return mSubExpirations.Set(aChunk); 87 }; 88 [[nodiscard]] nsresult NewAddPrefix(uint32_t aAddChunk, 89 const Prefix& aPrefix); 90 [[nodiscard]] nsresult NewSubPrefix(uint32_t aAddChunk, const Prefix& aPrefix, 91 uint32_t aSubChunk); 92 [[nodiscard]] nsresult NewAddComplete(uint32_t aChunk, 93 const Completion& aCompletion); 94 [[nodiscard]] nsresult NewSubComplete(uint32_t aAddChunk, 95 const Completion& aCompletion, 96 uint32_t aSubChunk); 97 [[nodiscard]] nsresult NewMissPrefix(const Prefix& aPrefix); 98 AddChunks()99 const ChunkSet& AddChunks() const { return mAddChunks; } SubChunks()100 const ChunkSet& SubChunks() const { return mSubChunks; } 101 102 // Expirations for chunks. AddExpirations()103 const ChunkSet& AddExpirations() const { return mAddExpirations; } SubExpirations()104 const ChunkSet& SubExpirations() const { return mSubExpirations; } 105 106 // Hashes associated with this chunk. AddPrefixes()107 AddPrefixArray& AddPrefixes() { return mAddPrefixes; } SubPrefixes()108 SubPrefixArray& SubPrefixes() { return mSubPrefixes; } AddCompletes()109 const AddCompleteArray& AddCompletes() const { return mAddCompletes; } AddCompletes()110 AddCompleteArray& AddCompletes() { return mAddCompletes; } SubCompletes()111 SubCompleteArray& SubCompletes() { return mSubCompletes; } 112 113 // Entries that cannot be completed. MissPrefixes()114 const MissPrefixArray& MissPrefixes() const { return mMissPrefixes; } 115 116 // For downcasting. 117 static const int TAG = 2; 118 119 private: 120 // The list of chunk numbers that we have for each of the type of chunks. 121 ChunkSet mAddChunks; 122 ChunkSet mSubChunks; 123 ChunkSet mAddExpirations; 124 ChunkSet mSubExpirations; 125 126 // 4-byte sha256 prefixes. 127 AddPrefixArray mAddPrefixes; 128 SubPrefixArray mSubPrefixes; 129 130 // This is only used by gethash so don't add this to Header. 131 MissPrefixArray mMissPrefixes; 132 133 // 32-byte hashes. 134 AddCompleteArray mAddCompletes; 135 SubCompleteArray mSubCompletes; 136 Tag()137 virtual int Tag() const override { return TAG; } 138 }; 139 140 // Structure for DBService/HashStore/Classifiers to update. 141 // It would contain the prefixes (both fixed and variable length) 142 // for addition and indices to removal. See Bug 1283009. 143 class TableUpdateV4 : public TableUpdate { 144 public: 145 typedef nsTArray<int32_t> RemovalIndiceArray; 146 147 public: TableUpdateV4(const nsACString & aTable)148 explicit TableUpdateV4(const nsACString& aTable) 149 : TableUpdate(aTable), mFullUpdate(false) {} 150 Empty()151 bool Empty() const override { 152 return mPrefixesMap.IsEmpty() && mRemovalIndiceArray.IsEmpty() && 153 mFullHashResponseMap.IsEmpty(); 154 } 155 IsFullUpdate()156 bool IsFullUpdate() const { return mFullUpdate; } Prefixes()157 const PrefixStringMap& Prefixes() const { return mPrefixesMap; } RemovalIndices()158 const RemovalIndiceArray& RemovalIndices() const { 159 return mRemovalIndiceArray; 160 } ClientState()161 const nsACString& ClientState() const { return mClientState; } SHA256()162 const nsACString& SHA256() const { return mSHA256; } FullHashResponse()163 const FullHashResponseMap& FullHashResponse() const { 164 return mFullHashResponseMap; 165 } 166 167 // For downcasting. 168 static const int TAG = 4; 169 SetFullUpdate(bool aIsFullUpdate)170 void SetFullUpdate(bool aIsFullUpdate) { mFullUpdate = aIsFullUpdate; } 171 void NewPrefixes(int32_t aSize, const nsACString& aPrefixes); SetNewClientState(const nsACString & aState)172 void SetNewClientState(const nsACString& aState) { mClientState = aState; } 173 void SetSHA256(const std::string& aSHA256); 174 175 nsresult NewRemovalIndices(const uint32_t* aIndices, size_t aNumOfIndices); 176 nsresult NewFullHashResponse(const Prefix& aPrefix, 177 const CachedFullHashResponse& aResponse); 178 179 // Clear Prefixes & Removal indice. 180 void Clear(); 181 182 private: Tag()183 virtual int Tag() const override { return TAG; } 184 185 bool mFullUpdate; 186 PrefixStringMap mPrefixesMap; 187 RemovalIndiceArray mRemovalIndiceArray; 188 nsCString mClientState; 189 nsCString mSHA256; 190 191 // This is used to store response from fullHashes.find. 192 FullHashResponseMap mFullHashResponseMap; 193 }; 194 195 // There is one hash store per table. 196 class HashStore { 197 public: 198 HashStore(const nsACString& aTableName, const nsACString& aProvider, 199 nsIFile* aRootStoreFile); 200 ~HashStore(); 201 TableName()202 const nsCString& TableName() const { return mTableName; } 203 204 // Version is set to 0 by default, it is only used when we want to open 205 // a specific version of HashStore. Note that the intention of aVersion 206 // is only to pass SanityCheck, reading data from older version should 207 // be handled additionally. 208 nsresult Open(uint32_t aVersion = 0); 209 210 // Add Prefixes/Completes are stored partly in the PrefixSet (contains the 211 // Prefix data organized for fast lookup/low RAM usage) and partly in the 212 // HashStore (Add Chunk numbers - only used for updates, slow retrieval). 213 // AugmentAdds function joins the separate datasets into one complete 214 // prefixes+chunknumbers dataset. 215 nsresult AugmentAdds(const nsTArray<uint32_t>& aPrefixes, 216 const nsTArray<nsCString>& aCompletes); 217 218 ChunkSet& AddChunks(); 219 ChunkSet& SubChunks(); AddPrefixes()220 AddPrefixArray& AddPrefixes() { return mAddPrefixes; } SubPrefixes()221 SubPrefixArray& SubPrefixes() { return mSubPrefixes; } AddCompletes()222 AddCompleteArray& AddCompletes() { return mAddCompletes; } SubCompletes()223 SubCompleteArray& SubCompletes() { return mSubCompletes; } 224 225 // ======= 226 // Updates 227 // ======= 228 // Begin the update process. Reads the store into memory. 229 nsresult BeginUpdate(); 230 231 // Imports the data from a TableUpdate. 232 nsresult ApplyUpdate(RefPtr<TableUpdateV2> aUpdate); 233 234 // Process expired chunks 235 nsresult Expire(); 236 237 // Rebuild the store, Incorporating all the applied updates. 238 nsresult Rebuild(); 239 240 // Write the current state of the store to disk. 241 // If you call between ApplyUpdate() and Rebuild(), you'll 242 // have a mess on your hands. 243 nsresult WriteFile(); 244 245 nsresult ReadCompletionsLegacyV3(AddCompleteArray& aCompletes); 246 247 nsresult Reset(); 248 249 private: 250 nsresult ReadHeader(); 251 nsresult SanityCheck(uint32_t aVersion = 0) const; 252 nsresult CalculateChecksum(nsAutoCString& aChecksum, uint32_t aFileSize, 253 bool aChecksumPresent); 254 nsresult CheckChecksum(uint32_t aFileSize); 255 void UpdateHeader(); 256 257 nsresult ReadCompletions(); 258 nsresult ReadChunkNumbers(); 259 nsresult ReadHashes(); 260 261 nsresult ReadAddPrefixes(); 262 nsresult ReadSubPrefixes(); 263 nsresult ReadAddCompletes(); 264 265 nsresult WriteAddPrefixChunks(nsIOutputStream* aOut); 266 nsresult WriteSubPrefixes(nsIOutputStream* aOut); 267 nsresult WriteAddCompleteChunks(nsIOutputStream* aOut); 268 269 nsresult ProcessSubs(); 270 271 nsresult PrepareForUpdate(); 272 273 // This is used for checking that the database is correct and for figuring out 274 // the number of chunks, etc. to read from disk on restart. 275 struct Header { 276 uint32_t magic; 277 uint32_t version; 278 uint32_t numAddChunks; 279 uint32_t numSubChunks; 280 uint32_t numAddPrefixes; 281 uint32_t numSubPrefixes; 282 uint32_t numAddCompletes; 283 uint32_t numSubCompletes; 284 }; 285 286 Header mHeader; 287 288 // The name of the table (must end in -shavar or -digest256, or evidently 289 // -simple for unittesting. 290 const nsCString mTableName; 291 nsCOMPtr<nsIFile> mStoreDirectory; 292 293 bool mInUpdate; 294 295 nsCOMPtr<nsIInputStream> mInputStream; 296 297 // Chunk numbers, stored as uint32_t arrays. 298 ChunkSet mAddChunks; 299 ChunkSet mSubChunks; 300 301 ChunkSet mAddExpirations; 302 ChunkSet mSubExpirations; 303 304 // Chunk data for shavar tables. See Entries.h for format. 305 AddPrefixArray mAddPrefixes; 306 SubPrefixArray mSubPrefixes; 307 308 // See bug 806422 for background. We must be able to distinguish between 309 // updates from the completion server and updates from the regular server. 310 AddCompleteArray mAddCompletes; 311 SubCompleteArray mSubCompletes; 312 313 uint32_t mFileSize; 314 315 // For gtest to inspect private members. 316 friend class PerProviderDirectoryTestUtils; 317 }; 318 319 } // namespace safebrowsing 320 } // namespace mozilla 321 322 #endif 323