1 // Aleth: Ethereum C++ client, tools and libraries.
2 // Copyright 2017-2019 Aleth Authors.
3 // Licensed under the GNU General Public License, Version 3.
4
5 #include <libethereum/SnapshotImporter.h>
6 #include <libethereum/StateImporter.h>
7 #include <libethereum/BlockChainImporter.h>
8 #include <libethereum/SnapshotStorage.h>
9 #include <test/tools/libtesteth/TestHelper.h>
10
11 using namespace dev;
12 using namespace dev::eth;
13 using namespace dev::test;
14
15 namespace
16 {
17 struct ImportedAccount
18 {
19 u256 nonce;
20 u256 balance;
21 std::map<h256, bytes> storage;
22 h256 codeHash;
23 };
24
25 class MockStateImporter: public StateImporterFace
26 {
27 public:
importAccount(h256 const & _addressHash,u256 const & _nonce,u256 const & _balance,std::map<h256,bytes> const & _storage,h256 const & _codeHash)28 void importAccount(h256 const& _addressHash, u256 const& _nonce, u256 const& _balance, std::map<h256, bytes> const& _storage, h256 const& _codeHash) override
29 {
30 if (importedAccounts.count(_addressHash))
31 importedAccounts[_addressHash].storage.insert(_storage.begin(), _storage.end());
32 else
33 importedAccounts.emplace(std::make_pair(_addressHash, ImportedAccount{_nonce, _balance, _storage, _codeHash}));
34 }
35
importCode(bytesConstRef _code)36 h256 importCode(bytesConstRef _code) override
37 {
38 importedCodes.push_back(_code.toBytes());
39 return sha3(_code);
40 }
commitStateDatabase()41 void commitStateDatabase() override { ++commitCounter; }
isAccountImported(h256 const & _addressHash) const42 bool isAccountImported(h256 const& _addressHash) const override { return importedAccounts.count(_addressHash) != 0; }
stateRoot() const43 h256 stateRoot() const override { return h256{}; }
lookupCode(h256 const & _hash) const44 std::string lookupCode(h256 const& _hash) const override
45 {
46 auto it = std::find_if(importedCodes.begin(), importedCodes.end(), [&_hash](bytes const& _code) { return sha3(_code) == _hash; });
47 return it == importedCodes.end() ? std::string{} : std::string(it->begin(), it->end());
48 }
49
50 std::unordered_map<h256, ImportedAccount> importedAccounts;
51 std::vector<bytes> importedCodes;
52 int commitCounter = 0;
53 };
54
55
56 struct ImportedBlock
57 {
58 BlockHeader header;
59 bytes transactions;
60 bytes uncles;
61 bytes receipts;
62 u256 totalDifficulty;
63 };
64
65 class MockBlockChainImporter: public BlockChainImporterFace
66 {
67 public:
importBlock(BlockHeader const & _header,RLP _transactions,RLP _uncles,RLP _receipts,u256 const & _totalDifficulty)68 void importBlock(BlockHeader const& _header, RLP _transactions, RLP _uncles, RLP _receipts, u256 const& _totalDifficulty) override
69 {
70 importedBlocks.push_back({_header, _transactions.data().toBytes(), _uncles.data().toBytes(), _receipts.data().toBytes(), _totalDifficulty});
71 }
setChainStartBlockNumber(u256 const & _number)72 void setChainStartBlockNumber(u256 const& _number) override { chainStartBlockNumber = _number; }
73
74 std::vector<ImportedBlock> importedBlocks;
75 u256 chainStartBlockNumber;
76 };
77
78 class MockSnapshotStorage: public SnapshotStorageFace
79 {
80 public:
readManifest() const81 bytes readManifest() const override { return manifest; }
readCompressedChunk(h256 const &) const82 std::string readCompressedChunk(h256 const&) const override { return std::string(); }
readChunk(h256 const & _chunkHash) const83 std::string readChunk(h256 const& _chunkHash) const override
84 {
85 auto it = chunks.find(_chunkHash);
86 return it == chunks.end() ? std::string{} : std::string(it->second.begin(), it->second.end());
87 }
copyTo(boost::filesystem::path const &) const88 void copyTo(boost::filesystem::path const&) const override {}
89
90 bytes manifest;
91 std::map<h256, bytes> chunks;
92 };
93
94 class SnapshotImporterTestFixture: public TestOutputHelperFixture
95 {
96 public:
SnapshotImporterTestFixture()97 SnapshotImporterTestFixture(): snapshotImporter(stateImporter, blockChainImporter) {}
98
99 MockStateImporter stateImporter;
100 MockBlockChainImporter blockChainImporter;
101 MockSnapshotStorage snapshotStorage;
102 SnapshotImporter snapshotImporter;
103 };
104
createManifest(int _version,h256s const & _stateChunks,h256s const & _blockChunks,h256 const & _stateRoot,unsigned _blockNumber,h256 const & _blockHash)105 bytes createManifest(int _version, h256s const& _stateChunks, h256s const& _blockChunks, h256 const& _stateRoot, unsigned _blockNumber, h256 const& _blockHash)
106 {
107 RLPStream s(6);
108 s << _version;
109 s.appendList(_stateChunks.size());
110 for (auto chunk: _stateChunks)
111 s << chunk;
112 s.appendList(_blockChunks.size());
113 for (auto chunk: _blockChunks)
114 s << chunk;
115 s << _stateRoot << _blockNumber << _blockHash;
116 return s.out();
117 }
118
createAccount(u256 const & _nonce,u256 const & _balance,byte _codeFlag,bytes const & _code,std::map<h256,bytes> _storage)119 bytes createAccount(u256 const& _nonce, u256 const& _balance, byte _codeFlag, bytes const& _code, std::map<h256, bytes> _storage)
120 {
121 RLPStream s(5);
122 s << _nonce << _balance << _codeFlag << _code;
123 s.appendList(_storage.size());
124 for (auto& keyValue: _storage)
125 {
126 s.appendList(2);
127 s << keyValue.first << keyValue.second;
128 }
129
130 return s.out();
131 }
132
createStateChunk(std::vector<std::pair<h256,bytes>> const & _accounts)133 bytes createStateChunk(std::vector<std::pair<h256, bytes>> const& _accounts)
134 {
135 RLPStream s(_accounts.size());
136 for (auto& addressAccount: _accounts)
137 {
138 s.appendList(2);
139 s << addressAccount.first;
140 s.appendRaw(addressAccount.second);
141 }
142
143 return s.out();
144 }
145
createRlpSingleItemList(unsigned _item)146 bytes createRlpSingleItemList(unsigned _item)
147 {
148 RLPStream s;
149 s << _item;
150 return s.out();
151 }
152
createAbridgedBlock(Address const & _author,h256 const & _stateRoot,h2048 const & _logBloom,u256 const & _difficulty,u256 const & _gasLimit,u256 const & _gasUsed,u256 const & _timestamp,bytes const & _extraData,h256 const & _mixHash,h64 const & _nonce,bytes const & _transactions,bytes const & _uncles)153 bytes createAbridgedBlock(Address const& _author, h256 const& _stateRoot, h2048 const& _logBloom, u256 const& _difficulty, u256 const& _gasLimit,
154 u256 const& _gasUsed, u256 const& _timestamp, bytes const& _extraData, h256 const& _mixHash, h64 const& _nonce,
155 bytes const& _transactions, bytes const& _uncles)
156 {
157 RLPStream s(12);
158 s << _author << _stateRoot << _logBloom << _difficulty << _gasLimit << _gasUsed << _timestamp << _extraData;
159 s.appendRaw(_transactions);
160 s.appendRaw(_uncles);
161 s << _mixHash << _nonce;
162
163 return s.out();
164 }
165
createSingleBlockChunk(unsigned _number,h256 const & _hash,u256 const & _totalDifficulty,bytes const & _abridgedBlock,bytes const & _receipts)166 bytes createSingleBlockChunk(unsigned _number, h256 const& _hash, u256 const& _totalDifficulty, bytes const& _abridgedBlock, bytes const& _receipts)
167 {
168 RLPStream s(4);
169 s << _number << _hash << _totalDifficulty;
170 s.appendList(2);
171 s.appendRaw(_abridgedBlock);
172 s.appendRaw(_receipts);
173
174 return s.out();
175 }
176 }
177
BOOST_FIXTURE_TEST_SUITE(SnapshotImporterSuite,SnapshotImporterTestFixture)178 BOOST_FIXTURE_TEST_SUITE(SnapshotImporterSuite, SnapshotImporterTestFixture)
179
180 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importChecksManifestVersion)
181 {
182 RLPStream s(6);
183 s << 3;
184 s.appendList(0);
185 s.appendList(0);
186 s << h256{} << 0 << h256{};
187 snapshotStorage.manifest = createManifest(3, {}, {}, h256{}, 0, h256{});
188
189 BOOST_REQUIRE_THROW(
190 snapshotImporter.import(snapshotStorage, h256{}), UnsupportedSnapshotManifestVersion);
191 }
192
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importNonsplittedAccount)193 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importNonsplittedAccount)
194 {
195 h256 stateChunk = sha3("123");
196 snapshotStorage.manifest = createManifest(2, {stateChunk}, {}, h256{}, 0, h256{});
197
198 bytes account = createAccount(1, 10, 0, {0x80}, {});
199 h256 addressHash = sha3("456");
200 bytes chunkBytes = createStateChunk({{addressHash, account}});
201 snapshotStorage.chunks[stateChunk] = chunkBytes;
202
203 snapshotImporter.import(snapshotStorage, h256{});
204
205 BOOST_REQUIRE_EQUAL(stateImporter.importedAccounts.size(), 1);
206 h256 const& importedAddress = stateImporter.importedAccounts.begin()->first;
207 BOOST_CHECK_EQUAL(importedAddress, addressHash);
208 ImportedAccount const& importedAccount = stateImporter.importedAccounts.begin()->second;
209 BOOST_CHECK_EQUAL(importedAccount.nonce, 1);
210 BOOST_CHECK_EQUAL(importedAccount.balance, 10);
211 BOOST_CHECK_EQUAL(importedAccount.codeHash, EmptySHA3);
212 BOOST_CHECK(importedAccount.storage.empty());
213 }
214
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importSplittedAccount)215 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importSplittedAccount)
216 {
217 h256 stateChunk1 = sha3("123");
218 h256 stateChunk2 = sha3("789");
219 snapshotStorage.manifest = createManifest(2, {stateChunk1, stateChunk2}, {}, h256{}, 0, h256{});
220
221 h256 addressHash = sha3("456");
222 std::pair<h256, bytes> storagePair1{sha3("111"), {1}};
223 std::pair<h256, bytes> storagePair2{sha3("222"), {2}};
224 bytes accountPart1 = createAccount(2, 10, 0, {0x80}, {storagePair1, storagePair2});
225 bytes chunk1 = createStateChunk({{addressHash, accountPart1}});
226 snapshotStorage.chunks[stateChunk1] = chunk1;
227
228 std::pair<h256, bytes> storagePair3{sha3("333"),{3}};
229 std::pair<h256, bytes> storagePair4{sha3("444"),{4}};
230 bytes accountPart2 = createAccount(2, 10, 0, {0x80}, {storagePair3, storagePair4});
231 bytes chunk2 = createStateChunk({{addressHash, accountPart2}});
232 snapshotStorage.chunks[stateChunk2] = chunk2;
233
234 snapshotImporter.import(snapshotStorage, h256{});
235
236 BOOST_REQUIRE_EQUAL(stateImporter.importedAccounts.size(), 1);
237 h256 const& importedAddress = stateImporter.importedAccounts.begin()->first;
238 BOOST_CHECK_EQUAL(importedAddress, addressHash);
239 ImportedAccount const& importedAccount = stateImporter.importedAccounts.begin()->second;
240 BOOST_CHECK_EQUAL(importedAccount.nonce, 2);
241 BOOST_CHECK_EQUAL(importedAccount.balance, 10);
242 BOOST_CHECK_EQUAL(importedAccount.codeHash, EmptySHA3);
243
244 std::map<h256, bytes> expectedStorage{storagePair1, storagePair2, storagePair3, storagePair4};
245 BOOST_CHECK(importedAccount.storage == expectedStorage);
246 }
247
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importAccountWithCode)248 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importAccountWithCode)
249 {
250 h256 stateChunk = sha3("123");
251 snapshotStorage.manifest = createManifest(2, {stateChunk}, {}, h256{}, 0, h256{});
252
253 bytes code = {1, 2, 3};
254 bytes account = createAccount(1, 10, 1, code, {});
255 h256 addressHash = sha3("456");
256 bytes chunkBytes = createStateChunk({{addressHash, account}});
257 snapshotStorage.chunks[stateChunk] = chunkBytes;
258
259 snapshotImporter.import(snapshotStorage, h256{});
260
261 BOOST_REQUIRE_EQUAL(stateImporter.importedAccounts.size(), 1);
262 h256 const& importedAddress = stateImporter.importedAccounts.begin()->first;
263 BOOST_CHECK_EQUAL(importedAddress, addressHash);
264 ImportedAccount const& importedAccount = stateImporter.importedAccounts.begin()->second;
265 BOOST_CHECK_EQUAL(importedAccount.codeHash, sha3(code));
266
267 BOOST_REQUIRE_EQUAL(stateImporter.importedCodes.size(), 1);
268 bytes const& importedCode = stateImporter.importedCodes.front();
269 BOOST_CHECK_EQUAL_COLLECTIONS(importedCode.begin(), importedCode.end(), code.begin(), code.end());
270 }
271
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importAccountsWithEqualCode)272 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importAccountsWithEqualCode)
273 {
274 h256 stateChunk = sha3("123");
275 snapshotStorage.manifest = createManifest(2, {stateChunk}, {}, h256{}, 0, h256{});
276
277 bytes code = {1, 2, 3};
278 bytes account1 = createAccount(1, 10, 1, code, {});
279 h256 addressHash1 = sha3("456");
280
281 h256 codeHash = sha3(code);
282 bytes account2 = createAccount(1, 10, 2, codeHash.asBytes(), {});
283 h256 addressHash2 = sha3("789");
284
285 snapshotStorage.chunks[stateChunk] = createStateChunk({{addressHash1, account1}, {addressHash2, account2}});
286
287 snapshotImporter.import(snapshotStorage, h256{});
288
289 BOOST_REQUIRE_EQUAL(stateImporter.importedAccounts.size(), 2);
290 BOOST_CHECK(stateImporter.importedAccounts.count(addressHash1) > 0);
291 BOOST_CHECK_EQUAL(stateImporter.importedAccounts[addressHash1].codeHash, codeHash);
292 BOOST_CHECK(stateImporter.importedAccounts.count(addressHash2) > 0);
293 BOOST_CHECK_EQUAL(stateImporter.importedAccounts[addressHash2].codeHash, codeHash);
294
295 BOOST_REQUIRE_EQUAL(stateImporter.importedCodes.size(), 1);
296 bytes const& importedCode = stateImporter.importedCodes.front();
297 BOOST_CHECK_EQUAL_COLLECTIONS(importedCode.begin(), importedCode.end(), code.begin(), code.end());
298 }
299
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_commitStateOnceEveryChunk)300 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_commitStateOnceEveryChunk)
301 {
302 h256 stateChunk1 = sha3("123");
303 h256 stateChunk2 = sha3("789");
304 snapshotStorage.manifest = createManifest(2, {stateChunk1, stateChunk2}, {}, h256{}, 0, h256{});
305
306 h256 addressHash1 = sha3("456");
307 bytes accountPart1 = createAccount(2, 10, 0, {0x80}, {});
308 bytes chunk1 = createStateChunk({{addressHash1, accountPart1}});
309 snapshotStorage.chunks[stateChunk1] = chunk1;
310
311 h256 addressHash2 = sha3("345");
312 bytes accountPart2 = createAccount(2, 10, 0, {0x80}, {});
313 bytes chunk2 = createStateChunk({{addressHash2, accountPart2}});
314 snapshotStorage.chunks[stateChunk2] = chunk2;
315
316 snapshotImporter.import(snapshotStorage, h256{});
317
318 BOOST_REQUIRE_EQUAL(stateImporter.importedAccounts.size(), 2);
319 BOOST_REQUIRE_EQUAL(stateImporter.commitCounter, 2);
320 }
321
322
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importEmptyBlock)323 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importEmptyBlock)
324 {
325 h256 blockChunk = sha3("123");
326 snapshotStorage.manifest = createManifest(2, {}, {blockChunk}, h256{}, 0, h256{});
327
328 Address author("111");
329 h256 stateRoot(sha3("222"));
330 h2048 logBloom(333);
331 u256 difficulty = 444;
332 u256 gasLimit = 555;
333 u256 gasUsed = 666;
334 u256 timestamp = 777;
335 bytes extraData = {8, 8, 8};
336 h256 mixHash = sha3("999");
337 Nonce nonce(012);
338 bytes block = createAbridgedBlock(author, stateRoot, logBloom, difficulty, gasLimit, gasUsed, timestamp, extraData, mixHash, nonce, RLPEmptyList, RLPEmptyList);
339
340 unsigned parentNumber = 345;
341 h256 parentHash = sha3("678");
342 u256 parentTotalDifficulty = 910;
343 bytes chunkBytes = createSingleBlockChunk(parentNumber, parentHash, parentTotalDifficulty, block, RLPEmptyList);
344 snapshotStorage.chunks[blockChunk] = chunkBytes;
345
346 snapshotImporter.import(snapshotStorage, h256{});
347
348 BOOST_REQUIRE_EQUAL(blockChainImporter.importedBlocks.size(), 1);
349 ImportedBlock const& importedBlock = blockChainImporter.importedBlocks.front();
350 BlockHeader const& header = importedBlock.header;
351 BOOST_CHECK_EQUAL(header.author(), author);
352 BOOST_CHECK_EQUAL(header.stateRoot(), stateRoot);
353 BOOST_CHECK_EQUAL(header.logBloom(), logBloom);
354 BOOST_CHECK_EQUAL(header.difficulty(), difficulty);
355 BOOST_CHECK_EQUAL(header.gasLimit(), gasLimit);
356 BOOST_CHECK_EQUAL(header.gasUsed(), gasUsed);
357 BOOST_CHECK_EQUAL(header.timestamp(), timestamp);
358 BOOST_CHECK_EQUAL_COLLECTIONS(header.extraData().begin(), header.extraData().end(), extraData.begin(), extraData.end());
359 BOOST_CHECK_EQUAL(Ethash::mixHash(header), mixHash);
360 BOOST_CHECK_EQUAL(Ethash::nonce(header), nonce);
361 BOOST_CHECK_EQUAL(header.number(), parentNumber + 1);
362 BOOST_CHECK_EQUAL(header.parentHash(), parentHash);
363 BOOST_CHECK_EQUAL(importedBlock.totalDifficulty, parentTotalDifficulty + difficulty);
364
365 BOOST_CHECK_EQUAL(blockChainImporter.chainStartBlockNumber, parentNumber + 1);
366 }
367
BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importBlockWithTransactions)368 BOOST_AUTO_TEST_CASE(SnapshotImporterSuite_importBlockWithTransactions)
369 {
370 h256 blockChunk = sha3("123");
371 snapshotStorage.manifest = createManifest(2, {}, {blockChunk}, h256{}, 0, h256{});
372
373 bytes transactions = createRlpSingleItemList(123);
374 bytes uncles = createRlpSingleItemList(456);
375 bytes block = createAbridgedBlock(Address("111"), sha3("222"), h2048(333), 444, 555, 666, 777, {8, 8, 8}, sha3("999"), Nonce(012), transactions, uncles);
376
377 bytes receipts = createRlpSingleItemList(789);
378 bytes chunkBytes = createSingleBlockChunk(345, sha3("678"), 910, block, receipts);
379 snapshotStorage.chunks[blockChunk] = chunkBytes;
380
381 snapshotImporter.import(snapshotStorage, h256{});
382
383 BOOST_REQUIRE_EQUAL(blockChainImporter.importedBlocks.size(), 1);
384 ImportedBlock const& importedBlock = blockChainImporter.importedBlocks.front();
385 BOOST_CHECK_EQUAL_COLLECTIONS(importedBlock.transactions.begin(), importedBlock.transactions.end(), transactions.begin(), transactions.end());
386 BOOST_CHECK_EQUAL_COLLECTIONS(importedBlock.uncles.begin(), importedBlock.uncles.end(), uncles.begin(), uncles.end());
387 BOOST_CHECK_EQUAL_COLLECTIONS(importedBlock.receipts.begin(), importedBlock.receipts.end(), receipts.begin(), receipts.end());
388 }
389
390 BOOST_AUTO_TEST_SUITE_END()
391