1 // Aleth: Ethereum C++ client, tools and libraries.
2 // Copyright 2015-2019 Aleth Authors.
3 // Licensed under the GNU General Public License, Version 3.
4 
5 #include "SnapshotImporter.h"
6 #include "Client.h"
7 #include "SnapshotStorage.h"
8 
9 #include <libdevcore/FileSystem.h>
10 #include <libdevcore/RLP.h>
11 #include <libdevcore/TrieHash.h>
12 #include <libethashseal/Ethash.h>
13 
14 #include <snappy.h>
15 
16 namespace dev
17 {
18 namespace eth
19 {
20 
import(SnapshotStorageFace const & _snapshotStorage,h256 const &)21 void SnapshotImporter::import(SnapshotStorageFace const& _snapshotStorage, h256 const& /*_genesisHash*/)
22 {
23     bytes const manifestBytes = _snapshotStorage.readManifest();
24     RLP manifest(manifestBytes);
25 
26     // For Snapshot format see https://github.com/paritytech/parity/wiki/Warp-Sync-Snapshot-Format
27     u256 const version = manifest[0].toInt<u256>(RLP::VeryStrict);
28     if (version != 2)
29         BOOST_THROW_EXCEPTION(UnsupportedSnapshotManifestVersion());
30     if (manifest.itemCount() != 6)
31         BOOST_THROW_EXCEPTION(InvalidSnapshotManifest());
32 
33     u256 const blockNumber = manifest[4].toInt<u256>(RLP::VeryStrict);
34     h256 const blockHash = manifest[5].toHash<h256>(RLP::VeryStrict);
35     LOG(m_logger) << "Importing snapshot for block " << blockNumber << " block hash " << blockHash;
36 
37     h256s const stateChunkHashes = manifest[1].toVector<h256>(RLP::VeryStrict);
38     h256 const stateRoot = manifest[3].toHash<h256>(RLP::VeryStrict);
39     importStateChunks(_snapshotStorage, stateChunkHashes, stateRoot);
40 
41     h256s const blockChunkHashes = manifest[2].toVector<h256>(RLP::VeryStrict);
42     importBlockChunks(_snapshotStorage, blockChunkHashes);
43 }
44 
importStateChunks(SnapshotStorageFace const & _snapshotStorage,h256s const & _stateChunkHashes,h256 const & _stateRoot)45 void SnapshotImporter::importStateChunks(SnapshotStorageFace const& _snapshotStorage, h256s const& _stateChunkHashes, h256 const& _stateRoot)
46 {
47     size_t const stateChunkCount = _stateChunkHashes.size();
48 
49     size_t chunksImported = 0;
50     size_t accountsImported = 0;
51 
52     for (auto const& stateChunkHash: _stateChunkHashes)
53     {
54         std::string const chunkUncompressed = _snapshotStorage.readChunk(stateChunkHash);
55 
56         RLP const accounts(chunkUncompressed);
57         size_t const accountCount = accounts.itemCount();
58         for (size_t accountIndex = 0; accountIndex < accountCount; ++accountIndex)
59         {
60             RLP const addressAndAccount = accounts[accountIndex];
61             if (addressAndAccount.itemCount() != 2)
62                 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
63 
64             h256 const addressHash = addressAndAccount[0].toHash<h256>(RLP::VeryStrict);
65             if (!addressHash)
66                 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
67 
68             // splitted parts of account can be only first in chunk
69             if (accountIndex > 0 && m_stateImporter.isAccountImported(addressHash))
70                 BOOST_THROW_EXCEPTION(AccountAlreadyImported());
71 
72             RLP const account = addressAndAccount[1];
73             if (account.itemCount() != 5)
74                 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
75 
76             u256 const nonce = account[0].toInt<u256>(RLP::VeryStrict);
77             u256 const balance = account[1].toInt<u256>(RLP::VeryStrict);
78 
79             RLP const storage = account[4];
80             std::map<h256, bytes> storageMap;
81             for (auto hashAndValue: storage)
82             {
83                 if (hashAndValue.itemCount() != 2)
84                     BOOST_THROW_EXCEPTION(InvalidStateChunkData());
85 
86                 h256 const keyHash = hashAndValue[0].toHash<h256>(RLP::VeryStrict);
87                 if (!keyHash || storageMap.find(keyHash) != storageMap.end())
88                     BOOST_THROW_EXCEPTION(InvalidStateChunkData());
89 
90                 bytes value = hashAndValue[1].toBytes(RLP::VeryStrict);
91                 if (value.empty())
92                     BOOST_THROW_EXCEPTION(InvalidStateChunkData());
93 
94                 storageMap.emplace(keyHash, std::move(value));
95             }
96 
97             byte const codeFlag = account[2].toInt<byte>(RLP::VeryStrict);
98             h256 codeHash;
99             switch (codeFlag)
100             {
101             case 0:
102                 codeHash = EmptySHA3;
103                 break;
104             case 1:
105                 codeHash = m_stateImporter.importCode(account[3].toBytesConstRef(RLP::VeryStrict));
106                 break;
107             case 2:
108                 codeHash = account[3].toHash<h256>(RLP::VeryStrict);
109                 if (!codeHash || m_stateImporter.lookupCode(codeHash).empty())
110                     BOOST_THROW_EXCEPTION(InvalidStateChunkData());
111                 break;
112             default:
113                 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
114             }
115 
116             m_stateImporter.importAccount(addressHash, nonce, balance, storageMap, codeHash);
117         }
118         accountsImported += accountCount;
119 
120         m_stateImporter.commitStateDatabase();
121 
122         ++chunksImported;
123         LOG(m_logger) << "Imported chunk " << chunksImported << " (" << accounts.itemCount()
124                       << " account records) Total account records imported: " << accountsImported;
125         LOG(m_logger) << stateChunkCount - chunksImported << " chunks left to import";
126     }
127 
128     // check root
129     LOG(m_logger) << "Chunks imported: " << chunksImported;
130     LOG(m_logger) << "Account records imported: " << accountsImported;
131     LOG(m_logger) << "Reconstructed state root: " << m_stateImporter.stateRoot();
132     LOG(m_logger) << "Manifest state root:      " << _stateRoot;
133     if (m_stateImporter.stateRoot() != _stateRoot)
134         BOOST_THROW_EXCEPTION(StateTrieReconstructionFailed());
135 }
136 
importBlockChunks(SnapshotStorageFace const & _snapshotStorage,h256s const & _blockChunkHashes)137 void SnapshotImporter::importBlockChunks(SnapshotStorageFace const& _snapshotStorage, h256s const& _blockChunkHashes)
138 {
139     size_t const blockChunkCount = _blockChunkHashes.size();
140     size_t blockChunksImported = 0;
141     // chunks are in decreasing order of first block number, so we go backwards to start from the oldest block
142     for (auto chunk = _blockChunkHashes.rbegin(); chunk != _blockChunkHashes.rend(); ++chunk)
143     {
144         std::string const chunkUncompressed = _snapshotStorage.readChunk(*chunk);
145 
146         RLP blockChunk(chunkUncompressed);
147         if (blockChunk.itemCount() < 3)
148             BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
149 
150         int64_t const firstBlockNumber = blockChunk[0].toPositiveInt64(RLP::VeryStrict);
151         h256 const firstBlockHash = blockChunk[1].toHash<h256>(RLP::VeryStrict);
152         u256 const firstBlockDifficulty = blockChunk[2].toInt<u256>(RLP::VeryStrict);
153         if (!firstBlockNumber || !firstBlockHash || !firstBlockDifficulty)
154             BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
155 
156         LOG(m_logger) << "chunk first block " << firstBlockNumber << " first block hash "
157                       << firstBlockHash << " difficulty " << firstBlockDifficulty;
158 
159         size_t const itemCount = blockChunk.itemCount();
160         h256 parentHash = firstBlockHash;
161         int64_t number = firstBlockNumber + 1;
162         u256 totalDifficulty = firstBlockDifficulty;
163         for (size_t i = 3; i < itemCount; ++i, ++number)
164         {
165             RLP blockAndReceipts = blockChunk[i];
166             if (blockAndReceipts.itemCount() != 2)
167                 BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
168 
169             RLP abridgedBlock = blockAndReceipts[0];
170 
171             BlockHeader header;
172             header.setParentHash(parentHash);
173             header.setAuthor(abridgedBlock[0].toHash<Address>(RLP::VeryStrict));
174 
175             h256 const blockStateRoot = abridgedBlock[1].toHash<h256>(RLP::VeryStrict);
176             RLP transactions = abridgedBlock[8];
177             h256 const txRoot = trieRootOver(transactions.itemCount(), [&](unsigned i) { return rlp(i); }, [&](unsigned i) { return transactions[i].data().toBytes(); });
178             RLP uncles = abridgedBlock[9];
179             RLP receipts = blockAndReceipts[1];
180             std::vector<bytesConstRef> receiptsVector;
181             for (auto receipt: receipts)
182                 receiptsVector.push_back(receipt.data());
183             h256 const receiptsRoot = orderedTrieRoot(receiptsVector);
184             h256 const unclesHash = sha3(uncles.data());
185             header.setRoots(txRoot, receiptsRoot, unclesHash, blockStateRoot);
186 
187             header.setLogBloom(abridgedBlock[2].toHash<LogBloom>(RLP::VeryStrict));
188             u256 const difficulty = abridgedBlock[3].toInt<u256>(RLP::VeryStrict);
189             header.setDifficulty(difficulty);
190             header.setNumber(number);
191             header.setGasLimit(abridgedBlock[4].toInt<u256>(RLP::VeryStrict));
192             header.setGasUsed(abridgedBlock[5].toInt<u256>(RLP::VeryStrict));
193             header.setTimestamp(abridgedBlock[6].toPositiveInt64(RLP::VeryStrict));
194             header.setExtraData(abridgedBlock[7].toBytes(RLP::VeryStrict));
195 
196             Ethash::setMixHash(header, abridgedBlock[10].toHash<h256>(RLP::VeryStrict));
197             Ethash::setNonce(header, abridgedBlock[11].toHash<Nonce>(RLP::VeryStrict));
198 
199             totalDifficulty += difficulty;
200             m_blockChainImporter.importBlock(header, transactions, uncles, receipts, totalDifficulty);
201 
202             parentHash = header.hash();
203         }
204 
205         LOG(m_logger) << "Imported chunk " << *chunk << " (" << itemCount - 3 << " blocks)";
206         LOG(m_logger) << blockChunkCount - (++blockChunksImported) << " chunks left to import";
207 
208         if (chunk == _blockChunkHashes.rbegin())
209         {
210             LOG(m_logger) << "Setting chain start block: " << firstBlockNumber + 1;
211             m_blockChainImporter.setChainStartBlockNumber(firstBlockNumber + 1);
212         }
213     }
214 }
215 
216 }  // namespace eth
217 }  // namespace dev