1 // Aleth: Ethereum C++ client, tools and libraries.
2 // Copyright 2015-2019 Aleth Authors.
3 // Licensed under the GNU General Public License, Version 3.
4
5 #include "SnapshotImporter.h"
6 #include "Client.h"
7 #include "SnapshotStorage.h"
8
9 #include <libdevcore/FileSystem.h>
10 #include <libdevcore/RLP.h>
11 #include <libdevcore/TrieHash.h>
12 #include <libethashseal/Ethash.h>
13
14 #include <snappy.h>
15
16 namespace dev
17 {
18 namespace eth
19 {
20
import(SnapshotStorageFace const & _snapshotStorage,h256 const &)21 void SnapshotImporter::import(SnapshotStorageFace const& _snapshotStorage, h256 const& /*_genesisHash*/)
22 {
23 bytes const manifestBytes = _snapshotStorage.readManifest();
24 RLP manifest(manifestBytes);
25
26 // For Snapshot format see https://github.com/paritytech/parity/wiki/Warp-Sync-Snapshot-Format
27 u256 const version = manifest[0].toInt<u256>(RLP::VeryStrict);
28 if (version != 2)
29 BOOST_THROW_EXCEPTION(UnsupportedSnapshotManifestVersion());
30 if (manifest.itemCount() != 6)
31 BOOST_THROW_EXCEPTION(InvalidSnapshotManifest());
32
33 u256 const blockNumber = manifest[4].toInt<u256>(RLP::VeryStrict);
34 h256 const blockHash = manifest[5].toHash<h256>(RLP::VeryStrict);
35 LOG(m_logger) << "Importing snapshot for block " << blockNumber << " block hash " << blockHash;
36
37 h256s const stateChunkHashes = manifest[1].toVector<h256>(RLP::VeryStrict);
38 h256 const stateRoot = manifest[3].toHash<h256>(RLP::VeryStrict);
39 importStateChunks(_snapshotStorage, stateChunkHashes, stateRoot);
40
41 h256s const blockChunkHashes = manifest[2].toVector<h256>(RLP::VeryStrict);
42 importBlockChunks(_snapshotStorage, blockChunkHashes);
43 }
44
importStateChunks(SnapshotStorageFace const & _snapshotStorage,h256s const & _stateChunkHashes,h256 const & _stateRoot)45 void SnapshotImporter::importStateChunks(SnapshotStorageFace const& _snapshotStorage, h256s const& _stateChunkHashes, h256 const& _stateRoot)
46 {
47 size_t const stateChunkCount = _stateChunkHashes.size();
48
49 size_t chunksImported = 0;
50 size_t accountsImported = 0;
51
52 for (auto const& stateChunkHash: _stateChunkHashes)
53 {
54 std::string const chunkUncompressed = _snapshotStorage.readChunk(stateChunkHash);
55
56 RLP const accounts(chunkUncompressed);
57 size_t const accountCount = accounts.itemCount();
58 for (size_t accountIndex = 0; accountIndex < accountCount; ++accountIndex)
59 {
60 RLP const addressAndAccount = accounts[accountIndex];
61 if (addressAndAccount.itemCount() != 2)
62 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
63
64 h256 const addressHash = addressAndAccount[0].toHash<h256>(RLP::VeryStrict);
65 if (!addressHash)
66 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
67
68 // splitted parts of account can be only first in chunk
69 if (accountIndex > 0 && m_stateImporter.isAccountImported(addressHash))
70 BOOST_THROW_EXCEPTION(AccountAlreadyImported());
71
72 RLP const account = addressAndAccount[1];
73 if (account.itemCount() != 5)
74 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
75
76 u256 const nonce = account[0].toInt<u256>(RLP::VeryStrict);
77 u256 const balance = account[1].toInt<u256>(RLP::VeryStrict);
78
79 RLP const storage = account[4];
80 std::map<h256, bytes> storageMap;
81 for (auto hashAndValue: storage)
82 {
83 if (hashAndValue.itemCount() != 2)
84 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
85
86 h256 const keyHash = hashAndValue[0].toHash<h256>(RLP::VeryStrict);
87 if (!keyHash || storageMap.find(keyHash) != storageMap.end())
88 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
89
90 bytes value = hashAndValue[1].toBytes(RLP::VeryStrict);
91 if (value.empty())
92 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
93
94 storageMap.emplace(keyHash, std::move(value));
95 }
96
97 byte const codeFlag = account[2].toInt<byte>(RLP::VeryStrict);
98 h256 codeHash;
99 switch (codeFlag)
100 {
101 case 0:
102 codeHash = EmptySHA3;
103 break;
104 case 1:
105 codeHash = m_stateImporter.importCode(account[3].toBytesConstRef(RLP::VeryStrict));
106 break;
107 case 2:
108 codeHash = account[3].toHash<h256>(RLP::VeryStrict);
109 if (!codeHash || m_stateImporter.lookupCode(codeHash).empty())
110 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
111 break;
112 default:
113 BOOST_THROW_EXCEPTION(InvalidStateChunkData());
114 }
115
116 m_stateImporter.importAccount(addressHash, nonce, balance, storageMap, codeHash);
117 }
118 accountsImported += accountCount;
119
120 m_stateImporter.commitStateDatabase();
121
122 ++chunksImported;
123 LOG(m_logger) << "Imported chunk " << chunksImported << " (" << accounts.itemCount()
124 << " account records) Total account records imported: " << accountsImported;
125 LOG(m_logger) << stateChunkCount - chunksImported << " chunks left to import";
126 }
127
128 // check root
129 LOG(m_logger) << "Chunks imported: " << chunksImported;
130 LOG(m_logger) << "Account records imported: " << accountsImported;
131 LOG(m_logger) << "Reconstructed state root: " << m_stateImporter.stateRoot();
132 LOG(m_logger) << "Manifest state root: " << _stateRoot;
133 if (m_stateImporter.stateRoot() != _stateRoot)
134 BOOST_THROW_EXCEPTION(StateTrieReconstructionFailed());
135 }
136
importBlockChunks(SnapshotStorageFace const & _snapshotStorage,h256s const & _blockChunkHashes)137 void SnapshotImporter::importBlockChunks(SnapshotStorageFace const& _snapshotStorage, h256s const& _blockChunkHashes)
138 {
139 size_t const blockChunkCount = _blockChunkHashes.size();
140 size_t blockChunksImported = 0;
141 // chunks are in decreasing order of first block number, so we go backwards to start from the oldest block
142 for (auto chunk = _blockChunkHashes.rbegin(); chunk != _blockChunkHashes.rend(); ++chunk)
143 {
144 std::string const chunkUncompressed = _snapshotStorage.readChunk(*chunk);
145
146 RLP blockChunk(chunkUncompressed);
147 if (blockChunk.itemCount() < 3)
148 BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
149
150 int64_t const firstBlockNumber = blockChunk[0].toPositiveInt64(RLP::VeryStrict);
151 h256 const firstBlockHash = blockChunk[1].toHash<h256>(RLP::VeryStrict);
152 u256 const firstBlockDifficulty = blockChunk[2].toInt<u256>(RLP::VeryStrict);
153 if (!firstBlockNumber || !firstBlockHash || !firstBlockDifficulty)
154 BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
155
156 LOG(m_logger) << "chunk first block " << firstBlockNumber << " first block hash "
157 << firstBlockHash << " difficulty " << firstBlockDifficulty;
158
159 size_t const itemCount = blockChunk.itemCount();
160 h256 parentHash = firstBlockHash;
161 int64_t number = firstBlockNumber + 1;
162 u256 totalDifficulty = firstBlockDifficulty;
163 for (size_t i = 3; i < itemCount; ++i, ++number)
164 {
165 RLP blockAndReceipts = blockChunk[i];
166 if (blockAndReceipts.itemCount() != 2)
167 BOOST_THROW_EXCEPTION(InvalidBlockChunkData());
168
169 RLP abridgedBlock = blockAndReceipts[0];
170
171 BlockHeader header;
172 header.setParentHash(parentHash);
173 header.setAuthor(abridgedBlock[0].toHash<Address>(RLP::VeryStrict));
174
175 h256 const blockStateRoot = abridgedBlock[1].toHash<h256>(RLP::VeryStrict);
176 RLP transactions = abridgedBlock[8];
177 h256 const txRoot = trieRootOver(transactions.itemCount(), [&](unsigned i) { return rlp(i); }, [&](unsigned i) { return transactions[i].data().toBytes(); });
178 RLP uncles = abridgedBlock[9];
179 RLP receipts = blockAndReceipts[1];
180 std::vector<bytesConstRef> receiptsVector;
181 for (auto receipt: receipts)
182 receiptsVector.push_back(receipt.data());
183 h256 const receiptsRoot = orderedTrieRoot(receiptsVector);
184 h256 const unclesHash = sha3(uncles.data());
185 header.setRoots(txRoot, receiptsRoot, unclesHash, blockStateRoot);
186
187 header.setLogBloom(abridgedBlock[2].toHash<LogBloom>(RLP::VeryStrict));
188 u256 const difficulty = abridgedBlock[3].toInt<u256>(RLP::VeryStrict);
189 header.setDifficulty(difficulty);
190 header.setNumber(number);
191 header.setGasLimit(abridgedBlock[4].toInt<u256>(RLP::VeryStrict));
192 header.setGasUsed(abridgedBlock[5].toInt<u256>(RLP::VeryStrict));
193 header.setTimestamp(abridgedBlock[6].toPositiveInt64(RLP::VeryStrict));
194 header.setExtraData(abridgedBlock[7].toBytes(RLP::VeryStrict));
195
196 Ethash::setMixHash(header, abridgedBlock[10].toHash<h256>(RLP::VeryStrict));
197 Ethash::setNonce(header, abridgedBlock[11].toHash<Nonce>(RLP::VeryStrict));
198
199 totalDifficulty += difficulty;
200 m_blockChainImporter.importBlock(header, transactions, uncles, receipts, totalDifficulty);
201
202 parentHash = header.hash();
203 }
204
205 LOG(m_logger) << "Imported chunk " << *chunk << " (" << itemCount - 3 << " blocks)";
206 LOG(m_logger) << blockChunkCount - (++blockChunksImported) << " chunks left to import";
207
208 if (chunk == _blockChunkHashes.rbegin())
209 {
210 LOG(m_logger) << "Setting chain start block: " << firstBlockNumber + 1;
211 m_blockChainImporter.setChainStartBlockNumber(firstBlockNumber + 1);
212 }
213 }
214 }
215
216 } // namespace eth
217 } // namespace dev