1#!/usr/bin/env python3 2# Copyright (c) 2018-2020 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5"""upgradewallet RPC functional test 6 7Test upgradewallet RPC. Download node binaries: 8 9test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 10 11Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py 12""" 13 14import os 15import shutil 16import struct 17 18from io import BytesIO 19 20from test_framework.bdb import dump_bdb_kv 21from test_framework.messages import deser_compact_size, deser_string 22from test_framework.test_framework import BitcoinTestFramework 23from test_framework.util import ( 24 assert_equal, 25 assert_is_hex_string, 26 sha256sum_file, 27) 28 29 30UPGRADED_KEYMETA_VERSION = 12 31 32def deser_keymeta(f): 33 ver, create_time = struct.unpack('<Iq', f.read(12)) 34 kp_str = deser_string(f) 35 seed_id = f.read(20) 36 fpr = f.read(4) 37 path_len = 0 38 path = [] 39 has_key_orig = False 40 if ver == UPGRADED_KEYMETA_VERSION: 41 path_len = deser_compact_size(f) 42 for i in range(0, path_len): 43 path.append(struct.unpack('<I', f.read(4))[0]) 44 has_key_orig = bool(f.read(1)) 45 return ver, create_time, kp_str, seed_id, fpr, path_len, path, has_key_orig 46 47class UpgradeWalletTest(BitcoinTestFramework): 48 def set_test_params(self): 49 self.setup_clean_chain = True 50 self.num_nodes = 3 51 self.extra_args = [ 52 ["-addresstype=bech32", "-keypool=2"], # current wallet version 53 ["-usehd=1", "-keypool=2"], # v0.16.3 wallet 54 ["-usehd=0", "-keypool=2"] # v0.15.2 wallet 55 ] 56 self.wallet_names = [self.default_wallet_name, None, None] 57 58 def skip_test_if_missing_module(self): 59 self.skip_if_no_wallet() 60 self.skip_if_no_previous_releases() 61 62 def setup_network(self): 63 self.setup_nodes() 64 65 def setup_nodes(self): 66 self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ 67 None, 68 160300, 69 150200, 70 ]) 71 self.start_nodes() 72 self.import_deterministic_coinbase_privkeys() 73 74 def dumb_sync_blocks(self): 75 """ 76 Little helper to sync older wallets. 77 Notice that v0.15.2's regtest is hardforked, so there is 78 no sync for it. 79 v0.15.2 is only being used to test for version upgrade 80 and master hash key presence. 81 v0.16.3 is being used to test for version upgrade and balances. 82 Further info: https://github.com/bitcoin/bitcoin/pull/18774#discussion_r416967844 83 """ 84 node_from = self.nodes[0] 85 v16_3_node = self.nodes[1] 86 to_height = node_from.getblockcount() 87 height = self.nodes[1].getblockcount() 88 for i in range(height, to_height+1): 89 b = node_from.getblock(blockhash=node_from.getblockhash(i), verbose=0) 90 v16_3_node.submitblock(b) 91 assert_equal(v16_3_node.getblockcount(), to_height) 92 93 def test_upgradewallet(self, wallet, previous_version, requested_version=None, expected_version=None): 94 unchanged = expected_version == previous_version 95 new_version = previous_version if unchanged else expected_version if expected_version else requested_version 96 assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) 97 assert_equal(wallet.upgradewallet(requested_version), 98 { 99 "wallet_name": "", 100 "previous_version": previous_version, 101 "current_version": new_version, 102 "result": "Already at latest version. Wallet version unchanged." if unchanged else "Wallet upgraded successfully from version {} to version {}.".format(previous_version, new_version), 103 } 104 ) 105 assert_equal(wallet.getwalletinfo()["walletversion"], new_version) 106 107 def test_upgradewallet_error(self, wallet, previous_version, requested_version, msg): 108 assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) 109 assert_equal(wallet.upgradewallet(requested_version), 110 { 111 "wallet_name": "", 112 "previous_version": previous_version, 113 "current_version": previous_version, 114 "error": msg, 115 } 116 ) 117 assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) 118 119 def run_test(self): 120 self.nodes[0].generatetoaddress(101, self.nodes[0].getnewaddress()) 121 self.dumb_sync_blocks() 122 # # Sanity check the test framework: 123 res = self.nodes[0].getblockchaininfo() 124 assert_equal(res['blocks'], 101) 125 node_master = self.nodes[0] 126 v16_3_node = self.nodes[1] 127 v15_2_node = self.nodes[2] 128 129 # Send coins to old wallets for later conversion checks. 130 v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat') 131 v16_3_address = v16_3_wallet.getnewaddress() 132 node_master.generatetoaddress(101, v16_3_address) 133 self.dumb_sync_blocks() 134 v16_3_balance = v16_3_wallet.getbalance() 135 136 self.log.info("Test upgradewallet RPC...") 137 # Prepare for copying of the older wallet 138 node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets", self.default_wallet_name) 139 node_master_wallet = os.path.join(node_master_wallet_dir, self.default_wallet_name, self.wallet_data_filename) 140 v16_3_wallet = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat") 141 v15_2_wallet = os.path.join(v15_2_node.datadir, "regtest/wallet.dat") 142 split_hd_wallet = os.path.join(v15_2_node.datadir, "regtest/splithd") 143 self.stop_nodes() 144 145 # Make split hd wallet 146 self.start_node(2, ['-usehd=1', '-keypool=2', '-wallet=splithd']) 147 self.stop_node(2) 148 149 def copy_v16(): 150 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() 151 # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it: 152 shutil.rmtree(node_master_wallet_dir) 153 os.mkdir(node_master_wallet_dir) 154 shutil.copy( 155 v16_3_wallet, 156 node_master_wallet_dir 157 ) 158 node_master.loadwallet(self.default_wallet_name) 159 160 def copy_non_hd(): 161 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() 162 # Copy the 0.15.2 non hd wallet to the last Bitcoin Core version and open it: 163 shutil.rmtree(node_master_wallet_dir) 164 os.mkdir(node_master_wallet_dir) 165 shutil.copy( 166 v15_2_wallet, 167 node_master_wallet_dir 168 ) 169 node_master.loadwallet(self.default_wallet_name) 170 171 def copy_split_hd(): 172 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet() 173 # Copy the 0.15.2 split hd wallet to the last Bitcoin Core version and open it: 174 shutil.rmtree(node_master_wallet_dir) 175 os.mkdir(node_master_wallet_dir) 176 shutil.copy( 177 split_hd_wallet, 178 os.path.join(node_master_wallet_dir, 'wallet.dat') 179 ) 180 node_master.loadwallet(self.default_wallet_name) 181 182 self.restart_node(0) 183 copy_v16() 184 wallet = node_master.get_wallet_rpc(self.default_wallet_name) 185 self.log.info("Test upgradewallet without a version argument") 186 self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) 187 # wallet should still contain the same balance 188 assert_equal(wallet.getbalance(), v16_3_balance) 189 190 copy_non_hd() 191 wallet = node_master.get_wallet_rpc(self.default_wallet_name) 192 # should have no master key hash before conversion 193 assert_equal('hdseedid' in wallet.getwalletinfo(), False) 194 self.log.info("Test upgradewallet with explicit version number") 195 self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900) 196 # after conversion master key hash should be present 197 assert_is_hex_string(wallet.getwalletinfo()['hdseedid']) 198 199 self.log.info("Intermediary versions don't effect anything") 200 copy_non_hd() 201 # Wallet starts with 60000 202 assert_equal(60000, wallet.getwalletinfo()['walletversion']) 203 wallet.unloadwallet() 204 before_checksum = sha256sum_file(node_master_wallet) 205 node_master.loadwallet('') 206 # Test an "upgrade" from 60000 to 129999 has no effect, as the next version is 130000 207 self.test_upgradewallet(wallet, previous_version=60000, requested_version=129999, expected_version=60000) 208 wallet.unloadwallet() 209 assert_equal(before_checksum, sha256sum_file(node_master_wallet)) 210 node_master.loadwallet('') 211 212 self.log.info('Wallets cannot be downgraded') 213 copy_non_hd() 214 self.test_upgradewallet_error(wallet, previous_version=60000, requested_version=40000, msg="Cannot downgrade wallet") 215 wallet.unloadwallet() 216 assert_equal(before_checksum, sha256sum_file(node_master_wallet)) 217 node_master.loadwallet('') 218 219 self.log.info('Can upgrade to HD') 220 # Inspect the old wallet and make sure there is no hdchain 221 orig_kvs = dump_bdb_kv(node_master_wallet) 222 assert b'\x07hdchain' not in orig_kvs 223 # Upgrade to HD, no split 224 self.test_upgradewallet(wallet, previous_version=60000, requested_version=130000) 225 # Check that there is now a hd chain and it is version 1, no internal chain counter 226 new_kvs = dump_bdb_kv(node_master_wallet) 227 assert b'\x07hdchain' in new_kvs 228 hd_chain = new_kvs[b'\x07hdchain'] 229 assert_equal(28, len(hd_chain)) 230 hd_chain_version, external_counter, seed_id = struct.unpack('<iI20s', hd_chain) 231 assert_equal(1, hd_chain_version) 232 seed_id = bytearray(seed_id) 233 seed_id.reverse() 234 old_kvs = new_kvs 235 # First 2 keys should still be non-HD 236 for i in range(0, 2): 237 info = wallet.getaddressinfo(wallet.getnewaddress()) 238 assert 'hdkeypath' not in info 239 assert 'hdseedid' not in info 240 # Next key should be HD 241 info = wallet.getaddressinfo(wallet.getnewaddress()) 242 assert_equal(seed_id.hex(), info['hdseedid']) 243 assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) 244 prev_seed_id = info['hdseedid'] 245 # Change key should be the same keypool 246 info = wallet.getaddressinfo(wallet.getrawchangeaddress()) 247 assert_equal(prev_seed_id, info['hdseedid']) 248 assert_equal('m/0\'/0\'/1\'', info['hdkeypath']) 249 250 self.log.info('Cannot upgrade to HD Split, needs Pre Split Keypool') 251 for version in [139900, 159900, 169899]: 252 self.test_upgradewallet_error(wallet, previous_version=130000, requested_version=version, 253 msg="Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.") 254 255 self.log.info('Upgrade HD to HD chain split') 256 self.test_upgradewallet(wallet, previous_version=130000, requested_version=169900) 257 # Check that the hdchain updated correctly 258 new_kvs = dump_bdb_kv(node_master_wallet) 259 hd_chain = new_kvs[b'\x07hdchain'] 260 assert_equal(32, len(hd_chain)) 261 hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) 262 assert_equal(2, hd_chain_version) 263 assert_equal(0, internal_counter) 264 seed_id = bytearray(seed_id) 265 seed_id.reverse() 266 assert_equal(seed_id.hex(), prev_seed_id) 267 # Next change address is the same keypool 268 info = wallet.getaddressinfo(wallet.getrawchangeaddress()) 269 assert_equal(prev_seed_id, info['hdseedid']) 270 assert_equal('m/0\'/0\'/2\'', info['hdkeypath']) 271 # Next change address is the new keypool 272 info = wallet.getaddressinfo(wallet.getrawchangeaddress()) 273 assert_equal(prev_seed_id, info['hdseedid']) 274 assert_equal('m/0\'/1\'/0\'', info['hdkeypath']) 275 # External addresses use the same keypool 276 info = wallet.getaddressinfo(wallet.getnewaddress()) 277 assert_equal(prev_seed_id, info['hdseedid']) 278 assert_equal('m/0\'/0\'/3\'', info['hdkeypath']) 279 280 self.log.info('Upgrade non-HD to HD chain split') 281 copy_non_hd() 282 self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900) 283 # Check that the hdchain updated correctly 284 new_kvs = dump_bdb_kv(node_master_wallet) 285 hd_chain = new_kvs[b'\x07hdchain'] 286 assert_equal(32, len(hd_chain)) 287 hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) 288 assert_equal(2, hd_chain_version) 289 assert_equal(2, internal_counter) 290 # Drain the keypool by fetching one external key and one change key. Should still be the same keypool 291 info = wallet.getaddressinfo(wallet.getnewaddress()) 292 assert 'hdseedid' not in info 293 assert 'hdkeypath' not in info 294 info = wallet.getaddressinfo(wallet.getrawchangeaddress()) 295 assert 'hdseedid' not in info 296 assert 'hdkeypath' not in info 297 # The next addresses are HD and should be on different HD chains 298 info = wallet.getaddressinfo(wallet.getnewaddress()) 299 ext_id = info['hdseedid'] 300 assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) 301 info = wallet.getaddressinfo(wallet.getrawchangeaddress()) 302 assert_equal(ext_id, info['hdseedid']) 303 assert_equal('m/0\'/1\'/0\'', info['hdkeypath']) 304 305 self.log.info('KeyMetadata should upgrade when loading into master') 306 copy_v16() 307 old_kvs = dump_bdb_kv(v16_3_wallet) 308 new_kvs = dump_bdb_kv(node_master_wallet) 309 for k, old_v in old_kvs.items(): 310 if k.startswith(b'\x07keymeta'): 311 new_ver, new_create_time, new_kp_str, new_seed_id, new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k])) 312 old_ver, old_create_time, old_kp_str, old_seed_id, old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v)) 313 assert_equal(10, old_ver) 314 if old_kp_str == b"": # imported things that don't have keymeta (i.e. imported coinbase privkeys) won't be upgraded 315 assert_equal(new_kvs[k], old_v) 316 continue 317 assert_equal(12, new_ver) 318 assert_equal(new_create_time, old_create_time) 319 assert_equal(new_kp_str, old_kp_str) 320 assert_equal(new_seed_id, old_seed_id) 321 assert_equal(0, old_path_len) 322 assert_equal(new_path_len, len(new_path)) 323 assert_equal([], old_path) 324 assert_equal(False, old_has_key_orig) 325 assert_equal(True, new_has_key_orig) 326 327 # Check that the path is right 328 built_path = [] 329 for s in new_kp_str.decode().split('/')[1:]: 330 h = 0 331 if s[-1] == '\'': 332 s = s[:-1] 333 h = 0x80000000 334 p = int(s) | h 335 built_path.append(p) 336 assert_equal(new_path, built_path) 337 338 self.log.info('Upgrading to NO_DEFAULT_KEY should not remove the defaultkey') 339 copy_split_hd() 340 # Check the wallet has a default key initially 341 old_kvs = dump_bdb_kv(node_master_wallet) 342 defaultkey = old_kvs[b'\x0adefaultkey'] 343 self.log.info("Upgrade the wallet. Should still have the same default key.") 344 self.test_upgradewallet(wallet, previous_version=139900, requested_version=159900) 345 new_kvs = dump_bdb_kv(node_master_wallet) 346 up_defaultkey = new_kvs[b'\x0adefaultkey'] 347 assert_equal(defaultkey, up_defaultkey) 348 # 0.16.3 doesn't have a default key 349 v16_3_kvs = dump_bdb_kv(v16_3_wallet) 350 assert b'\x0adefaultkey' not in v16_3_kvs 351 352 353if __name__ == '__main__': 354 UpgradeWalletTest().main() 355