1#!/usr/bin/env python3
2# Copyright (c) 2018-2020 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Backwards compatibility functional test
6
7Test various backwards compatibility scenarios. Requires previous releases binaries,
8see test/README.md.
9
10v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
11Due to a hardfork in regtest, it can't be used to sync nodes.
12
13
14Due to RPC changes introduced in various versions the below tests
15won't work for older versions without some patches or workarounds.
16
17Use only the latest patch version of each release, unless a test specifically
18needs an older patch version.
19"""
20
21import os
22import shutil
23
24from test_framework.blocktools import COINBASE_MATURITY
25from test_framework.test_framework import BitcoinTestFramework
26from test_framework.descriptors import descsum_create
27
28from test_framework.util import (
29    assert_equal,
30    assert_raises_rpc_error,
31)
32
33
34class BackwardsCompatibilityTest(BitcoinTestFramework):
35    def set_test_params(self):
36        self.setup_clean_chain = True
37        self.num_nodes = 6
38        # Add new version after each release:
39        self.extra_args = [
40            ["-addresstype=bech32"], # Pre-release: use to mine blocks
41            ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # Pre-release: use to receive coins, swap wallets, etc
42            ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.1
43            ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.18.1
44            ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.2
45            ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-wallet=wallet.dat"], # v0.16.3
46        ]
47        self.wallet_names = [self.default_wallet_name]
48
49    def skip_test_if_missing_module(self):
50        self.skip_if_no_wallet()
51        self.skip_if_no_previous_releases()
52
53    def setup_nodes(self):
54        self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[
55            None,
56            None,
57            190100,
58            180100,
59            170200,
60            160300,
61        ])
62
63        self.start_nodes()
64        self.import_deterministic_coinbase_privkeys()
65
66    def run_test(self):
67        self.nodes[0].generatetoaddress(COINBASE_MATURITY + 1, self.nodes[0].getnewaddress())
68
69        self.sync_blocks()
70
71        # Sanity check the test framework:
72        res = self.nodes[self.num_nodes - 1].getblockchaininfo()
73        assert_equal(res['blocks'], COINBASE_MATURITY + 1)
74
75        node_master = self.nodes[self.num_nodes - 5]
76        node_v19 = self.nodes[self.num_nodes - 4]
77        node_v18 = self.nodes[self.num_nodes - 3]
78        node_v17 = self.nodes[self.num_nodes - 2]
79        node_v16 = self.nodes[self.num_nodes - 1]
80
81        self.log.info("Test wallet backwards compatibility...")
82        # Create a number of wallets and open them in older versions:
83
84        # w1: regular wallet, created on master: update this test when default
85        #     wallets can no longer be opened by older versions.
86        node_master.createwallet(wallet_name="w1")
87        wallet = node_master.get_wallet_rpc("w1")
88        info = wallet.getwalletinfo()
89        assert info['private_keys_enabled']
90        assert info['keypoolsize'] > 0
91        # Create a confirmed transaction, receiving coins
92        address = wallet.getnewaddress()
93        self.nodes[0].sendtoaddress(address, 10)
94        self.sync_mempools()
95        self.nodes[0].generate(1)
96        self.sync_blocks()
97        # Create a conflicting transaction using RBF
98        return_address = self.nodes[0].getnewaddress()
99        tx1_id = self.nodes[1].sendtoaddress(return_address, 1)
100        tx2_id = self.nodes[1].bumpfee(tx1_id)["txid"]
101        # Confirm the transaction
102        self.sync_mempools()
103        self.nodes[0].generate(1)
104        self.sync_blocks()
105        # Create another conflicting transaction using RBF
106        tx3_id = self.nodes[1].sendtoaddress(return_address, 1)
107        tx4_id = self.nodes[1].bumpfee(tx3_id)["txid"]
108        # Abandon transaction, but don't confirm
109        self.nodes[1].abandontransaction(tx3_id)
110
111        # w1_v19: regular wallet, created with v0.19
112        node_v19.rpc.createwallet(wallet_name="w1_v19")
113        wallet = node_v19.get_wallet_rpc("w1_v19")
114        info = wallet.getwalletinfo()
115        assert info['private_keys_enabled']
116        assert info['keypoolsize'] > 0
117        # Use addmultisigaddress (see #18075)
118        address_18075 = wallet.rpc.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"]
119        assert wallet.getaddressinfo(address_18075)["solvable"]
120
121        # w1_v18: regular wallet, created with v0.18
122        node_v18.rpc.createwallet(wallet_name="w1_v18")
123        wallet = node_v18.get_wallet_rpc("w1_v18")
124        info = wallet.getwalletinfo()
125        assert info['private_keys_enabled']
126        assert info['keypoolsize'] > 0
127
128        # w2: wallet with private keys disabled, created on master: update this
129        #     test when default wallets private keys disabled can no longer be
130        #     opened by older versions.
131        node_master.createwallet(wallet_name="w2", disable_private_keys=True)
132        wallet = node_master.get_wallet_rpc("w2")
133        info = wallet.getwalletinfo()
134        assert info['private_keys_enabled'] == False
135        assert info['keypoolsize'] == 0
136
137        # w2_v19: wallet with private keys disabled, created with v0.19
138        node_v19.rpc.createwallet(wallet_name="w2_v19", disable_private_keys=True)
139        wallet = node_v19.get_wallet_rpc("w2_v19")
140        info = wallet.getwalletinfo()
141        assert info['private_keys_enabled'] == False
142        assert info['keypoolsize'] == 0
143
144        # w2_v18: wallet with private keys disabled, created with v0.18
145        node_v18.rpc.createwallet(wallet_name="w2_v18", disable_private_keys=True)
146        wallet = node_v18.get_wallet_rpc("w2_v18")
147        info = wallet.getwalletinfo()
148        assert info['private_keys_enabled'] == False
149        assert info['keypoolsize'] == 0
150
151        # w3: blank wallet, created on master: update this
152        #     test when default blank wallets can no longer be opened by older versions.
153        node_master.createwallet(wallet_name="w3", blank=True)
154        wallet = node_master.get_wallet_rpc("w3")
155        info = wallet.getwalletinfo()
156        assert info['private_keys_enabled']
157        assert info['keypoolsize'] == 0
158
159        # w3_v19: blank wallet, created with v0.19
160        node_v19.rpc.createwallet(wallet_name="w3_v19", blank=True)
161        wallet = node_v19.get_wallet_rpc("w3_v19")
162        info = wallet.getwalletinfo()
163        assert info['private_keys_enabled']
164        assert info['keypoolsize'] == 0
165
166        # w3_v18: blank wallet, created with v0.18
167        node_v18.rpc.createwallet(wallet_name="w3_v18", blank=True)
168        wallet = node_v18.get_wallet_rpc("w3_v18")
169        info = wallet.getwalletinfo()
170        assert info['private_keys_enabled']
171        assert info['keypoolsize'] == 0
172
173        # Copy the wallets to older nodes:
174        node_master_wallets_dir = os.path.join(node_master.datadir, "regtest/wallets")
175        node_v19_wallets_dir = os.path.join(node_v19.datadir, "regtest/wallets")
176        node_v18_wallets_dir = os.path.join(node_v18.datadir, "regtest/wallets")
177        node_v17_wallets_dir = os.path.join(node_v17.datadir, "regtest/wallets")
178        node_v16_wallets_dir = os.path.join(node_v16.datadir, "regtest")
179        node_master.unloadwallet("w1")
180        node_master.unloadwallet("w2")
181        node_v19.unloadwallet("w1_v19")
182        node_v19.unloadwallet("w2_v19")
183        node_v18.unloadwallet("w1_v18")
184        node_v18.unloadwallet("w2_v18")
185
186        # Copy wallets to v0.16
187        for wallet in os.listdir(node_master_wallets_dir):
188            shutil.copytree(
189                os.path.join(node_master_wallets_dir, wallet),
190                os.path.join(node_v16_wallets_dir, wallet)
191            )
192
193        # Copy wallets to v0.17
194        for wallet in os.listdir(node_master_wallets_dir):
195            shutil.copytree(
196                os.path.join(node_master_wallets_dir, wallet),
197                os.path.join(node_v17_wallets_dir, wallet)
198            )
199        for wallet in os.listdir(node_v18_wallets_dir):
200            shutil.copytree(
201                os.path.join(node_v18_wallets_dir, wallet),
202                os.path.join(node_v17_wallets_dir, wallet)
203            )
204
205        # Copy wallets to v0.18
206        for wallet in os.listdir(node_master_wallets_dir):
207            shutil.copytree(
208                os.path.join(node_master_wallets_dir, wallet),
209                os.path.join(node_v18_wallets_dir, wallet)
210            )
211
212        # Copy wallets to v0.19
213        for wallet in os.listdir(node_master_wallets_dir):
214            shutil.copytree(
215                os.path.join(node_master_wallets_dir, wallet),
216                os.path.join(node_v19_wallets_dir, wallet)
217            )
218
219        if not self.options.descriptors:
220            # Descriptor wallets break compatibility, only run this test for legacy wallet
221            # Open the wallets in v0.19
222            node_v19.loadwallet("w1")
223            wallet = node_v19.get_wallet_rpc("w1")
224            info = wallet.getwalletinfo()
225            assert info['private_keys_enabled']
226            assert info['keypoolsize'] > 0
227            txs = wallet.listtransactions()
228            assert_equal(len(txs), 5)
229            assert_equal(txs[1]["txid"], tx1_id)
230            assert_equal(txs[2]["walletconflicts"], [tx1_id])
231            assert_equal(txs[1]["replaced_by_txid"], tx2_id)
232            assert not(txs[1]["abandoned"])
233            assert_equal(txs[1]["confirmations"], -1)
234            assert_equal(txs[2]["blockindex"], 1)
235            assert txs[3]["abandoned"]
236            assert_equal(txs[4]["walletconflicts"], [tx3_id])
237            assert_equal(txs[3]["replaced_by_txid"], tx4_id)
238            assert not(hasattr(txs[3], "blockindex"))
239
240            node_v19.loadwallet("w2")
241            wallet = node_v19.get_wallet_rpc("w2")
242            info = wallet.getwalletinfo()
243            assert info['private_keys_enabled'] == False
244            assert info['keypoolsize'] == 0
245
246            node_v19.loadwallet("w3")
247            wallet = node_v19.get_wallet_rpc("w3")
248            info = wallet.getwalletinfo()
249            assert info['private_keys_enabled']
250            assert info['keypoolsize'] == 0
251
252            # Open the wallets in v0.18
253            node_v18.loadwallet("w1")
254            wallet = node_v18.get_wallet_rpc("w1")
255            info = wallet.getwalletinfo()
256            assert info['private_keys_enabled']
257            assert info['keypoolsize'] > 0
258            txs = wallet.listtransactions()
259            assert_equal(len(txs), 5)
260            assert_equal(txs[1]["txid"], tx1_id)
261            assert_equal(txs[2]["walletconflicts"], [tx1_id])
262            assert_equal(txs[1]["replaced_by_txid"], tx2_id)
263            assert not(txs[1]["abandoned"])
264            assert_equal(txs[1]["confirmations"], -1)
265            assert_equal(txs[2]["blockindex"], 1)
266            assert txs[3]["abandoned"]
267            assert_equal(txs[4]["walletconflicts"], [tx3_id])
268            assert_equal(txs[3]["replaced_by_txid"], tx4_id)
269            assert not(hasattr(txs[3], "blockindex"))
270
271            node_v18.loadwallet("w2")
272            wallet = node_v18.get_wallet_rpc("w2")
273            info = wallet.getwalletinfo()
274            assert info['private_keys_enabled'] == False
275            assert info['keypoolsize'] == 0
276
277            node_v18.loadwallet("w3")
278            wallet = node_v18.get_wallet_rpc("w3")
279            info = wallet.getwalletinfo()
280            assert info['private_keys_enabled']
281            assert info['keypoolsize'] == 0
282
283            node_v17.loadwallet("w1")
284            wallet = node_v17.get_wallet_rpc("w1")
285            info = wallet.getwalletinfo()
286            assert info['private_keys_enabled']
287            assert info['keypoolsize'] > 0
288
289            node_v17.loadwallet("w2")
290            wallet = node_v17.get_wallet_rpc("w2")
291            info = wallet.getwalletinfo()
292            assert info['private_keys_enabled'] == False
293            assert info['keypoolsize'] == 0
294        else:
295            # Descriptor wallets appear to be corrupted wallets to old software
296            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w1")
297            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w2")
298            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v19.loadwallet, "w3")
299            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w1")
300            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w2")
301            assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v18.loadwallet, "w3")
302
303        # Open the wallets in v0.17
304        node_v17.loadwallet("w1_v18")
305        wallet = node_v17.get_wallet_rpc("w1_v18")
306        info = wallet.getwalletinfo()
307        assert info['private_keys_enabled']
308        assert info['keypoolsize'] > 0
309
310        node_v17.loadwallet("w2_v18")
311        wallet = node_v17.get_wallet_rpc("w2_v18")
312        info = wallet.getwalletinfo()
313        assert info['private_keys_enabled'] == False
314        assert info['keypoolsize'] == 0
315
316        # RPC loadwallet failure causes bitcoind to exit, in addition to the RPC
317        # call failure, so the following test won't work:
318        # assert_raises_rpc_error(-4, "Wallet loading failed.", node_v17.loadwallet, 'w3_v18')
319
320        # Instead, we stop node and try to launch it with the wallet:
321        self.stop_node(4)
322        node_v17.assert_start_raises_init_error(["-wallet=w3_v18"], "Error: Error loading w3_v18: Wallet requires newer version of Bitcoin Core")
323        if self.options.descriptors:
324            # Descriptor wallets appear to be corrupted wallets to old software
325            node_v17.assert_start_raises_init_error(["-wallet=w1"], "Error: wallet.dat corrupt, salvage failed")
326            node_v17.assert_start_raises_init_error(["-wallet=w2"], "Error: wallet.dat corrupt, salvage failed")
327            node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: wallet.dat corrupt, salvage failed")
328        else:
329            node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core")
330        self.start_node(4)
331
332        if not self.options.descriptors:
333            # Descriptor wallets break compatibility, only run this test for legacy wallets
334            # Open most recent wallet in v0.16 (no loadwallet RPC)
335            self.restart_node(5, extra_args=["-wallet=w2"])
336            wallet = node_v16.get_wallet_rpc("w2")
337            info = wallet.getwalletinfo()
338            assert info['keypoolsize'] == 1
339
340        # Create upgrade wallet in v0.16
341        self.restart_node(-1, extra_args=["-wallet=u1_v16"])
342        wallet = node_v16.get_wallet_rpc("u1_v16")
343        v16_addr = wallet.getnewaddress('', "bech32")
344        v16_info = wallet.validateaddress(v16_addr)
345        v16_pubkey = v16_info['pubkey']
346        self.stop_node(-1)
347
348        self.log.info("Test wallet upgrade path...")
349        # u1: regular wallet, created with v0.17
350        node_v17.rpc.createwallet(wallet_name="u1_v17")
351        wallet = node_v17.get_wallet_rpc("u1_v17")
352        address = wallet.getnewaddress("bech32")
353        v17_info = wallet.getaddressinfo(address)
354        hdkeypath = v17_info["hdkeypath"]
355        pubkey = v17_info["pubkey"]
356
357        if self.is_bdb_compiled():
358            # Old wallets are BDB and will only work if BDB is compiled
359            # Copy the 0.16 wallet to the last Bitcoin Core version and open it:
360            shutil.copyfile(
361                os.path.join(node_v16_wallets_dir, "wallets/u1_v16"),
362                os.path.join(node_master_wallets_dir, "u1_v16")
363            )
364            load_res = node_master.loadwallet("u1_v16")
365            # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054
366            assert_equal(load_res['warning'], '')
367            wallet = node_master.get_wallet_rpc("u1_v16")
368            info = wallet.getaddressinfo(v16_addr)
369            descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")"
370            assert_equal(info["desc"], descsum_create(descriptor))
371
372            # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it
373            os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16"))
374            shutil.copyfile(
375                os.path.join(node_master_wallets_dir, "u1_v16"),
376                os.path.join(node_v16_wallets_dir, "wallets/u1_v16")
377            )
378            self.start_node(-1, extra_args=["-wallet=u1_v16"])
379            wallet = node_v16.get_wallet_rpc("u1_v16")
380            info = wallet.validateaddress(v16_addr)
381            assert_equal(info, v16_info)
382
383            # Copy the 0.17 wallet to the last Bitcoin Core version and open it:
384            node_v17.unloadwallet("u1_v17")
385            shutil.copytree(
386                os.path.join(node_v17_wallets_dir, "u1_v17"),
387                os.path.join(node_master_wallets_dir, "u1_v17")
388            )
389            node_master.loadwallet("u1_v17")
390            wallet = node_master.get_wallet_rpc("u1_v17")
391            info = wallet.getaddressinfo(address)
392            descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")"
393            assert_equal(info["desc"], descsum_create(descriptor))
394
395            # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it
396            node_master.unloadwallet("u1_v17")
397            shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17"))
398            shutil.copytree(
399                os.path.join(node_master_wallets_dir, "u1_v17"),
400                os.path.join(node_v17_wallets_dir, "u1_v17")
401            )
402            node_v17.loadwallet("u1_v17")
403            wallet = node_v17.get_wallet_rpc("u1_v17")
404            info = wallet.getaddressinfo(address)
405            assert_equal(info, v17_info)
406
407            # Copy the 0.19 wallet to the last Bitcoin Core version and open it:
408            shutil.copytree(
409                os.path.join(node_v19_wallets_dir, "w1_v19"),
410                os.path.join(node_master_wallets_dir, "w1_v19")
411            )
412            node_master.loadwallet("w1_v19")
413            wallet = node_master.get_wallet_rpc("w1_v19")
414            assert wallet.getaddressinfo(address_18075)["solvable"]
415
416            # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it
417            node_master.unloadwallet("w1_v19")
418            shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19"))
419            shutil.copytree(
420                os.path.join(node_master_wallets_dir, "w1_v19"),
421                os.path.join(node_v19_wallets_dir, "w1_v19")
422            )
423            node_v19.loadwallet("w1_v19")
424            wallet = node_v19.get_wallet_rpc("w1_v19")
425            assert wallet.getaddressinfo(address_18075)["solvable"]
426
427if __name__ == '__main__':
428    BackwardsCompatibilityTest().main()
429