1#!/usr/bin/env python3
2# Copyright (c) 2017-2019 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test multiwallet.
6
7Verify that a bitcoind node can load multiple wallet files
8"""
9from threading import Thread
10import os
11import shutil
12import time
13
14from test_framework.authproxy import JSONRPCException
15from test_framework.test_framework import BitcoinTestFramework
16from test_framework.test_node import ErrorMatch
17from test_framework.util import (
18    assert_equal,
19    assert_raises_rpc_error,
20    get_rpc_proxy,
21)
22from test_framework.qtumconfig import COINBASE_MATURITY, INITIAL_BLOCK_REWARD
23
24FEATURE_LATEST = 169900
25
26got_loading_error = False
27def test_load_unload(node, name):
28    global got_loading_error
29    for i in range(10):
30        if got_loading_error:
31            return
32        try:
33            node.loadwallet(name)
34            node.unloadwallet(name)
35        except JSONRPCException as e:
36            if e.error['code'] == -4 and 'Wallet already being loading' in e.error['message']:
37                got_loading_error = True
38                return
39
40
41class MultiWalletTest(BitcoinTestFramework):
42    def set_test_params(self):
43        self.setup_clean_chain = True
44        self.num_nodes = 2
45        self.rpc_timeout = 120
46
47    def skip_test_if_missing_module(self):
48        self.skip_if_no_wallet()
49
50    def add_options(self, parser):
51        parser.add_argument(
52            '--data_wallets_dir',
53            default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/wallets/'),
54            help='Test data with wallet directories (default: %(default)s)',
55        )
56
57    def run_test(self):
58        node = self.nodes[0]
59
60        data_dir = lambda *p: os.path.join(node.datadir, self.chain, *p)
61        wallet_dir = lambda *p: data_dir('wallets', *p)
62        wallet = lambda name: node.get_wallet_rpc(name)
63
64        def wallet_file(name):
65            if os.path.isdir(wallet_dir(name)):
66                return wallet_dir(name, "wallet.dat")
67            return wallet_dir(name)
68
69        assert_equal(self.nodes[0].listwalletdir(), { 'wallets': [{ 'name': '' }] })
70
71        # check wallet.dat is created
72        self.stop_nodes()
73        assert_equal(os.path.isfile(wallet_dir('wallet.dat')), True)
74
75        # create symlink to verify wallet directory path can be referenced
76        # through symlink
77        os.mkdir(wallet_dir('w7'))
78        os.symlink('w7', wallet_dir('w7_symlink'))
79
80        # rename wallet.dat to make sure plain wallet file paths (as opposed to
81        # directory paths) can be loaded
82        os.rename(wallet_dir("wallet.dat"), wallet_dir("w8"))
83
84        # create another dummy wallet for use in testing backups later
85        self.start_node(0, [])
86        self.stop_nodes()
87        empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat')
88        os.rename(wallet_dir("wallet.dat"), empty_wallet)
89
90        # restart node with a mix of wallet names:
91        #   w1, w2, w3 - to verify new wallets created when non-existing paths specified
92        #   w          - to verify wallet name matching works when one wallet path is prefix of another
93        #   sub/w5     - to verify relative wallet path is created correctly
94        #   extern/w6  - to verify absolute wallet path is created correctly
95        #   w7_symlink - to verify symlinked wallet path is initialized correctly
96        #   w8         - to verify existing wallet file is loaded correctly
97        #   ''         - to verify default wallet file is created correctly
98        wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', '']
99        extra_args = ['-wallet={}'.format(n) for n in wallet_names]
100        self.start_node(0, extra_args)
101        assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8'])
102
103        assert_equal(set(node.listwallets()), set(wallet_names))
104
105        # check that all requested wallets were created
106        self.stop_node(0)
107        for wallet_name in wallet_names:
108            assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
109
110        # should not initialize if wallet path can't be created
111        exp_stderr = "boost::filesystem::create_directory:"
112        self.nodes[0].assert_start_raises_init_error(['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
113
114        self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist')
115        self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir())
116        self.nodes[0].assert_start_raises_init_error(['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir())
117
118        # should not initialize if there are duplicate wallets
119        self.nodes[0].assert_start_raises_init_error(['-wallet=w1', '-wallet=w1'], 'Error: Error loading wallet w1. Duplicate -wallet filename specified.')
120
121        # should not initialize if one wallet is a copy of another
122        shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy'))
123        exp_stderr = r"BerkeleyBatch: Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
124        self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
125
126        # should not initialize if wallet file is a symlink
127        os.symlink('w8', wallet_dir('w8_symlink'))
128        self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
129
130        # should not initialize if the specified walletdir does not exist
131        self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
132        # should not initialize if the specified walletdir is not a directory
133        not_a_dir = wallet_dir('notadir')
134        open(not_a_dir, 'a', encoding="utf8").close()
135        self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory')
136
137        self.log.info("Do not allow -zapwallettxes with multiwallet")
138        self.nodes[0].assert_start_raises_init_error(['-zapwallettxes', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file")
139        self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=1', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file")
140        self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=2', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file")
141
142        self.log.info("Do not allow -salvagewallet with multiwallet")
143        self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file")
144        self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file")
145
146        self.log.info("Do not allow -upgradewallet with multiwallet")
147        self.nodes[0].assert_start_raises_init_error(['-upgradewallet', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file")
148        self.nodes[0].assert_start_raises_init_error(['-upgradewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file")
149
150        # if wallets/ doesn't exist, datadir should be the default wallet dir
151        wallet_dir2 = data_dir('walletdir')
152        os.rename(wallet_dir(), wallet_dir2)
153        self.start_node(0, ['-wallet=w4', '-wallet=w5'])
154        assert_equal(set(node.listwallets()), {"w4", "w5"})
155        w5 = wallet("w5")
156        node.generatetoaddress(nblocks=1, address=w5.getnewaddress())
157
158        # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded
159        os.rename(wallet_dir2, wallet_dir())
160        self.restart_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + data_dir()])
161        assert_equal(set(node.listwallets()), {"w4", "w5"})
162        w5 = wallet("w5")
163        w5_info = w5.getwalletinfo()
164        assert_equal(w5_info['immature_balance'], INITIAL_BLOCK_REWARD)
165
166        competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir')
167        os.mkdir(competing_wallet_dir)
168        self.restart_node(0, ['-walletdir=' + competing_wallet_dir])
169        exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
170        self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
171
172        self.restart_node(0, extra_args)
173
174        assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy'])
175
176        wallets = [wallet(w) for w in wallet_names]
177        wallet_bad = wallet("bad")
178
179        # check wallet names and balances
180        node.generatetoaddress(nblocks=1, address=wallets[0].getnewaddress())
181        for wallet_name, wallet in zip(wallet_names, wallets):
182            info = wallet.getwalletinfo()
183            assert_equal(info['immature_balance'], INITIAL_BLOCK_REWARD if wallet is wallets[0] else 0)
184            assert_equal(info['walletname'], wallet_name)
185
186        # accessing invalid wallet fails
187        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo)
188
189        # accessing wallet RPC without using wallet endpoint fails
190        assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo)
191
192        w1, w2, w3, w4, *_ = wallets
193        w1.generatetoaddress(nblocks=COINBASE_MATURITY+1, address=w1.getnewaddress())
194        assert_equal(w1.getbalance(), 2*INITIAL_BLOCK_REWARD)
195        assert_equal(w2.getbalance(), 0)
196        assert_equal(w3.getbalance(), 0)
197        assert_equal(w4.getbalance(), 0)
198
199        w1.sendtoaddress(w2.getnewaddress(), 1)
200        w1.sendtoaddress(w3.getnewaddress(), 2)
201        w1.sendtoaddress(w4.getnewaddress(), 3)
202        node.generatetoaddress(nblocks=1, address=w1.getnewaddress())
203        assert_equal(w2.getbalance(), 1)
204        assert_equal(w3.getbalance(), 2)
205        assert_equal(w4.getbalance(), 3)
206
207        batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()])
208        assert_equal(batch[0]["result"]["chain"], self.chain)
209        assert_equal(batch[1]["result"]["walletname"], "w1")
210
211        self.log.info('Check for per-wallet settxfee call')
212        assert_equal(w1.getwalletinfo()['paytxfee'], 0)
213        assert_equal(w2.getwalletinfo()['paytxfee'], 0)
214        w2.settxfee(4.0)
215        assert_equal(w1.getwalletinfo()['paytxfee'], 0)
216        assert_equal(w2.getwalletinfo()['paytxfee'], 4.0)
217
218        self.log.info("Test dynamic wallet loading")
219
220        self.restart_node(0, ['-nowallet'])
221        assert_equal(node.listwallets(), [])
222        assert_raises_rpc_error(-32601, "Method not found", node.getwalletinfo)
223
224        self.log.info("Load first wallet")
225        loadwallet_name = node.loadwallet(wallet_names[0])
226        assert_equal(loadwallet_name['name'], wallet_names[0])
227        assert_equal(node.listwallets(), wallet_names[0:1])
228        node.getwalletinfo()
229        w1 = node.get_wallet_rpc(wallet_names[0])
230        w1.getwalletinfo()
231
232        self.log.info("Load second wallet")
233        loadwallet_name = node.loadwallet(wallet_names[1])
234        assert_equal(loadwallet_name['name'], wallet_names[1])
235        assert_equal(node.listwallets(), wallet_names[0:2])
236        assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo)
237        w2 = node.get_wallet_rpc(wallet_names[1])
238        w2.getwalletinfo()
239
240        self.log.info("Concurrent wallet loading")
241        threads = []
242        for _ in range(3):
243            n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir)
244            t = Thread(target=test_load_unload, args=(n, wallet_names[2], ))
245            t.start()
246            threads.append(t)
247        for t in threads:
248            t.join()
249        global got_loading_error
250        assert_equal(got_loading_error, True)
251
252        self.log.info("Load remaining wallets")
253        for wallet_name in wallet_names[2:]:
254            loadwallet_name = self.nodes[0].loadwallet(wallet_name)
255            assert_equal(loadwallet_name['name'], wallet_name)
256
257        assert_equal(set(self.nodes[0].listwallets()), set(wallet_names))
258
259        # Fail to load if wallet doesn't exist
260        assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets')
261
262        # Fail to load duplicate wallets
263        assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0])
264
265        # Fail to load duplicate wallets by different ways (directory and filepath)
266        assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat')
267
268        # Fail to load if one wallet is a copy of another
269        assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
270
271        # Fail to load if one wallet is a copy of another, test this twice to make sure that we don't re-introduce #14304
272        assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
273
274
275        # Fail to load if wallet file is a symlink
276        assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink')
277
278        # Fail to load if a directory is specified that doesn't contain a wallet
279        os.mkdir(wallet_dir('empty_wallet_dir'))
280        assert_raises_rpc_error(-18, "Directory empty_wallet_dir does not contain a wallet.dat file", self.nodes[0].loadwallet, 'empty_wallet_dir')
281
282        self.log.info("Test dynamic wallet creation.")
283
284        # Fail to create a wallet if it already exists.
285        assert_raises_rpc_error(-4, "Wallet w2 already exists.", self.nodes[0].createwallet, 'w2')
286
287        # Successfully create a wallet with a new name
288        loadwallet_name = self.nodes[0].createwallet('w9')
289        assert_equal(loadwallet_name['name'], 'w9')
290        w9 = node.get_wallet_rpc('w9')
291        assert_equal(w9.getwalletinfo()['walletname'], 'w9')
292
293        assert 'w9' in self.nodes[0].listwallets()
294
295        # Successfully create a wallet using a full path
296        new_wallet_dir = os.path.join(self.options.tmpdir, 'new_walletdir')
297        new_wallet_name = os.path.join(new_wallet_dir, 'w10')
298        loadwallet_name = self.nodes[0].createwallet(new_wallet_name)
299        assert_equal(loadwallet_name['name'], new_wallet_name)
300        w10 = node.get_wallet_rpc(new_wallet_name)
301        assert_equal(w10.getwalletinfo()['walletname'], new_wallet_name)
302
303        assert new_wallet_name in self.nodes[0].listwallets()
304
305        self.log.info("Test dynamic wallet unloading")
306
307        # Test `unloadwallet` errors
308        assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet)
309        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy")
310        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet)
311        assert_raises_rpc_error(-8, "Cannot unload the requested wallet", w1.unloadwallet, "w2"),
312
313        # Successfully unload the specified wallet name
314        self.nodes[0].unloadwallet("w1")
315        assert 'w1' not in self.nodes[0].listwallets()
316
317        # Successfully unload the wallet referenced by the request endpoint
318        # Also ensure unload works during walletpassphrase timeout
319        w2.encryptwallet('test')
320        w2.walletpassphrase('test', 1)
321        w2.unloadwallet()
322        time.sleep(1.1)
323        assert 'w2' not in self.nodes[0].listwallets()
324
325        # Successfully unload all wallets
326        for wallet_name in self.nodes[0].listwallets():
327            self.nodes[0].unloadwallet(wallet_name)
328        assert_equal(self.nodes[0].listwallets(), [])
329        assert_raises_rpc_error(-32601, "Method not found (wallet method is disabled because no wallet is loaded)", self.nodes[0].getwalletinfo)
330
331        # Successfully load a previously unloaded wallet
332        self.nodes[0].loadwallet('w1')
333        assert_equal(self.nodes[0].listwallets(), ['w1'])
334        assert_equal(w1.getwalletinfo()['walletname'], 'w1')
335
336        assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), ['', os.path.join('sub', 'w5'), 'w', 'w1', 'w2', 'w3', 'w7', 'w7_symlink', 'w8', 'w8_copy', 'w9'])
337
338        # Test backing up and restoring wallets
339        self.log.info("Test wallet backup")
340        self.restart_node(0, ['-nowallet'])
341        for wallet_name in wallet_names:
342            self.nodes[0].loadwallet(wallet_name)
343        for wallet_name in wallet_names:
344            rpc = self.nodes[0].get_wallet_rpc(wallet_name)
345            addr = rpc.getnewaddress()
346            backup = os.path.join(self.options.tmpdir, 'backup.dat')
347            rpc.backupwallet(backup)
348            self.nodes[0].unloadwallet(wallet_name)
349            shutil.copyfile(empty_wallet, wallet_file(wallet_name))
350            self.nodes[0].loadwallet(wallet_name)
351            assert_equal(rpc.getaddressinfo(addr)['ismine'], False)
352            self.nodes[0].unloadwallet(wallet_name)
353            shutil.copyfile(backup, wallet_file(wallet_name))
354            self.nodes[0].loadwallet(wallet_name)
355            assert_equal(rpc.getaddressinfo(addr)['ismine'], True)
356
357        # Test .walletlock file is closed
358        self.start_node(1)
359        wallet = os.path.join(self.options.tmpdir, 'my_wallet')
360        self.nodes[0].createwallet(wallet)
361        assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet)
362        self.nodes[0].unloadwallet(wallet)
363        self.nodes[1].loadwallet(wallet)
364
365        # Fail to load if wallet is downgraded
366        shutil.copytree(os.path.join(self.options.data_wallets_dir, 'high_minversion'), wallet_dir('high_minversion'))
367        self.restart_node(0, extra_args=['-upgradewallet={}'.format(FEATURE_LATEST)])
368        assert {'name': 'high_minversion'} in self.nodes[0].listwalletdir()['wallets']
369        self.log.info("Fail -upgradewallet that results in downgrade")
370        assert_raises_rpc_error(
371            -4,
372            'Wallet loading failed: Error loading {}: Wallet requires newer version of {}'.format(
373                wallet_dir('high_minversion', 'wallet.dat'), self.config['environment']['PACKAGE_NAME']),
374            lambda: self.nodes[0].loadwallet(filename='high_minversion'),
375        )
376
377
378if __name__ == '__main__':
379    MultiWalletTest().main()
380