1#!/usr/bin/env python3
2# Copyright (c) 2017-2020 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test multiwallet.
6
7Verify that a bitcoind node can load multiple wallet files
8"""
9from decimal import Decimal
10from threading import Thread
11import os
12import shutil
13import stat
14import time
15
16from test_framework.authproxy import JSONRPCException
17from test_framework.test_framework import BitcoinTestFramework
18from test_framework.test_node import ErrorMatch
19from test_framework.util import (
20    assert_equal,
21    assert_raises_rpc_error,
22    get_rpc_proxy,
23)
24
25got_loading_error = False
26def test_load_unload(node, name):
27    global got_loading_error
28    for _ in range(10):
29        if got_loading_error:
30            return
31        try:
32            node.loadwallet(name)
33            node.unloadwallet(name)
34        except JSONRPCException as e:
35            if e.error['code'] == -4 and 'Wallet already being loading' in e.error['message']:
36                got_loading_error = True
37                return
38
39
40class MultiWalletTest(BitcoinTestFramework):
41    def set_test_params(self):
42        self.setup_clean_chain = True
43        self.num_nodes = 2
44        self.rpc_timeout = 120
45        self.extra_args = [["-nowallet"], []]
46
47    def skip_test_if_missing_module(self):
48        self.skip_if_no_wallet()
49
50    def add_options(self, parser):
51        parser.add_argument(
52            '--data_wallets_dir',
53            default=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/wallets/'),
54            help='Test data with wallet directories (default: %(default)s)',
55        )
56
57    def run_test(self):
58        node = self.nodes[0]
59
60        data_dir = lambda *p: os.path.join(node.datadir, self.chain, *p)
61        wallet_dir = lambda *p: data_dir('wallets', *p)
62        wallet = lambda name: node.get_wallet_rpc(name)
63
64        def wallet_file(name):
65            if name == self.default_wallet_name:
66                return wallet_dir(self.default_wallet_name, self.wallet_data_filename)
67            if os.path.isdir(wallet_dir(name)):
68                return wallet_dir(name, "wallet.dat")
69            return wallet_dir(name)
70
71        assert_equal(self.nodes[0].listwalletdir(), { 'wallets': [{ 'name': self.default_wallet_name }] })
72
73        # check wallet.dat is created
74        self.stop_nodes()
75        assert_equal(os.path.isfile(wallet_dir(self.default_wallet_name, self.wallet_data_filename)), True)
76
77        # create symlink to verify wallet directory path can be referenced
78        # through symlink
79        os.mkdir(wallet_dir('w7'))
80        os.symlink('w7', wallet_dir('w7_symlink'))
81
82        os.symlink('..', wallet_dir('recursive_dir_symlink'))
83
84        os.mkdir(wallet_dir('self_walletdat_symlink'))
85        os.symlink('wallet.dat', wallet_dir('self_walletdat_symlink/wallet.dat'))
86
87        # rename wallet.dat to make sure plain wallet file paths (as opposed to
88        # directory paths) can be loaded
89        # create another dummy wallet for use in testing backups later
90        self.start_node(0)
91        node.createwallet("empty")
92        node.createwallet("plain")
93        node.createwallet("created")
94        self.stop_nodes()
95        empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat')
96        os.rename(wallet_file("empty"), empty_wallet)
97        shutil.rmtree(wallet_dir("empty"))
98        empty_created_wallet = os.path.join(self.options.tmpdir, 'empty.created.dat')
99        os.rename(wallet_dir("created", self.wallet_data_filename), empty_created_wallet)
100        shutil.rmtree(wallet_dir("created"))
101        os.rename(wallet_file("plain"), wallet_dir("w8"))
102        shutil.rmtree(wallet_dir("plain"))
103
104        # restart node with a mix of wallet names:
105        #   w1, w2, w3 - to verify new wallets created when non-existing paths specified
106        #   w          - to verify wallet name matching works when one wallet path is prefix of another
107        #   sub/w5     - to verify relative wallet path is created correctly
108        #   extern/w6  - to verify absolute wallet path is created correctly
109        #   w7_symlink - to verify symlinked wallet path is initialized correctly
110        #   w8         - to verify existing wallet file is loaded correctly. Not tested for SQLite wallets as this is a deprecated BDB behavior.
111        #   ''         - to verify default wallet file is created correctly
112        to_create = ['w1', 'w2', 'w3', 'w', 'sub/w5', 'w7_symlink']
113        in_wallet_dir = [w.replace('/', os.path.sep) for w in to_create]  # Wallets in the wallet dir
114        in_wallet_dir.append('w7')  # w7 is not loaded or created, but will be listed by listwalletdir because w7_symlink
115        to_create.append(os.path.join(self.options.tmpdir, 'extern/w6'))  # External, not in the wallet dir, so we need to avoid adding it to in_wallet_dir
116        to_load = [self.default_wallet_name]
117        if not self.options.descriptors:
118            to_load.append('w8')
119        wallet_names = to_create + to_load  # Wallet names loaded in the wallet
120        in_wallet_dir += to_load  # The loaded wallets are also in the wallet dir
121        self.start_node(0)
122        for wallet_name in to_create:
123            self.nodes[0].createwallet(wallet_name)
124        for wallet_name in to_load:
125            self.nodes[0].loadwallet(wallet_name)
126
127        os.mkdir(wallet_dir('no_access'))
128        os.chmod(wallet_dir('no_access'), 0)
129        try:
130            with self.nodes[0].assert_debug_log(expected_msgs=['Too many levels of symbolic links', 'Error scanning']):
131                walletlist = self.nodes[0].listwalletdir()['wallets']
132        finally:
133            # Need to ensure access is restored for cleanup
134            os.chmod(wallet_dir('no_access'), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
135        assert_equal(sorted(map(lambda w: w['name'], walletlist)), sorted(in_wallet_dir))
136
137        assert_equal(set(node.listwallets()), set(wallet_names))
138
139        # should raise rpc error if wallet path can't be created
140        err_code = -4 if self.options.descriptors else -1
141        assert_raises_rpc_error(err_code, "boost::filesystem::create_directory:", self.nodes[0].createwallet, "w8/bad")
142
143        # check that all requested wallets were created
144        self.stop_node(0)
145        for wallet_name in wallet_names:
146            assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
147
148        self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist')
149        self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" is a relative path', cwd=data_dir())
150        self.nodes[0].assert_start_raises_init_error(['-walletdir=debug.log'], 'Error: Specified -walletdir "debug.log" is not a directory', cwd=data_dir())
151
152        self.start_node(0, ['-wallet=w1', '-wallet=w1'])
153        self.stop_node(0, 'Warning: Ignoring duplicate -wallet w1.')
154
155        if not self.options.descriptors:
156            # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary
157            # should not initialize if one wallet is a copy of another
158            shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy'))
159            in_wallet_dir.append('w8_copy')
160            exp_stderr = r"BerkeleyDatabase: Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
161            self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
162
163        # should not initialize if wallet file is a symlink
164        os.symlink('w8', wallet_dir('w8_symlink'))
165        self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
166
167        # should not initialize if the specified walletdir does not exist
168        self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
169        # should not initialize if the specified walletdir is not a directory
170        not_a_dir = wallet_dir('notadir')
171        open(not_a_dir, 'a', encoding="utf8").close()
172        self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory')
173
174        self.log.info("Do not allow -upgradewallet with multiwallet")
175        self.nodes[0].assert_start_raises_init_error(['-upgradewallet'], "Error: Error parsing command line arguments: Invalid parameter -upgradewallet")
176
177        # if wallets/ doesn't exist, datadir should be the default wallet dir
178        wallet_dir2 = data_dir('walletdir')
179        os.rename(wallet_dir(), wallet_dir2)
180        self.start_node(0)
181        self.nodes[0].createwallet("w4")
182        self.nodes[0].createwallet("w5")
183        assert_equal(set(node.listwallets()), {"w4", "w5"})
184        w5 = wallet("w5")
185        node.generatetoaddress(nblocks=1, address=w5.getnewaddress())
186
187        # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded
188        os.rename(wallet_dir2, wallet_dir())
189        self.restart_node(0, ['-nowallet', '-walletdir=' + data_dir()])
190        self.nodes[0].loadwallet("w4")
191        self.nodes[0].loadwallet("w5")
192        assert_equal(set(node.listwallets()), {"w4", "w5"})
193        w5 = wallet("w5")
194        w5_info = w5.getwalletinfo()
195        assert_equal(w5_info['immature_balance'], 50)
196
197        competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir')
198        os.mkdir(competing_wallet_dir)
199        self.restart_node(0, ['-nowallet', '-walletdir=' + competing_wallet_dir])
200        self.nodes[0].createwallet(self.default_wallet_name)
201        if self.options.descriptors:
202            exp_stderr = r"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?"
203        else:
204            exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!"
205        self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
206
207        self.restart_node(0)
208        for wallet_name in wallet_names:
209            self.nodes[0].loadwallet(wallet_name)
210
211        assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), sorted(in_wallet_dir))
212
213        wallets = [wallet(w) for w in wallet_names]
214        wallet_bad = wallet("bad")
215
216        # check wallet names and balances
217        node.generatetoaddress(nblocks=1, address=wallets[0].getnewaddress())
218        for wallet_name, wallet in zip(wallet_names, wallets):
219            info = wallet.getwalletinfo()
220            assert_equal(info['immature_balance'], 50 if wallet is wallets[0] else 0)
221            assert_equal(info['walletname'], wallet_name)
222
223        # accessing invalid wallet fails
224        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo)
225
226        # accessing wallet RPC without using wallet endpoint fails
227        assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo)
228
229        w1, w2, w3, w4, *_ = wallets
230        node.generatetoaddress(nblocks=101, address=w1.getnewaddress())
231        assert_equal(w1.getbalance(), 100)
232        assert_equal(w2.getbalance(), 0)
233        assert_equal(w3.getbalance(), 0)
234        assert_equal(w4.getbalance(), 0)
235
236        w1.sendtoaddress(w2.getnewaddress(), 1)
237        w1.sendtoaddress(w3.getnewaddress(), 2)
238        w1.sendtoaddress(w4.getnewaddress(), 3)
239        node.generatetoaddress(nblocks=1, address=w1.getnewaddress())
240        assert_equal(w2.getbalance(), 1)
241        assert_equal(w3.getbalance(), 2)
242        assert_equal(w4.getbalance(), 3)
243
244        batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()])
245        assert_equal(batch[0]["result"]["chain"], self.chain)
246        assert_equal(batch[1]["result"]["walletname"], "w1")
247
248        self.log.info('Check for per-wallet settxfee call')
249        assert_equal(w1.getwalletinfo()['paytxfee'], 0)
250        assert_equal(w2.getwalletinfo()['paytxfee'], 0)
251        w2.settxfee(0.001)
252        assert_equal(w1.getwalletinfo()['paytxfee'], 0)
253        assert_equal(w2.getwalletinfo()['paytxfee'], Decimal('0.00100000'))
254
255        self.log.info("Test dynamic wallet loading")
256
257        self.restart_node(0, ['-nowallet'])
258        assert_equal(node.listwallets(), [])
259        assert_raises_rpc_error(-18, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)", node.getwalletinfo)
260
261        self.log.info("Load first wallet")
262        loadwallet_name = node.loadwallet(wallet_names[0])
263        assert_equal(loadwallet_name['name'], wallet_names[0])
264        assert_equal(node.listwallets(), wallet_names[0:1])
265        node.getwalletinfo()
266        w1 = node.get_wallet_rpc(wallet_names[0])
267        w1.getwalletinfo()
268
269        self.log.info("Load second wallet")
270        loadwallet_name = node.loadwallet(wallet_names[1])
271        assert_equal(loadwallet_name['name'], wallet_names[1])
272        assert_equal(node.listwallets(), wallet_names[0:2])
273        assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo)
274        w2 = node.get_wallet_rpc(wallet_names[1])
275        w2.getwalletinfo()
276
277        self.log.info("Concurrent wallet loading")
278        threads = []
279        for _ in range(3):
280            n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir)
281            t = Thread(target=test_load_unload, args=(n, wallet_names[2], ))
282            t.start()
283            threads.append(t)
284        for t in threads:
285            t.join()
286        global got_loading_error
287        assert_equal(got_loading_error, True)
288
289        self.log.info("Load remaining wallets")
290        for wallet_name in wallet_names[2:]:
291            loadwallet_name = self.nodes[0].loadwallet(wallet_name)
292            assert_equal(loadwallet_name['name'], wallet_name)
293
294        assert_equal(set(self.nodes[0].listwallets()), set(wallet_names))
295
296        # Fail to load if wallet doesn't exist
297        path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallets")
298        assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path), self.nodes[0].loadwallet, 'wallets')
299
300        # Fail to load duplicate wallets
301        path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", "wallet.dat")
302        if self.options.descriptors:
303            assert_raises_rpc_error(-4, "Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?", self.nodes[0].loadwallet, wallet_names[0])
304        else:
305            assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0])
306
307            # This tests the default wallet that BDB makes, so SQLite wallet doesn't need to test this
308            # Fail to load duplicate wallets by different ways (directory and filepath)
309            path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat")
310            assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat')
311
312            # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary
313            # Fail to load if one wallet is a copy of another
314            assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
315
316            # Fail to load if one wallet is a copy of another, test this twice to make sure that we don't re-introduce #14304
317            assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
318
319        # Fail to load if wallet file is a symlink
320        assert_raises_rpc_error(-4, "Wallet file verification failed. Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink')
321
322        # Fail to load if a directory is specified that doesn't contain a wallet
323        os.mkdir(wallet_dir('empty_wallet_dir'))
324        path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "empty_wallet_dir")
325        assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Data is not in recognized format.".format(path), self.nodes[0].loadwallet, 'empty_wallet_dir')
326
327        self.log.info("Test dynamic wallet creation.")
328
329        # Fail to create a wallet if it already exists.
330        path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w2")
331        assert_raises_rpc_error(-4, "Failed to create database path '{}'. Database already exists.".format(path), self.nodes[0].createwallet, 'w2')
332
333        # Successfully create a wallet with a new name
334        loadwallet_name = self.nodes[0].createwallet('w9')
335        in_wallet_dir.append('w9')
336        assert_equal(loadwallet_name['name'], 'w9')
337        w9 = node.get_wallet_rpc('w9')
338        assert_equal(w9.getwalletinfo()['walletname'], 'w9')
339
340        assert 'w9' in self.nodes[0].listwallets()
341
342        # Successfully create a wallet using a full path
343        new_wallet_dir = os.path.join(self.options.tmpdir, 'new_walletdir')
344        new_wallet_name = os.path.join(new_wallet_dir, 'w10')
345        loadwallet_name = self.nodes[0].createwallet(new_wallet_name)
346        assert_equal(loadwallet_name['name'], new_wallet_name)
347        w10 = node.get_wallet_rpc(new_wallet_name)
348        assert_equal(w10.getwalletinfo()['walletname'], new_wallet_name)
349
350        assert new_wallet_name in self.nodes[0].listwallets()
351
352        self.log.info("Test dynamic wallet unloading")
353
354        # Test `unloadwallet` errors
355        assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet)
356        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy")
357        assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet)
358        assert_raises_rpc_error(-8, "Both the RPC endpoint wallet and wallet_name parameter were provided (only one allowed)", w1.unloadwallet, "w2"),
359        assert_raises_rpc_error(-8, "Both the RPC endpoint wallet and wallet_name parameter were provided (only one allowed)", w1.unloadwallet, "w1"),
360
361        # Successfully unload the specified wallet name
362        self.nodes[0].unloadwallet("w1")
363        assert 'w1' not in self.nodes[0].listwallets()
364
365        # Successfully unload the wallet referenced by the request endpoint
366        # Also ensure unload works during walletpassphrase timeout
367        w2.encryptwallet('test')
368        w2.walletpassphrase('test', 1)
369        w2.unloadwallet()
370        time.sleep(1.1)
371        assert 'w2' not in self.nodes[0].listwallets()
372
373        # Successfully unload all wallets
374        for wallet_name in self.nodes[0].listwallets():
375            self.nodes[0].unloadwallet(wallet_name)
376        assert_equal(self.nodes[0].listwallets(), [])
377        assert_raises_rpc_error(-18, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)", self.nodes[0].getwalletinfo)
378
379        # Successfully load a previously unloaded wallet
380        self.nodes[0].loadwallet('w1')
381        assert_equal(self.nodes[0].listwallets(), ['w1'])
382        assert_equal(w1.getwalletinfo()['walletname'], 'w1')
383
384        assert_equal(sorted(map(lambda w: w['name'], self.nodes[0].listwalletdir()['wallets'])), sorted(in_wallet_dir))
385
386        # Test backing up and restoring wallets
387        self.log.info("Test wallet backup")
388        self.restart_node(0, ['-nowallet'])
389        for wallet_name in wallet_names:
390            self.nodes[0].loadwallet(wallet_name)
391        for wallet_name in wallet_names:
392            rpc = self.nodes[0].get_wallet_rpc(wallet_name)
393            addr = rpc.getnewaddress()
394            backup = os.path.join(self.options.tmpdir, 'backup.dat')
395            if os.path.exists(backup):
396                os.unlink(backup)
397            rpc.backupwallet(backup)
398            self.nodes[0].unloadwallet(wallet_name)
399            shutil.copyfile(empty_created_wallet if wallet_name == self.default_wallet_name else empty_wallet, wallet_file(wallet_name))
400            self.nodes[0].loadwallet(wallet_name)
401            assert_equal(rpc.getaddressinfo(addr)['ismine'], False)
402            self.nodes[0].unloadwallet(wallet_name)
403            shutil.copyfile(backup, wallet_file(wallet_name))
404            self.nodes[0].loadwallet(wallet_name)
405            assert_equal(rpc.getaddressinfo(addr)['ismine'], True)
406
407        # Test .walletlock file is closed
408        self.start_node(1)
409        wallet = os.path.join(self.options.tmpdir, 'my_wallet')
410        self.nodes[0].createwallet(wallet)
411        if self.options.descriptors:
412            assert_raises_rpc_error(-4, "Unable to obtain an exclusive lock", self.nodes[1].loadwallet, wallet)
413        else:
414            assert_raises_rpc_error(-4, "Error initializing wallet database environment", self.nodes[1].loadwallet, wallet)
415        self.nodes[0].unloadwallet(wallet)
416        self.nodes[1].loadwallet(wallet)
417
418
419if __name__ == '__main__':
420    MultiWalletTest().main()
421