1#!/usr/bin/env python3
2# Copyright (c) 2014-2020 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test descendant package tracking code."""
6
7from decimal import Decimal
8
9from test_framework.blocktools import COINBASE_MATURITY
10from test_framework.messages import COIN
11from test_framework.p2p import P2PTxInvStore
12from test_framework.test_framework import BitcoinTestFramework
13from test_framework.util import (
14    assert_equal,
15    assert_raises_rpc_error,
16    chain_transaction,
17    satoshi_round,
18)
19
20# default limits
21MAX_ANCESTORS = 25
22MAX_DESCENDANTS = 25
23# custom limits for node1
24MAX_ANCESTORS_CUSTOM = 5
25MAX_DESCENDANTS_CUSTOM = 10
26assert MAX_DESCENDANTS_CUSTOM >= MAX_ANCESTORS_CUSTOM
27
28class MempoolPackagesTest(BitcoinTestFramework):
29    def set_test_params(self):
30        self.num_nodes = 2
31        self.extra_args = [
32            [
33                "-maxorphantx=1000",
34                "-whitelist=noban@127.0.0.1",  # immediate tx relay
35            ],
36            [
37                "-maxorphantx=1000",
38                "-limitancestorcount={}".format(MAX_ANCESTORS_CUSTOM),
39                "-limitdescendantcount={}".format(MAX_DESCENDANTS_CUSTOM),
40            ],
41        ]
42
43    def skip_test_if_missing_module(self):
44        self.skip_if_no_wallet()
45
46    def run_test(self):
47        # Mine some blocks and have them mature.
48        peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs
49        self.nodes[0].generate(COINBASE_MATURITY + 1)
50        utxo = self.nodes[0].listunspent(10)
51        txid = utxo[0]['txid']
52        vout = utxo[0]['vout']
53        value = utxo[0]['amount']
54
55        fee = Decimal("0.0001")
56        # MAX_ANCESTORS transactions off a confirmed tx should be fine
57        chain = []
58        witness_chain = []
59        for _ in range(MAX_ANCESTORS):
60            (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1)
61            value = sent_value
62            chain.append(txid)
63            # We need the wtxids to check P2P announcements
64            fulltx = self.nodes[0].getrawtransaction(txid)
65            witnesstx = self.nodes[0].decoderawtransaction(fulltx, True)
66            witness_chain.append(witnesstx['hash'])
67
68        # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata)
69        # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between
70        peer_inv_store.wait_for_broadcast(witness_chain)
71
72        # Check mempool has MAX_ANCESTORS transactions in it, and descendant and ancestor
73        # count and fees should look correct
74        mempool = self.nodes[0].getrawmempool(True)
75        assert_equal(len(mempool), MAX_ANCESTORS)
76        descendant_count = 1
77        descendant_fees = 0
78        descendant_vsize = 0
79
80        ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool])
81        ancestor_count = MAX_ANCESTORS
82        ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool])
83
84        descendants = []
85        ancestors = list(chain)
86        for x in reversed(chain):
87            # Check that getmempoolentry is consistent with getrawmempool
88            entry = self.nodes[0].getmempoolentry(x)
89            assert_equal(entry, mempool[x])
90
91            # Check that the descendant calculations are correct
92            assert_equal(mempool[x]['descendantcount'], descendant_count)
93            descendant_fees += mempool[x]['fee']
94            assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee'])
95            assert_equal(mempool[x]['fees']['base'], mempool[x]['fee'])
96            assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee'])
97            assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN)
98            assert_equal(mempool[x]['fees']['descendant'], descendant_fees)
99            descendant_vsize += mempool[x]['vsize']
100            assert_equal(mempool[x]['descendantsize'], descendant_vsize)
101            descendant_count += 1
102
103            # Check that ancestor calculations are correct
104            assert_equal(mempool[x]['ancestorcount'], ancestor_count)
105            assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN)
106            assert_equal(mempool[x]['ancestorsize'], ancestor_vsize)
107            ancestor_vsize -= mempool[x]['vsize']
108            ancestor_fees -= mempool[x]['fee']
109            ancestor_count -= 1
110
111            # Check that parent/child list is correct
112            assert_equal(mempool[x]['spentby'], descendants[-1:])
113            assert_equal(mempool[x]['depends'], ancestors[-2:-1])
114
115            # Check that getmempooldescendants is correct
116            assert_equal(sorted(descendants), sorted(self.nodes[0].getmempooldescendants(x)))
117
118            # Check getmempooldescendants verbose output is correct
119            for descendant, dinfo in self.nodes[0].getmempooldescendants(x, True).items():
120                assert_equal(dinfo['depends'], [chain[chain.index(descendant)-1]])
121                if dinfo['descendantcount'] > 1:
122                    assert_equal(dinfo['spentby'], [chain[chain.index(descendant)+1]])
123                else:
124                    assert_equal(dinfo['spentby'], [])
125            descendants.append(x)
126
127            # Check that getmempoolancestors is correct
128            ancestors.remove(x)
129            assert_equal(sorted(ancestors), sorted(self.nodes[0].getmempoolancestors(x)))
130
131            # Check that getmempoolancestors verbose output is correct
132            for ancestor, ainfo in self.nodes[0].getmempoolancestors(x, True).items():
133                assert_equal(ainfo['spentby'], [chain[chain.index(ancestor)+1]])
134                if ainfo['ancestorcount'] > 1:
135                    assert_equal(ainfo['depends'], [chain[chain.index(ancestor)-1]])
136                else:
137                    assert_equal(ainfo['depends'], [])
138
139
140        # Check that getmempoolancestors/getmempooldescendants correctly handle verbose=true
141        v_ancestors = self.nodes[0].getmempoolancestors(chain[-1], True)
142        assert_equal(len(v_ancestors), len(chain)-1)
143        for x in v_ancestors.keys():
144            assert_equal(mempool[x], v_ancestors[x])
145        assert chain[-1] not in v_ancestors.keys()
146
147        v_descendants = self.nodes[0].getmempooldescendants(chain[0], True)
148        assert_equal(len(v_descendants), len(chain)-1)
149        for x in v_descendants.keys():
150            assert_equal(mempool[x], v_descendants[x])
151        assert chain[0] not in v_descendants.keys()
152
153        # Check that ancestor modified fees includes fee deltas from
154        # prioritisetransaction
155        self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=1000)
156        mempool = self.nodes[0].getrawmempool(True)
157        ancestor_fees = 0
158        for x in chain:
159            ancestor_fees += mempool[x]['fee']
160            assert_equal(mempool[x]['fees']['ancestor'], ancestor_fees + Decimal('0.00001'))
161            assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN + 1000)
162
163        # Undo the prioritisetransaction for later tests
164        self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=-1000)
165
166        # Check that descendant modified fees includes fee deltas from
167        # prioritisetransaction
168        self.nodes[0].prioritisetransaction(txid=chain[-1], fee_delta=1000)
169        mempool = self.nodes[0].getrawmempool(True)
170
171        descendant_fees = 0
172        for x in reversed(chain):
173            descendant_fees += mempool[x]['fee']
174            assert_equal(mempool[x]['fees']['descendant'], descendant_fees + Decimal('0.00001'))
175            assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000)
176
177        # Adding one more transaction on to the chain should fail.
178        assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [txid], [vout], value, fee, 1)
179
180        # Check that prioritising a tx before it's added to the mempool works
181        # First clear the mempool by mining a block.
182        self.nodes[0].generate(1)
183        self.sync_blocks()
184        assert_equal(len(self.nodes[0].getrawmempool()), 0)
185        # Prioritise a transaction that has been mined, then add it back to the
186        # mempool by using invalidateblock.
187        self.nodes[0].prioritisetransaction(txid=chain[-1], fee_delta=2000)
188        self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash())
189        # Keep node1's tip synced with node0
190        self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash())
191
192        # Now check that the transaction is in the mempool, with the right modified fee
193        mempool = self.nodes[0].getrawmempool(True)
194
195        descendant_fees = 0
196        for x in reversed(chain):
197            descendant_fees += mempool[x]['fee']
198            if (x == chain[-1]):
199                assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']+satoshi_round(0.00002))
200                assert_equal(mempool[x]['fees']['modified'], mempool[x]['fee']+satoshi_round(0.00002))
201            assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000)
202            assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002))
203
204        # Check that node1's mempool is as expected (-> custom ancestor limit)
205        mempool0 = self.nodes[0].getrawmempool(False)
206        mempool1 = self.nodes[1].getrawmempool(False)
207        assert_equal(len(mempool1), MAX_ANCESTORS_CUSTOM)
208        assert set(mempool1).issubset(set(mempool0))
209        for tx in chain[:MAX_ANCESTORS_CUSTOM]:
210            assert tx in mempool1
211        # TODO: more detailed check of node1's mempool (fees etc.)
212        # check transaction unbroadcast info (should be false if in both mempools)
213        mempool = self.nodes[0].getrawmempool(True)
214        for tx in mempool:
215            assert_equal(mempool[tx]['unbroadcast'], False)
216
217        # TODO: test ancestor size limits
218
219        # Now test descendant chain limits
220        txid = utxo[1]['txid']
221        value = utxo[1]['amount']
222        vout = utxo[1]['vout']
223
224        transaction_package = []
225        tx_children = []
226        # First create one parent tx with 10 children
227        (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 10)
228        parent_transaction = txid
229        for i in range(10):
230            transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value})
231
232        # Sign and send up to MAX_DESCENDANT transactions chained off the parent tx
233        chain = [] # save sent txs for the purpose of checking node1's mempool later (see below)
234        for _ in range(MAX_DESCENDANTS - 1):
235            utxo = transaction_package.pop(0)
236            (txid, sent_value) = chain_transaction(self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10)
237            chain.append(txid)
238            if utxo['txid'] is parent_transaction:
239                tx_children.append(txid)
240            for j in range(10):
241                transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value})
242
243        mempool = self.nodes[0].getrawmempool(True)
244        assert_equal(mempool[parent_transaction]['descendantcount'], MAX_DESCENDANTS)
245        assert_equal(sorted(mempool[parent_transaction]['spentby']), sorted(tx_children))
246
247        for child in tx_children:
248            assert_equal(mempool[child]['depends'], [parent_transaction])
249
250        # Sending one more chained transaction will fail
251        utxo = transaction_package.pop(0)
252        assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [utxo['txid']], [utxo['vout']], utxo['amount'], fee, 10)
253
254        # Check that node1's mempool is as expected, containing:
255        # - txs from previous ancestor test (-> custom ancestor limit)
256        # - parent tx for descendant test
257        # - txs chained off parent tx (-> custom descendant limit)
258        self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) ==
259                                MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10)
260        mempool0 = self.nodes[0].getrawmempool(False)
261        mempool1 = self.nodes[1].getrawmempool(False)
262        assert set(mempool1).issubset(set(mempool0))
263        assert parent_transaction in mempool1
264        for tx in chain[:MAX_DESCENDANTS_CUSTOM]:
265            assert tx in mempool1
266        for tx in chain[MAX_DESCENDANTS_CUSTOM:]:
267            assert tx not in mempool1
268        # TODO: more detailed check of node1's mempool (fees etc.)
269
270        # TODO: test descendant size limits
271
272        # Test reorg handling
273        # First, the basics:
274        self.nodes[0].generate(1)
275        self.sync_blocks()
276        self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash())
277        self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash())
278
279        # Now test the case where node1 has a transaction T in its mempool that
280        # depends on transactions A and B which are in a mined block, and the
281        # block containing A and B is disconnected, AND B is not accepted back
282        # into node1's mempool because its ancestor count is too high.
283
284        # Create 8 transactions, like so:
285        # Tx0 -> Tx1 (vout0)
286        #   \--> Tx2 (vout1) -> Tx3 -> Tx4 -> Tx5 -> Tx6 -> Tx7
287        #
288        # Mine them in the next block, then generate a new tx8 that spends
289        # Tx1 and Tx7, and add to node1's mempool, then disconnect the
290        # last block.
291
292        # Create tx0 with 2 outputs
293        utxo = self.nodes[0].listunspent()
294        txid = utxo[0]['txid']
295        value = utxo[0]['amount']
296        vout = utxo[0]['vout']
297
298        send_value = satoshi_round((value - fee)/2)
299        inputs = [ {'txid' : txid, 'vout' : vout} ]
300        outputs = {}
301        for _ in range(2):
302            outputs[self.nodes[0].getnewaddress()] = send_value
303        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
304        signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx)
305        txid = self.nodes[0].sendrawtransaction(signedtx['hex'])
306        tx0_id = txid
307        value = send_value
308
309        # Create tx1
310        tx1_id, _ = chain_transaction(self.nodes[0], [tx0_id], [0], value, fee, 1)
311
312        # Create tx2-7
313        vout = 1
314        txid = tx0_id
315        for _ in range(6):
316            (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 1)
317            vout = 0
318            value = sent_value
319
320        # Mine these in a block
321        self.nodes[0].generate(1)
322        self.sync_all()
323
324        # Now generate tx8, with a big fee
325        inputs = [ {'txid' : tx1_id, 'vout': 0}, {'txid' : txid, 'vout': 0} ]
326        outputs = { self.nodes[0].getnewaddress() : send_value + value - 4*fee }
327        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
328        signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx)
329        txid = self.nodes[0].sendrawtransaction(signedtx['hex'])
330        self.sync_mempools()
331
332        # Now try to disconnect the tip on each node...
333        self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash())
334        self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash())
335        self.sync_blocks()
336
337if __name__ == '__main__':
338    MempoolPackagesTest().main()
339