1#!/usr/bin/env python3 2# Copyright (c) 2014-2020 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5"""Test behavior of headers messages to announce blocks. 6 7Setup: 8 9- Two nodes: 10 - node0 is the node-under-test. We create two p2p connections to it. The 11 first p2p connection is a control and should only ever receive inv's. The 12 second p2p connection tests the headers sending logic. 13 - node1 is used to create reorgs. 14 15test_null_locators 16================== 17 18Sends two getheaders requests with null locator values. First request's hashstop 19value refers to validated block, while second request's hashstop value refers to 20a block which hasn't been validated. Verifies only the first request returns 21headers. 22 23test_nonnull_locators 24===================== 25 26Part 1: No headers announcements before "sendheaders" 27a. node mines a block [expect: inv] 28 send getdata for the block [expect: block] 29b. node mines another block [expect: inv] 30 send getheaders and getdata [expect: headers, then block] 31c. node mines another block [expect: inv] 32 peer mines a block, announces with header [expect: getdata] 33d. node mines another block [expect: inv] 34 35Part 2: After "sendheaders", headers announcements should generally work. 36a. peer sends sendheaders [expect: no response] 37 peer sends getheaders with current tip [expect: no response] 38b. node mines a block [expect: tip header] 39c. for N in 1, ..., 10: 40 * for announce-type in {inv, header} 41 - peer mines N blocks, announces with announce-type 42 [ expect: getheaders/getdata or getdata, deliver block(s) ] 43 - node mines a block [ expect: 1 header ] 44 45Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer. 46- For response-type in {inv, getheaders} 47 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ] 48 * node mines an 8-block reorg [ expect: inv at tip ] 49 * peer responds with getblocks/getdata [expect: inv, blocks ] 50 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ] 51 * node mines another block at tip [ expect: inv ] 52 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers] 53 * peer requests block [ expect: block ] 54 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ] 55 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block] 56 * node mines 1 block [expect: 1 header, peer responds with getdata] 57 58Part 4: Test direct fetch behavior 59a. Announce 2 old block headers. 60 Expect: no getdata requests. 61b. Announce 3 new blocks via 1 headers message. 62 Expect: one getdata request for all 3 blocks. 63 (Send blocks.) 64c. Announce 1 header that forks off the last two blocks. 65 Expect: no response. 66d. Announce 1 more header that builds on that fork. 67 Expect: one getdata request for two blocks. 68e. Announce 16 more headers that build on that fork. 69 Expect: getdata request for 14 more blocks. 70f. Announce 1 more header that builds on that fork. 71 Expect: no response. 72 73Part 5: Test handling of headers that don't connect. 74a. Repeat 10 times: 75 1. Announce a header that doesn't connect. 76 Expect: getheaders message 77 2. Send headers chain. 78 Expect: getdata for the missing blocks, tip update. 79b. Then send 9 more headers that don't connect. 80 Expect: getheaders message each time. 81c. Announce a header that does connect. 82 Expect: no response. 83d. Announce 49 headers that don't connect. 84 Expect: getheaders message each time. 85e. Announce one more that doesn't connect. 86 Expect: disconnect. 87""" 88from test_framework.blocktools import create_block, create_coinbase 89from test_framework.messages import CInv 90from test_framework.p2p import ( 91 CBlockHeader, 92 NODE_WITNESS, 93 P2PInterface, 94 p2p_lock, 95 MSG_BLOCK, 96 msg_block, 97 msg_getblocks, 98 msg_getdata, 99 msg_getheaders, 100 msg_headers, 101 msg_inv, 102 msg_sendheaders, 103) 104from test_framework.test_framework import BitcoinTestFramework 105from test_framework.util import ( 106 assert_equal, 107) 108 109DIRECT_FETCH_RESPONSE_TIME = 0.05 110 111class BaseNode(P2PInterface): 112 def __init__(self): 113 super().__init__() 114 115 self.block_announced = False 116 self.last_blockhash_announced = None 117 self.recent_headers_announced = [] 118 119 def send_get_data(self, block_hashes): 120 """Request data for a list of block hashes.""" 121 msg = msg_getdata() 122 for x in block_hashes: 123 msg.inv.append(CInv(MSG_BLOCK, x)) 124 self.send_message(msg) 125 126 def send_get_headers(self, locator, hashstop): 127 msg = msg_getheaders() 128 msg.locator.vHave = locator 129 msg.hashstop = hashstop 130 self.send_message(msg) 131 132 def send_block_inv(self, blockhash): 133 msg = msg_inv() 134 msg.inv = [CInv(MSG_BLOCK, blockhash)] 135 self.send_message(msg) 136 137 def send_header_for_blocks(self, new_blocks): 138 headers_message = msg_headers() 139 headers_message.headers = [CBlockHeader(b) for b in new_blocks] 140 self.send_message(headers_message) 141 142 def send_getblocks(self, locator): 143 getblocks_message = msg_getblocks() 144 getblocks_message.locator.vHave = locator 145 self.send_message(getblocks_message) 146 147 def wait_for_block_announcement(self, block_hash, timeout=60): 148 test_function = lambda: self.last_blockhash_announced == block_hash 149 self.wait_until(test_function, timeout=timeout) 150 151 def on_inv(self, message): 152 self.block_announced = True 153 self.last_blockhash_announced = message.inv[-1].hash 154 155 def on_headers(self, message): 156 if len(message.headers): 157 self.block_announced = True 158 for x in message.headers: 159 x.calc_sha256() 160 # append because headers may be announced over multiple messages. 161 self.recent_headers_announced.append(x.sha256) 162 self.last_blockhash_announced = message.headers[-1].sha256 163 164 def clear_block_announcements(self): 165 with p2p_lock: 166 self.block_announced = False 167 self.last_message.pop("inv", None) 168 self.last_message.pop("headers", None) 169 self.recent_headers_announced = [] 170 171 172 def check_last_headers_announcement(self, headers): 173 """Test whether the last headers announcements received are right. 174 Headers may be announced across more than one message.""" 175 test_function = lambda: (len(self.recent_headers_announced) >= len(headers)) 176 self.wait_until(test_function) 177 with p2p_lock: 178 assert_equal(self.recent_headers_announced, headers) 179 self.block_announced = False 180 self.last_message.pop("headers", None) 181 self.recent_headers_announced = [] 182 183 def check_last_inv_announcement(self, inv): 184 """Test whether the last announcement received had the right inv. 185 inv should be a list of block hashes.""" 186 187 test_function = lambda: self.block_announced 188 self.wait_until(test_function) 189 190 with p2p_lock: 191 compare_inv = [] 192 if "inv" in self.last_message: 193 compare_inv = [x.hash for x in self.last_message["inv"].inv] 194 assert_equal(compare_inv, inv) 195 self.block_announced = False 196 self.last_message.pop("inv", None) 197 198class SendHeadersTest(BitcoinTestFramework): 199 def set_test_params(self): 200 self.setup_clean_chain = True 201 self.num_nodes = 2 202 203 def mine_blocks(self, count): 204 """Mine count blocks and return the new tip.""" 205 206 # Clear out block announcements from each p2p listener 207 [x.clear_block_announcements() for x in self.nodes[0].p2ps] 208 self.nodes[0].generatetoaddress(count, self.nodes[0].get_deterministic_priv_key().address) 209 return int(self.nodes[0].getbestblockhash(), 16) 210 211 def mine_reorg(self, length): 212 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). 213 214 Note: we clear the state of our p2p connections after the 215 to-be-reorged-out blocks are mined, so that we don't break later tests. 216 return the list of block hashes newly mined.""" 217 218 # make sure all invalidated blocks are node0's 219 self.nodes[0].generatetoaddress(length, self.nodes[0].get_deterministic_priv_key().address) 220 self.sync_blocks(self.nodes, wait=0.1) 221 for x in self.nodes[0].p2ps: 222 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) 223 x.clear_block_announcements() 224 225 tip_height = self.nodes[1].getblockcount() 226 hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) 227 self.nodes[1].invalidateblock(hash_to_invalidate) 228 all_hashes = self.nodes[1].generatetoaddress(length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain 229 self.sync_blocks(self.nodes, wait=0.1) 230 return [int(x, 16) for x in all_hashes] 231 232 def run_test(self): 233 # Setup the p2p connections 234 inv_node = self.nodes[0].add_p2p_connection(BaseNode()) 235 # Make sure NODE_NETWORK is not set for test_node, so no block download 236 # will occur outside of direct fetching 237 test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) 238 239 self.test_null_locators(test_node, inv_node) 240 self.test_nonnull_locators(test_node, inv_node) 241 242 def test_null_locators(self, test_node, inv_node): 243 tip = self.nodes[0].getblockheader(self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)[0]) 244 tip_hash = int(tip["hash"], 16) 245 246 inv_node.check_last_inv_announcement(inv=[tip_hash]) 247 test_node.check_last_inv_announcement(inv=[tip_hash]) 248 249 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") 250 test_node.clear_block_announcements() 251 test_node.send_get_headers(locator=[], hashstop=tip_hash) 252 test_node.check_last_headers_announcement(headers=[tip_hash]) 253 254 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") 255 block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) 256 block.solve() 257 test_node.send_header_for_blocks([block]) 258 test_node.clear_block_announcements() 259 test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) 260 test_node.sync_with_ping() 261 assert_equal(test_node.block_announced, False) 262 inv_node.clear_block_announcements() 263 test_node.send_message(msg_block(block)) 264 inv_node.check_last_inv_announcement(inv=[int(block.hash, 16)]) 265 266 def test_nonnull_locators(self, test_node, inv_node): 267 tip = int(self.nodes[0].getbestblockhash(), 16) 268 269 # PART 1 270 # 1. Mine a block; expect inv announcements each time 271 self.log.info("Part 1: headers don't start before sendheaders message...") 272 for i in range(4): 273 self.log.debug("Part 1.{}: starting...".format(i)) 274 old_tip = tip 275 tip = self.mine_blocks(1) 276 inv_node.check_last_inv_announcement(inv=[tip]) 277 test_node.check_last_inv_announcement(inv=[tip]) 278 # Try a few different responses; none should affect next announcement 279 if i == 0: 280 # first request the block 281 test_node.send_get_data([tip]) 282 test_node.wait_for_block(tip) 283 elif i == 1: 284 # next try requesting header and block 285 test_node.send_get_headers(locator=[old_tip], hashstop=tip) 286 test_node.send_get_data([tip]) 287 test_node.wait_for_block(tip) 288 test_node.clear_block_announcements() # since we requested headers... 289 elif i == 2: 290 # this time announce own block via headers 291 inv_node.clear_block_announcements() 292 height = self.nodes[0].getblockcount() 293 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] 294 block_time = last_time + 1 295 new_block = create_block(tip, create_coinbase(height + 1), block_time) 296 new_block.solve() 297 test_node.send_header_for_blocks([new_block]) 298 test_node.wait_for_getdata([new_block.sha256]) 299 test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed 300 inv_node.wait_until(lambda: inv_node.block_announced) 301 inv_node.clear_block_announcements() 302 test_node.clear_block_announcements() 303 304 self.log.info("Part 1: success!") 305 self.log.info("Part 2: announce blocks with headers after sendheaders message...") 306 # PART 2 307 # 2. Send a sendheaders message and test that headers announcements 308 # commence and keep working. 309 test_node.send_message(msg_sendheaders()) 310 prev_tip = int(self.nodes[0].getbestblockhash(), 16) 311 test_node.send_get_headers(locator=[prev_tip], hashstop=0) 312 test_node.sync_with_ping() 313 314 # Now that we've synced headers, headers announcements should work 315 tip = self.mine_blocks(1) 316 inv_node.check_last_inv_announcement(inv=[tip]) 317 test_node.check_last_headers_announcement(headers=[tip]) 318 319 height = self.nodes[0].getblockcount() + 1 320 block_time += 10 # Advance far enough ahead 321 for i in range(10): 322 self.log.debug("Part 2.{}: starting...".format(i)) 323 # Mine i blocks, and alternate announcing either via 324 # inv (of tip) or via headers. After each, new blocks 325 # mined by the node should successfully be announced 326 # with block header, even though the blocks are never requested 327 for j in range(2): 328 self.log.debug("Part 2.{}.{}: starting...".format(i, j)) 329 blocks = [] 330 for _ in range(i + 1): 331 blocks.append(create_block(tip, create_coinbase(height), block_time)) 332 blocks[-1].solve() 333 tip = blocks[-1].sha256 334 block_time += 1 335 height += 1 336 if j == 0: 337 # Announce via inv 338 test_node.send_block_inv(tip) 339 test_node.wait_for_getheaders() 340 # Should have received a getheaders now 341 test_node.send_header_for_blocks(blocks) 342 # Test that duplicate inv's won't result in duplicate 343 # getdata requests, or duplicate headers announcements 344 [inv_node.send_block_inv(x.sha256) for x in blocks] 345 test_node.wait_for_getdata([x.sha256 for x in blocks]) 346 inv_node.sync_with_ping() 347 else: 348 # Announce via headers 349 test_node.send_header_for_blocks(blocks) 350 test_node.wait_for_getdata([x.sha256 for x in blocks]) 351 # Test that duplicate headers won't result in duplicate 352 # getdata requests (the check is further down) 353 inv_node.send_header_for_blocks(blocks) 354 inv_node.sync_with_ping() 355 [test_node.send_message(msg_block(x)) for x in blocks] 356 test_node.sync_with_ping() 357 inv_node.sync_with_ping() 358 # This block should not be announced to the inv node (since it also 359 # broadcast it) 360 assert "inv" not in inv_node.last_message 361 assert "headers" not in inv_node.last_message 362 tip = self.mine_blocks(1) 363 inv_node.check_last_inv_announcement(inv=[tip]) 364 test_node.check_last_headers_announcement(headers=[tip]) 365 height += 1 366 block_time += 1 367 368 self.log.info("Part 2: success!") 369 370 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...") 371 372 # PART 3. Headers announcements can stop after large reorg, and resume after 373 # getheaders or inv from peer. 374 for j in range(2): 375 self.log.debug("Part 3.{}: starting...".format(j)) 376 # First try mining a reorg that can propagate with header announcement 377 new_block_hashes = self.mine_reorg(length=7) 378 tip = new_block_hashes[-1] 379 inv_node.check_last_inv_announcement(inv=[tip]) 380 test_node.check_last_headers_announcement(headers=new_block_hashes) 381 382 block_time += 8 383 384 # Mine a too-large reorg, which should be announced with a single inv 385 new_block_hashes = self.mine_reorg(length=8) 386 tip = new_block_hashes[-1] 387 inv_node.check_last_inv_announcement(inv=[tip]) 388 test_node.check_last_inv_announcement(inv=[tip]) 389 390 block_time += 9 391 392 fork_point = self.nodes[0].getblock("%064x" % new_block_hashes[0])["previousblockhash"] 393 fork_point = int(fork_point, 16) 394 395 # Use getblocks/getdata 396 test_node.send_getblocks(locator=[fork_point]) 397 test_node.check_last_inv_announcement(inv=new_block_hashes) 398 test_node.send_get_data(new_block_hashes) 399 test_node.wait_for_block(new_block_hashes[-1]) 400 401 for i in range(3): 402 self.log.debug("Part 3.{}.{}: starting...".format(j, i)) 403 404 # Mine another block, still should get only an inv 405 tip = self.mine_blocks(1) 406 inv_node.check_last_inv_announcement(inv=[tip]) 407 test_node.check_last_inv_announcement(inv=[tip]) 408 if i == 0: 409 # Just get the data -- shouldn't cause headers announcements to resume 410 test_node.send_get_data([tip]) 411 test_node.wait_for_block(tip) 412 elif i == 1: 413 # Send a getheaders message that shouldn't trigger headers announcements 414 # to resume (best header sent will be too old) 415 test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) 416 test_node.send_get_data([tip]) 417 test_node.wait_for_block(tip) 418 elif i == 2: 419 # This time, try sending either a getheaders to trigger resumption 420 # of headers announcements, or mine a new block and inv it, also 421 # triggering resumption of headers announcements. 422 test_node.send_get_data([tip]) 423 test_node.wait_for_block(tip) 424 if j == 0: 425 test_node.send_get_headers(locator=[tip], hashstop=0) 426 test_node.sync_with_ping() 427 else: 428 test_node.send_block_inv(tip) 429 test_node.sync_with_ping() 430 # New blocks should now be announced with header 431 tip = self.mine_blocks(1) 432 inv_node.check_last_inv_announcement(inv=[tip]) 433 test_node.check_last_headers_announcement(headers=[tip]) 434 435 self.log.info("Part 3: success!") 436 437 self.log.info("Part 4: Testing direct fetch behavior...") 438 tip = self.mine_blocks(1) 439 height = self.nodes[0].getblockcount() + 1 440 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] 441 block_time = last_time + 1 442 443 # Create 2 blocks. Send the blocks, then send the headers. 444 blocks = [] 445 for _ in range(2): 446 blocks.append(create_block(tip, create_coinbase(height), block_time)) 447 blocks[-1].solve() 448 tip = blocks[-1].sha256 449 block_time += 1 450 height += 1 451 inv_node.send_message(msg_block(blocks[-1])) 452 453 inv_node.sync_with_ping() # Make sure blocks are processed 454 test_node.last_message.pop("getdata", None) 455 test_node.send_header_for_blocks(blocks) 456 test_node.sync_with_ping() 457 # should not have received any getdata messages 458 with p2p_lock: 459 assert "getdata" not in test_node.last_message 460 461 # This time, direct fetch should work 462 blocks = [] 463 for _ in range(3): 464 blocks.append(create_block(tip, create_coinbase(height), block_time)) 465 blocks[-1].solve() 466 tip = blocks[-1].sha256 467 block_time += 1 468 height += 1 469 470 test_node.send_header_for_blocks(blocks) 471 test_node.sync_with_ping() 472 test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) 473 474 [test_node.send_message(msg_block(x)) for x in blocks] 475 476 test_node.sync_with_ping() 477 478 # Now announce a header that forks the last two blocks 479 tip = blocks[0].sha256 480 height -= 2 481 blocks = [] 482 483 # Create extra blocks for later 484 for _ in range(20): 485 blocks.append(create_block(tip, create_coinbase(height), block_time)) 486 blocks[-1].solve() 487 tip = blocks[-1].sha256 488 block_time += 1 489 height += 1 490 491 # Announcing one block on fork should not trigger direct fetch 492 # (less work than tip) 493 test_node.last_message.pop("getdata", None) 494 test_node.send_header_for_blocks(blocks[0:1]) 495 test_node.sync_with_ping() 496 with p2p_lock: 497 assert "getdata" not in test_node.last_message 498 499 # Announcing one more block on fork should trigger direct fetch for 500 # both blocks (same work as tip) 501 test_node.send_header_for_blocks(blocks[1:2]) 502 test_node.sync_with_ping() 503 test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) 504 505 # Announcing 16 more headers should trigger direct fetch for 14 more 506 # blocks 507 test_node.send_header_for_blocks(blocks[2:18]) 508 test_node.sync_with_ping() 509 test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) 510 511 # Announcing 1 more header should not trigger any response 512 test_node.last_message.pop("getdata", None) 513 test_node.send_header_for_blocks(blocks[18:19]) 514 test_node.sync_with_ping() 515 with p2p_lock: 516 assert "getdata" not in test_node.last_message 517 518 self.log.info("Part 4: success!") 519 520 # Now deliver all those blocks we announced. 521 [test_node.send_message(msg_block(x)) for x in blocks] 522 523 self.log.info("Part 5: Testing handling of unconnecting headers") 524 # First we test that receipt of an unconnecting header doesn't prevent 525 # chain sync. 526 for i in range(10): 527 self.log.debug("Part 5.{}: starting...".format(i)) 528 test_node.last_message.pop("getdata", None) 529 blocks = [] 530 # Create two more blocks. 531 for _ in range(2): 532 blocks.append(create_block(tip, create_coinbase(height), block_time)) 533 blocks[-1].solve() 534 tip = blocks[-1].sha256 535 block_time += 1 536 height += 1 537 # Send the header of the second block -> this won't connect. 538 with p2p_lock: 539 test_node.last_message.pop("getheaders", None) 540 test_node.send_header_for_blocks([blocks[1]]) 541 test_node.wait_for_getheaders() 542 test_node.send_header_for_blocks(blocks) 543 test_node.wait_for_getdata([x.sha256 for x in blocks]) 544 [test_node.send_message(msg_block(x)) for x in blocks] 545 test_node.sync_with_ping() 546 assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) 547 548 blocks = [] 549 # Now we test that if we repeatedly don't send connecting headers, we 550 # don't go into an infinite loop trying to get them to connect. 551 MAX_UNCONNECTING_HEADERS = 10 552 for _ in range(MAX_UNCONNECTING_HEADERS + 1): 553 blocks.append(create_block(tip, create_coinbase(height), block_time)) 554 blocks[-1].solve() 555 tip = blocks[-1].sha256 556 block_time += 1 557 height += 1 558 559 for i in range(1, MAX_UNCONNECTING_HEADERS): 560 # Send a header that doesn't connect, check that we get a getheaders. 561 with p2p_lock: 562 test_node.last_message.pop("getheaders", None) 563 test_node.send_header_for_blocks([blocks[i]]) 564 test_node.wait_for_getheaders() 565 566 # Next header will connect, should re-set our count: 567 test_node.send_header_for_blocks([blocks[0]]) 568 569 # Remove the first two entries (blocks[1] would connect): 570 blocks = blocks[2:] 571 572 # Now try to see how many unconnecting headers we can send 573 # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS 574 for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): 575 # Send a header that doesn't connect, check that we get a getheaders. 576 with p2p_lock: 577 test_node.last_message.pop("getheaders", None) 578 test_node.send_header_for_blocks([blocks[i % len(blocks)]]) 579 test_node.wait_for_getheaders() 580 581 # Eventually this stops working. 582 test_node.send_header_for_blocks([blocks[-1]]) 583 584 # Should get disconnected 585 test_node.wait_for_disconnect() 586 587 self.log.info("Part 5: success!") 588 589 # Finally, check that the inv node never received a getdata request, 590 # throughout the test 591 assert "getdata" not in inv_node.last_message 592 593if __name__ == '__main__': 594 SendHeadersTest().main() 595