1# -*- coding: utf-8 -*- 2''' 3Tests to try out keeping. Potentially ephemeral 4 5''' 6# pylint: skip-file 7# pylint: disable=C0103 8import sys 9if sys.version_info < (2, 7): 10 import unittest2 as unittest 11else: 12 import unittest 13 14import os 15import time 16import tempfile 17import shutil 18from collections import deque 19 20from ioflo.aid.odicting import odict 21from ioflo.aid.timing import Timer, StoreTimer 22from ioflo.base.storing import Store 23from ioflo.base.consoling import getConsole 24console = getConsole() 25 26# Import raet libs 27from raet.abiding import * # import globals 28from raet import raeting, nacling 29from raet.road import estating, keeping, stacking, packeting, transacting 30 31if sys.platform == 'win32': 32 TEMPDIR = 'c:/temp' 33 if not os.path.exists(TEMPDIR): 34 os.mkdir(TEMPDIR) 35else: 36 TEMPDIR = '/tmp' 37 38def setUpModule(): 39 console.reinit(verbosity=console.Wordage.concise) 40 41def tearDownModule(): 42 pass 43 44class BasicTestCase(unittest.TestCase): 45 """""" 46 47 def setUp(self): 48 self.store = Store(stamp=0.0) 49 self.timer = StoreTimer(store=self.store, duration=1.0) 50 51 self.base = tempfile.mkdtemp(prefix="raet", suffix="base", dir=TEMPDIR) 52 53 def tearDown(self): 54 if os.path.exists(self.base): 55 shutil.rmtree(self.base) 56 57 def createRoadData(self, 58 base, 59 name='', 60 ha=None, 61 main=None, 62 auto=raeting.AutoMode.never.value, 63 role=None, 64 kind=None, ): 65 ''' 66 Creates odict and populates with data to setup road stack 67 68 ''' 69 data = odict() 70 data['name'] = name 71 data['ha'] = ha 72 data['main'] = main 73 data['auto'] = auto 74 data['role'] = role if role is not None else name 75 data['kind'] = kind 76 data['dirpath'] = os.path.join(base, 'road', 'keep', name) 77 signer = nacling.Signer() 78 data['sighex'] = signer.keyhex 79 data['verhex'] = signer.verhex 80 privateer = nacling.Privateer() 81 data['prihex'] = privateer.keyhex 82 data['pubhex'] = privateer.pubhex 83 84 return data 85 86 def createRoadStack(self, 87 data, 88 uid=None, 89 ha=None, 90 main=None, 91 auto=None, 92 role=None, 93 kind=None, 94 period=None, 95 offset=None,): 96 ''' 97 Creates stack and local estate from data with 98 and overrides with parameters 99 100 returns stack 101 102 ''' 103 stack = stacking.RoadStack(store=self.store, 104 name=data['name'], 105 uid=uid, 106 ha=ha or data['ha'], 107 main=main if main is not None else data['main'], 108 role=role if role is not None else data['role'], 109 sigkey=data['sighex'], 110 prikey=data['prihex'], 111 auto=auto if auto is not None else data['auto'], 112 kind=kind if kind is not None else data['kind'], 113 dirpath=data['dirpath'], 114 period=period, 115 offset=offset,) 116 117 return stack 118 119 def join(self, initiator, correspondent, deid=None, duration=1.0, 120 cascade=False): 121 ''' 122 Utility method to do join. Call from test method. 123 ''' 124 console.terse("\nJoin Transaction **************\n") 125 if not initiator.remotes: 126 initiator.addRemote(estating.RemoteEstate(stack=initiator, 127 fuid=0, # vacuous join 128 sid=0, # always 0 for join 129 ha=correspondent.local.ha)) 130 initiator.join(uid=deid, cascade=cascade) 131 self.service(correspondent, initiator, duration=duration) 132 133 def allow(self, initiator, correspondent, deid=None, duration=1.0, 134 cascade=False): 135 ''' 136 Utility method to do allow. Call from test method. 137 ''' 138 console.terse("\nAllow Transaction **************\n") 139 initiator.allow(uid=deid, cascade=cascade) 140 self.service(correspondent, initiator, duration=duration) 141 142 def alive(self, initiator, correspondent, duid=None, duration=1.0, 143 cascade=False): 144 ''' 145 Utility method to do alive. Call from test method. 146 ''' 147 console.terse("\nAlive Transaction **************\n") 148 initiator.alive(uid=duid, cascade=cascade) 149 self.service(correspondent, initiator, duration=duration) 150 151 def message(self, msgs, initiator, correspondent, duration=2.0): 152 ''' 153 Utility to send messages both ways 154 ''' 155 for msg in msgs: 156 initiator.transmit(msg) 157 158 self.service(initiator, correspondent, duration=duration) 159 160 def flushReceives(self, stack): 161 ''' 162 Flush any queued up udp packets in receive buffer 163 ''' 164 stack.serviceReceives() 165 stack.rxes.clear() 166 167 def dupReceives(self, stack): 168 ''' 169 Duplicate each queued up udp packet in receive buffer 170 ''' 171 stack.serviceReceives() 172 rxes = stack.rxes 173 stack.rxes = deque() 174 for rx in rxes: 175 stack.rxes.append(rx) # one 176 stack.rxes.append(rx) # and one more 177 178 def service(self, main, other, duration=1.0): 179 ''' 180 Utility method to service queues. Call from test method. 181 ''' 182 self.timer.restart(duration=duration) 183 while not self.timer.expired: 184 other.serviceAll() 185 main.serviceAll() 186 if not (main.transactions or other.transactions): 187 break 188 self.store.advanceStamp(0.0625) 189 time.sleep(0.0625) 190 191 def serviceStack(self, stack, duration=1.0): 192 ''' 193 Utility method to service queues for one stack. Call from test method. 194 ''' 195 self.timer.restart(duration=duration) 196 while not self.timer.expired: 197 stack.serviceAll() 198 if not (stack.transactions): 199 break 200 self.store.advanceStamp(0.0625) 201 time.sleep(0.0625) 202 203 def serviceStacks(self, stacks, duration=1.0): 204 ''' 205 Utility method to service queues for list of stacks. Call from test method. 206 ''' 207 self.timer.restart(duration=duration) 208 while not self.timer.expired: 209 for stack in stacks: 210 stack.serviceAll() 211 if all([not stack.transactions for stack in stacks]): 212 break 213 self.store.advanceStamp(0.0625) 214 time.sleep(0.0625) 215 216 def serviceStacksDropRx(self, stacks, duration=1.0): 217 ''' 218 Utility method to service queues for list of stacks. Call from test method. 219 ''' 220 self.timer.restart(duration=duration) 221 while not self.timer.expired: 222 for stack in stacks: 223 stack.serviceReceives() 224 stack.rxes.clear() 225 stack.serviceRxes() 226 stack.process() 227 stack.serviceAllTx() 228 if all([not stack.transactions for stack in stacks]): 229 break 230 self.store.advanceStamp(0.1) 231 time.sleep(0.1) 232 233 def serviceManageStacks(self, stacks, duration=1.0, cascade=False): 234 ''' 235 Utility method to service queues and manage presence for list of stacks. 236 Call from test method. 237 ''' 238 self.timer.restart(duration=duration) 239 while not self.timer.expired: 240 for stack in stacks: 241 stack.serviceAllRx() 242 stack.manage(cascade=cascade) 243 stack.serviceAllTx() 244 if all([not stack.transactions for stack in stacks]): 245 break 246 self.store.advanceStamp(0.1) 247 time.sleep(0.1) 248 249 def answerAlive(self, stack, deid=None, kind=raeting.PcktKind.nack.value, dataMod=None): 250 ''' 251 Utility method to receive a packet in the given stack and send alive nack as a responce 252 Call from test method. 253 :param stack: correspondent stack that have to receive and nack a transaction 254 :param deid: remote estate id in the stack 255 :param kind: nack kind (nack, reject, ...) 256 :rtype : None 257 ''' 258 stack.serviceReceives() 259 # process rx 260 raw, sa = stack.rxes.popleft() 261 console.verbose("{0} received packet\n{1}\n".format(stack.name, raw)) 262 packet = packeting.RxPacket(stack=stack, packed=raw) 263 packet.parseOuter() 264 sh, sp = sa 265 packet.data.update(sh=sh, sp=sp) 266 data = odict(hk=stack.Hk, bk=stack.Bk, fk=stack.Fk, ck=stack.Ck) 267 if dataMod: 268 data.update(dataMod) 269 remote = stack.retrieveRemote(deid) 270 alivent = transacting.Alivent(stack=stack, 271 remote=remote, 272 bcst=packet.data['bf'], 273 sid=packet.data['si'], 274 tid=packet.data['ti'], 275 txData=data, 276 rxPacket=packet) 277 if kind == raeting.PcktKind.ack: 278 alivent.alive() 279 else: 280 alivent.nack(kind=kind) 281 282 def aliveBrokenInner(self, stack, uid=None, timeout=None, cascade=False): 283 ''' 284 Initiate alive transaction 285 If duid is None then create remote at ha 286 ''' 287 remote = stack.retrieveRemote(uid=uid) 288 self.assertIsNotNone(remote) 289 data = odict(hk=stack.Hk, bk=stack.Bk, fk=stack.Fk, ck=stack.Ck) 290 data['ck'] = -1 291 aliver = transacting.Aliver(stack=stack, 292 remote=remote, 293 timeout=timeout, 294 txData=data, 295 cascade=cascade) 296 aliver.alive() 297 298 def testJoinNameRoleDiffer(self): 299 ''' 300 Test join from other where name and role are different 301 ''' 302 console.terse("{0}\n".format(self.testJoinNameRoleDiffer.__doc__)) 303 304 mainData = self.createRoadData(name='main_stack', 305 role='main', 306 base=self.base, 307 auto=raeting.AutoMode.once.value) 308 keeping.clearAllKeep(mainData['dirpath']) 309 main = self.createRoadStack(data=mainData, 310 main=True, 311 auto=mainData['auto'], 312 ha=None) 313 314 otherData = self.createRoadData(name='other_stack', 315 role='other', 316 base=self.base, 317 auto=raeting.AutoMode.once.value) 318 keeping.clearAllKeep(otherData['dirpath']) 319 other = self.createRoadStack(data=otherData, 320 main=None, 321 ha=("", raeting.RAET_TEST_PORT)) 322 323 self.assertNotEqual(main.local.name, main.local.role) 324 self.assertNotEqual(other.local.name, other.local.role) 325 self.assertIs(other.main, None) 326 self.assertIs(other.keep.auto, raeting.AutoMode.once.value) 327 328 console.terse("\nJoin Other to Main *********\n") 329 self.join(other, main) 330 for stack in [main, other]: 331 self.assertEqual(len(stack.transactions), 0) 332 self.assertEqual(len(stack.remotes), 1) 333 self.assertEqual(len(stack.nameRemotes), 1) 334 remote = stack.remotes.values()[0] 335 self.assertIs(remote.joined, True) 336 self.assertIs(remote.allowed, None) 337 self.assertIs(remote.alived, None) 338 339 console.terse("\nAllow Other to Main *********\n") 340 self.allow(other, main) 341 for stack in [main, other]: 342 self.assertEqual(len(stack.transactions), 0) 343 self.assertEqual(len(stack.remotes), 1) 344 self.assertEqual(len(stack.nameRemotes), 1) 345 remote = stack.remotes.values()[0] 346 self.assertIs(remote.joined, True) 347 self.assertIs(remote.allowed, True) 348 self.assertIs(remote.alived, True) # fast alive 349 350 console.terse("\nAlive Other to Main *********\n") 351 self.alive(other, main) 352 for stack in [main, other]: 353 self.assertEqual(len(stack.transactions), 0) 354 self.assertEqual(len(stack.remotes), 1) 355 self.assertEqual(len(stack.nameRemotes), 1) 356 remote = stack.remotes.values()[0] 357 self.assertIs(remote.joined, True) 358 self.assertIs(remote.allowed, True) 359 self.assertIs(remote.alived, True) 360 361 console.terse("\nAlive Main to Other *********\n") 362 self.alive(main, other) 363 for stack in [main, other]: 364 self.assertEqual(len(stack.transactions), 0) 365 self.assertEqual(len(stack.remotes), 1) 366 self.assertEqual(len(stack.nameRemotes), 1) 367 remote = stack.remotes.values()[0] 368 self.assertIs(remote.joined, True) 369 self.assertIs(remote.allowed, True) 370 self.assertIs(remote.alived, True) 371 372 for stack in [main, other]: 373 stack.server.close() 374 stack.clearAllKeeps() 375 376 def testJoinFromMain(self): 377 ''' 378 Test join,initiated by main 379 ''' 380 console.terse("{0}\n".format(self.testJoinFromMain.__doc__)) 381 382 mainData = self.createRoadData(name='main', 383 base=self.base, 384 auto=raeting.AutoMode.once.value) 385 keeping.clearAllKeep(mainData['dirpath']) 386 main = self.createRoadStack(data=mainData, 387 main=True, 388 auto=mainData['auto'], 389 ha=None) 390 391 otherData = self.createRoadData(name='other', 392 base=self.base, 393 auto=raeting.AutoMode.never.value) 394 keeping.clearAllKeep(otherData['dirpath']) 395 other = self.createRoadStack(data=otherData, 396 main=None, 397 ha=("", raeting.RAET_TEST_PORT)) 398 399 self.assertIs(other.keep.auto, raeting.AutoMode.never.value) 400 self.assertIs(other.main, None) 401 402 console.terse("\nJoin Main to Other *********\n") 403 self.join(main, other) # vacuous join fails because other not main 404 for stack in [main, other]: 405 self.assertEqual(len(stack.transactions), 0) 406 self.assertEqual(len(stack.remotes), 0) 407 self.assertEqual(len(stack.nameRemotes), 0) 408 409 # now fix it so other can accept vacuous joins 410 other.main = True 411 other.keep.auto = raeting.AutoMode.once.value 412 self.assertIs(other.main, True) 413 self.assertIs(other.keep.auto, raeting.AutoMode.once.value) 414 415 self.join(main, other) 416 for stack in [main, other]: 417 self.assertEqual(len(stack.transactions), 0) 418 self.assertEqual(len(stack.remotes), 1) 419 self.assertEqual(len(stack.nameRemotes), 1) 420 remote = stack.remotes.values()[0] 421 self.assertIs(remote.joined, True) 422 self.assertIs(remote.allowed, None) 423 self.assertIs(remote.alived, None) 424 425 console.terse("\nAllow Main to Other *********\n") 426 self.allow(main, other) 427 for stack in [main, other]: 428 self.assertEqual(len(stack.transactions), 0) 429 self.assertEqual(len(stack.remotes), 1) 430 self.assertEqual(len(stack.nameRemotes), 1) 431 remote = stack.remotes.values()[0] 432 self.assertIs(remote.joined, True) 433 self.assertIs(remote.allowed, True) 434 self.assertIs(remote.alived, True) # fast alive 435 436 main.remotes.values()[0].alived = None # reset alived 437 other.remotes.values()[0].alived = None # reset alived 438 439 console.terse("\nAlive Main to other *********\n") 440 self.alive(main, other) 441 for stack in [main, other]: 442 self.assertEqual(len(stack.transactions), 0) 443 self.assertEqual(len(stack.remotes), 1) 444 self.assertEqual(len(stack.nameRemotes), 1) 445 remote = stack.remotes.values()[0] 446 self.assertIs(remote.joined, True) 447 self.assertIs(remote.allowed, True) 448 self.assertIs(remote.alived, True) 449 450 main.remotes.values()[0].alived = None # reset alived 451 other.remotes.values()[0].alived = None # reset alived 452 453 console.terse("\nAlive Other to Main *********\n") 454 self.alive(other, main) 455 for stack in [main, other]: 456 self.assertEqual(len(stack.transactions), 0) 457 self.assertEqual(len(stack.remotes), 1) 458 self.assertEqual(len(stack.nameRemotes), 1) 459 remote = stack.remotes.values()[0] 460 self.assertIs(remote.joined, True) 461 self.assertIs(remote.allowed, True) 462 self.assertIs(remote.alived, True) 463 464 465 for stack in [main, other]: 466 stack.server.close() 467 stack.clearAllKeeps() 468 469 def testJoinFromMainNameRoleDiffer(self): 470 ''' 471 Test join from main where name and role are different 472 ''' 473 console.terse("{0}\n".format(self.testJoinFromMainNameRoleDiffer.__doc__)) 474 475 mainData = self.createRoadData(name='main_stack', 476 role='main', 477 base=self.base, 478 auto=raeting.AutoMode.once.value) 479 keeping.clearAllKeep(mainData['dirpath']) 480 main = self.createRoadStack(data=mainData, 481 main=True, 482 auto=mainData['auto'], 483 ha=None) 484 485 otherData = self.createRoadData(name='other_stack', 486 role='other', 487 base=self.base, 488 auto=raeting.AutoMode.never.value) 489 keeping.clearAllKeep(otherData['dirpath']) 490 other = self.createRoadStack(data=otherData, 491 main=None, 492 ha=("", raeting.RAET_TEST_PORT)) 493 494 self.assertNotEqual(main.local.name, main.local.role) 495 self.assertNotEqual(other.local.name, other.local.role) 496 self.assertIs(other.keep.auto, raeting.AutoMode.never.value) 497 self.assertIs(other.main, None) 498 499 console.terse("\nJoin Main to Other *********\n") 500 self.join(main, other) # vacuous join fails because other not main 501 for stack in [main, other]: 502 self.assertEqual(len(stack.transactions), 0) 503 self.assertEqual(len(stack.remotes), 0) 504 self.assertEqual(len(stack.nameRemotes), 0) 505 506 # now fix it so other can accept vacuous joins 507 other.main = True 508 other.keep.auto = raeting.AutoMode.once.value 509 self.assertIs(other.main, True) 510 self.assertIs(other.keep.auto, raeting.AutoMode.once.value) 511 512 513 self.join(main, other) 514 for stack in [main, other]: 515 self.assertEqual(len(stack.transactions), 0) 516 self.assertEqual(len(stack.remotes), 1) 517 self.assertEqual(len(stack.nameRemotes), 1) 518 remote = stack.remotes.values()[0] 519 self.assertIs(remote.joined, True) 520 self.assertIs(remote.allowed, None) 521 self.assertIs(remote.alived, None) 522 523 console.terse("\nAllow Main to Other *********\n") 524 self.allow(main, other) 525 for stack in [main, other]: 526 self.assertEqual(len(stack.transactions), 0) 527 self.assertEqual(len(stack.remotes), 1) 528 self.assertEqual(len(stack.nameRemotes), 1) 529 remote = stack.remotes.values()[0] 530 self.assertIs(remote.joined, True) 531 self.assertIs(remote.allowed, True) 532 self.assertIs(remote.alived, True) # fast alive 533 534 main.remotes.values()[0].alived = None # reset alived 535 other.remotes.values()[0].alived = None # reset alived 536 537 console.terse("\nAlive Main to other *********\n") 538 self.alive(main, other) 539 for stack in [main, other]: 540 self.assertEqual(len(stack.transactions), 0) 541 self.assertEqual(len(stack.remotes), 1) 542 self.assertEqual(len(stack.nameRemotes), 1) 543 remote = stack.remotes.values()[0] 544 self.assertIs(remote.joined, True) 545 self.assertIs(remote.allowed, True) 546 self.assertIs(remote.alived, True) 547 548 main.remotes.values()[0].alived = None # reset alived 549 other.remotes.values()[0].alived = None # reset alived 550 551 console.terse("\nAlive Other to Main *********\n") 552 self.alive(other, main) 553 for stack in [main, other]: 554 self.assertEqual(len(stack.transactions), 0) 555 self.assertEqual(len(stack.remotes), 1) 556 self.assertEqual(len(stack.nameRemotes), 1) 557 remote = stack.remotes.values()[0] 558 self.assertIs(remote.joined, True) 559 self.assertIs(remote.allowed, True) 560 self.assertIs(remote.alived, True) 561 562 for stack in [main, other]: 563 stack.server.close() 564 stack.clearAllKeeps() 565 566 def testJoinFromMainKindChange(self): 567 ''' 568 Test allow from main where name changed from join 569 This reproduces what happens if name changed after successful join 570 and reboot 571 so joined is persisted so allow fails 572 ''' 573 console.terse("{0}\n".format(self.testJoinFromMainKindChange.__doc__)) 574 575 mainData = self.createRoadData(name='main_stack', 576 role='main', 577 base=self.base, 578 auto=raeting.AutoMode.once.value) 579 keeping.clearAllKeep(mainData['dirpath']) 580 main = self.createRoadStack(data=mainData, 581 main=True, 582 auto=mainData['auto'], 583 ha=None) 584 585 otherData = self.createRoadData(name='other_stack', 586 role='other', 587 base=self.base, 588 auto=raeting.AutoMode.once.value) 589 keeping.clearAllKeep(otherData['dirpath']) 590 other = self.createRoadStack(data=otherData, 591 main=True, 592 ha=("", raeting.RAET_TEST_PORT)) 593 594 self.assertNotEqual(main.local.name, main.local.role) 595 self.assertNotEqual(other.local.name, other.local.role) 596 self.assertIs(other.main, True) 597 self.assertIs(other.keep.auto, raeting.AutoMode.once.value) 598 599 console.terse("\nJoin Main to Other *********\n") 600 self.join(main, other) 601 for stack in [main, other]: 602 self.assertEqual(len(stack.transactions), 0) 603 self.assertEqual(len(stack.remotes), 1) 604 self.assertEqual(len(stack.nameRemotes), 1) 605 remote = stack.remotes.values()[0] 606 self.assertIs(remote.joined, True) 607 self.assertIs(remote.allowed, None) 608 self.assertIs(remote.alived, None) 609 610 # Change kind 611 other.kind = 1 612 self.assertNotEqual(other.kind, otherData['kind']) 613 main.kind = 1 614 self.assertNotEqual(main.kind, mainData['kind']) 615 616 self.join(main, other) # fails because not same all and immutable road 617 for stack in [main, other]: 618 self.assertEqual(len(stack.transactions), 0) 619 620 self.assertEqual(len(main.remotes), 0) 621 self.assertEqual(len(main.nameRemotes), 0) 622 623 self.assertEqual(len(other.remotes), 1) 624 self.assertEqual(len(other.nameRemotes), 1) 625 remote = other.remotes.values()[0] 626 self.assertIs(remote.joined, True) 627 self.assertIs(remote.allowed, None) 628 self.assertIs(remote.alived, None) 629 630 other.mutable = True 631 self.assertIs(other.mutable, True) 632 self.join(main, other) # fails because not same all and immutable road 633 for stack in [main, other]: 634 self.assertEqual(len(stack.transactions), 0) 635 self.assertEqual(len(stack.remotes), 1) 636 self.assertEqual(len(stack.nameRemotes), 1) 637 remote = stack.remotes.values()[0] 638 self.assertIs(remote.joined, True) 639 self.assertIs(remote.allowed, None) 640 self.assertIs(remote.alived, None) 641 642 console.terse("\nAllow Main to Other *********\n") 643 self.allow(main, other) 644 for stack in [main, other]: 645 self.assertEqual(len(stack.transactions), 0) 646 self.assertEqual(len(stack.remotes), 1) 647 self.assertEqual(len(stack.nameRemotes), 1) 648 remote = stack.remotes.values()[0] 649 self.assertIs(remote.joined, True) 650 self.assertIs(remote.allowed, True) 651 self.assertIs(remote.alived, True) # fast alive 652 653 main.remotes.values()[0].alived = None # reset alived 654 other.remotes.values()[0].alived = None # reset alived 655 656 console.terse("\nAlive Main to other *********\n") 657 self.alive(main, other) 658 for stack in [main, other]: 659 self.assertEqual(len(stack.transactions), 0) 660 self.assertEqual(len(stack.remotes), 1) 661 self.assertEqual(len(stack.nameRemotes), 1) 662 remote = stack.remotes.values()[0] 663 self.assertIs(remote.joined, True) 664 self.assertIs(remote.allowed, True) 665 self.assertIs(remote.alived, True) 666 667 main.remotes.values()[0].alived = None # reset alived 668 other.remotes.values()[0].alived = None # reset alived 669 670 console.terse("\nAlive Other to Main *********\n") 671 self.alive(other, main) 672 for stack in [main, other]: 673 self.assertEqual(len(stack.transactions), 0) 674 self.assertEqual(len(stack.remotes), 1) 675 self.assertEqual(len(stack.nameRemotes), 1) 676 remote = stack.remotes.values()[0] 677 self.assertIs(remote.joined, True) 678 self.assertIs(remote.allowed, True) 679 self.assertIs(remote.alived, True) 680 681 682 for stack in [main, other]: 683 stack.server.close() 684 stack.clearAllKeeps() 685 686 def testAliveDead(self): 687 ''' 688 Test basic alive transaction given already joined and allowed 689 ''' 690 console.terse("{0}\n".format(self.testAliveDead.__doc__)) 691 692 mainData = self.createRoadData(name='main', 693 base=self.base, 694 auto=raeting.AutoMode.once.value) 695 keeping.clearAllKeep(mainData['dirpath']) 696 main = self.createRoadStack(data=mainData, 697 main=True, 698 auto=mainData['auto'], 699 ha=None) 700 701 otherData = self.createRoadData(name='other', base=self.base) 702 keeping.clearAllKeep(otherData['dirpath']) 703 other = self.createRoadStack(data=otherData, 704 main=None, 705 auto=raeting.AutoMode.once.value, 706 ha=("", raeting.RAET_TEST_PORT)) 707 708 self.join(other, main) 709 remotes = [] 710 for stack in [main, other]: 711 self.assertEqual(len(stack.transactions), 0) 712 self.assertEqual(len(stack.remotes), 1) 713 remote = stack.remotes.values()[0] 714 remotes.append(remote) 715 self.assertIs(remote.joined, True) 716 self.assertNotEqual(remote.nuid, 0) 717 self.assertNotEqual(remote.fuid, 0) 718 719 self.assertEqual(remotes[0].fuid, remotes[1].nuid) 720 self.assertEqual(remotes[1].fuid, remotes[0].nuid) 721 722 for stack in [main, other]: 723 self.assertIs(stack.remotes.values()[0].allowed, None) 724 self.allow(other, main) 725 for stack in [main, other]: 726 self.assertEqual(len(stack.transactions), 0) 727 self.assertEqual(len(stack.remotes), 1) 728 remote = stack.remotes.values()[0] 729 self.assertIs(remote.joined, True) 730 self.assertIs(remote.allowed, True) 731 self.assertIs(remote.alived, True) 732 733 console.terse("\nAlive Other to Main *********\n") 734 for stack in [main, other]: 735 remote = stack.remotes.values()[0] 736 remote.alived = None 737 self.assertIs(stack.remotes.values()[0].alived, None) 738 self.alive(other, main, duid=other.remotes.values()[0].uid) 739 for stack in [main, other]: 740 self.assertEqual(len(stack.transactions), 0) 741 self.assertEqual(len(stack.remotes), 1) 742 remote = stack.remotes.values()[0] 743 self.assertIs(remote.joined, True) 744 self.assertIs(remote.allowed, True) 745 self.assertIs(remote.alived, True) 746 747 console.terse("\nAlive Main to Other *********\n") 748 for stack in [main, other]: 749 stack.remotes.values()[0].alived = None 750 self.assertIs(stack.remotes.values()[0].alived, None) 751 self.alive(main, other, duid=main.remotes.values()[0].uid) 752 for stack in [main, other]: 753 self.assertEqual(len(stack.transactions), 0) 754 self.assertEqual(len(stack.remotes), 1) 755 remote = stack.remotes.values()[0] 756 self.assertIs(remote.joined, True) 757 self.assertIs(remote.allowed, True) 758 self.assertIs(remote.alived, True) 759 760 console.terse("\nDead Other from Main *********\n") 761 # start alive from main to other but do not service responses from other 762 # so main alive trans times out and other appears dead to main 763 for stack in [main, other]: 764 self.assertIs(stack.remotes.values()[0].alived, True) 765 main.alive() 766 self.serviceStack(main, duration=3.0) # only service main 767 self.assertEqual(len(main.transactions), 0) # timed out 768 self.assertIs(main.remotes.values()[0].alived, False) 769 self.serviceStack(other, duration=3.0) # now service other side 770 self.assertEqual(len(other.transactions), 0) # gets queued requests 771 self.assertIs(other.remotes.values()[0].alived, True) 772 self.assertIs(main.remotes.values()[0].alived, False) 773 774 console.terse("\nDead Main from Other *********\n") 775 # start alive from other to main but do not service responses from main 776 # so other alive trans times out and main appears dead to other 777 for stack in [main, other]: 778 self.assertEqual(len(stack.transactions), 0) 779 self.assertEqual(len(stack.remotes), 1) 780 self.assertIs(other.remotes.values()[0].alived, True) 781 other.alive() 782 self.serviceStack(other, duration=3.0) # only service other 783 self.assertEqual(len(other.transactions), 0) # timed out 784 self.assertIs(other.remotes.values()[0].alived, False) 785 self.serviceStack(main, duration=3.0) # now service main side 786 self.assertEqual(len(main.transactions), 0) # gets queued requests 787 self.assertIs(main.remotes.values()[0].alived, True) 788 self.assertIs(other.remotes.values()[0].alived, False) 789 790 for stack in [main, other]: 791 stack.server.close() 792 stack.clearAllKeeps() 793 794 def testAliveDeadMultiple(self): 795 ''' 796 Test alive transaction with multiple remotes given already joined and allowed 797 ''' 798 console.terse("{0}\n".format(self.testAliveDeadMultiple.__doc__)) 799 800 mainData = self.createRoadData(name='main', 801 base=self.base, 802 auto=raeting.AutoMode.once.value) 803 keeping.clearAllKeep(mainData['dirpath']) 804 main = self.createRoadStack(data=mainData, 805 main=True, 806 auto=mainData['auto'], 807 ha=None) 808 809 otherData = self.createRoadData(name='other', 810 base=self.base, 811 auto=raeting.AutoMode.once.value) 812 keeping.clearAllKeep(otherData['dirpath']) 813 other = self.createRoadStack(data=otherData, 814 main=None, 815 auto=otherData['auto'], 816 ha=("", raeting.RAET_TEST_PORT)) 817 818 819 self.join(other, main) 820 for stack in [main, other]: 821 self.assertEqual(len(stack.transactions), 0) 822 self.assertEqual(len(stack.remotes), 1) 823 remote = stack.remotes.values()[0] 824 self.assertIs(remote.joined, True) 825 self.assertIs(remote.allowed, None) 826 self.assertIs(remote.alived, None) 827 828 self.allow(other, main) 829 for stack in [main, other]: 830 self.assertEqual(len(stack.transactions), 0) 831 self.assertEqual(len(stack.remotes), 1) 832 remote = stack.remotes.values()[0] 833 self.assertIs(remote.joined, True) 834 self.assertIs(remote.allowed, True) 835 self.assertIs(remote.alived, True) 836 837 other1Data = self.createRoadData(name='other1', 838 base=self.base, 839 auto=raeting.AutoMode.once.value) 840 keeping.clearAllKeep(other1Data['dirpath']) 841 other1 = self.createRoadStack(data=other1Data, 842 main=None, 843 auto=other1Data['auto'], 844 ha=("", 7533)) 845 846 self.join(other1, main) 847 for stack in [main, other1]: 848 self.assertEqual(len(stack.transactions), 0) 849 850 self.assertEqual(len(main.remotes), 2) 851 self.assertEqual(len(other1.remotes), 1) 852 853 for remote in [main.remotes.values()[1], 854 other1.remotes.values()[0]]: 855 self.assertIs(remote.joined, True) 856 self.assertIs(remote.allowed, None) 857 self.assertIs(remote.alived, None) 858 859 self.allow(other1, main) 860 for stack in [main, other1]: 861 self.assertEqual(len(stack.transactions), 0) 862 for remote in [main.remotes.values()[1], 863 other1.remotes.values()[0]]: 864 self.assertIs(remote.joined, True) 865 self.assertIs(remote.allowed, True) 866 self.assertIs(remote.alived, True) 867 868 console.terse("\nAlive Other to Main *********\n") 869 for remote in [main.remotes.values()[0], 870 other.remotes.values()[0]]: 871 self.assertIs(remote.joined, True) 872 self.assertIs(remote.allowed, True) 873 remote.alived = None # force not alived 874 self.assertIs(remote.alived, None) 875 876 self.alive(other, main) 877 for stack in [main, other]: 878 self.assertEqual(len(stack.transactions), 0) 879 for remote in [main.remotes.values()[0], 880 other.remotes.values()[0]]: 881 self.assertIs(remote.joined, True) 882 self.assertIs(remote.allowed, True) 883 self.assertIs(remote.alived, True) 884 885 console.terse("\nAlive Main to Other *********\n") 886 for remote in [main.remotes.values()[0], 887 other.remotes.values()[0]]: 888 remote.alived = None 889 self.assertIs(remote.alived, None) 890 891 self.alive(main, other, duid=main.remotes.values()[0].uid) 892 for stack in [main, other]: 893 self.assertEqual(len(stack.transactions), 0) 894 for remote in [main.remotes.values()[0], 895 other.remotes.values()[0]]: 896 self.assertIs(remote.joined, True) 897 self.assertIs(remote.allowed, True) 898 self.assertIs(remote.alived, True) 899 900 console.terse("\nAlive Other1 to Main *********\n") 901 for remote in [main.remotes.values()[1], 902 other1.remotes.values()[0]]: 903 remote.alived = None 904 self.assertIs(remote.alived, None) 905 906 self.alive(other1, main, duid=other1.remotes.values()[0].uid) 907 for stack in [main, other1]: 908 self.assertEqual(len(stack.transactions), 0) 909 for remote in [main.remotes.values()[1], 910 other1.remotes.values()[0]]: 911 self.assertIs(remote.joined, True) 912 self.assertIs(remote.allowed, True) 913 self.assertIs(remote.alived, True) 914 915 console.terse("\nAlive Main to Other1 *********\n") 916 for remote in [main.remotes.values()[1], 917 other1.remotes.values()[0]]: 918 remote.alived = None 919 self.assertIs(remote.alived, None) 920 921 self.alive(main, other1, duid=main.remotes.values()[1].uid) 922 for stack in [main, other1]: 923 self.assertEqual(len(stack.transactions), 0) 924 for remote in [main.remotes.values()[1], 925 other1.remotes.values()[0]]: 926 self.assertIs(remote.joined, True) 927 self.assertIs(remote.allowed, True) 928 self.assertIs(remote.alived, True) 929 930 console.terse("\nDead Other to Main from Main *********\n") 931 for remote in main.remotes.values(): 932 self.assertIs(remote.alived, True) 933 main.alive(uid=remote.nuid) 934 # do not service other stack so other appears dead to main 935 self.serviceStacks([main, other1], duration=4.0) 936 for stack in [main, other, other1]: 937 self.assertEqual(len(stack.transactions), 0) 938 self.assertIs(main.remotes.values()[0].alived, False) 939 self.assertIs(main.remotes.values()[1].alived, True) 940 self.assertIs(other.remotes.values()[0].alived, True) 941 self.assertIs(other1.remotes.values()[0].alived, True) 942 943 console.terse("\nDead Main to Other From Other *********\n") 944 self.flushReceives(other) # flush packets sent by main to other 945 # do not service main stack so main appears dead to other 946 other.alive(uid=other.remotes.values()[0].nuid) 947 self.serviceStacks([other], duration=3.0) # other 948 for stack in [main, other, other1]: 949 self.assertEqual(len(stack.transactions), 0) 950 self.assertIs(main.remotes.values()[0].alived, False) 951 self.assertIs(main.remotes.values()[1].alived, True) 952 self.assertIs(other.remotes.values()[0].alived, False) 953 self.assertIs(other1.remotes.values()[0].alived, True) 954 self.flushReceives(main) # flush buffered packets on main 955 956 #bring other back to life 957 console.terse("\nReliven main to other from other *********\n") 958 for remote in [main.remotes.values()[0], 959 other.remotes.values()[0]]: 960 self.assertIs(remote.alived, False) 961 other.alive(cascade=True) 962 self.serviceStacks([other, main], duration=3.0) 963 for remote in [main.remotes.values()[0], 964 other.remotes.values()[0]]: 965 self.assertIs(remote.alived, True) 966 967 console.terse("\nAlive Dead Other 1 to Main from Main *********\n") 968 for remote in main.remotes.values(): 969 self.assertIs(remote.alived, True) 970 main.alive(uid=remote.nuid) 971 # do not service other1 stack so other appears dead to main 972 self.serviceStacks([main, other], duration=3.0) 973 for stack in [main, other]: 974 self.assertEqual(len(stack.transactions), 0) 975 self.assertIs(main.remotes.values()[0].alived, True) 976 self.assertIs(main.remotes.values()[1].alived, False) 977 self.assertIs(other.remotes.values()[0].alived, True) 978 self.assertIs(other1.remotes.values()[0].alived, True) 979 980 console.terse("\nDead Main to Other1 From Other1 *********\n") 981 # do not service main stack so main appears dead to other1 982 self.flushReceives(other1) # flush packets sent by main to other 983 other1.alive(uid=other.remotes.values()[0].nuid) 984 self.serviceStacks([other1], duration=3.0) # other 985 for stack in [other1]: 986 self.assertEqual(len(stack.transactions), 0) 987 self.assertIs(main.remotes.values()[0].alived, True) 988 self.assertIs(main.remotes.values()[1].alived, False) 989 self.assertIs(other.remotes.values()[0].alived, True) 990 self.assertIs(other1.remotes.values()[0].alived, False) 991 self.flushReceives(main) # flush packets sent other1 to main 992 993 994 console.terse("\nReliven main to other1 from other 1 *********\n") 995 #bring main back to life from other1 996 for remote in [main.remotes.values()[1], 997 other1.remotes.values()[0]]: 998 self.assertIs(remote.alived, False) 999 other1.alive(cascade=True) 1000 self.serviceStacks([other1, main], duration=3.0) 1001 for remote in [main.remotes.values()[1], 1002 other1.remotes.values()[0]]: 1003 self.assertIs(remote.alived, True) 1004 1005 for stack in [main, other, other1]: 1006 stack.server.close() 1007 stack.clearAllKeeps() 1008 time.sleep(0.1) 1009 1010 1011 def testAliveUnjoinedUnallowedBoth(self): 1012 ''' 1013 Test alive transaction for other to main given 1014 unjoined and/or unallowed on both main and other 1015 1016 Merely set the joined, allowed, and alived status 1017 Do not create fresh stacks. 1018 ''' 1019 console.terse("{0}\n".format(self.testAliveUnjoinedUnallowedBoth.__doc__)) 1020 1021 mainData = self.createRoadData(name='main', 1022 base=self.base, 1023 auto=raeting.AutoMode.once.value) 1024 keeping.clearAllKeep(mainData['dirpath']) 1025 main = self.createRoadStack(data=mainData, 1026 #uid=1, 1027 main=True, 1028 auto=mainData['auto'], 1029 ha=None) 1030 1031 otherData = self.createRoadData(name='other', 1032 base=self.base, 1033 auto=raeting.AutoMode.once.value) 1034 keeping.clearAllKeep(otherData['dirpath']) 1035 other = self.createRoadStack(data=otherData, 1036 #uid=1, 1037 main=None, 1038 auto=otherData['auto'], 1039 ha=("", raeting.RAET_TEST_PORT)) 1040 1041 self.join(other, main) # bootstrap channel 1042 for stack in [main, other]: 1043 self.assertEqual(len(stack.transactions), 0) 1044 self.assertEqual(len(stack.remotes), 1) 1045 remote = stack.remotes.values()[0] 1046 self.assertIs(remote.joined, True) 1047 1048 console.terse("\nAllow Other to Main *********\n") 1049 # force unjoined already unallowed unalived 1050 for stack in [main, other]: 1051 stack.remotes.values()[0].joined = None 1052 1053 for stack in [main, other]: 1054 remote = stack.remotes.values()[0] 1055 self.assertIs(remote.joined, None) 1056 self.assertIs(remote.allowed, None) 1057 self.assertIs(remote.alived, None) 1058 self.allow(other, main) # will join instead since unjoined 1059 for stack in [main, other]: 1060 self.assertEqual(len(stack.transactions), 0) 1061 self.assertEqual(len(stack.remotes), 1) 1062 remote = stack.remotes.values()[0] 1063 self.assertIs(remote.joined, True) 1064 self.assertIs(remote.allowed, None) 1065 1066 self.allow(other, main) # now try to allow again 1067 for stack in [main, other]: 1068 self.assertEqual(len(stack.transactions), 0) 1069 self.assertEqual(len(stack.remotes), 1) 1070 remote = stack.remotes.values()[0] 1071 self.assertIs(remote.joined, True) 1072 self.assertIs(remote.allowed, True) 1073 self.assertIs(remote.alived, True) 1074 1075 1076 console.terse("\nAllow Main to Other *********\n") 1077 # force unjoined already unallowed unalived 1078 for stack in [main, other]: 1079 stack.remotes.values()[0].joined = None 1080 stack.remotes.values()[0].allowed = None 1081 stack.remotes.values()[0].alived = None 1082 1083 for stack in [main, other]: 1084 remote = stack.remotes.values()[0] 1085 self.assertIs(remote.joined, None) 1086 self.assertIs(remote.allowed, None) 1087 self.assertIs(remote.alived, None) 1088 self.allow(main, other) # will join instead since unjoined 1089 for stack in [main, other]: 1090 self.assertEqual(len(stack.transactions), 0) 1091 self.assertEqual(len(stack.remotes), 1) 1092 remote = stack.remotes.values()[0] 1093 self.assertIs(remote.joined, True) 1094 self.assertIs(remote.allowed, None) 1095 1096 self.allow(main, other) # now try to allow again 1097 for stack in [main, other]: 1098 self.assertEqual(len(stack.transactions), 0) 1099 self.assertEqual(len(stack.remotes), 1) 1100 remote = stack.remotes.values()[0] 1101 self.assertIs(remote.joined, True) 1102 self.assertIs(remote.allowed, True) 1103 self.assertIs(remote.alived, True) 1104 1105 console.terse("\nAlive Other to Main *********\n") 1106 # force unjoined unallowed unalived 1107 for stack in [main, other]: 1108 stack.remotes.values()[0].joined = None 1109 stack.remotes.values()[0].allowed = None 1110 stack.remotes.values()[0].alived = None 1111 1112 # alive checks for joined and if not then joins 1113 for stack in [main, other]: 1114 remote = stack.remotes.values()[0] 1115 self.assertIs(remote.joined, None) 1116 self.assertIs(remote.allowed, None) 1117 self.assertIs(remote.alived, None) 1118 self.alive(other, main) 1119 for stack in [main, other]: 1120 self.assertEqual(len(stack.transactions), 0) 1121 self.assertEqual(len(stack.remotes), 1) 1122 remote = stack.remotes.values()[0] 1123 self.assertIs(remote.joined, True) 1124 self.assertIs(remote.allowed, None) 1125 self.assertIs(remote.alived, None) 1126 1127 # Alive checks for allowed and if not allows 1128 self.alive(other, main) 1129 for stack in [main, other]: 1130 self.assertEqual(len(stack.transactions), 0) 1131 self.assertEqual(len(stack.remotes), 1) 1132 remote = stack.remotes.values()[0] 1133 self.assertIs(remote.joined, True) 1134 self.assertIs(remote.allowed, True) 1135 self.assertIs(remote.alived, True) 1136 1137 # Alive should complete now 1138 for stack in [main, other]: 1139 self.assertEqual(len(stack.remotes), 1) 1140 remote = stack.remotes.values()[0] 1141 remote.alived = None 1142 self.assertIs(remote.alived, None) 1143 1144 self.alive(other, main) 1145 for stack in [main, other]: 1146 self.assertEqual(len(stack.transactions), 0) 1147 self.assertEqual(len(stack.remotes), 1) 1148 remote = stack.remotes.values()[0] 1149 self.assertIs(remote.joined, True) 1150 self.assertIs(remote.allowed, True) 1151 self.assertIs(remote.alived, True) 1152 1153 console.terse("\nAlive Main to Other *********\n") 1154 # now do same from main to other 1155 # force unjoined unallowed unalived 1156 for stack in [main, other]: 1157 remote = stack.remotes.values()[0] 1158 remote.joined = None 1159 remote.allowed = None 1160 remote.alived = None 1161 1162 for stack in [main, other]: 1163 remote = stack.remotes.values()[0] 1164 self.assertIs(remote.joined, None) 1165 self.assertIs(remote.allowed, None) 1166 self.assertIs(remote.alived, None) 1167 self.alive(main, other) 1168 for stack in [main, other]: 1169 self.assertEqual(len(stack.transactions), 0) 1170 self.assertEqual(len(stack.remotes), 1) 1171 remote = stack.remotes.values()[0] 1172 self.assertIs(remote.joined, True) 1173 self.assertIs(remote.allowed, None) 1174 self.assertIs(remote.alived, None) 1175 1176 # Alive checks for allowed and if not allows 1177 self.alive(main, other) 1178 for stack in [main, other]: 1179 self.assertEqual(len(stack.transactions), 0) 1180 self.assertEqual(len(stack.remotes), 1) 1181 remote = stack.remotes.values()[0] 1182 self.assertIs(remote.joined, True) 1183 self.assertIs(remote.allowed, True) 1184 self.assertIs(remote.alived, True) 1185 1186 # Alive should complete now 1187 for stack in [main, other]: 1188 self.assertEqual(len(stack.remotes), 1) 1189 remote = stack.remotes.values()[0] 1190 remote.alived = None 1191 self.assertIs(remote.alived, None) 1192 1193 self.alive(main, other) 1194 for stack in [main, other]: 1195 self.assertEqual(len(stack.transactions), 0) 1196 self.assertEqual(len(stack.remotes), 1) 1197 remote = stack.remotes.values()[0] 1198 self.assertIs(remote.joined, True) 1199 self.assertIs(remote.allowed, True) 1200 self.assertIs(remote.alived, True) 1201 1202 for stack in [main, other]: 1203 stack.server.close() 1204 stack.clearAllKeeps() 1205 1206 def testCascadeBoth(self): 1207 ''' 1208 Test allow and alive cascades for other to main unjoined and/or unallowed on both 1209 but with cascade = True so it repairs joined and allowed status and redos 1210 alive 1211 1212 create fresh stacks on each cascade. 1213 ''' 1214 console.terse("{0}\n".format(self.testCascadeBoth.__doc__)) 1215 1216 mainData = self.createRoadData(name='main', 1217 base=self.base, 1218 auto=raeting.AutoMode.once.value) 1219 keeping.clearAllKeep(mainData['dirpath']) 1220 main = self.createRoadStack(data=mainData, 1221 main=True, 1222 auto=mainData['auto'], 1223 ha=None) 1224 1225 otherData = self.createRoadData(name='other', 1226 base=self.base, 1227 auto=raeting.AutoMode.once.value) 1228 keeping.clearAllKeep(otherData['dirpath']) 1229 other = self.createRoadStack(data=otherData, 1230 main=None, 1231 auto=otherData['auto'], 1232 ha=("", raeting.RAET_TEST_PORT)) 1233 1234 console.terse("\nJoin Other to Main Cascade *********\n") 1235 for stack in [main, other]: 1236 self.assertEqual(len(stack.remotes), 0) 1237 1238 self.join(other, main, cascade=True) # now join cascade w 1239 for stack in [main, other]: 1240 self.assertEqual(len(stack.transactions), 0) 1241 self.assertEqual(len(stack.remotes), 1) 1242 remote = stack.remotes.values()[0] 1243 self.assertIs(remote.joined, True) 1244 self.assertIs(remote.allowed, True) # cascade will include alive 1245 self.assertIs(remote.alived, True) # cascade will include alive 1246 1247 console.terse("\Join Main to Other Cascade *********\n") 1248 main.server.close() 1249 other.server.close() 1250 keeping.clearAllKeep(mainData['dirpath']) 1251 main = self.createRoadStack(data=mainData, 1252 main=True, 1253 auto=mainData['auto'], 1254 ha=None) 1255 1256 keeping.clearAllKeep(otherData['dirpath']) 1257 other = self.createRoadStack(data=otherData, 1258 main=None, 1259 auto=raeting.AutoMode.never.value, 1260 ha=("", raeting.RAET_TEST_PORT)) 1261 1262 for stack in [main, other]: 1263 self.assertEqual(len(stack.remotes), 0) 1264 1265 self.join(main, other) # bootstrap channel fails because other not main 1266 for stack in [main, other]: 1267 self.assertEqual(len(stack.transactions), 0) 1268 self.assertEqual(len(stack.remotes), 0) 1269 1270 # now fix it so other can accept vacuous joins 1271 other.main = True 1272 other.keep.auto = raeting.AutoMode.once.value 1273 1274 self.join(main, other) # bootstrap channel 1275 for stack in [main, other]: 1276 self.assertEqual(len(stack.transactions), 0) 1277 self.assertEqual(len(stack.remotes), 1) 1278 remote = stack.remotes.values()[0] 1279 self.assertIs(remote.joined, True) 1280 1281 # force unjoined already unallowed unalived 1282 for stack in [main, other]: 1283 stack.remotes.values()[0].joined = None 1284 1285 for stack in [main, other]: 1286 remote = stack.remotes.values()[0] 1287 self.assertIs(remote.joined, None) 1288 self.assertIs(remote.allowed, None) 1289 self.assertIs(remote.alived, None) 1290 self.join(main, other, cascade=True) # now alive cascade 1291 for stack in [main, other]: 1292 self.assertEqual(len(stack.transactions), 0) 1293 self.assertEqual(len(stack.remotes), 1) 1294 remote = stack.remotes.values()[0] 1295 self.assertIs(remote.joined, True) 1296 self.assertIs(remote.allowed, True) 1297 self.assertIs(remote.alived, True) # cascade will include alive 1298 1299 console.terse("\nAllow Other to Main Cascade *********\n") 1300 main.server.close() 1301 other.server.close() 1302 keeping.clearAllKeep(mainData['dirpath']) 1303 main = self.createRoadStack(data=mainData, 1304 main=True, 1305 auto=mainData['auto'], 1306 ha=None) 1307 1308 keeping.clearAllKeep(otherData['dirpath']) 1309 other = self.createRoadStack(data=otherData, 1310 main=None, 1311 auto=otherData['auto'], 1312 ha=("", raeting.RAET_TEST_PORT)) 1313 1314 for stack in [main, other]: 1315 self.assertEqual(len(stack.remotes), 0) 1316 1317 self.join(other, main) # bootstrap channel since allow requires remote 1318 for stack in [main, other]: 1319 self.assertEqual(len(stack.transactions), 0) 1320 self.assertEqual(len(stack.remotes), 1) 1321 remote = stack.remotes.values()[0] 1322 self.assertIs(remote.joined, True) 1323 1324 # force unjoined already unallowed unalived 1325 for stack in [main, other]: 1326 stack.remotes.values()[0].joined = None 1327 1328 for stack in [main, other]: 1329 remote = stack.remotes.values()[0] 1330 self.assertIs(remote.joined, None) 1331 self.assertIs(remote.allowed, None) 1332 self.assertIs(remote.alived, None) 1333 self.allow(other, main, cascade=True) # now allow cascade so join then allow 1334 for stack in [main, other]: 1335 self.assertEqual(len(stack.transactions), 0) 1336 self.assertEqual(len(stack.remotes), 1) 1337 remote = stack.remotes.values()[0] 1338 self.assertIs(remote.joined, True) 1339 self.assertIs(remote.allowed, True) 1340 self.assertIs(remote.alived, True) # cascade will include alive 1341 1342 console.terse("\nAllow Main to Other Cascade *********\n") 1343 main.server.close() 1344 other.server.close() 1345 keeping.clearAllKeep(mainData['dirpath']) 1346 main = self.createRoadStack(data=mainData, 1347 main=True, 1348 auto=mainData['auto'], 1349 ha=None) 1350 1351 keeping.clearAllKeep(otherData['dirpath']) 1352 other = self.createRoadStack(data=otherData, 1353 main=None, 1354 auto=raeting.AutoMode.never.value, 1355 ha=("", raeting.RAET_TEST_PORT)) 1356 1357 for stack in [main, other]: 1358 self.assertEqual(len(stack.remotes), 0) 1359 1360 self.join(main, other) # bootstrap channel fails because other not main 1361 for stack in [main, other]: 1362 self.assertEqual(len(stack.transactions), 0) 1363 self.assertEqual(len(stack.remotes), 0) 1364 1365 # now fix it so other can accept vacuous joins 1366 other.main = True 1367 other.keep.auto = raeting.AutoMode.once.value 1368 1369 self.join(main, other) # bootstrap channel since allow requires remote 1370 for stack in [main, other]: 1371 self.assertEqual(len(stack.transactions), 0) 1372 self.assertEqual(len(stack.remotes), 1) 1373 remote = stack.remotes.values()[0] 1374 self.assertIs(remote.joined, True) 1375 1376 # force unjoined already unallowed unalived 1377 for stack in [main, other]: 1378 stack.remotes.values()[0].joined = None 1379 1380 for stack in [main, other]: 1381 remote = stack.remotes.values()[0] 1382 self.assertIs(remote.joined, None) 1383 self.assertIs(remote.allowed, None) 1384 self.assertIs(remote.alived, None) 1385 self.allow(main, other, cascade=True) # now allow cascade so join then allow 1386 for stack in [main, other]: 1387 self.assertEqual(len(stack.transactions), 0) 1388 self.assertEqual(len(stack.remotes), 1) 1389 remote = stack.remotes.values()[0] 1390 self.assertIs(remote.joined, True) 1391 self.assertIs(remote.allowed, True) 1392 self.assertIs(remote.alived, True) # cascade will include alive 1393 1394 console.terse("\nAlive Other to Main Cascade *********\n") 1395 main.server.close() 1396 other.server.close() 1397 keeping.clearAllKeep(mainData['dirpath']) 1398 main = self.createRoadStack(data=mainData, 1399 main=True, 1400 auto=mainData['auto'], 1401 ha=None) 1402 1403 keeping.clearAllKeep(otherData['dirpath']) 1404 other = self.createRoadStack(data=otherData, 1405 main=None, 1406 auto=otherData['auto'], 1407 ha=("", raeting.RAET_TEST_PORT)) 1408 1409 for stack in [main, other]: 1410 self.assertEqual(len(stack.remotes), 0) 1411 1412 self.join(other, main) # bootstrap channel 1413 for stack in [main, other]: 1414 self.assertEqual(len(stack.transactions), 0) 1415 self.assertEqual(len(stack.remotes), 1) 1416 remote = stack.remotes.values()[0] 1417 self.assertIs(remote.joined, True) 1418 1419 # force unjoined already unallowed unalived 1420 for stack in [main, other]: 1421 stack.remotes.values()[0].joined = None 1422 1423 for stack in [main, other]: 1424 remote = stack.remotes.values()[0] 1425 self.assertIs(remote.joined, None) 1426 self.assertIs(remote.allowed, None) 1427 self.assertIs(remote.alived, None) 1428 self.alive(other, main, cascade=True) # now alive cascade 1429 for stack in [main, other]: 1430 self.assertEqual(len(stack.transactions), 0) 1431 self.assertEqual(len(stack.remotes), 1) 1432 remote = stack.remotes.values()[0] 1433 self.assertIs(remote.joined, True) 1434 self.assertIs(remote.allowed, True) 1435 self.assertIs(remote.alived, True) # cascade will include alive 1436 1437 console.terse("\nAlive Main to Other Cascade *********\n") 1438 main.server.close() 1439 other.server.close() 1440 keeping.clearAllKeep(mainData['dirpath']) 1441 main = self.createRoadStack(data=mainData, 1442 main=True, 1443 auto=mainData['auto'], 1444 ha=None) 1445 1446 keeping.clearAllKeep(otherData['dirpath']) 1447 other = self.createRoadStack(data=otherData, 1448 main=None, 1449 auto=raeting.AutoMode.never.value, 1450 ha=("", raeting.RAET_TEST_PORT)) 1451 1452 for stack in [main, other]: 1453 self.assertEqual(len(stack.remotes), 0) 1454 1455 self.join(main, other) # bootstrap channel fails because other not main 1456 for stack in [main, other]: 1457 self.assertEqual(len(stack.transactions), 0) 1458 self.assertEqual(len(stack.remotes), 0) 1459 1460 # now fix it so other can accept vacuous joins 1461 other.main = True 1462 other.keep.auto = raeting.AutoMode.once.value 1463 1464 self.join(main, other) # bootstrap channel 1465 for stack in [main, other]: 1466 self.assertEqual(len(stack.transactions), 0) 1467 self.assertEqual(len(stack.remotes), 1) 1468 remote = stack.remotes.values()[0] 1469 self.assertIs(remote.joined, True) 1470 1471 # force unjoined already unallowed unalived 1472 for stack in [main, other]: 1473 stack.remotes.values()[0].joined = None 1474 1475 for stack in [main, other]: 1476 remote = stack.remotes.values()[0] 1477 self.assertIs(remote.joined, None) 1478 self.assertIs(remote.allowed, None) 1479 self.assertIs(remote.alived, None) 1480 self.alive(main, other, cascade=True) # now alive cascade 1481 for stack in [main, other]: 1482 self.assertEqual(len(stack.transactions), 0) 1483 self.assertEqual(len(stack.remotes), 1) 1484 remote = stack.remotes.values()[0] 1485 self.assertIs(remote.joined, True) 1486 self.assertIs(remote.allowed, True) 1487 self.assertIs(remote.alived, True) # cascade will include alive 1488 1489 for stack in [main, other]: 1490 stack.server.close() 1491 stack.clearAllKeeps() 1492 1493 def testManageOneSide(self): 1494 ''' 1495 Test stack manage remotes 1496 ''' 1497 console.terse("{0}\n".format(self.testManageOneSide.__doc__)) 1498 1499 mainData = self.createRoadData(name='main', 1500 base=self.base, 1501 auto=raeting.AutoMode.once.value) 1502 keeping.clearAllKeep(mainData['dirpath']) 1503 main = self.createRoadStack(data=mainData, 1504 main=True, 1505 auto=mainData['auto'], 1506 ha=None) 1507 1508 otherData = self.createRoadData(name='other', 1509 base=self.base, 1510 auto=raeting.AutoMode.once.value) 1511 keeping.clearAllKeep(otherData['dirpath']) 1512 other = self.createRoadStack(data=otherData, 1513 main=None, 1514 auto=otherData['auto'], 1515 ha=("", raeting.RAET_TEST_PORT)) 1516 1517 other1Data = self.createRoadData(name='other1', 1518 base=self.base, 1519 auto=raeting.AutoMode.once.value,) 1520 keeping.clearAllKeep(other1Data['dirpath']) 1521 other1 = self.createRoadStack(data=other1Data, 1522 main=None, 1523 auto=other1Data['auto'], 1524 ha=("", 7533)) 1525 1526 for stack in [other, other1]: 1527 self.join(stack, main) 1528 self.allow(stack, main) 1529 1530 console.terse("\nTest manage remotes presence *********\n") 1531 console.terse("\nMake all alive *********\n") 1532 stacks = [main, other, other1] 1533 for remote in main.remotes.values(): #make all alive 1534 main.alive(uid=remote.uid) 1535 self.serviceStacks(stacks, duration=3.0) 1536 for remote in main.remotes.values(): 1537 self.assertTrue(remote.alived) 1538 1539 main.manage() 1540 for stack in stacks: # no alive transactions started 1541 self.assertEqual(len(stack.transactions), 0) 1542 1543 console.terse("\nMake all expired so send alive *********\n") 1544 # advance clock so remote keep alive timers expire 1545 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 1546 main.manage() 1547 for remote in main.remotes.values(): # should start 1548 self.assertIs(remote.alived, True) 1549 1550 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1551 1552 self.serviceStacks(stacks, duration=3.0) 1553 for stack in stacks: 1554 self.assertEqual(len(stack.transactions), 0) 1555 for remote in main.remotes.values(): 1556 self.assertTrue(remote.alived) 1557 1558 1559 for stack in [main, other, other1]: 1560 stack.server.close() 1561 stack.clearAllKeeps() 1562 time.sleep(0.1) 1563 1564 def testManageBothSides(self): 1565 ''' 1566 Test stack manage remotes main and others 1567 ''' 1568 console.terse("{0}\n".format(self.testManageBothSides.__doc__)) 1569 1570 mainData = self.createRoadData(name='main', 1571 base=self.base, 1572 auto=raeting.AutoMode.once.value) 1573 keeping.clearAllKeep(mainData['dirpath']) 1574 main = self.createRoadStack(data=mainData, 1575 main=True, 1576 auto=mainData['auto'], 1577 ha=None) 1578 1579 otherData = self.createRoadData(name='other', 1580 base=self.base, 1581 auto=raeting.AutoMode.once.value) 1582 keeping.clearAllKeep(otherData['dirpath']) 1583 other = self.createRoadStack(data=otherData, 1584 main=None, 1585 auto=otherData['auto'], 1586 ha=("", raeting.RAET_TEST_PORT)) 1587 1588 other1Data = self.createRoadData(name='other1', 1589 base=self.base, 1590 auto=raeting.AutoMode.once.value) 1591 keeping.clearAllKeep(other1Data['dirpath']) 1592 other1 = self.createRoadStack(data=other1Data, 1593 main=None, 1594 auto=other1Data['auto'], 1595 ha=("", 7533)) 1596 1597 for stack in [other, other1]: 1598 self.join(stack, main) 1599 self.allow(stack, main) 1600 1601 console.terse("\nTest manage remotes presence both ways *********\n") 1602 console.terse("\nMake all alive *********\n") 1603 stacks = [main, other, other1] 1604 for remote in main.remotes.values(): #make all alive 1605 main.alive(uid=remote.uid) 1606 self.serviceStacks(stacks, duration=3.0) 1607 for remote in main.remotes.values(): 1608 self.assertTrue(remote.alived) 1609 1610 for stack in stacks: 1611 stack.manage() 1612 1613 for stack in stacks: # no alive transactions started 1614 self.assertEqual(len(stack.transactions), 0) 1615 1616 console.terse("\nMake all expired so send alive *********\n") 1617 # advance clock so remote keep alive timers expire 1618 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 1619 for stack in stacks: 1620 stack.manage() 1621 1622 for stack in stacks: 1623 for remote in stack.remotes.values(): 1624 self.assertIs(remote.alived, True) 1625 1626 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1627 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 1628 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 1629 1630 self.serviceManageStacks(stacks, duration=3.0) 1631 for stack in stacks: 1632 self.assertEqual(len(stack.transactions), 0) 1633 1634 for stack in stacks: 1635 for remote in stack.remotes.values(): 1636 self.assertIs(remote.alived, True) 1637 1638 1639 for stack in [main, other, other1]: 1640 stack.server.close() 1641 stack.clearAllKeeps() 1642 time.sleep(0.1) 1643 1644 def testManageMainRebootCascade(self): 1645 ''' 1646 Test stack manage remotes as if main were rebooted 1647 ''' 1648 console.terse("{0}\n".format(self.testManageMainRebootCascade.__doc__)) 1649 1650 mainData = self.createRoadData(name='main', 1651 base=self.base, 1652 auto=raeting.AutoMode.once.value) 1653 mainDirpath = mainData['dirpath'] 1654 keeping.clearAllKeep(mainData['dirpath']) 1655 main = self.createRoadStack(data=mainData, 1656 main=True, 1657 auto=mainData['auto'], 1658 ha=None) 1659 1660 otherData = self.createRoadData(name='other', 1661 base=self.base, 1662 auto=raeting.AutoMode.once.value) 1663 otherDirpath = otherData['dirpath'] 1664 keeping.clearAllKeep(otherData['dirpath']) 1665 other = self.createRoadStack(data=otherData, 1666 main=None, 1667 auto=otherData['auto'], 1668 ha=("", raeting.RAET_TEST_PORT)) 1669 1670 other1Data = self.createRoadData(name='other1', 1671 base=self.base, 1672 auto=raeting.AutoMode.once.value) 1673 other1Dirpath = other1Data['dirpath'] 1674 keeping.clearAllKeep(other1Data['dirpath']) 1675 other1 = self.createRoadStack(data=other1Data, 1676 main=None, 1677 auto=other1Data['auto'], 1678 ha=("", 7533)) 1679 1680 for stack in [other, other1]: 1681 self.join(stack, main) 1682 self.allow(stack, main) 1683 1684 console.terse("\nTest manage remotes presence *********\n") 1685 console.terse("\nMake all alive *********\n") 1686 stacks = [main, other, other1] 1687 for remote in main.remotes.values(): #make all alive 1688 main.alive(uid=remote.uid) 1689 self.serviceStacks(stacks, duration=3.0) 1690 for remote in main.remotes.values(): 1691 self.assertTrue(remote.alived) 1692 1693 main.manage(immediate=True) 1694 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1695 for remote in main.remotes.values(): 1696 self.assertIs(remote.alived, True) 1697 1698 self.serviceStacks(stacks, duration=3.0) 1699 for stack in stacks: 1700 self.assertEqual(len(stack.transactions), 0) 1701 for remote in main.remotes.values(): 1702 self.assertTrue(remote.alived) 1703 1704 # now close down main and reload from saved data and manage 1705 console.terse("\nMake all alive with cascade after main reboots *********\n") 1706 main.server.close() 1707 main = stacking.RoadStack(store=self.store, 1708 main=True, 1709 dirpath=mainDirpath, 1710 ) 1711 stacks = [main, other, other1] 1712 1713 for remote in main.remotes.values(): 1714 self.assertIs(remote.joined, True) #joined status is persisted 1715 self.assertIs(remote.allowed, None) 1716 self.assertIs(remote.alived, None) 1717 1718 main.manage(immediate=True, cascade=True) 1719 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1720 for remote in main.remotes.values(): # should reset alive to None 1721 self.assertIs(remote.alived, None) 1722 self.serviceStacks(stacks, duration=3.0) 1723 for stack in stacks: 1724 self.assertEqual(len(stack.transactions), 0) 1725 for remote in main.remotes.values(): 1726 self.assertIs(remote.joined, True) 1727 self.assertIs(remote.allowed, True) 1728 self.assertIs(remote.alived, True) 1729 1730 # Now test as if others are rebooted 1731 console.terse("\nMake all alive with cascade after others reboot *********\n") 1732 other.server.close() 1733 other1.server.close() 1734 other = stacking.RoadStack(dirpath=otherDirpath, 1735 store=self.store) 1736 other1 = stacking.RoadStack(dirpath=other1Dirpath, 1737 store=self.store) 1738 stacks = [main, other, other1] 1739 1740 for stack in [other, other1]: 1741 for remote in stack.remotes.values(): 1742 self.assertIs(remote.joined, True) 1743 self.assertIs(remote.allowed, None) 1744 self.assertIs(remote.alived, None) 1745 1746 main.manage(immediate=True, cascade=True) 1747 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1748 for remote in main.remotes.values(): 1749 self.assertIs(remote.alived, True) 1750 self.serviceStacks(stacks, duration=3.0) 1751 for stack in stacks: 1752 self.assertEqual(len(stack.transactions), 0) 1753 for stack in [other, other1]: 1754 for remote in stack.remotes.values(): 1755 self.assertIs(remote.joined, True) 1756 self.assertIs(remote.allowed, True) 1757 self.assertIs(remote.alived, True) 1758 1759 for stack in [main, other, other1]: 1760 stack.server.close() 1761 stack.clearAllKeeps() 1762 time.sleep(0.1) 1763 1764 def testManageRebootCascadeBothSides(self): 1765 ''' 1766 Test stack manage remotes as if main were rebooted 1767 ''' 1768 console.terse("{0}\n".format(self.testManageRebootCascadeBothSides.__doc__)) 1769 1770 mainData = self.createRoadData(name='main', 1771 base=self.base, 1772 auto=raeting.AutoMode.once.value) 1773 mainDirpath = mainData['dirpath'] 1774 keeping.clearAllKeep(mainData['dirpath']) 1775 main = self.createRoadStack(data=mainData, 1776 main=True, 1777 auto=mainData['auto'], 1778 ha=None) 1779 1780 otherData = self.createRoadData(name='other', 1781 base=self.base, 1782 auto=raeting.AutoMode.once.value) 1783 otherDirpath = otherData['dirpath'] 1784 keeping.clearAllKeep(otherData['dirpath']) 1785 other = self.createRoadStack(data=otherData, 1786 main=None, 1787 auto=otherData['auto'], 1788 ha=("", raeting.RAET_TEST_PORT)) 1789 1790 other1Data = self.createRoadData(name='other1', 1791 base=self.base, 1792 auto=raeting.AutoMode.once.value) 1793 other1Dirpath = other1Data['dirpath'] 1794 keeping.clearAllKeep(other1Data['dirpath']) 1795 other1 = self.createRoadStack(data=other1Data, 1796 main=None, 1797 auto=other1Data['auto'], 1798 ha=("", 7533)) 1799 1800 for stack in (other, other1): 1801 self.join(stack, main) 1802 self.allow(stack, main) 1803 1804 console.terse("\nTest manage remotes presence *********\n") 1805 console.terse("\nMake all alive *********\n") 1806 stacks = [main, other, other1] 1807 for remote in main.remotes.values(): #make all alive 1808 main.alive(uid=remote.uid) 1809 self.serviceStacks(stacks, duration=3.0) 1810 for stack in stacks: 1811 for remote in stack.remotes.values(): 1812 self.assertIs(remote.alived, True) 1813 1814 main.manage(immediate=True) 1815 other.manage(immediate=True) 1816 other1.manage(immediate=True) 1817 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1818 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 1819 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 1820 1821 for stack in stacks: 1822 for remote in stack.remotes.values(): 1823 self.assertIs(remote.alived, True) 1824 1825 self.serviceManageStacks(stacks, duration=3.0) 1826 for stack in stacks: 1827 self.assertEqual(len(stack.transactions), 0) 1828 for stack in stacks: 1829 for remote in stack.remotes.values(): 1830 self.assertIs(remote.alived, True) 1831 1832 # now test if main rebooted 1833 console.terse("\nMake all alive with cascade after main reboots *********\n") 1834 main.server.close() 1835 main = stacking.RoadStack(store=self.store, 1836 main=True, 1837 dirpath=mainDirpath,) 1838 stacks = [main, other, other1] 1839 1840 for stack in [main]: 1841 for remote in stack.remotes.values(): 1842 self.assertIs(remote.joined, True) #joined status is persisted 1843 self.assertIs(remote.allowed, None) 1844 self.assertIs(remote.alived, None) 1845 1846 for stack in [other, other1]: 1847 for remote in stack.remotes.values(): 1848 self.assertIs(remote.joined, True) #joined status is persisted 1849 self.assertIs(remote.allowed, True) 1850 self.assertIs(remote.alived, True) 1851 1852 main.manage(immediate=True, cascade=True) 1853 other.manage(cascade=True) 1854 other1.manage(cascade=True) 1855 self.assertEqual(len(main.transactions), 2) 1856 self.assertEqual(len(other.transactions), 0) 1857 self.assertEqual(len(other1.transactions), 0) 1858 1859 for stack in [main]: 1860 for remote in stack.remotes.values(): 1861 self.assertIs(remote.alived, None) 1862 1863 for stack in [other, other1]: 1864 for remote in stack.remotes.values(): 1865 self.assertIs(remote.alived, True) 1866 1867 self.serviceManageStacks(stacks, duration=3.0, cascade=True) 1868 for stack in stacks: 1869 self.assertEqual(len(stack.transactions), 0) 1870 for stack in stacks: 1871 for remote in stack.remotes.values(): 1872 self.assertIs(remote.joined, True) #joined status is persisted 1873 self.assertIs(remote.allowed, True) 1874 self.assertIs(remote.alived, True) 1875 1876 # Now test as if others are rebooted but not main 1877 console.terse("\nMake all alive with cascade after others reboot *********\n") 1878 other.server.close() 1879 other1.server.close() 1880 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 1881 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 1882 stacks = [main, other, other1] 1883 1884 for stack in [main]: 1885 for remote in stack.remotes.values(): 1886 self.assertIs(remote.alived, True) 1887 1888 for stack in [other, other1]: 1889 for remote in stack.remotes.values(): 1890 self.assertIs(remote.joined, True) 1891 self.assertIs(remote.allowed, None) 1892 self.assertIs(remote.alived, None) 1893 1894 main.manage(cascade=True) 1895 other.manage(immediate=True,cascade=True) 1896 other1.manage(immediate=True,cascade=True) 1897 self.assertEqual(len(main.transactions), 0) 1898 self.assertEqual(len(other.transactions), 1) 1899 self.assertEqual(len(other1.transactions), 1) 1900 1901 for stack in [main]: 1902 for remote in stack.remotes.values(): 1903 self.assertIs(remote.alived, True) 1904 1905 for stack in [other, other1]: 1906 for remote in stack.remotes.values(): 1907 self.assertIs(remote.joined, True) 1908 self.assertIs(remote.allowed, None) 1909 self.assertIs(remote.alived, None) 1910 1911 self.serviceManageStacks(stacks, duration=3.0, cascade=True) 1912 for stack in stacks: 1913 self.assertEqual(len(stack.transactions), 0) 1914 for stack in stacks: 1915 for remote in stack.remotes.values(): 1916 self.assertIs(remote.joined, True) 1917 self.assertIs(remote.allowed, True) 1918 self.assertIs(remote.alived, True) 1919 1920 # now close down all and reload from saved data and manage 1921 console.terse("\nMake all alive with cascade after all reboot *********\n") 1922 main.server.close() 1923 main = stacking.RoadStack(store=self.store, 1924 main=True, 1925 dirpath=mainDirpath,) 1926 other.server.close() 1927 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 1928 other1.server.close() 1929 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 1930 1931 stacks = [main, other, other1] 1932 1933 for stack in stacks: 1934 for remote in stack.remotes.values(): 1935 self.assertIs(remote.joined, True) #joined status is persisted 1936 self.assertIs(remote.allowed, None) # None on reload from file 1937 self.assertIs(remote.alived, None) # None on reload from file 1938 1939 main.manage(immediate=True, cascade=True) 1940 other.manage(immediate=True, cascade=True) 1941 other1.manage(immediate=True, cascade=True) 1942 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1943 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 1944 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 1945 for stack in stacks: 1946 for remote in stack.remotes.values(): 1947 self.assertIs(remote.joined, True) #joined status is persisted 1948 self.assertIs(remote.allowed, None) # None on reload from file 1949 self.assertIs(remote.alived, None) # None on reload from file 1950 1951 self.serviceManageStacks(stacks, duration=3.0) 1952 for stack in stacks: 1953 self.assertEqual(len(stack.transactions), 0) 1954 for stack in stacks: 1955 for remote in stack.remotes.values(): 1956 self.assertIs(remote.joined, True) 1957 self.assertIs(remote.allowed, True) 1958 self.assertIs(remote.alived, True) 1959 1960 # now close down all and reload from saved data but loose joined status 1961 console.terse("\nMake all alive with cascade after all reboot and lose join *********\n") 1962 main.server.close() 1963 main = stacking.RoadStack(store=self.store, 1964 main=True, 1965 dirpath=mainDirpath,) 1966 other.server.close() 1967 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 1968 other1.server.close() 1969 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 1970 1971 stacks = [main, other, other1] 1972 1973 for stack in stacks: 1974 for remote in stack.remotes.values(): 1975 remote.joined = None 1976 self.assertIs(remote.joined, None) #joined status is persisted 1977 self.assertIs(remote.allowed, None) # None on reload from file 1978 self.assertIs(remote.alived, None) # None on reload from file 1979 1980 main.manage(immediate=True, cascade=True) 1981 other.manage(immediate=True, cascade=True) 1982 other1.manage(immediate=True, cascade=True) 1983 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 1984 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 1985 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 1986 for stack in stacks: 1987 for remote in stack.remotes.values(): 1988 self.assertIs(remote.joined, None) #joined status is persisted 1989 self.assertIs(remote.allowed, None) # None on reload from file 1990 self.assertIs(remote.alived, None) # None on reload from file 1991 1992 self.serviceManageStacks(stacks, duration=3.0) 1993 for stack in stacks: 1994 self.assertEqual(len(stack.transactions), 0) 1995 for stack in stacks: 1996 for remote in stack.remotes.values(): 1997 self.assertIs(remote.joined, True) 1998 self.assertIs(remote.allowed, True) 1999 self.assertIs(remote.alived, True) 2000 2001 for stack in [main, other, other1]: 2002 stack.server.close() 2003 stack.clearAllKeeps() 2004 time.sleep(0.1) 2005 2006 2007 def testManageRebootCascadeBothSidesAlt(self): 2008 ''' 2009 Test stack manage remotes as if main were rebooted and main name 2010 means it loses simultaneous join name resolution 2011 ''' 2012 console.terse("{0}\n".format(self.testManageRebootCascadeBothSidesAlt.__doc__)) 2013 2014 mainData = self.createRoadData(name='zmain', 2015 base=self.base, 2016 auto=raeting.AutoMode.once.value) 2017 mainDirpath = mainData['dirpath'] 2018 keeping.clearAllKeep(mainData['dirpath']) 2019 main = self.createRoadStack(data=mainData, 2020 main=True, 2021 auto=mainData['auto'], 2022 ha=None) 2023 2024 otherData = self.createRoadData(name='other', 2025 base=self.base, 2026 auto=raeting.AutoMode.once.value) 2027 otherDirpath = otherData['dirpath'] 2028 keeping.clearAllKeep(otherData['dirpath']) 2029 other = self.createRoadStack(data=otherData, 2030 main=None, 2031 auto=otherData['auto'], 2032 ha=("", raeting.RAET_TEST_PORT)) 2033 2034 other1Data = self.createRoadData(name='other1', 2035 base=self.base, 2036 auto=raeting.AutoMode.once.value) 2037 other1Dirpath = other1Data['dirpath'] 2038 keeping.clearAllKeep(other1Data['dirpath']) 2039 other1 = self.createRoadStack(data=other1Data, 2040 main=None, 2041 auto=other1Data['auto'], 2042 ha=("", 7533)) 2043 2044 2045 for stack in (other, other1): 2046 self.join(stack, main, cascade=True) 2047 self.allow(stack, main, cascade=True) 2048 2049 stacks = [main, other, other1] 2050 self.serviceStacks(stacks, duration=3.0) 2051 for stack in stacks: 2052 for remote in stack.remotes.values(): 2053 self.assertIs(remote.joined, True) 2054 self.assertIs(remote.allowed, True) 2055 self.assertIs(remote.alived, True) 2056 2057 2058 console.terse("\nTest manage remotes presence *********\n") 2059 main.manage(immediate=True) 2060 other.manage(immediate=True) 2061 other1.manage(immediate=True) 2062 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 2063 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 2064 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 2065 2066 for stack in stacks: 2067 for remote in stack.remotes.values(): 2068 self.assertIs(remote.alived, True) 2069 2070 self.serviceManageStacks(stacks, duration=3.0) 2071 for stack in stacks: 2072 self.assertEqual(len(stack.transactions), 0) 2073 for stack in stacks: 2074 for remote in stack.remotes.values(): 2075 self.assertIs(remote.alived, True) 2076 2077 # now test if main rebooted 2078 console.terse("\nMake all alive with cascade after main reboots *********\n") 2079 main.server.close() 2080 main = stacking.RoadStack(store=self.store, 2081 main=True, 2082 dirpath=mainDirpath,) 2083 stacks = [main, other, other1] 2084 2085 for stack in [main]: 2086 for remote in stack.remotes.values(): 2087 self.assertIs(remote.joined, True) #joined status is persisted 2088 self.assertIs(remote.allowed, None) 2089 self.assertIs(remote.alived, None) 2090 2091 for stack in [other, other1]: 2092 for remote in stack.remotes.values(): 2093 self.assertIs(remote.joined, True) #joined status is persisted 2094 self.assertIs(remote.allowed, True) 2095 self.assertIs(remote.alived, True) 2096 2097 main.manage(immediate=True, cascade=True) 2098 other.manage(cascade=True) 2099 other1.manage(cascade=True) 2100 self.assertEqual(len(main.transactions), 2) 2101 self.assertEqual(len(other.transactions), 0) 2102 self.assertEqual(len(other1.transactions), 0) 2103 2104 for stack in [main]: 2105 for remote in stack.remotes.values(): 2106 self.assertIs(remote.alived, None) 2107 2108 for stack in [other, other1]: 2109 for remote in stack.remotes.values(): 2110 self.assertIs(remote.alived, True) 2111 2112 self.serviceManageStacks(stacks, duration=3.0, cascade=True) 2113 for stack in stacks: 2114 self.assertEqual(len(stack.transactions), 0) 2115 for stack in stacks: 2116 for remote in stack.remotes.values(): 2117 self.assertIs(remote.joined, True) #joined status is persisted 2118 self.assertIs(remote.allowed, True) 2119 self.assertIs(remote.alived, True) 2120 2121 # Now test as if others are rebooted but not main 2122 console.terse("\nMake all alive with cascade after others reboot *********\n") 2123 other.server.close() 2124 other1.server.close() 2125 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 2126 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 2127 stacks = [main, other, other1] 2128 2129 for stack in [main]: 2130 for remote in stack.remotes.values(): 2131 self.assertIs(remote.alived, True) 2132 2133 for stack in [other, other1]: 2134 for remote in stack.remotes.values(): 2135 self.assertIs(remote.joined, True) 2136 self.assertIs(remote.allowed, None) 2137 self.assertIs(remote.alived, None) 2138 2139 main.manage(cascade=True) 2140 other.manage(immediate=True,cascade=True) 2141 other1.manage(immediate=True,cascade=True) 2142 self.assertEqual(len(main.transactions), 0) 2143 self.assertEqual(len(other.transactions), 1) 2144 self.assertEqual(len(other1.transactions), 1) 2145 2146 for stack in [main]: 2147 for remote in stack.remotes.values(): 2148 self.assertIs(remote.alived, True) 2149 2150 for stack in [other, other1]: 2151 for remote in stack.remotes.values(): 2152 self.assertIs(remote.joined, True) 2153 self.assertIs(remote.allowed, None) 2154 self.assertIs(remote.alived, None) 2155 2156 self.serviceManageStacks(stacks, duration=3.0, cascade=True) 2157 for stack in stacks: 2158 self.assertEqual(len(stack.transactions), 0) 2159 for stack in stacks: 2160 for remote in stack.remotes.values(): 2161 self.assertIs(remote.joined, True) 2162 self.assertIs(remote.allowed, True) 2163 self.assertIs(remote.alived, True) 2164 2165 # now close down all and reload from saved data and manage 2166 console.terse("\nMake all alive with cascade after all reboot *********\n") 2167 main.server.close() 2168 main = stacking.RoadStack(store=self.store, 2169 main=True, 2170 dirpath=mainDirpath,) 2171 other.server.close() 2172 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 2173 other1.server.close() 2174 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 2175 2176 stacks = [main, other, other1] 2177 2178 for stack in stacks: 2179 for remote in stack.remotes.values(): 2180 self.assertIs(remote.joined, True) #joined status is persisted 2181 self.assertIs(remote.allowed, None) # None on reload from file 2182 self.assertIs(remote.alived, None) # None on reload from file 2183 2184 main.manage(immediate=True, cascade=True) 2185 other.manage(immediate=True, cascade=True) 2186 other1.manage(immediate=True, cascade=True) 2187 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 2188 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 2189 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 2190 for stack in stacks: 2191 for remote in stack.remotes.values(): 2192 self.assertIs(remote.joined, True) #joined status is persisted 2193 self.assertIs(remote.allowed, None) # None on reload from file 2194 self.assertIs(remote.alived, None) # None on reload from file 2195 2196 self.serviceManageStacks(stacks, duration=3.0) 2197 for stack in stacks: 2198 self.assertEqual(len(stack.transactions), 0) 2199 for stack in stacks: 2200 for remote in stack.remotes.values(): 2201 self.assertIs(remote.joined, True) 2202 self.assertIs(remote.allowed, True) 2203 self.assertIs(remote.alived, True) 2204 2205 # now close down all and reload from saved data but loose joined status 2206 console.terse("\nMake all alive with cascade after all reboot and lose join *********\n") 2207 main.server.close() 2208 main = stacking.RoadStack(store=self.store, 2209 main=True, 2210 dirpath=mainDirpath,) 2211 other.server.close() 2212 other = stacking.RoadStack(dirpath=otherDirpath, store=self.store) 2213 other1.server.close() 2214 other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store) 2215 2216 stacks = [main, other, other1] 2217 2218 for stack in stacks: 2219 for remote in stack.remotes.values(): 2220 remote.joined = None 2221 self.assertIs(remote.joined, None) #joined status is persisted 2222 self.assertIs(remote.allowed, None) # None on reload from file 2223 self.assertIs(remote.alived, None) # None on reload from file 2224 2225 main.manage(immediate=True, cascade=True) 2226 other.manage(immediate=True, cascade=True) 2227 other1.manage(immediate=True, cascade=True) 2228 self.assertEqual(len(main.transactions), 2) # started 2 alive transactions 2229 self.assertEqual(len(other.transactions), 1) # started 1 alive transactions 2230 self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions 2231 for stack in stacks: 2232 for remote in stack.remotes.values(): 2233 self.assertIs(remote.joined, None) #joined status is persisted 2234 self.assertIs(remote.allowed, None) # None on reload from file 2235 self.assertIs(remote.alived, None) # None on reload from file 2236 2237 self.serviceManageStacks(stacks, duration=3.0) 2238 for stack in stacks: 2239 self.assertEqual(len(stack.transactions), 0) 2240 for stack in stacks: 2241 for remote in stack.remotes.values(): 2242 self.assertIs(remote.joined, True) 2243 self.assertIs(remote.allowed, True) 2244 self.assertIs(remote.alived, True) 2245 2246 for stack in [main, other, other1]: 2247 stack.server.close() 2248 stack.clearAllKeeps() 2249 time.sleep(0.1) 2250 2251 def testManageUnjoinedAllTimersExpired(self): 2252 ''' 2253 Test stack manage unjoined remotes as if both presence and reap timers expired (A1) 2254 ''' 2255 console.terse("{0}\n".format(self.testManageUnjoinedAllTimersExpired.__doc__)) 2256 2257 mainData = self.createRoadData(name='main', 2258 base=self.base, 2259 auto=raeting.AutoMode.once.value) 2260 keeping.clearAllKeep(mainData['dirpath']) 2261 main = self.createRoadStack(data=mainData, 2262 main=True, 2263 auto=mainData['auto'], 2264 ha=None) 2265 2266 otherData = self.createRoadData(name='other', 2267 base=self.base, 2268 auto=raeting.AutoMode.once.value) 2269 keeping.clearAllKeep(otherData['dirpath']) 2270 other = self.createRoadStack(data=otherData, 2271 main=None, 2272 auto=otherData['auto'], 2273 ha=("", raeting.RAET_TEST_PORT)) 2274 2275 stacks = [main, other] 2276 2277 console.terse("\nBootstrap channel *********\n") 2278 self.join(other, main) # bootstrap channel 2279 for stack in [main, other]: 2280 self.assertEqual(len(stack.transactions), 0) 2281 self.assertEqual(len(stack.remotes), 1) 2282 remote = stack.remotes.values()[0] 2283 self.assertTrue(remote.joined) 2284 2285 # force unjoined already unallowed unalived 2286 for stack in [main, other]: 2287 stack.remotes.values()[0].joined = None 2288 2289 for stack in [main, other]: 2290 remote = stack.remotes.values()[0] 2291 self.assertIs(remote.joined, None) 2292 self.assertIs(remote.allowed, None) 2293 self.assertIs(remote.alived, None) 2294 self.assertIs(remote.reaped, None) 2295 2296 console.terse("\nTest manage the unjoined remote *********\n") 2297 console.terse("\nMake all expired so send alive and reap *********\n") 2298 # advance clock so both remote timers expire 2299 self.store.advanceStamp(stacking.RoadStack.Interim) 2300 remote = main.remotes.values()[0] 2301 self.assertTrue(remote.timer.expired) 2302 self.assertTrue(remote.reapTimer.expired) 2303 2304 # Manage 2305 main.clearStats() 2306 main.manage() 2307 remote = main.remotes.values()[0] 2308 2309 self.assertEqual(len(main.transactions), 1) # started 1 join transactions 2310 self.assertIs(remote.joined, None) 2311 self.assertIs(remote.allowed, None) 2312 self.assertIs(remote.alived, None) 2313 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 2314 2315 # Service both sides 2316 self.serviceStacks(stacks) 2317 for stack in stacks: 2318 self.assertEqual(len(stack.transactions), 0) # join is done 2319 remote = main.remotes.values()[0] 2320 self.assertTrue(remote.joined) # join success 2321 self.assertIs(remote.allowed, None) 2322 self.assertIs(remote.alived, None) 2323 self.assertTrue(remote.reaped) # still reaped 2324 2325 for stack in [main, other]: 2326 stack.server.close() 2327 stack.clearAllKeeps() 2328 2329 def testManageUnallowedAllTimersExpired(self): 2330 ''' 2331 Test stack manage unallowed remotes as if both presence and reap timers expired (A2) 2332 ''' 2333 console.terse("{0}\n".format(self.testManageUnallowedAllTimersExpired.__doc__)) 2334 2335 mainData = self.createRoadData(name='main', 2336 base=self.base, 2337 auto=raeting.AutoMode.once.value) 2338 keeping.clearAllKeep(mainData['dirpath']) 2339 main = self.createRoadStack(data=mainData, 2340 main=True, 2341 auto=mainData['auto'], 2342 ha=None) 2343 2344 otherData = self.createRoadData(name='other', 2345 base=self.base, 2346 auto=raeting.AutoMode.once.value) 2347 keeping.clearAllKeep(otherData['dirpath']) 2348 other = self.createRoadStack(data=otherData, 2349 main=None, 2350 auto=otherData['auto'], 2351 ha=("", raeting.RAET_TEST_PORT)) 2352 2353 stacks = [main, other] 2354 2355 console.terse("\nBootstrap channel *********\n") 2356 self.join(other, main) # bootstrap channel 2357 for stack in [main, other]: 2358 self.assertEqual(len(stack.transactions), 0) 2359 self.assertEqual(len(stack.remotes), 1) 2360 remote = stack.remotes.values()[0] 2361 self.assertTrue(remote.joined) 2362 self.assertIs(remote.allowed, None) 2363 self.assertIs(remote.alived, None) 2364 self.assertIs(remote.reaped, None) 2365 2366 console.terse("\nTest manage the unallowed remote *********\n") 2367 console.terse("\nMake all expired so send alive and reap *********\n") 2368 # advance clock so both remote timers expire 2369 self.store.advanceStamp(stacking.RoadStack.Interim) 2370 remote = main.remotes.values()[0] 2371 self.assertTrue(remote.timer.expired) 2372 self.assertTrue(remote.reapTimer.expired) 2373 2374 # Manage 2375 main.manage() 2376 remote = main.remotes.values()[0] 2377 2378 self.assertEqual(len(main.transactions), 1) # started 1 allow transactions 2379 self.assertTrue(remote.joined) 2380 self.assertIs(remote.allowed, None) 2381 self.assertIs(remote.alived, None) 2382 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 2383 2384 # Service both sides 2385 self.serviceStacks(stacks) 2386 for stack in stacks: 2387 self.assertEqual(len(stack.transactions), 0) # allow is done 2388 remote = main.remotes.values()[0] 2389 self.assertTrue(remote.joined) 2390 self.assertTrue(remote.allowed) # allow success 2391 self.assertTrue(remote.alived) # fast alive 2392 self.assertFalse(remote.reaped) # unreaped by first non-join transaction 2393 2394 for stack in [main, other]: 2395 stack.server.close() 2396 stack.clearAllKeeps() 2397 2398 def testManageCascadeUnjoinedAllTimersExpired(self): 2399 ''' 2400 Test stack manage cascade unjoined remotes as if both presence and reap timers expired (A3) 2401 ''' 2402 console.terse("{0}\n".format(self.testManageCascadeUnjoinedAllTimersExpired.__doc__)) 2403 2404 mainData = self.createRoadData(name='main', 2405 base=self.base, 2406 auto=raeting.AutoMode.once.value) 2407 keeping.clearAllKeep(mainData['dirpath']) 2408 main = self.createRoadStack(data=mainData, 2409 main=True, 2410 auto=mainData['auto'], 2411 ha=None) 2412 2413 otherData = self.createRoadData(name='other', 2414 base=self.base, 2415 auto=raeting.AutoMode.once.value) 2416 keeping.clearAllKeep(otherData['dirpath']) 2417 other = self.createRoadStack(data=otherData, 2418 main=None, 2419 auto=otherData['auto'], 2420 ha=("", raeting.RAET_TEST_PORT)) 2421 2422 stacks = [main, other] 2423 2424 console.terse("\nBootstrap channel *********\n") 2425 self.join(other, main) # bootstrap channel 2426 for stack in [main, other]: 2427 self.assertEqual(len(stack.transactions), 0) 2428 self.assertEqual(len(stack.remotes), 1) 2429 remote = stack.remotes.values()[0] 2430 self.assertTrue(remote.joined) 2431 2432 # force unjoined already unallowed unalived 2433 for stack in [main, other]: 2434 stack.remotes.values()[0].joined = None 2435 2436 for stack in [main, other]: 2437 remote = stack.remotes.values()[0] 2438 self.assertIs(remote.joined, None) 2439 self.assertIs(remote.allowed, None) 2440 self.assertIs(remote.alived, None) 2441 self.assertIs(remote.reaped, None) 2442 2443 console.terse("\nTest manage cascade the unjoined remote *********\n") 2444 console.terse("\nMake all expired so send alive and reap *********\n") 2445 # advance clock so both remote timers expire 2446 self.store.advanceStamp(stacking.RoadStack.Interim) 2447 remote = main.remotes.values()[0] 2448 self.assertTrue(remote.timer.expired) 2449 self.assertTrue(remote.reapTimer.expired) 2450 2451 # Manage 2452 main.manage(cascade=True) 2453 remote = main.remotes.values()[0] 2454 2455 self.assertEqual(len(main.transactions), 1) # started 1 join transactions 2456 self.assertIs(remote.joined, None) 2457 self.assertIs(remote.allowed, None) 2458 self.assertIs(remote.alived, None) 2459 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 2460 2461 # Service both sides 2462 self.serviceStacks(stacks, duration=3.0) 2463 for stack in stacks: 2464 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2465 remote = main.remotes.values()[0] 2466 self.assertTrue(remote.joined) # join success 2467 self.assertTrue(remote.allowed) # allow success 2468 self.assertTrue(remote.alived) # alive success 2469 self.assertFalse(remote.reaped) # unreaped 2470 2471 for stack in [main, other]: 2472 stack.server.close() 2473 stack.clearAllKeeps() 2474 2475 def testManageCascadeUnallowedAllTimersExpired(self): 2476 ''' 2477 Test stack manage cascade unallowed remotes as if both presence and reap timers expired (A4) 2478 ''' 2479 console.terse("{0}\n".format(self.testManageCascadeUnallowedAllTimersExpired.__doc__)) 2480 2481 mainData = self.createRoadData(name='main', 2482 base=self.base, 2483 auto=raeting.AutoMode.once.value) 2484 keeping.clearAllKeep(mainData['dirpath']) 2485 main = self.createRoadStack(data=mainData, 2486 main=True, 2487 auto=mainData['auto'], 2488 ha=None) 2489 2490 otherData = self.createRoadData(name='other', 2491 base=self.base, 2492 auto=raeting.AutoMode.once.value) 2493 keeping.clearAllKeep(otherData['dirpath']) 2494 other = self.createRoadStack(data=otherData, 2495 main=None, 2496 auto=otherData['auto'], 2497 ha=("", raeting.RAET_TEST_PORT)) 2498 2499 stacks = [main, other] 2500 2501 console.terse("\nBootstrap channel *********\n") 2502 self.join(other, main) # bootstrap channel 2503 for stack in [main, other]: 2504 self.assertEqual(len(stack.transactions), 0) 2505 self.assertEqual(len(stack.remotes), 1) 2506 remote = stack.remotes.values()[0] 2507 self.assertTrue(remote.joined) 2508 self.assertIs(remote.allowed, None) 2509 self.assertIs(remote.alived, None) 2510 self.assertIs(remote.reaped, None) 2511 2512 console.terse("\nTest manage cascade the unallowed remote *********\n") 2513 console.terse("\nMake all expired so send alive and reap *********\n") 2514 # advance clock so both remote timers expire 2515 self.store.advanceStamp(stacking.RoadStack.Interim) 2516 remote = main.remotes.values()[0] 2517 self.assertTrue(remote.timer.expired) 2518 self.assertTrue(remote.reapTimer.expired) 2519 2520 # Manage 2521 main.manage(cascade=True) 2522 remote = main.remotes.values()[0] 2523 2524 self.assertEqual(len(main.transactions), 1) # started 1 allow transactions 2525 self.assertTrue(remote.joined) 2526 self.assertIs(remote.allowed, None) 2527 self.assertIs(remote.alived, None) 2528 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 2529 2530 # Service both sides 2531 self.serviceStacks(stacks) 2532 for stack in stacks: 2533 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2534 remote = main.remotes.values()[0] 2535 self.assertTrue(remote.joined) 2536 self.assertTrue(remote.allowed) # allow success 2537 self.assertTrue(remote.alived) # alive success 2538 self.assertFalse(remote.reaped) # unreaped 2539 2540 for stack in [main, other]: 2541 stack.server.close() 2542 stack.clearAllKeeps() 2543 2544 def testManageUnalivedAllTimersExpired(self): 2545 ''' 2546 Test stack manage not alived remotes as if both presence and reap timers expired (A5) 2547 ''' 2548 console.terse("{0}\n".format(self.testManageUnalivedAllTimersExpired.__doc__)) 2549 2550 mainData = self.createRoadData(name='main', 2551 base=self.base, 2552 auto=raeting.AutoMode.once.value) 2553 keeping.clearAllKeep(mainData['dirpath']) 2554 main = self.createRoadStack(data=mainData, 2555 main=True, 2556 auto=mainData['auto'], 2557 ha=None) 2558 2559 otherData = self.createRoadData(name='other', 2560 base=self.base, 2561 auto=raeting.AutoMode.once.value) 2562 keeping.clearAllKeep(otherData['dirpath']) 2563 other = self.createRoadStack(data=otherData, 2564 main=None, 2565 auto=otherData['auto'], 2566 ha=("", raeting.RAET_TEST_PORT)) 2567 2568 stacks = [main, other] 2569 2570 console.terse("\nBootstrap channel *********\n") 2571 self.join(other, main) # bootstrap channel 2572 self.allow(other, main) 2573 for stack in [main, other]: 2574 self.assertEqual(len(stack.transactions), 0) 2575 self.assertEqual(len(stack.remotes), 1) 2576 remote = stack.remotes.values()[0] 2577 self.assertTrue(remote.joined) 2578 self.assertTrue(remote.allowed) 2579 self.assertTrue(remote.alived) # fast alive 2580 self.assertIs(remote.reaped, None) 2581 2582 console.terse("\nTest manage the joined and allowed remote *********\n") 2583 console.terse("\nMake all expired so send alive and reap *********\n") 2584 # advance clock so both remote timers expire 2585 self.store.advanceStamp(stacking.RoadStack.Interim) 2586 remote = main.remotes.values()[0] 2587 self.assertTrue(remote.timer.expired) 2588 self.assertTrue(remote.reapTimer.expired) 2589 main.remotes.values()[0].alived = None # reset alived 2590 2591 # Manage 2592 main.manage() 2593 remote = main.remotes.values()[0] 2594 2595 self.assertEqual(len(main.transactions), 1) # started 1 alive transactions 2596 self.assertTrue(remote.joined) 2597 self.assertTrue(remote.allowed) 2598 self.assertIsNone(remote.alived) # alived status isn't changed 2599 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 2600 2601 # Service both sides 2602 self.serviceStacks(stacks) 2603 for stack in stacks: 2604 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2605 remote = main.remotes.values()[0] 2606 self.assertTrue(remote.joined) 2607 self.assertTrue(remote.allowed) 2608 self.assertTrue(remote.alived) # alive success 2609 self.assertFalse(remote.reaped) # unreaped 2610 2611 for stack in [main, other]: 2612 stack.server.close() 2613 stack.clearAllKeeps() 2614 2615 def testManageUnjoinedPresenceTimerExpired(self): 2616 ''' 2617 Test stack manage unjoined remotes as if presence timer expired (B1) 2618 ''' 2619 console.terse("{0}\n".format(self.testManageUnjoinedPresenceTimerExpired.__doc__)) 2620 2621 mainData = self.createRoadData(name='main', 2622 base=self.base, 2623 auto=raeting.AutoMode.once.value) 2624 keeping.clearAllKeep(mainData['dirpath']) 2625 main = self.createRoadStack(data=mainData, 2626 main=True, 2627 auto=mainData['auto'], 2628 ha=None) 2629 2630 otherData = self.createRoadData(name='other', 2631 base=self.base, 2632 auto=raeting.AutoMode.once.value) 2633 keeping.clearAllKeep(otherData['dirpath']) 2634 other = self.createRoadStack(data=otherData, 2635 main=None, 2636 auto=otherData['auto'], 2637 ha=("", raeting.RAET_TEST_PORT)) 2638 2639 stacks = [main, other] 2640 2641 console.terse("\nBootstrap channel *********\n") 2642 self.join(other, main) # bootstrap channel 2643 for stack in [main, other]: 2644 self.assertEqual(len(stack.transactions), 0) 2645 self.assertEqual(len(stack.remotes), 1) 2646 remote = stack.remotes.values()[0] 2647 self.assertTrue(remote.joined) 2648 2649 # force unjoined already unallowed unalived 2650 for stack in [main, other]: 2651 stack.remotes.values()[0].joined = None 2652 2653 for stack in [main, other]: 2654 remote = stack.remotes.values()[0] 2655 self.assertIs(remote.joined, None) 2656 self.assertIs(remote.allowed, None) 2657 self.assertIs(remote.alived, None) 2658 self.assertIs(remote.reaped, None) 2659 2660 console.terse("\nTest manage the unjoined remote *********\n") 2661 console.terse("\nMake all expired so send alive and reap *********\n") 2662 # advance clock so remote keep alive timers expire 2663 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 2664 remote = main.remotes.values()[0] 2665 self.assertTrue(remote.timer.expired) 2666 self.assertFalse(remote.reapTimer.expired) 2667 2668 # Manage 2669 main.clearStats() 2670 main.manage() 2671 remote = main.remotes.values()[0] 2672 2673 self.assertEqual(len(main.transactions), 1) # started 1 join transactions 2674 self.assertIs(remote.joined, None) 2675 self.assertIs(remote.allowed, None) 2676 self.assertIs(remote.alived, None) 2677 self.assertIs(remote.reaped, None) 2678 2679 # Service both sides 2680 self.serviceStacks(stacks) 2681 for stack in stacks: 2682 self.assertEqual(len(stack.transactions), 0) # join is done 2683 remote = main.remotes.values()[0] 2684 self.assertTrue(remote.joined) # join success 2685 self.assertIs(remote.allowed, None) 2686 self.assertIs(remote.alived, None) 2687 self.assertIs(remote.reaped, None) 2688 2689 for stack in [main, other]: 2690 stack.server.close() 2691 stack.clearAllKeeps() 2692 2693 def testManageUnallowedPresenceTimerExpired(self): 2694 ''' 2695 Test stack manage unallowed remotes as if presence timer expired (B2) 2696 ''' 2697 console.terse("{0}\n".format(self.testManageUnallowedPresenceTimerExpired.__doc__)) 2698 2699 mainData = self.createRoadData(name='main', 2700 base=self.base, 2701 auto=raeting.AutoMode.once.value) 2702 keeping.clearAllKeep(mainData['dirpath']) 2703 main = self.createRoadStack(data=mainData, 2704 main=True, 2705 auto=mainData['auto'], 2706 ha=None) 2707 2708 otherData = self.createRoadData(name='other', 2709 base=self.base, 2710 auto=raeting.AutoMode.once.value) 2711 keeping.clearAllKeep(otherData['dirpath']) 2712 other = self.createRoadStack(data=otherData, 2713 main=None, 2714 auto=otherData['auto'], 2715 ha=("", raeting.RAET_TEST_PORT)) 2716 2717 stacks = [main, other] 2718 2719 console.terse("\nBootstrap channel *********\n") 2720 self.join(other, main) # bootstrap channel 2721 for stack in [main, other]: 2722 self.assertEqual(len(stack.transactions), 0) 2723 self.assertEqual(len(stack.remotes), 1) 2724 remote = stack.remotes.values()[0] 2725 self.assertTrue(remote.joined) 2726 self.assertIs(remote.allowed, None) 2727 self.assertIs(remote.alived, None) 2728 self.assertIs(remote.reaped, None) 2729 2730 console.terse("\nTest manage the unallowed remote *********\n") 2731 console.terse("\nMake all expired so send alive and reap *********\n") 2732 # advance clock so remote keep alive timers expire 2733 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 2734 remote = main.remotes.values()[0] 2735 self.assertTrue(remote.timer.expired) 2736 self.assertFalse(remote.reapTimer.expired) 2737 2738 # Manage 2739 main.manage() 2740 remote = main.remotes.values()[0] 2741 2742 self.assertEqual(len(main.transactions), 1) # started 1 allow transactions 2743 self.assertTrue(remote.joined) 2744 self.assertIs(remote.allowed, None) 2745 self.assertIs(remote.alived, None) 2746 self.assertIs(remote.reaped, None) 2747 2748 # Service both sides 2749 self.serviceStacks(stacks) 2750 for stack in stacks: 2751 self.assertEqual(len(stack.transactions), 0) # allow is done 2752 remote = main.remotes.values()[0] 2753 self.assertTrue(remote.joined) 2754 self.assertTrue(remote.allowed) # allow success 2755 self.assertTrue(remote.alived) # fast alive 2756 self.assertIs(remote.reaped, None) 2757 2758 for stack in [main, other]: 2759 stack.server.close() 2760 stack.clearAllKeeps() 2761 2762 def testManageCascadeUnjoinedPresenceTimerExpired(self): 2763 ''' 2764 Test stack manage cascade unjoined remotes as if presence timer expired (B3) 2765 ''' 2766 console.terse("{0}\n".format(self.testManageCascadeUnjoinedPresenceTimerExpired.__doc__)) 2767 2768 mainData = self.createRoadData(name='main', 2769 base=self.base, 2770 auto=raeting.AutoMode.once.value) 2771 keeping.clearAllKeep(mainData['dirpath']) 2772 main = self.createRoadStack(data=mainData, 2773 main=True, 2774 auto=mainData['auto'], 2775 ha=None) 2776 2777 otherData = self.createRoadData(name='other', 2778 base=self.base, 2779 auto=raeting.AutoMode.once.value) 2780 keeping.clearAllKeep(otherData['dirpath']) 2781 other = self.createRoadStack(data=otherData, 2782 main=None, 2783 auto=otherData['auto'], 2784 ha=("", raeting.RAET_TEST_PORT)) 2785 2786 stacks = [main, other] 2787 2788 console.terse("\nBootstrap channel *********\n") 2789 self.join(other, main) # bootstrap channel 2790 for stack in [main, other]: 2791 self.assertEqual(len(stack.transactions), 0) 2792 self.assertEqual(len(stack.remotes), 1) 2793 remote = stack.remotes.values()[0] 2794 self.assertTrue(remote.joined) 2795 2796 # force unjoined already unallowed unalived 2797 for stack in [main, other]: 2798 stack.remotes.values()[0].joined = None 2799 2800 for stack in [main, other]: 2801 remote = stack.remotes.values()[0] 2802 self.assertIs(remote.joined, None) 2803 self.assertIs(remote.allowed, None) 2804 self.assertIs(remote.alived, None) 2805 self.assertIs(remote.reaped, None) 2806 2807 console.terse("\nTest manage cascade the unjoined remote *********\n") 2808 console.terse("\nMake all expired so send alive and reap *********\n") 2809 # advance clock so remote keep alive timers expire 2810 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 2811 remote = main.remotes.values()[0] 2812 self.assertTrue(remote.timer.expired) 2813 self.assertFalse(remote.reapTimer.expired) 2814 2815 # Manage 2816 main.manage(cascade=True) 2817 remote = main.remotes.values()[0] 2818 2819 self.assertEqual(len(main.transactions), 1) # started 1 join transactions 2820 self.assertIs(remote.joined, None) 2821 self.assertIs(remote.allowed, None) 2822 self.assertIs(remote.alived, None) 2823 self.assertIs(remote.reaped, None) 2824 2825 # Service both sides 2826 self.serviceStacks(stacks, duration=3.0) 2827 for stack in stacks: 2828 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2829 remote = main.remotes.values()[0] 2830 self.assertTrue(remote.joined) # join success 2831 self.assertTrue(remote.allowed) # allow success 2832 self.assertTrue(remote.alived) # alive success 2833 self.assertIs(remote.reaped, None) 2834 2835 for stack in [main, other]: 2836 stack.server.close() 2837 stack.clearAllKeeps() 2838 2839 def testManageCascadeUnallowedPresenceTimerExpired(self): 2840 ''' 2841 Test stack manage cascade unallowed remotes as if presence timer expired (B4) 2842 ''' 2843 console.terse("{0}\n".format(self.testManageCascadeUnallowedPresenceTimerExpired.__doc__)) 2844 2845 mainData = self.createRoadData(name='main', 2846 base=self.base, 2847 auto=raeting.AutoMode.once.value) 2848 keeping.clearAllKeep(mainData['dirpath']) 2849 main = self.createRoadStack(data=mainData, 2850 main=True, 2851 auto=mainData['auto'], 2852 ha=None) 2853 2854 otherData = self.createRoadData(name='other', 2855 base=self.base, 2856 auto=raeting.AutoMode.once.value) 2857 keeping.clearAllKeep(otherData['dirpath']) 2858 other = self.createRoadStack(data=otherData, 2859 main=None, 2860 auto=otherData['auto'], 2861 ha=("", raeting.RAET_TEST_PORT)) 2862 2863 stacks = [main, other] 2864 2865 console.terse("\nBootstrap channel *********\n") 2866 self.join(other, main) # bootstrap channel 2867 for stack in [main, other]: 2868 self.assertEqual(len(stack.transactions), 0) 2869 self.assertEqual(len(stack.remotes), 1) 2870 remote = stack.remotes.values()[0] 2871 self.assertTrue(remote.joined) 2872 self.assertIs(remote.allowed, None) 2873 self.assertIs(remote.alived, None) 2874 self.assertIs(remote.reaped, None) 2875 2876 console.terse("\nTest manage cascade the unallowed remote *********\n") 2877 console.terse("\nMake all expired so send alive and reap *********\n") 2878 # advance clock so remote keep alive timers expire 2879 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 2880 remote = main.remotes.values()[0] 2881 self.assertTrue(remote.timer.expired) 2882 self.assertFalse(remote.reapTimer.expired) 2883 2884 # Manage 2885 main.manage(cascade=True) 2886 remote = main.remotes.values()[0] 2887 2888 self.assertEqual(len(main.transactions), 1) # started 1 allow transactions 2889 self.assertTrue(remote.joined) 2890 self.assertIs(remote.allowed, None) 2891 self.assertIs(remote.alived, None) 2892 self.assertIs(remote.reaped, None) 2893 2894 # Service both sides 2895 self.serviceStacks(stacks) 2896 for stack in stacks: 2897 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2898 remote = main.remotes.values()[0] 2899 self.assertTrue(remote.joined) 2900 self.assertTrue(remote.allowed) # allow success 2901 self.assertTrue(remote.alived) # alive success 2902 self.assertIs(remote.reaped, None) 2903 2904 for stack in [main, other]: 2905 stack.server.close() 2906 stack.clearAllKeeps() 2907 2908 def testManageUnalivedPresenceTimerExpired(self): 2909 ''' 2910 Test stack manage not alived remotes as if presence timer expired (B5) 2911 ''' 2912 console.terse("{0}\n".format(self.testManageUnalivedPresenceTimerExpired.__doc__)) 2913 2914 mainData = self.createRoadData(name='main', 2915 base=self.base, 2916 auto=raeting.AutoMode.once.value) 2917 keeping.clearAllKeep(mainData['dirpath']) 2918 main = self.createRoadStack(data=mainData, 2919 main=True, 2920 auto=mainData['auto'], 2921 ha=None) 2922 2923 otherData = self.createRoadData(name='other', 2924 base=self.base, 2925 auto=raeting.AutoMode.once.value) 2926 keeping.clearAllKeep(otherData['dirpath']) 2927 other = self.createRoadStack(data=otherData, 2928 main=None, 2929 auto=otherData['auto'], 2930 ha=("", raeting.RAET_TEST_PORT)) 2931 2932 stacks = [main, other] 2933 2934 console.terse("\nBootstrap channel *********\n") 2935 self.join(other, main) # bootstrap channel 2936 self.allow(other, main) 2937 for stack in [main, other]: 2938 self.assertEqual(len(stack.transactions), 0) 2939 self.assertEqual(len(stack.remotes), 1) 2940 remote = stack.remotes.values()[0] 2941 self.assertTrue(remote.joined) 2942 self.assertTrue(remote.allowed) 2943 self.assertTrue(remote.alived) # fast alive 2944 self.assertIs(remote.reaped, None) 2945 2946 console.terse("\nTest manage the joined and allowed remote *********\n") 2947 console.terse("\nMake timer expired so send alive *********\n") 2948 # advance clock so remote keep alive timers expire 2949 self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset) 2950 remote = main.remotes.values()[0] 2951 self.assertTrue(remote.timer.expired) 2952 self.assertFalse(remote.reapTimer.expired) 2953 main.remotes.values()[0].alived = None # reset alived 2954 2955 # Manage 2956 main.manage() 2957 remote = main.remotes.values()[0] 2958 2959 self.assertEqual(len(main.transactions), 1) # started 1 alive transactions 2960 self.assertTrue(remote.joined) 2961 self.assertTrue(remote.allowed) 2962 self.assertIs(remote.alived, None) 2963 self.assertIs(remote.reaped, None) 2964 2965 # Service both sides 2966 self.serviceStacks(stacks) 2967 for stack in stacks: 2968 self.assertEqual(len(stack.transactions), 0) # all transactions finished 2969 remote = main.remotes.values()[0] 2970 self.assertTrue(remote.joined) 2971 self.assertTrue(remote.allowed) 2972 self.assertTrue(remote.alived) # alive success 2973 self.assertIs(remote.reaped, None) 2974 2975 for stack in [main, other]: 2976 stack.server.close() 2977 stack.clearAllKeeps() 2978 2979 def testManageReapTimerExpired(self): 2980 ''' 2981 Test stack manage remotes as if reap timer expired (C1) 2982 ''' 2983 console.terse("{0}\n".format(self.testManageReapTimerExpired.__doc__)) 2984 2985 mainData = self.createRoadData(name='main', 2986 base=self.base, 2987 auto=raeting.AutoMode.once.value) 2988 keeping.clearAllKeep(mainData['dirpath']) 2989 main = self.createRoadStack(data=mainData, 2990 main=True, 2991 auto=mainData['auto'], 2992 ha=None, 2993 period=stacking.RoadStack.Interim*2) # set presence timeout longer than reap 2994 2995 otherData = self.createRoadData(name='other', 2996 base=self.base, 2997 auto=raeting.AutoMode.once.value) 2998 keeping.clearAllKeep(otherData['dirpath']) 2999 other = self.createRoadStack(data=otherData, 3000 main=None, 3001 auto=otherData['auto'], 3002 ha=("", raeting.RAET_TEST_PORT)) 3003 3004 stacks = [main, other] 3005 3006 console.terse("\nBootstrap channel *********\n") 3007 self.join(other, main) # bootstrap channel 3008 for stack in [main, other]: 3009 self.assertEqual(len(stack.transactions), 0) 3010 self.assertEqual(len(stack.remotes), 1) 3011 remote = stack.remotes.values()[0] 3012 self.assertTrue(remote.joined) 3013 self.assertIs(remote.allowed, None) 3014 self.assertIs(remote.alived, None) 3015 self.assertIs(remote.reaped, None) 3016 3017 console.terse("\nTest manage remote *********\n") 3018 console.terse("\nMake reap timer expired so send nothing just reap *********\n") 3019 # advance clock so reap timer expire 3020 self.store.advanceStamp(stacking.RoadStack.Interim) 3021 remote = main.remotes.values()[0] 3022 self.assertFalse(remote.timer.expired) 3023 self.assertTrue(remote.reapTimer.expired) 3024 3025 # Manage 3026 main.clearStats() 3027 main.manage() 3028 remote = main.remotes.values()[0] 3029 3030 self.assertEqual(len(main.transactions), 0) # started no transactions because presence timer isn't expired 3031 self.assertTrue(remote.joined) 3032 self.assertIs(remote.allowed, None) 3033 self.assertIs(remote.alived, None) 3034 self.assertTrue(remote.reaped) # reaped the remote because reap timer expired 3035 3036 # Service both sides 3037 self.serviceStacks(stacks) 3038 for stack in stacks: 3039 self.assertEqual(len(stack.transactions), 0) 3040 remote = main.remotes.values()[0] 3041 self.assertTrue(remote.joined) 3042 self.assertIs(remote.allowed, None) 3043 self.assertIs(remote.alived, None) 3044 self.assertTrue(remote.reaped) # still reaped since there was no transactions 3045 3046 for stack in [main, other]: 3047 stack.server.close() 3048 stack.clearAllKeeps() 3049 3050 def testManageNoTimerExpired(self): 3051 ''' 3052 Test stack manage remotes as if no timer expired (C2) 3053 ''' 3054 console.terse("{0}\n".format(self.testManageNoTimerExpired.__doc__)) 3055 3056 mainData = self.createRoadData(name='main', 3057 base=self.base, 3058 auto=raeting.AutoMode.once.value) 3059 keeping.clearAllKeep(mainData['dirpath']) 3060 main = self.createRoadStack(data=mainData, 3061 main=True, 3062 auto=mainData['auto'], 3063 ha=None) 3064 3065 otherData = self.createRoadData(name='other', 3066 base=self.base, 3067 auto=raeting.AutoMode.once.value) 3068 keeping.clearAllKeep(otherData['dirpath']) 3069 other = self.createRoadStack(data=otherData, 3070 main=None, 3071 auto=otherData['auto'], 3072 ha=("", raeting.RAET_TEST_PORT)) 3073 3074 stacks = [main, other] 3075 3076 console.terse("\nBootstrap channel *********\n") 3077 self.join(other, main) # bootstrap channel 3078 for stack in [main, other]: 3079 self.assertEqual(len(stack.transactions), 0) 3080 self.assertEqual(len(stack.remotes), 1) 3081 remote = stack.remotes.values()[0] 3082 self.assertTrue(remote.joined) 3083 self.assertIs(remote.allowed, None) 3084 self.assertIs(remote.alived, None) 3085 self.assertIs(remote.reaped, None) 3086 3087 console.terse("\nTest manage remote *********\n") 3088 # ensure no timer expired 3089 remote = main.remotes.values()[0] 3090 self.assertFalse(remote.timer.expired) 3091 self.assertFalse(remote.reapTimer.expired) 3092 3093 # Manage 3094 main.clearStats() 3095 main.manage() 3096 remote = main.remotes.values()[0] 3097 3098 self.assertEqual(len(main.transactions), 0) # started no transactions because presence timer isn't expired 3099 self.assertTrue(remote.joined) 3100 self.assertIs(remote.allowed, None) 3101 self.assertIs(remote.alived, None) 3102 self.assertIs(remote.reaped, None) 3103 3104 # Service both sides 3105 self.serviceStacks(stacks) 3106 for stack in stacks: 3107 self.assertEqual(len(stack.transactions), 0) 3108 remote = main.remotes.values()[0] 3109 self.assertTrue(remote.joined) 3110 self.assertIs(remote.allowed, None) 3111 self.assertIs(remote.alived, None) 3112 self.assertIs(remote.reaped, None) 3113 3114 for stack in [main, other]: 3115 stack.server.close() 3116 stack.clearAllKeeps() 3117 3118 def testManageReaped(self): 3119 ''' 3120 Test stack manage reaped remotes (D1) 3121 ''' 3122 console.terse("{0}\n".format(self.testManageReaped.__doc__)) 3123 3124 mainData = self.createRoadData(name='main', 3125 base=self.base, 3126 auto=raeting.AutoMode.once.value) 3127 keeping.clearAllKeep(mainData['dirpath']) 3128 main = self.createRoadStack(data=mainData, 3129 main=True, 3130 auto=mainData['auto'], 3131 ha=None) 3132 3133 otherData = self.createRoadData(name='other', 3134 base=self.base, 3135 auto=raeting.AutoMode.once.value) 3136 keeping.clearAllKeep(otherData['dirpath']) 3137 other = self.createRoadStack(data=otherData, 3138 main=None, 3139 auto=otherData['auto'], 3140 ha=("", raeting.RAET_TEST_PORT)) 3141 3142 stacks = [main, other] 3143 3144 console.terse("\nBootstrap channel *********\n") 3145 self.join(other, main) # bootstrap channel 3146 for stack in [main, other]: 3147 self.assertEqual(len(stack.transactions), 0) 3148 self.assertEqual(len(stack.remotes), 1) 3149 remote = stack.remotes.values()[0] 3150 self.assertTrue(remote.joined) 3151 self.assertIs(remote.allowed, None) 3152 self.assertIs(remote.alived, None) 3153 self.assertIs(remote.reaped, None) 3154 3155 console.terse("\nTest manage the unallowed remote *********\n") 3156 console.terse("\nMake all expired so send alive and reap *********\n") 3157 # advance clock so both remote timers expire 3158 self.store.advanceStamp(stacking.RoadStack.Interim) 3159 remote = main.remotes.values()[0] 3160 self.assertTrue(remote.timer.expired) 3161 self.assertTrue(remote.reapTimer.expired) 3162 # force reap the remote 3163 remote.reaped = True 3164 3165 # Manage 3166 main.manage() 3167 remote = main.remotes.values()[0] 3168 3169 # check nothing has changed 3170 self.assertEqual(len(main.transactions), 0) # started no transactions 3171 self.assertTrue(remote.joined) 3172 self.assertIs(remote.allowed, None) 3173 self.assertIs(remote.alived, None) 3174 self.assertTrue(remote.reaped) 3175 3176 # Service both sides 3177 self.serviceStacks(stacks) 3178 # check nothing has changed 3179 for stack in stacks: 3180 self.assertEqual(len(stack.transactions), 0) 3181 remote = main.remotes.values()[0] 3182 self.assertTrue(remote.joined) 3183 self.assertIs(remote.allowed, None) 3184 self.assertIs(remote.alived, None) 3185 self.assertTrue(remote.reaped) 3186 3187 for stack in [main, other]: 3188 stack.server.close() 3189 stack.clearAllKeeps() 3190 3191 def testAliventUnjoined(self): 3192 ''' 3193 Test unjoined alivent receives alive (A1) 3194 Refresh no change alive, nack unjoined, remove 3195 ''' 3196 console.terse("{0}\n".format(self.testAliventUnjoined.__doc__)) 3197 3198 mainData = self.createRoadData(name='main', 3199 base=self.base, 3200 auto=raeting.AutoMode.once.value) 3201 keeping.clearAllKeep(mainData['dirpath']) 3202 main = self.createRoadStack(data=mainData, 3203 main=True, 3204 auto=mainData['auto'], 3205 ha=None) 3206 3207 otherData = self.createRoadData(name='other', 3208 base=self.base, 3209 auto=raeting.AutoMode.once.value) 3210 keeping.clearAllKeep(otherData['dirpath']) 3211 other = self.createRoadStack(data=otherData, 3212 main=None, 3213 auto=otherData['auto'], 3214 ha=("", raeting.RAET_TEST_PORT)) 3215 3216 stacks = [main, other] 3217 3218 console.terse("\nBootstrap channel *********\n") 3219 # remote on joiner side have to be joined and allowed 3220 self.join(other, main) # bootstrap channel 3221 self.allow(other, main) 3222 for stack in [main, other]: 3223 self.assertEqual(len(stack.transactions), 0) 3224 self.assertEqual(len(stack.remotes), 1) 3225 remote = stack.remotes.values()[0] 3226 self.assertTrue(remote.joined) 3227 self.assertTrue(remote.allowed) 3228 self.assertTrue(remote.alived) # fast alive 3229 3230 # force unjoin, unallow and unalive other side 3231 other.remotes.values()[0].joined = None 3232 other.remotes.values()[0].allowed = None 3233 other.remotes.values()[0].alived = None 3234 3235 self.assertTrue(main.remotes.values()[0].joined) 3236 self.assertTrue(main.remotes.values()[0].allowed) 3237 self.assertTrue(main.remotes.values()[0].alived) # fast alive 3238 self.assertIs(other.remotes.values()[0].joined, None) 3239 self.assertIs(other.remotes.values()[0].allowed, None) 3240 self.assertIs(other.remotes.values()[0].alived, None) 3241 for stack in [main, other]: 3242 remote = stack.remotes.values()[0] 3243 self.assertIs(remote.reaped, None) 3244 3245 console.terse("\nTest unjoined alivent *********\n") 3246 main.alive() # no cascade alive, unjoined so would be joined as a final result 3247 self.serviceStack(main, duration=0.1) 3248 self.serviceStack(other, duration=0.1) 3249 self.assertEqual(len(other.transactions), 0) # transaction was removed 3250 self.assertIn('unjoined_alive_attempt', other.stats) # the reason is 'unjoined' 3251 self.assertEqual(other.stats['unjoined_alive_attempt'], 1) 3252 self.assertEqual(len(main.transactions), 1) 3253 self.assertEqual(len(other.remotes), 1) 3254 remote = other.remotes.values()[0] 3255 self.assertIs(remote.joined, None) 3256 self.assertIs(remote.allowed, None) 3257 self.assertIs(remote.alived, None) # not changed 3258 self.assertIs(remote.reaped, None) 3259 3260 # alive nacked as 'unjoined', the result would be join attempt from aliver side 3261 self.serviceStacks(stacks) 3262 for stack in stacks: 3263 self.assertEqual(len(stack.transactions), 0) 3264 self.assertEqual(len(stack.remotes), 1) 3265 remote = other.remotes.values()[0] 3266 self.assertTrue(remote.joined) 3267 self.assertIs(remote.allowed, None) 3268 self.assertIs(remote.alived, None) 3269 self.assertIs(remote.reaped, None) 3270 3271 for stack in [main, other]: 3272 stack.server.close() 3273 stack.clearAllKeeps() 3274 3275 def testAliventUnallowed(self): 3276 ''' 3277 Test unallowed alivent receives alive (A2) 3278 Refresh no change alive, nack unallowed, remove 3279 ''' 3280 console.terse("{0}\n".format(self.testAliventUnallowed.__doc__)) 3281 3282 mainData = self.createRoadData(name='main', 3283 base=self.base, 3284 auto=raeting.AutoMode.once.value) 3285 keeping.clearAllKeep(mainData['dirpath']) 3286 main = self.createRoadStack(data=mainData, 3287 main=True, 3288 auto=mainData['auto'], 3289 ha=None) 3290 3291 otherData = self.createRoadData(name='other', 3292 base=self.base, 3293 auto=raeting.AutoMode.once.value) 3294 keeping.clearAllKeep(otherData['dirpath']) 3295 other = self.createRoadStack(data=otherData, 3296 main=None, 3297 auto=otherData['auto'], 3298 ha=("", raeting.RAET_TEST_PORT)) 3299 3300 stacks = [main, other] 3301 3302 console.terse("\nBootstrap channel *********\n") 3303 # remote on joiner side have to be joined and allowed 3304 self.join(other, main) # bootstrap channel 3305 self.allow(other, main) 3306 for stack in [main, other]: 3307 self.assertEqual(len(stack.transactions), 0) 3308 self.assertEqual(len(stack.remotes), 1) 3309 remote = stack.remotes.values()[0] 3310 self.assertTrue(remote.joined) 3311 self.assertTrue(remote.allowed) 3312 self.assertTrue(remote.alived) # fast alive 3313 3314 # force unallow and unalive other side 3315 other.remotes.values()[0].allowed = None 3316 other.remotes.values()[0].alived = None 3317 3318 self.assertTrue(main.remotes.values()[0].allowed) 3319 self.assertTrue(main.remotes.values()[0].alived) 3320 self.assertIs(other.remotes.values()[0].allowed, None) 3321 self.assertIs(other.remotes.values()[0].alived, None) 3322 for stack in [main, other]: 3323 remote = stack.remotes.values()[0] 3324 self.assertTrue(remote.joined) 3325 self.assertIs(remote.reaped, None) 3326 3327 console.terse("\nTest unallowed alivent *********\n") 3328 main.alive() # no cascade alive, unallowed so would be allowed as a final result 3329 self.serviceStack(main, duration=0.1) 3330 self.serviceStack(other, duration=0.1) 3331 self.assertEqual(len(other.transactions), 0) # transaction was removed 3332 self.assertIn('unallowed_alive_attempt', other.stats) # the reason is 'unallowed' 3333 self.assertEqual(other.stats['unallowed_alive_attempt'], 1) 3334 self.assertEqual(len(main.transactions), 1) 3335 self.assertEqual(len(other.remotes), 1) 3336 remote = other.remotes.values()[0] 3337 self.assertTrue(remote.joined) 3338 self.assertIs(remote.allowed, None) 3339 self.assertIs(remote.alived, None) # not changed 3340 self.assertIs(remote.reaped, None) 3341 3342 # alive nacked as 'unallowed', the result would be allow attempt from aliver side 3343 self.serviceStacks(stacks) 3344 for stack in stacks: 3345 self.assertEqual(len(stack.transactions), 0) 3346 self.assertEqual(len(stack.remotes), 1) 3347 remote = other.remotes.values()[0] 3348 self.assertTrue(remote.joined) 3349 self.assertTrue(remote.allowed) 3350 self.assertTrue(remote.alived) # fast alive 3351 self.assertIs(remote.reaped, None) 3352 3353 for stack in [main, other]: 3354 stack.server.close() 3355 stack.clearAllKeeps() 3356 3357 def testAliventJoinedAllowed(self): 3358 ''' 3359 Test joined and allowed alivent receives alive (C1) 3360 Ack, refresh alive, unreap 3361 ''' 3362 console.terse("{0}\n".format(self.testAliventUnallowed.__doc__)) 3363 3364 mainData = self.createRoadData(name='main', 3365 base=self.base, 3366 auto=raeting.AutoMode.once.value) 3367 keeping.clearAllKeep(mainData['dirpath']) 3368 main = self.createRoadStack(data=mainData, 3369 main=True, 3370 auto=mainData['auto'], 3371 ha=None) 3372 3373 otherData = self.createRoadData(name='other', 3374 base=self.base, 3375 auto=raeting.AutoMode.once.value) 3376 keeping.clearAllKeep(otherData['dirpath']) 3377 other = self.createRoadStack(data=otherData, 3378 main=True, # only main can reap and unreap 3379 auto=otherData['auto'], 3380 ha=("", raeting.RAET_TEST_PORT)) 3381 3382 stacks = [main, other] 3383 3384 console.terse("\nBootstrap channel *********\n") 3385 # remote on joiner side have to be joined and allowed 3386 self.join(other, main) # bootstrap channel 3387 self.allow(other, main) 3388 for stack in [main, other]: 3389 self.assertEqual(len(stack.transactions), 0) 3390 self.assertEqual(len(stack.remotes), 1) 3391 remote = stack.remotes.values()[0] 3392 self.assertTrue(remote.joined) 3393 self.assertTrue(remote.allowed) 3394 self.assertTrue(remote.alived) # fast alive 3395 self.assertIs(remote.reaped, None) 3396 3397 # force unalive and reap to check it would be unreaped as result 3398 other.remotes.values()[0].alived = None 3399 other.remotes.values()[0].reaped = True 3400 3401 console.terse("\nTest joined allowed alivent *********\n") 3402 main.alive() # no cascade alive, joined and allowed so ack, refresh alive and unreap 3403 self.serviceStack(main, duration=0.1) 3404 self.serviceStack(other,duration=0.1) 3405 self.assertEqual(len(other.transactions), 0) # transaction finished 3406 self.assertIn('alive_complete', other.stats) # the reason is 'unallowed' 3407 self.assertEqual(other.stats['alive_complete'], 1) 3408 self.assertEqual(len(main.transactions), 1) 3409 self.assertEqual(len(other.remotes), 1) 3410 remote = other.remotes.values()[0] 3411 self.assertTrue(remote.joined) 3412 self.assertTrue(remote.allowed) 3413 self.assertTrue(remote.alived) # alive updated 3414 self.assertFalse(remote.reaped) # unreaped 3415 3416 # alive acked 3417 self.serviceStacks(stacks) 3418 for stack in stacks: 3419 self.assertEqual(len(stack.transactions), 0) 3420 self.assertEqual(len(stack.remotes), 1) 3421 remote = other.remotes.values()[0] 3422 self.assertTrue(remote.joined) 3423 self.assertTrue(remote.allowed) 3424 self.assertTrue(remote.alived) 3425 self.assertIs(main.remotes.values()[0].reaped, None) 3426 self.assertFalse(other.remotes.values()[0].reaped) 3427 3428 for stack in [main, other]: 3429 stack.server.close() 3430 stack.clearAllKeeps() 3431 3432 def testAliverReceiveNack(self): 3433 ''' 3434 Test aliver receive nack: nack, refuse (A2) and reject (A5) 3435 Other cases which are ack (A1), unjoined (A3) and unallowed (A4) are covered by other tests 3436 ''' 3437 console.terse("{0}\n".format(self.testAliverReceiveNack.__doc__)) 3438 3439 mainData = self.createRoadData(name='main', 3440 base=self.base, 3441 auto=raeting.AutoMode.once.value) 3442 keeping.clearAllKeep(mainData['dirpath']) 3443 main = self.createRoadStack(data=mainData, 3444 main=True, 3445 auto=mainData['auto'], 3446 ha=None) 3447 3448 otherData = self.createRoadData(name='other', 3449 base=self.base, 3450 auto=raeting.AutoMode.once.value) 3451 keeping.clearAllKeep(otherData['dirpath']) 3452 other = self.createRoadStack(data=otherData, 3453 main=None, 3454 auto=otherData['auto'], 3455 ha=("", raeting.RAET_TEST_PORT)) 3456 3457 stacks = [main, other] 3458 3459 console.terse("\nBootstrap channel *********\n") 3460 # remote on joiner side have to be joined and allowed 3461 self.join(other, main) # bootstrap channel 3462 self.allow(other, main) 3463 self.alive(main, other) 3464 for stack in [main, other]: 3465 self.assertEqual(len(stack.transactions), 0) 3466 self.assertEqual(len(stack.remotes), 1) 3467 remote = stack.remotes.values()[0] 3468 self.assertTrue(remote.joined) 3469 self.assertTrue(remote.allowed) 3470 self.assertTrue(remote.alived) 3471 self.assertIs(remote.reaped, None) 3472 3473 console.terse("\nTest aliver receive nack *********\n") 3474 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3475 self.serviceStack(main, duration=0.1) 3476 self.answerAlive(other, kind=raeting.PcktKind.nack.value) 3477 3478 self.serviceStacks(stacks) 3479 for stack in stacks: 3480 self.assertEqual(len(stack.transactions), 0) 3481 self.assertEqual(len(stack.remotes), 1) 3482 remote = other.remotes.values()[0] 3483 self.assertTrue(remote.joined) 3484 self.assertTrue(remote.allowed) 3485 self.assertTrue(remote.alived) # alived status isn't changed 3486 self.assertIs(remote.reaped, None) 3487 self.assertIn('aliver_transaction_failure', main.stats) # transaction failed 3488 self.assertEqual(main.stats['aliver_transaction_failure'], 1) 3489 3490 main.clearStats() 3491 console.terse("\nTest aliver receive refuse *********\n") 3492 main.alive() # no cascade alive, aliver will set alive to False and remove transaction 3493 self.serviceStack(main, duration=0.1) 3494 self.answerAlive(other, kind=raeting.PcktKind.refuse.value) 3495 3496 self.serviceStacks(stacks) 3497 for stack in stacks: 3498 self.assertEqual(len(stack.transactions), 0) 3499 self.assertEqual(len(stack.remotes), 1) 3500 remote = other.remotes.values()[0] 3501 self.assertTrue(remote.joined) 3502 self.assertTrue(remote.allowed) 3503 self.assertTrue(remote.alived) # alived status isn't changed 3504 self.assertIs(remote.reaped, None) 3505 self.assertIn('aliver_transaction_failure', main.stats) # transaction failed 3506 self.assertEqual(main.stats['aliver_transaction_failure'], 1) 3507 3508 main.clearStats() 3509 console.terse("\nTest aliver receive reject *********\n") 3510 main.alive() # no cascade alive, aliver will set alive to False and remove transaction 3511 self.serviceStack(main, duration=0.1) 3512 self.answerAlive(other, kind=raeting.PcktKind.reject.value) 3513 3514 self.serviceStacks(stacks) 3515 for stack in stacks: 3516 self.assertEqual(len(stack.transactions), 0) 3517 self.assertEqual(len(stack.remotes), 1) 3518 remote = other.remotes.values()[0] 3519 self.assertTrue(remote.joined) 3520 self.assertTrue(remote.allowed) 3521 self.assertIs(remote.reaped, None) 3522 self.assertIs(main.remotes.values()[0].alived, False) # alive is set to false 3523 self.assertIn('aliver_transaction_failure', main.stats) # transaction failed 3524 self.assertEqual(main.stats['aliver_transaction_failure'], 1) 3525 3526 main.clearStats() 3527 console.terse("\nTest invalid (unexpected) nack kind *********\n") 3528 main.alive() # no cascade alive, aliver will set alive to False and remove transaction 3529 self.serviceStack(main, duration=0.1) 3530 self.answerAlive(other, kind=raeting.PcktKind.unknown.value) # unknown is invalid kind for nack. it will be set to nack 3531 self.serviceStack(other) # send nack 3532 self.serviceStack(main) # receive nack 3533 3534 self.assertEqual(len(main.transactions), 1) # transaction wasn't dropped, will attempt to redo 3535 self.assertEqual(len(other.transactions), 0) 3536 3537 self.serviceStacks(stacks) 3538 for stack in stacks: 3539 self.assertEqual(len(stack.transactions), 0) 3540 self.assertEqual(len(stack.remotes), 1) 3541 remote = other.remotes.values()[0] 3542 self.assertTrue(remote.joined) 3543 self.assertTrue(remote.allowed) 3544 self.assertIs(remote.reaped, None) 3545 self.assertTrue(main.remotes.values()[0].alived) # alived after redo 3546 for stack in [main, other]: 3547 stack.server.close() 3548 stack.clearAllKeeps() 3549 3550 def testAliverAlivePackError(self): 3551 ''' 3552 Test packet.pack() throws error in aliver.alive() (coverage) 3553 ''' 3554 console.terse("{0}\n".format(self.testAliverAlivePackError.__doc__)) 3555 3556 mainData = self.createRoadData(name='main', 3557 base=self.base, 3558 auto=raeting.AutoMode.once.value) 3559 keeping.clearAllKeep(mainData['dirpath']) 3560 main = self.createRoadStack(data=mainData, 3561 main=True, 3562 auto=mainData['auto'], 3563 ha=None) 3564 3565 otherData = self.createRoadData(name='other', 3566 base=self.base, 3567 auto=raeting.AutoMode.once.value) 3568 keeping.clearAllKeep(otherData['dirpath']) 3569 other = self.createRoadStack(data=otherData, 3570 main=None, 3571 auto=otherData['auto'], 3572 ha=("", raeting.RAET_TEST_PORT)) 3573 3574 stacks = [main, other] 3575 3576 console.terse("\nBootstrap channel *********\n") 3577 # remote on joiner side have to be joined and allowed 3578 self.join(other, main) # bootstrap channel 3579 self.allow(other, main) 3580 self.alive(main, other) 3581 for stack in [main, other]: 3582 self.assertEqual(len(stack.transactions), 0) 3583 self.assertEqual(len(stack.remotes), 1) 3584 remote = stack.remotes.values()[0] 3585 self.assertTrue(remote.joined) 3586 self.assertTrue(remote.allowed) 3587 self.assertTrue(remote.alived) 3588 self.assertIs(remote.reaped, None) 3589 3590 main.clearStats() 3591 console.terse("\nTest aliver packet.pack error *********\n") 3592 default_size = raeting.UDP_MAX_PACKET_SIZE 3593 raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will throw PacketError 3594 main.alive() # will fail with packing error 3595 raeting.UDP_MAX_PACKET_SIZE = default_size 3596 3597 self.assertEqual(len(main.transactions), 0) 3598 self.assertEqual(len(main.remotes), 1) 3599 remote = other.remotes.values()[0] 3600 self.assertTrue(remote.joined) 3601 self.assertTrue(remote.allowed) 3602 self.assertTrue(remote.alived) # alived status isn't changed 3603 self.assertIs(remote.reaped, None) 3604 self.assertIn('packing_error', main.stats) # transaction failed 3605 self.assertEqual(main.stats['packing_error'], 1) 3606 3607 for stack in [main, other]: 3608 stack.server.close() 3609 stack.clearAllKeeps() 3610 3611 def testAliverErrorParseInner(self): 3612 ''' 3613 Test aliver behavior if packet.parseInner() fail (coverage) 3614 ''' 3615 console.terse("{0}\n".format(self.testAliverErrorParseInner.__doc__)) 3616 3617 mainData = self.createRoadData(name='main', 3618 base=self.base, 3619 auto=raeting.AutoMode.once.value) 3620 keeping.clearAllKeep(mainData['dirpath']) 3621 main = self.createRoadStack(data=mainData, 3622 main=True, 3623 auto=mainData['auto'], 3624 ha=None) 3625 3626 otherData = self.createRoadData(name='other', 3627 base=self.base, 3628 auto=raeting.AutoMode.once.value) 3629 keeping.clearAllKeep(otherData['dirpath']) 3630 other = self.createRoadStack(data=otherData, 3631 main=None, 3632 auto=otherData['auto'], 3633 ha=("", raeting.RAET_TEST_PORT)) 3634 3635 stacks = [main, other] 3636 3637 console.terse("\nBootstrap channel *********\n") 3638 # remote on joiner side have to be joined and allowed 3639 self.join(other, main) # bootstrap channel 3640 self.allow(other, main) 3641 self.alive(main, other) 3642 for stack in [main, other]: 3643 self.assertEqual(len(stack.transactions), 0) 3644 self.assertEqual(len(stack.remotes), 1) 3645 remote = stack.remotes.values()[0] 3646 self.assertTrue(remote.joined) 3647 self.assertTrue(remote.allowed) 3648 self.assertTrue(remote.alived) 3649 self.assertIs(remote.reaped, None) 3650 3651 main.clearStats() 3652 console.terse("\nTest aliver receive broken ack *********\n") 3653 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3654 self.serviceStack(main, duration=0.125) # main send alive to other 3655 self.answerAlive(other, kind=raeting.PcktKind.ack.value, dataMod={'ck': -1}) # other receive and answer 3656 self.serviceStack(other, duration=0.125) # other send the answer to main 3657 self.serviceStack(main, duration=0.125) # main handle the answer 3658 3659 self.assertEqual(len(main.transactions), 1) # transaction wasn't handled 3660 self.assertEqual(len(main.remotes), 1) 3661 self.assertIn('parsing_inner_error', main.stats) # transaction failed 3662 self.assertEqual(main.stats['parsing_inner_error'], 1) 3663 3664 self.serviceStacks(stacks) # retry and finish transaction 3665 for stack in stacks: 3666 self.assertEqual(len(stack.transactions), 0) 3667 self.assertEqual(len(stack.remotes), 1) 3668 remote = other.remotes.values()[0] 3669 self.assertTrue(remote.joined) 3670 self.assertTrue(remote.allowed) 3671 self.assertTrue(remote.alived) # alived status isn't changed 3672 self.assertIs(remote.reaped, None) 3673 self.assertIn('alive_complete', main.stats) # transaction failed 3674 self.assertEqual(main.stats['alive_complete'], 1) 3675 3676 main.clearStats() 3677 console.terse("\nTest aliver receive broken refuse *********\n") 3678 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3679 self.serviceStack(main, duration=0.125) # main send alive to other 3680 self.answerAlive(other, kind=raeting.PcktKind.refuse.value, dataMod={'ck': -1}) # other receive and answer 3681 self.serviceStack(other, duration=0.125) # other send the answer to main 3682 self.serviceStack(main, duration=0.125) # main handle the answer 3683 3684 self.assertEqual(len(main.transactions), 1) # transaction wasn't handled 3685 self.assertEqual(len(main.remotes), 1) 3686 self.assertIn('parsing_inner_error', main.stats) # transaction failed 3687 self.assertEqual(main.stats['parsing_inner_error'], 1) 3688 3689 self.serviceStacks(stacks) # retry and finish transaction 3690 for stack in stacks: 3691 self.assertEqual(len(stack.transactions), 0) 3692 self.assertEqual(len(stack.remotes), 1) 3693 remote = other.remotes.values()[0] 3694 self.assertTrue(remote.joined) 3695 self.assertTrue(remote.allowed) 3696 self.assertTrue(remote.alived) # alived status isn't changed 3697 self.assertIs(remote.reaped, None) 3698 self.assertIn('alive_complete', main.stats) # transaction failed 3699 self.assertEqual(main.stats['alive_complete'], 1) 3700 3701 main.clearStats() 3702 console.terse("\nTest aliver receive broken reject *********\n") 3703 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3704 self.serviceStack(main, duration=0.125) # main send alive to other 3705 self.answerAlive(other, kind=raeting.PcktKind.reject.value, dataMod={'ck': -1}) # other receive and answer 3706 self.serviceStack(other, duration=0.125) # other send the answer to main 3707 self.serviceStack(main, duration=0.125) # main handle the answer 3708 3709 self.assertEqual(len(main.transactions), 1) # transaction wasn't handled 3710 self.assertEqual(len(main.remotes), 1) 3711 self.assertIn('parsing_inner_error', main.stats) # transaction failed 3712 self.assertEqual(main.stats['parsing_inner_error'], 1) 3713 3714 self.serviceStacks(stacks) # retry and finish transaction 3715 for stack in stacks: 3716 self.assertEqual(len(stack.transactions), 0) 3717 self.assertEqual(len(stack.remotes), 1) 3718 remote = other.remotes.values()[0] 3719 self.assertTrue(remote.joined) 3720 self.assertTrue(remote.allowed) 3721 self.assertTrue(remote.alived) # alived status isn't changed 3722 self.assertIs(remote.reaped, None) 3723 self.assertIn('alive_complete', main.stats) # transaction failed 3724 self.assertEqual(main.stats['alive_complete'], 1) 3725 3726 main.clearStats() 3727 console.terse("\nTest aliver receive broken unjoined *********\n") 3728 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3729 self.serviceStack(main, duration=0.125) # main send alive to other 3730 self.answerAlive(other, kind=raeting.PcktKind.unjoined.value, dataMod={'ck': -1}) # other receive and answer 3731 self.serviceStack(other, duration=0.125) # other send the answer to main 3732 self.serviceStack(main, duration=0.125) # main handle the answer 3733 3734 self.assertEqual(len(main.transactions), 1) # transaction wasn't handled 3735 self.assertEqual(len(main.remotes), 1) 3736 self.assertIn('parsing_inner_error', main.stats) # transaction failed 3737 self.assertEqual(main.stats['parsing_inner_error'], 1) 3738 3739 self.serviceStacks(stacks) # retry and finish transaction 3740 for stack in stacks: 3741 self.assertEqual(len(stack.transactions), 0) 3742 self.assertEqual(len(stack.remotes), 1) 3743 remote = other.remotes.values()[0] 3744 self.assertTrue(remote.joined) 3745 self.assertTrue(remote.allowed) 3746 self.assertTrue(remote.alived) # alived status isn't changed 3747 self.assertIs(remote.reaped, None) 3748 self.assertIn('alive_complete', main.stats) # transaction failed 3749 self.assertEqual(main.stats['alive_complete'], 1) 3750 3751 main.clearStats() 3752 console.terse("\nTest aliver receive broken unallowed *********\n") 3753 main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction 3754 self.serviceStack(main, duration=0.125) # main send alive to other 3755 self.answerAlive(other, kind=raeting.PcktKind.unallowed.value, dataMod={'ck': -1}) # other receive and answer 3756 self.serviceStack(other, duration=0.125) # other send the answer to main 3757 self.serviceStack(main, duration=0.125) # main handle the answer 3758 3759 self.assertEqual(len(main.transactions), 1) # transaction wasn't handled 3760 self.assertEqual(len(main.remotes), 1) 3761 self.assertIn('parsing_inner_error', main.stats) # transaction failed 3762 self.assertEqual(main.stats['parsing_inner_error'], 1) 3763 3764 self.serviceStacks(stacks) # retry and finish transaction 3765 for stack in stacks: 3766 self.assertEqual(len(stack.transactions), 0) 3767 self.assertEqual(len(stack.remotes), 1) 3768 remote = other.remotes.values()[0] 3769 self.assertTrue(remote.joined) 3770 self.assertTrue(remote.allowed) 3771 self.assertTrue(remote.alived) # alived status isn't changed 3772 self.assertIs(remote.reaped, None) 3773 self.assertIn('alive_complete', main.stats) # transaction failed 3774 self.assertEqual(main.stats['alive_complete'], 1) 3775 3776 for stack in [main, other]: 3777 stack.server.close() 3778 stack.clearAllKeeps() 3779 3780 def testAliventAliveErrorParseInner(self): 3781 ''' 3782 Test for Alivent.alive() error on parse packet inner. (coverage) 3783 ''' 3784 console.terse("{0}\n".format(self.testAliventAliveErrorParseInner.__doc__)) 3785 3786 mainData = self.createRoadData(name='main', 3787 base=self.base, 3788 auto=raeting.AutoMode.once.value) 3789 keeping.clearAllKeep(mainData['dirpath']) 3790 main = self.createRoadStack(data=mainData, 3791 main=True, 3792 auto=mainData['auto'], 3793 ha=None) 3794 3795 otherData = self.createRoadData(name='other', 3796 base=self.base, 3797 auto=raeting.AutoMode.once.value) 3798 keeping.clearAllKeep(otherData['dirpath']) 3799 other = self.createRoadStack(data=otherData, 3800 main=True, # only main can reap and unreap 3801 auto=otherData['auto'], 3802 ha=("", raeting.RAET_TEST_PORT)) 3803 3804 stacks = [main, other] 3805 3806 console.terse("\nBootstrap channel *********\n") 3807 # remote on joiner side have to be joined and allowed 3808 self.join(other, main) # bootstrap channel 3809 self.allow(other, main) 3810 for stack in [main, other]: 3811 self.assertEqual(len(stack.transactions), 0) 3812 self.assertEqual(len(stack.remotes), 1) 3813 remote = stack.remotes.values()[0] 3814 self.assertTrue(remote.joined) 3815 self.assertTrue(remote.allowed) 3816 self.assertTrue(remote.alived) # fast alive 3817 self.assertIs(remote.reaped, None) 3818 3819 other.remotes.values()[0].alived = None # reset alived 3820 other.clearStats() 3821 console.terse("\nTest alive with broken coat kind *********\n") 3822 self.aliveBrokenInner(main) # alive with broken coat kind 3823 self.serviceStack(main, duration=0.1) 3824 self.serviceStack(other,duration=0.1) 3825 self.assertEqual(len(other.transactions), 0) # transaction finished 3826 self.assertIn('parsing_inner_error', other.stats) # Error ocured 3827 self.assertEqual(other.stats['parsing_inner_error'], 1) 3828 self.assertEqual(len(main.transactions), 1) 3829 self.assertEqual(len(other.remotes), 1) 3830 remote = other.remotes.values()[0] 3831 self.assertTrue(remote.joined) 3832 self.assertTrue(remote.allowed) 3833 self.assertIsNone(remote.alived) # alived status isn't changed 3834 self.assertIsNone(remote.reaped) 3835 3836 # redo the broken packet then drop it 3837 self.serviceStacks(stacks, duration=10.0) 3838 for stack in stacks: 3839 self.assertEqual(len(stack.transactions), 0) 3840 self.assertEqual(len(stack.remotes), 1) 3841 remote = other.remotes.values()[0] 3842 self.assertTrue(remote.joined) 3843 self.assertTrue(remote.allowed) 3844 self.assertFalse(remote.alived) # dead 3845 self.assertIs(main.remotes.values()[0].reaped, None) 3846 self.assertFalse(other.remotes.values()[0].reaped) 3847 3848 for stack in [main, other]: 3849 stack.server.close() 3850 stack.clearAllKeeps() 3851 3852 def testAliventAliveErrorPacketPack(self): 3853 ''' 3854 Test for Alivent.alive() error on packing a packet. (coverage) 3855 ''' 3856 console.terse("{0}\n".format(self.testAliventAliveErrorPacketPack.__doc__)) 3857 3858 mainData = self.createRoadData(name='main', 3859 base=self.base, 3860 auto=raeting.AutoMode.once.value) 3861 keeping.clearAllKeep(mainData['dirpath']) 3862 main = self.createRoadStack(data=mainData, 3863 main=True, 3864 auto=mainData['auto'], 3865 ha=None) 3866 3867 otherData = self.createRoadData(name='other', 3868 base=self.base, 3869 auto=raeting.AutoMode.once.value) 3870 keeping.clearAllKeep(otherData['dirpath']) 3871 other = self.createRoadStack(data=otherData, 3872 main=True, # only main can reap and unreap 3873 auto=otherData['auto'], 3874 ha=("", raeting.RAET_TEST_PORT)) 3875 3876 stacks = [main, other] 3877 3878 console.terse("\nBootstrap channel *********\n") 3879 # remote on joiner side have to be joined and allowed 3880 self.join(other, main) # bootstrap channel 3881 self.allow(other, main) 3882 for stack in [main, other]: 3883 self.assertEqual(len(stack.transactions), 0) 3884 self.assertEqual(len(stack.remotes), 1) 3885 remote = stack.remotes.values()[0] 3886 self.assertTrue(remote.joined) 3887 self.assertTrue(remote.allowed) 3888 self.assertTrue(remote.alived) 3889 self.assertIs(remote.reaped, None) 3890 3891 other.remotes.values()[0].alived = None 3892 other.clearStats() 3893 console.terse("\nTest alivent alive with packet pack error *********\n") 3894 main.alive() # alive 3895 self.serviceStack(main, duration=0.1) # aliver send 3896 default_size = raeting.UDP_MAX_PACKET_SIZE # backup 3897 raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will fail 3898 self.serviceStack(other,duration=0.1) # alivent receive 3899 raeting.UDP_MAX_PACKET_SIZE = default_size 3900 3901 self.assertEqual(len(other.transactions), 0) # transaction removed 3902 self.assertIn('packing_error', other.stats) # Error ocured 3903 self.assertEqual(other.stats['packing_error'], 1) 3904 self.assertEqual(len(main.transactions), 1) 3905 self.assertEqual(len(other.remotes), 1) 3906 remote = other.remotes.values()[0] 3907 self.assertTrue(remote.joined) 3908 self.assertTrue(remote.allowed) 3909 self.assertIsNone(remote.alived) 3910 self.assertIsNone(remote.reaped) 3911 3912 # redo with normal udp packet limit 3913 self.serviceStacks(stacks) 3914 for stack in stacks: 3915 self.assertEqual(len(stack.transactions), 0) 3916 self.assertEqual(len(stack.remotes), 1) 3917 remote = other.remotes.values()[0] 3918 self.assertTrue(remote.joined) 3919 self.assertTrue(remote.allowed) 3920 self.assertTrue(remote.alived) # redo 3921 self.assertIs(main.remotes.values()[0].reaped, None) 3922 self.assertFalse(other.remotes.values()[0].reaped) 3923 3924 for stack in [main, other]: 3925 stack.server.close() 3926 stack.clearAllKeeps() 3927 3928 def testAliventNackErrorPacketPack(self): 3929 ''' 3930 Test for Alivent.nack() error on packing a packet. (coverage) 3931 ''' 3932 console.terse("{0}\n".format(self.testAliventNackErrorPacketPack.__doc__)) 3933 3934 mainData = self.createRoadData(name='main', 3935 base=self.base, 3936 auto=raeting.AutoMode.once.value) 3937 keeping.clearAllKeep(mainData['dirpath']) 3938 main = self.createRoadStack(data=mainData, 3939 main=True, 3940 auto=mainData['auto'], 3941 ha=None) 3942 3943 otherData = self.createRoadData(name='other', 3944 base=self.base, 3945 auto=raeting.AutoMode.once.value) 3946 keeping.clearAllKeep(otherData['dirpath']) 3947 other = self.createRoadStack(data=otherData, 3948 main=True, # only main can reap and unreap 3949 auto=otherData['auto'], 3950 ha=("", raeting.RAET_TEST_PORT)) 3951 3952 stacks = [main, other] 3953 3954 console.terse("\nBootstrap channel *********\n") 3955 # remote on joiner side have to be joined and allowed 3956 self.join(other, main) # bootstrap channel 3957 self.allow(other, main) 3958 for stack in [main, other]: 3959 self.assertEqual(len(stack.transactions), 0) 3960 self.assertEqual(len(stack.remotes), 1) 3961 remote = stack.remotes.values()[0] 3962 self.assertTrue(remote.joined) 3963 self.assertTrue(remote.allowed) 3964 self.assertTrue(remote.alived) # fast alive 3965 self.assertIs(remote.reaped, None) 3966 3967 # force unallow and unalive joinent 3968 other.remotes.values()[0].allowed = None 3969 other.remotes.values()[0].alived = None 3970 self.assertIsNone(other.remotes.values()[0].allowed) 3971 self.assertIsNone(other.remotes.values()[0].alived) 3972 3973 other.clearStats() 3974 console.terse("\nTest alivent nack with packet pack error *********\n") 3975 main.alive() # alive 3976 self.serviceStack(main, duration=0.1) # aliver send 3977 default_size = raeting.UDP_MAX_PACKET_SIZE # backup 3978 raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will fail 3979 self.serviceStack(other,duration=0.1) # alivent receive 3980 raeting.UDP_MAX_PACKET_SIZE = default_size 3981 3982 self.assertEqual(len(other.transactions), 0) # transaction removed 3983 self.assertIn('packing_error', other.stats) # Error ocured 3984 self.assertEqual(other.stats['packing_error'], 1) 3985 self.assertEqual(len(main.transactions), 1) 3986 self.assertEqual(len(other.remotes), 1) 3987 remote = other.remotes.values()[0] 3988 self.assertTrue(remote.joined) 3989 self.assertIsNone(remote.allowed) 3990 self.assertIsNone(remote.alived) 3991 self.assertIsNone(remote.reaped) 3992 3993 # redo with normal udp packet limit 3994 self.serviceStacks(stacks) 3995 for stack in stacks: 3996 self.assertEqual(len(stack.transactions), 0) 3997 self.assertEqual(len(stack.remotes), 1) 3998 remote = other.remotes.values()[0] 3999 self.assertTrue(remote.joined) 4000 self.assertTrue(remote.allowed) 4001 self.assertTrue(remote.alived) # fast alive 4002 self.assertIs(main.remotes.values()[0].reaped, None) 4003 self.assertFalse(other.remotes.values()[0].reaped) 4004 4005 for stack in [main, other]: 4006 stack.server.close() 4007 stack.clearAllKeeps() 4008 4009 def testAliventReceiveRequest(self): 4010 ''' 4011 Test for Alivent.receive() (coverage) 4012 ''' 4013 console.terse("{0}\n".format(self.testAliventReceiveRequest.__doc__)) 4014 4015 mainData = self.createRoadData(name='main', 4016 base=self.base, 4017 auto=raeting.AutoMode.once.value) 4018 keeping.clearAllKeep(mainData['dirpath']) 4019 main = self.createRoadStack(data=mainData, 4020 main=True, 4021 auto=mainData['auto'], 4022 ha=None) 4023 4024 otherData = self.createRoadData(name='other', 4025 base=self.base, 4026 auto=raeting.AutoMode.once.value) 4027 keeping.clearAllKeep(otherData['dirpath']) 4028 other = self.createRoadStack(data=otherData, 4029 main=True, # only main can reap and unreap 4030 auto=otherData['auto'], 4031 ha=("", raeting.RAET_TEST_PORT)) 4032 4033 stacks = [main, other] 4034 4035 console.terse("\nBootstrap channel *********\n") 4036 # remote on joiner side have to be joined and allowed 4037 self.join(other, main) # bootstrap channel 4038 self.allow(other, main) 4039 for stack in [main, other]: 4040 self.assertEqual(len(stack.transactions), 0) 4041 self.assertEqual(len(stack.remotes), 1) 4042 remote = stack.remotes.values()[0] 4043 self.assertTrue(remote.joined) 4044 self.assertTrue(remote.allowed) 4045 self.assertTrue(remote.alived) # fast alive 4046 self.assertIs(remote.reaped, None) 4047 4048 other.remotes.values()[0].alived = None # reset alived 4049 other.clearStats() 4050 console.terse("\nTest alivent receive request *********\n") 4051 main.alive() # alive from main to other 4052 self.serviceStack(main, duration=0.1) # Send alive 4053 # receive alive on other in custom way: use transaction receive instead of stack.correspond() 4054 other.serviceReceives() 4055 # handle rx 4056 raw, sa = other.rxes.popleft() 4057 console.verbose("{0} received packet\n{1}\n".format(other.name, raw)) 4058 packet = packeting.RxPacket(stack=other, packed=raw) 4059 packet.parseOuter() 4060 sh, sp = sa 4061 packet.data.update(sh=sh, sp=sp) 4062 # process rx 4063 nuid = packet.data['de'] 4064 self.assertNotEqual(nuid, 0) 4065 remote = other.remotes.get(nuid, None) 4066 self.assertIsNotNone(remote) 4067 rsid = packet.data['si'] 4068 self.assertNotEqual(rsid, 0) 4069 remote.rsid = rsid 4070 # create transaction 4071 data = odict(hk=other.Hk, bk=other.Bk, fk=other.Fk, ck=other.Ck) 4072 alivent = transacting.Alivent(stack=other, 4073 remote=remote, 4074 bcst=packet.data['bf'], 4075 sid=packet.data['si'], 4076 tid=packet.data['ti'], 4077 txData=data, 4078 rxPacket=packet) 4079 # receive 4080 alivent.receive(packet) 4081 self.serviceStacks(stacks) # Process alive 4082 4083 for stack in stacks: 4084 self.assertEqual(len(stack.transactions), 0) 4085 self.assertEqual(len(stack.remotes), 1) 4086 remote = other.remotes.values()[0] 4087 self.assertTrue(remote.joined) 4088 self.assertTrue(remote.allowed) 4089 self.assertTrue(remote.alived) 4090 4091 for stack in [main, other]: 4092 stack.server.close() 4093 stack.clearAllKeeps() 4094 4095 def testAliventProcessTimeout(self): 4096 ''' 4097 Test for Alivent.process() (coverage) 4098 ''' 4099 console.terse("{0}\n".format(self.testAliventProcessTimeout.__doc__)) 4100 4101 mainData = self.createRoadData(name='main', 4102 base=self.base, 4103 auto=raeting.AutoMode.once.value) 4104 keeping.clearAllKeep(mainData['dirpath']) 4105 main = self.createRoadStack(data=mainData, 4106 main=True, 4107 auto=mainData['auto'], 4108 ha=None) 4109 4110 otherData = self.createRoadData(name='other', 4111 base=self.base, 4112 auto=raeting.AutoMode.once.value) 4113 keeping.clearAllKeep(otherData['dirpath']) 4114 other = self.createRoadStack(data=otherData, 4115 main=True, # only main can reap and unreap 4116 auto=otherData['auto'], 4117 ha=("", raeting.RAET_TEST_PORT)) 4118 4119 stacks = [main, other] 4120 4121 console.terse("\nBootstrap channel *********\n") 4122 # remote on joiner side have to be joined and allowed 4123 self.join(other, main) # bootstrap channel 4124 self.allow(other, main) 4125 for stack in [main, other]: 4126 self.assertEqual(len(stack.transactions), 0) 4127 self.assertEqual(len(stack.remotes), 1) 4128 remote = stack.remotes.values()[0] 4129 self.assertTrue(remote.joined) 4130 self.assertTrue(remote.allowed) 4131 self.assertTrue(remote.alived) # fast alive 4132 self.assertIs(remote.reaped, None) 4133 4134 other.remotes.values()[0].alived = None # reset alived 4135 other.clearStats() 4136 console.terse("\nTest alivent process alive transaction timeout *********\n") 4137 main.alive() # alive from main to other 4138 self.serviceStack(main, duration=0.1) # Send alive 4139 # receive alive on other in custom way: use transaction receive instead of stack.correspond() 4140 other.serviceReceives() 4141 # handle rx 4142 raw, sa = other.rxes.popleft() 4143 console.verbose("{0} received packet\n{1}\n".format(other.name, raw)) 4144 packet = packeting.RxPacket(stack=other, packed=raw) 4145 packet.parseOuter() 4146 sh, sp = sa 4147 packet.data.update(sh=sh, sp=sp) 4148 # process rx 4149 nuid = packet.data['de'] 4150 self.assertNotEqual(nuid, 0) 4151 remote = other.remotes.get(nuid, None) 4152 self.assertIsNotNone(remote) 4153 rsid = packet.data['si'] 4154 self.assertNotEqual(rsid, 0) 4155 remote.rsid = rsid 4156 # create transaction 4157 data = odict(hk=other.Hk, bk=other.Bk, fk=other.Fk, ck=other.Ck) 4158 alivent = transacting.Alivent(stack=other, 4159 remote=remote, 4160 bcst=packet.data['bf'], 4161 sid=packet.data['si'], 4162 tid=packet.data['ti'], 4163 txData=data, 4164 rxPacket=packet) 4165 # timeout 4166 self.assertTrue(alivent.timeout > 0.0) 4167 self.store.advanceStamp(alivent.timeout) 4168 alivent.process() 4169 self.serviceStacks(stacks) # Process alive 4170 4171 self.assertIn('alivent_transaction_failure', stack.stats) 4172 self.assertEqual(stack.stats['alivent_transaction_failure'], 1) 4173 4174 for stack in stacks: 4175 self.assertEqual(len(stack.transactions), 0) 4176 self.assertEqual(len(stack.remotes), 1) 4177 remote = other.remotes.values()[0] 4178 self.assertTrue(remote.joined) 4179 self.assertTrue(remote.allowed) 4180 self.assertIsNone(remote.alived) 4181 4182 for stack in [main, other]: 4183 stack.server.close() 4184 stack.clearAllKeeps() 4185 4186 def testFirstAliveRequestDropped(self): 4187 ''' 4188 Test network dropped first alive request (redo timeout) 4189 ''' 4190 console.terse("{0}\n".format(self.testFirstAliveRequestDropped.__doc__)) 4191 4192 mainData = self.createRoadData(name='main', 4193 base=self.base, 4194 auto=raeting.AutoMode.once.value) 4195 keeping.clearAllKeep(mainData['dirpath']) 4196 main = self.createRoadStack(data=mainData, 4197 main=True, 4198 auto=mainData['auto'], 4199 ha=None) 4200 4201 otherData = self.createRoadData(name='other', 4202 base=self.base, 4203 auto=raeting.AutoMode.once.value) 4204 keeping.clearAllKeep(otherData['dirpath']) 4205 other = self.createRoadStack(data=otherData, 4206 main=True, # only main can reap and unreap 4207 auto=otherData['auto'], 4208 ha=("", raeting.RAET_TEST_PORT)) 4209 4210 stacks = [main, other] 4211 4212 console.terse("\nBootstrap channel *********\n") 4213 # remote on joiner side have to be joined and allowed 4214 self.join(other, main) # bootstrap channel 4215 self.allow(other, main) 4216 for stack in [main, other]: 4217 self.assertEqual(len(stack.transactions), 0) 4218 self.assertEqual(len(stack.remotes), 1) 4219 remote = stack.remotes.values()[0] 4220 self.assertTrue(remote.joined) 4221 self.assertTrue(remote.allowed) 4222 self.assertTrue(remote.alived) # fast alive 4223 self.assertIs(remote.reaped, None) 4224 4225 main.remotes.values()[0].alived = None # reset alived 4226 main.clearStats() 4227 console.terse("\nTest aliver didn't received response, redo *********\n") 4228 main.alive() # alive from main to other 4229 self.serviceStack(main, duration=0.1) # Send alive 4230 self.flushReceives(other) 4231 self.serviceStacks(stacks) # timeout, redo, alive 4232 self.serviceStacks(stacks) # fix race condition 4233 4234 self.assertIn('redo_alive', main.stats) 4235 self.assertEqual(main.stats['redo_alive'], 1) # 1 redo 4236 for stack in stacks: 4237 self.assertEqual(len(stack.transactions), 0) 4238 self.assertEqual(len(stack.remotes), 1) 4239 remote = stack.remotes.values()[0] 4240 self.assertTrue(remote.joined) 4241 self.assertTrue(remote.allowed) 4242 self.assertTrue(remote.alived) # Alive complete 4243 4244 for stack in [main, other]: 4245 stack.server.close() 4246 stack.clearAllKeeps() 4247 4248 def testAllAliveRequestsDropped(self): 4249 ''' 4250 Test network dropped all alive requests (transaction timeout) 4251 ''' 4252 console.terse("{0}\n".format(self.testAllAliveRequestsDropped.__doc__)) 4253 4254 mainData = self.createRoadData(name='main', 4255 base=self.base, 4256 auto=raeting.AutoMode.once.value) 4257 keeping.clearAllKeep(mainData['dirpath']) 4258 main = self.createRoadStack(data=mainData, 4259 main=True, 4260 auto=mainData['auto'], 4261 ha=None) 4262 4263 otherData = self.createRoadData(name='other', 4264 base=self.base, 4265 auto=raeting.AutoMode.once.value) 4266 keeping.clearAllKeep(otherData['dirpath']) 4267 other = self.createRoadStack(data=otherData, 4268 main=True, # only main can reap and unreap 4269 auto=otherData['auto'], 4270 ha=("", raeting.RAET_TEST_PORT)) 4271 4272 stacks = [main, other] 4273 4274 console.terse("\nBootstrap channel *********\n") 4275 # remote on joiner side have to be joined and allowed 4276 self.join(other, main) # bootstrap channel 4277 self.allow(other, main) 4278 for stack in [main, other]: 4279 self.assertEqual(len(stack.transactions), 0) 4280 self.assertEqual(len(stack.remotes), 1) 4281 remote = stack.remotes.values()[0] 4282 self.assertTrue(remote.joined) 4283 self.assertTrue(remote.allowed) 4284 self.assertTrue(remote.alived) # fast alive 4285 self.assertIs(remote.reaped, None) 4286 4287 other.remotes.values()[0].alived = None # reset alived 4288 main.clearStats() 4289 console.terse("\nTest aliver received no response, transaction timeout *********\n") 4290 main.alive() # alive from main to other 4291 self.serviceStacksDropRx(stacks, duration=3.0) # redo timeout, packet timeout, drop 4292 4293 self.assertIn('redo_alive', main.stats) 4294 self.assertEqual(main.stats['redo_alive'], 3) # 3 redo 4295 for stack in stacks: 4296 self.assertEqual(len(stack.transactions), 0) 4297 self.assertEqual(len(stack.remotes), 1) 4298 remote = stack.remotes.values()[0] 4299 self.assertTrue(remote.joined) 4300 self.assertTrue(remote.allowed) 4301 self.assertFalse(remote.alived) 4302 4303 for stack in [main, other]: 4304 stack.server.close() 4305 stack.clearAllKeeps() 4306 4307 def testFirstAliveRequestDelayed(self): 4308 ''' 4309 Test network delayed response so it has been received after redo. 4310 ''' 4311 console.terse("{0}\n".format(self.testFirstAliveRequestDelayed.__doc__)) 4312 4313 mainData = self.createRoadData(name='main', 4314 base=self.base, 4315 auto=raeting.AutoMode.once.value) 4316 keeping.clearAllKeep(mainData['dirpath']) 4317 main = self.createRoadStack(data=mainData, 4318 main=True, 4319 auto=mainData['auto'], 4320 ha=None) 4321 4322 otherData = self.createRoadData(name='other', 4323 base=self.base, 4324 auto=raeting.AutoMode.once.value) 4325 keeping.clearAllKeep(otherData['dirpath']) 4326 other = self.createRoadStack(data=otherData, 4327 main=True, # only main can reap and unreap 4328 auto=otherData['auto'], 4329 ha=("", raeting.RAET_TEST_PORT)) 4330 4331 stacks = [main, other] 4332 4333 console.terse("\nBootstrap channel *********\n") 4334 # remote on joiner side have to be joined and allowed 4335 self.join(other, main) # bootstrap channel 4336 self.allow(other, main) 4337 for stack in [main, other]: 4338 self.assertEqual(len(stack.transactions), 0) 4339 self.assertEqual(len(stack.remotes), 1) 4340 remote = stack.remotes.values()[0] 4341 self.assertTrue(remote.joined) 4342 self.assertTrue(remote.allowed) 4343 self.assertTrue(remote.alived) # fast alive 4344 self.assertIs(remote.reaped, None) 4345 4346 main.remotes.values()[0].alived = None # reset alived 4347 other.remotes.values()[0].alived = None # reset alived 4348 main.clearStats() 4349 other.clearStats() 4350 console.terse("\nTest alivent received both request and redo *********\n") 4351 main.alive() # alive from main to other 4352 self.serviceStack(main, duration=0.5) # Send alive and redo 4353 self.serviceStacks(stacks) # service delayed messages 4354 self.serviceStacks(stacks) # fix race condition 4355 4356 self.assertIn('redo_alive', main.stats) 4357 self.assertEqual(main.stats['redo_alive'], 1) # 1 redo 4358 self.assertIn('stale_correspondent_attempt', main.stats) 4359 self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt 4360 self.assertIn('stale_correspondent_nack', main.stats) 4361 self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer 4362 self.assertIn('alive_complete', main.stats) 4363 self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main 4364 self.assertIn('stale_packet', other.stats) 4365 self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other 4366 self.assertIn('alive_complete', other.stats) 4367 self.assertEqual(other.stats['alive_complete'], 2) # 2 complete on other 4368 for stack in stacks: 4369 self.assertEqual(len(stack.transactions), 0) 4370 self.assertEqual(len(stack.remotes), 1) 4371 remote = stack.remotes.values()[0] 4372 self.assertTrue(remote.joined) 4373 self.assertTrue(remote.allowed) 4374 self.assertTrue(remote.alived) # Alive complete 4375 4376 for stack in [main, other]: 4377 stack.server.close() 4378 stack.clearAllKeeps() 4379 4380 def testAllAliveRequestsDelayed(self): 4381 ''' 4382 Test network delayed all alive requests (aliver receive response after transaction dropped) 4383 ''' 4384 console.terse("{0}\n".format(self.testAllAliveRequestsDelayed.__doc__)) 4385 4386 mainData = self.createRoadData(name='main', 4387 base=self.base, 4388 auto=raeting.AutoMode.once.value) 4389 keeping.clearAllKeep(mainData['dirpath']) 4390 main = self.createRoadStack(data=mainData, 4391 main=True, 4392 auto=mainData['auto'], 4393 ha=None) 4394 4395 otherData = self.createRoadData(name='other', 4396 base=self.base, 4397 auto=raeting.AutoMode.once.value) 4398 keeping.clearAllKeep(otherData['dirpath']) 4399 other = self.createRoadStack(data=otherData, 4400 main=True, # only main can reap and unreap 4401 auto=otherData['auto'], 4402 ha=("", raeting.RAET_TEST_PORT)) 4403 4404 stacks = [main, other] 4405 4406 console.terse("\nBootstrap channel *********\n") 4407 # remote on joiner side have to be joined and allowed 4408 self.join(other, main) # bootstrap channel 4409 self.allow(other, main) 4410 for stack in [main, other]: 4411 self.assertEqual(len(stack.transactions), 0) 4412 self.assertEqual(len(stack.remotes), 1) 4413 remote = stack.remotes.values()[0] 4414 self.assertTrue(remote.joined) 4415 self.assertTrue(remote.allowed) 4416 self.assertTrue(remote.alived) # fast alive 4417 self.assertIs(remote.reaped, None) 4418 4419 main.remotes.values()[0].alived = None # reset alived 4420 other.remotes.values()[0].alived = None # reset alived 4421 main.clearStats() 4422 other.clearStats() 4423 console.terse("\nTest alivent received alive and redo after aliver transaction timeout *********\n") 4424 main.alive() # alive from main to other 4425 self.serviceStack(main, duration=3.0) # redo timeout, packet timeout, drop 4426 self.serviceStacks(stacks) # ack 4 alives on other 4427 self.serviceStacks(stacks) # stale nack 4 alives on main 4428 for stack in stacks: 4429 self.assertEqual(len(stack.txes), 0) # ensure both stacks done 4430 4431 self.assertIn('redo_alive', main.stats) 4432 self.assertEqual(main.stats['redo_alive'], 3) # 3 redo 4433 self.assertIn('stale_correspondent_attempt', main.stats) 4434 self.assertEqual(main.stats['stale_correspondent_attempt'], 4) # 4 stale attempt 4435 self.assertIn('stale_correspondent_nack', main.stats) 4436 self.assertEqual(main.stats['stale_correspondent_nack'], 4) # 4 stale nack answer 4437 self.assertIn('stale_packet', other.stats) 4438 self.assertEqual(other.stats['stale_packet'], 4) # 4 stale nack on other 4439 self.assertIn('alive_complete', other.stats) 4440 self.assertEqual(other.stats['alive_complete'], 4) # 2 complete on other 4441 for stack in stacks: 4442 self.assertEqual(len(stack.transactions), 0) 4443 self.assertEqual(len(stack.remotes), 1) 4444 remote = stack.remotes.values()[0] 4445 self.assertTrue(remote.joined) 4446 self.assertTrue(remote.allowed) 4447 self.assertFalse(main.remotes.values()[0].alived) # main didn't received correct alive ack 4448 self.assertTrue(other.remotes.values()[0].alived) # other received correct alive request 4449 4450 for stack in [main, other]: 4451 stack.server.close() 4452 stack.clearAllKeeps() 4453 4454 def testFirstAliveRequestDuplicated(self): 4455 ''' 4456 Test network duplicated first alive request (aliver ack both, alivent stale nack second) 4457 ''' 4458 console.terse("{0}\n".format(self.testFirstAliveRequestDuplicated.__doc__)) 4459 4460 mainData = self.createRoadData(name='main', 4461 base=self.base, 4462 auto=raeting.AutoMode.once.value) 4463 keeping.clearAllKeep(mainData['dirpath']) 4464 main = self.createRoadStack(data=mainData, 4465 main=True, 4466 auto=mainData['auto'], 4467 ha=None) 4468 4469 otherData = self.createRoadData(name='other', 4470 base=self.base, 4471 auto=raeting.AutoMode.once.value) 4472 keeping.clearAllKeep(otherData['dirpath']) 4473 other = self.createRoadStack(data=otherData, 4474 main=True, # only main can reap and unreap 4475 auto=otherData['auto'], 4476 ha=("", raeting.RAET_TEST_PORT)) 4477 4478 stacks = [main, other] 4479 4480 console.terse("\nBootstrap channel *********\n") 4481 # remote on joiner side have to be joined and allowed 4482 self.join(other, main) # bootstrap channel 4483 self.allow(other, main) 4484 for stack in [main, other]: 4485 self.assertEqual(len(stack.transactions), 0) 4486 self.assertEqual(len(stack.remotes), 1) 4487 remote = stack.remotes.values()[0] 4488 self.assertTrue(remote.joined) 4489 self.assertTrue(remote.allowed) 4490 self.assertTrue(remote.alived) # fast alive 4491 self.assertIs(remote.reaped, None) 4492 4493 main.remotes.values()[0].alived = None # reset alived 4494 main.clearStats() 4495 console.terse("\nTest alivent received duplicated request *********\n") 4496 main.alive() # alive from main to other 4497 self.serviceStack(main, duration=0.1) # Send alive 4498 self.dupReceives(other) 4499 self.serviceStacks(stacks) # 1 req, 2 ack, 1st accept, 2nd nack 4500 self.serviceStacks(stacks) # fix race condition 4501 4502 self.assertIn('stale_correspondent_attempt', main.stats) 4503 self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt (dup) 4504 self.assertIn('stale_correspondent_nack', main.stats) 4505 self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer (dup) 4506 self.assertIn('alive_complete', main.stats) 4507 self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main 4508 self.assertIn('stale_packet', other.stats) 4509 self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other (dup) 4510 self.assertIn('alive_complete', other.stats) 4511 self.assertEqual(other.stats['alive_complete'], 2) # 2 complete on other 4512 for stack in stacks: 4513 self.assertEqual(len(stack.transactions), 0) 4514 self.assertEqual(len(stack.remotes), 1) 4515 remote = stack.remotes.values()[0] 4516 self.assertTrue(remote.joined) 4517 self.assertTrue(remote.allowed) 4518 self.assertTrue(remote.alived) # Alive complete 4519 4520 for stack in [main, other]: 4521 stack.server.close() 4522 stack.clearAllKeeps() 4523 4524 def testAliveAckDuplicated(self): 4525 ''' 4526 Test network duplicated alive ack response (stale nack the second one) 4527 ''' 4528 console.terse("{0}\n".format(self.testAliveAckDuplicated.__doc__)) 4529 4530 mainData = self.createRoadData(name='main', 4531 base=self.base, 4532 auto=raeting.AutoMode.once.value) 4533 keeping.clearAllKeep(mainData['dirpath']) 4534 main = self.createRoadStack(data=mainData, 4535 main=True, 4536 auto=mainData['auto'], 4537 ha=None) 4538 4539 otherData = self.createRoadData(name='other', 4540 base=self.base, 4541 auto=raeting.AutoMode.once.value) 4542 keeping.clearAllKeep(otherData['dirpath']) 4543 other = self.createRoadStack(data=otherData, 4544 main=True, # only main can reap and unreap 4545 auto=otherData['auto'], 4546 ha=("", raeting.RAET_TEST_PORT)) 4547 4548 stacks = [main, other] 4549 4550 console.terse("\nBootstrap channel *********\n") 4551 # remote on joiner side have to be joined and allowed 4552 self.join(other, main) # bootstrap channel 4553 self.allow(other, main) 4554 for stack in [main, other]: 4555 self.assertEqual(len(stack.transactions), 0) 4556 self.assertEqual(len(stack.remotes), 1) 4557 remote = stack.remotes.values()[0] 4558 self.assertTrue(remote.joined) 4559 self.assertTrue(remote.allowed) 4560 self.assertIs(remote.alived, True) 4561 self.assertIs(remote.reaped, None) 4562 4563 main.clearStats() 4564 other.clearStats() 4565 console.terse("\nTest aliver received response twice *********\n") 4566 for stack in stacks: 4567 self.assertEqual(len(stack.remotes), 1) 4568 remote = stack.remotes.values()[0] 4569 remote.alived = None 4570 self.assertIs(remote.alived, None) # Reset alived 4571 4572 main.alive() # alive from main to other 4573 self.serviceStack(main, duration=0.125) # Send alive 4574 self.serviceStack(other, duration=0.125) # Send ack 4575 time.sleep(0.3) 4576 self.dupReceives(main) # duplicate response 4577 self.serviceStacks(stacks, duration=0.25) # 1st accept, 2nd stale nack 4578 self.serviceStacks(stacks, duration=1.0) # fix race condition 4579 4580 self.assertIn('stale_correspondent_attempt', main.stats) 4581 self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt (dup) 4582 self.assertIn('stale_correspondent_nack', main.stats) 4583 self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer (dup) 4584 self.assertIn('alive_complete', main.stats) 4585 self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main 4586 self.assertIn('stale_packet', other.stats) 4587 self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other (dup) 4588 self.assertIn('alive_complete', other.stats) 4589 self.assertEqual(other.stats['alive_complete'], 1) # 1 complete on other 4590 for stack in stacks: 4591 self.assertEqual(len(stack.transactions), 0) 4592 self.assertEqual(len(stack.remotes), 1) 4593 remote = stack.remotes.values()[0] 4594 self.assertTrue(remote.joined) 4595 self.assertTrue(remote.allowed) 4596 self.assertTrue(remote.alived) # Alive complete 4597 4598 for stack in [main, other]: 4599 stack.server.close() 4600 stack.clearAllKeeps() 4601 4602def runOne(test): 4603 ''' 4604 Unittest Runner 4605 ''' 4606 test = BasicTestCase(test) 4607 suite = unittest.TestSuite([test]) 4608 unittest.TextTestRunner(verbosity=2).run(suite) 4609 4610def runSome(): 4611 ''' 4612 Unittest runner 4613 ''' 4614 tests = [] 4615 names = [ 4616 'testJoinNameRoleDiffer', 4617 'testJoinFromMain', 4618 'testJoinFromMainNameRoleDiffer', 4619 'testJoinFromMainKindChange', 4620 'testAliveDead', 4621 'testAliveDeadMultiple', 4622 'testAliveUnjoinedUnallowedBoth', 4623 'testCascadeBoth', 4624 'testManageOneSide', 4625 'testManageBothSides', 4626 'testManageMainRebootCascade', 4627 'testManageRebootCascadeBothSides', 4628 'testManageRebootCascadeBothSidesAlt', 4629 'testManageUnjoinedAllTimersExpired', 4630 'testManageUnallowedAllTimersExpired', 4631 'testManageCascadeUnjoinedAllTimersExpired', 4632 'testManageCascadeUnallowedAllTimersExpired', 4633 'testManageUnalivedAllTimersExpired', 4634 'testManageUnjoinedPresenceTimerExpired', 4635 'testManageUnallowedPresenceTimerExpired', 4636 'testManageCascadeUnjoinedPresenceTimerExpired', 4637 'testManageCascadeUnallowedPresenceTimerExpired', 4638 'testManageUnalivedPresenceTimerExpired', 4639 'testManageReapTimerExpired', 4640 'testManageNoTimerExpired', 4641 'testManageReaped', 4642 'testAliventUnjoined', 4643 'testAliventUnallowed', 4644 'testAliventJoinedAllowed', 4645 'testAliverReceiveNack', 4646 'testAliverAlivePackError', 4647 'testAliverErrorParseInner', 4648 'testAliventAliveErrorParseInner', 4649 'testAliventAliveErrorPacketPack', 4650 'testAliventNackErrorPacketPack', 4651 'testAliventReceiveRequest', 4652 'testAliventProcessTimeout', 4653 'testFirstAliveRequestDropped', 4654 'testAllAliveRequestsDropped', 4655 'testFirstAliveRequestDelayed', 4656 'testAllAliveRequestsDelayed', 4657 'testFirstAliveRequestDuplicated', 4658 'testAliveAckDuplicated', 4659 4660 ] 4661 4662 tests.extend(map(BasicTestCase, names)) 4663 4664 suite = unittest.TestSuite(tests) 4665 unittest.TextTestRunner(verbosity=2).run(suite) 4666 4667def runAll(): 4668 ''' 4669 Unittest runner 4670 ''' 4671 suite = unittest.TestSuite() 4672 suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase)) 4673 4674 unittest.TextTestRunner(verbosity=2).run(suite) 4675 4676if __name__ == '__main__' and __package__ is None: 4677 4678 #console.reinit(verbosity=console.Wordage.concise) 4679 4680 runAll() #run all unittests 4681 4682 #runSome()#only run some 4683 4684 #runOne('testAliveAckDuplicated') 4685