1# -*- coding: utf-8 -*-
2'''
3Tests to try out keeping. Potentially ephemeral
4
5'''
6# pylint: skip-file
7# pylint: disable=C0103
8import sys
9if sys.version_info < (2, 7):
10    import unittest2 as unittest
11else:
12    import unittest
13
14import os
15import time
16import tempfile
17import shutil
18from collections import deque
19
20from ioflo.aid.odicting import odict
21from ioflo.aid.timing import Timer, StoreTimer
22from ioflo.base.storing import Store
23from ioflo.base.consoling import getConsole
24console = getConsole()
25
26# Import raet libs
27from raet.abiding import *  # import globals
28from raet import raeting, nacling
29from raet.road import estating, keeping, stacking, packeting, transacting
30
31if sys.platform == 'win32':
32    TEMPDIR = 'c:/temp'
33    if not os.path.exists(TEMPDIR):
34        os.mkdir(TEMPDIR)
35else:
36    TEMPDIR = '/tmp'
37
38def setUpModule():
39    console.reinit(verbosity=console.Wordage.concise)
40
41def tearDownModule():
42    pass
43
44class BasicTestCase(unittest.TestCase):
45    """"""
46
47    def setUp(self):
48        self.store = Store(stamp=0.0)
49        self.timer = StoreTimer(store=self.store, duration=1.0)
50
51        self.base = tempfile.mkdtemp(prefix="raet",  suffix="base", dir=TEMPDIR)
52
53    def tearDown(self):
54        if os.path.exists(self.base):
55            shutil.rmtree(self.base)
56
57    def createRoadData(self,
58                       base,
59                       name='',
60                       ha=None,
61                       main=None,
62                       auto=raeting.AutoMode.never.value,
63                       role=None,
64                       kind=None, ):
65        '''
66        Creates odict and populates with data to setup road stack
67
68        '''
69        data = odict()
70        data['name'] = name
71        data['ha'] = ha
72        data['main'] =  main
73        data['auto'] = auto
74        data['role'] = role if role is not None else name
75        data['kind'] = kind
76        data['dirpath'] = os.path.join(base, 'road', 'keep', name)
77        signer = nacling.Signer()
78        data['sighex'] = signer.keyhex
79        data['verhex'] = signer.verhex
80        privateer = nacling.Privateer()
81        data['prihex'] = privateer.keyhex
82        data['pubhex'] = privateer.pubhex
83
84        return data
85
86    def createRoadStack(self,
87                        data,
88                        uid=None,
89                        ha=None,
90                        main=None,
91                        auto=None,
92                        role=None,
93                        kind=None,
94                        period=None,
95                        offset=None,):
96        '''
97        Creates stack and local estate from data with
98        and overrides with parameters
99
100        returns stack
101
102        '''
103        stack = stacking.RoadStack(store=self.store,
104                                   name=data['name'],
105                                   uid=uid,
106                                   ha=ha or data['ha'],
107                                   main=main if main is not None else data['main'],
108                                   role=role if role is not None else data['role'],
109                                   sigkey=data['sighex'],
110                                   prikey=data['prihex'],
111                                   auto=auto if auto is not None else data['auto'],
112                                   kind=kind if kind is not None else data['kind'],
113                                   dirpath=data['dirpath'],
114                                   period=period,
115                                   offset=offset,)
116
117        return stack
118
119    def join(self, initiator, correspondent, deid=None, duration=1.0,
120                cascade=False):
121        '''
122        Utility method to do join. Call from test method.
123        '''
124        console.terse("\nJoin Transaction **************\n")
125        if not initiator.remotes:
126            initiator.addRemote(estating.RemoteEstate(stack=initiator,
127                                                      fuid=0, # vacuous join
128                                                      sid=0, # always 0 for join
129                                                      ha=correspondent.local.ha))
130        initiator.join(uid=deid, cascade=cascade)
131        self.service(correspondent, initiator, duration=duration)
132
133    def allow(self, initiator, correspondent, deid=None, duration=1.0,
134                cascade=False):
135        '''
136        Utility method to do allow. Call from test method.
137        '''
138        console.terse("\nAllow Transaction **************\n")
139        initiator.allow(uid=deid, cascade=cascade)
140        self.service(correspondent, initiator, duration=duration)
141
142    def alive(self, initiator, correspondent, duid=None, duration=1.0,
143                cascade=False):
144        '''
145        Utility method to do alive. Call from test method.
146        '''
147        console.terse("\nAlive Transaction **************\n")
148        initiator.alive(uid=duid, cascade=cascade)
149        self.service(correspondent, initiator, duration=duration)
150
151    def message(self, msgs, initiator, correspondent, duration=2.0):
152        '''
153        Utility to send messages both ways
154        '''
155        for msg in msgs:
156            initiator.transmit(msg)
157
158        self.service(initiator, correspondent, duration=duration)
159
160    def flushReceives(self, stack):
161        '''
162        Flush any queued up udp packets in receive buffer
163        '''
164        stack.serviceReceives()
165        stack.rxes.clear()
166
167    def dupReceives(self, stack):
168        '''
169        Duplicate each queued up udp packet in receive buffer
170        '''
171        stack.serviceReceives()
172        rxes = stack.rxes
173        stack.rxes = deque()
174        for rx in rxes:
175            stack.rxes.append(rx) # one
176            stack.rxes.append(rx) # and one more
177
178    def service(self, main, other, duration=1.0):
179        '''
180        Utility method to service queues. Call from test method.
181        '''
182        self.timer.restart(duration=duration)
183        while not self.timer.expired:
184            other.serviceAll()
185            main.serviceAll()
186            if not (main.transactions or other.transactions):
187                break
188            self.store.advanceStamp(0.0625)
189            time.sleep(0.0625)
190
191    def serviceStack(self, stack, duration=1.0):
192        '''
193        Utility method to service queues for one stack. Call from test method.
194        '''
195        self.timer.restart(duration=duration)
196        while not self.timer.expired:
197            stack.serviceAll()
198            if not (stack.transactions):
199                break
200            self.store.advanceStamp(0.0625)
201            time.sleep(0.0625)
202
203    def serviceStacks(self, stacks, duration=1.0):
204        '''
205        Utility method to service queues for list of stacks. Call from test method.
206        '''
207        self.timer.restart(duration=duration)
208        while not self.timer.expired:
209            for stack in stacks:
210                stack.serviceAll()
211            if all([not stack.transactions for stack in stacks]):
212                break
213            self.store.advanceStamp(0.0625)
214            time.sleep(0.0625)
215
216    def serviceStacksDropRx(self, stacks, duration=1.0):
217        '''
218        Utility method to service queues for list of stacks. Call from test method.
219        '''
220        self.timer.restart(duration=duration)
221        while not self.timer.expired:
222            for stack in stacks:
223                stack.serviceReceives()
224                stack.rxes.clear()
225                stack.serviceRxes()
226                stack.process()
227                stack.serviceAllTx()
228            if all([not stack.transactions for stack in stacks]):
229                break
230            self.store.advanceStamp(0.1)
231            time.sleep(0.1)
232
233    def serviceManageStacks(self, stacks, duration=1.0, cascade=False):
234        '''
235        Utility method to service queues and manage presence for list of stacks.
236        Call from test method.
237        '''
238        self.timer.restart(duration=duration)
239        while not self.timer.expired:
240            for stack in stacks:
241                stack.serviceAllRx()
242                stack.manage(cascade=cascade)
243                stack.serviceAllTx()
244            if all([not stack.transactions for stack in stacks]):
245                break
246            self.store.advanceStamp(0.1)
247            time.sleep(0.1)
248
249    def answerAlive(self, stack, deid=None, kind=raeting.PcktKind.nack.value, dataMod=None):
250        '''
251        Utility method to receive a packet in the given stack and send alive nack as a responce
252        Call from test method.
253        :param stack: correspondent stack that have to receive and nack a transaction
254        :param deid: remote estate id in the stack
255        :param kind: nack kind (nack, reject, ...)
256        :rtype : None
257        '''
258        stack.serviceReceives()
259        # process rx
260        raw, sa = stack.rxes.popleft()
261        console.verbose("{0} received packet\n{1}\n".format(stack.name, raw))
262        packet = packeting.RxPacket(stack=stack, packed=raw)
263        packet.parseOuter()
264        sh, sp = sa
265        packet.data.update(sh=sh, sp=sp)
266        data = odict(hk=stack.Hk, bk=stack.Bk, fk=stack.Fk, ck=stack.Ck)
267        if dataMod:
268            data.update(dataMod)
269        remote = stack.retrieveRemote(deid)
270        alivent = transacting.Alivent(stack=stack,
271                                      remote=remote,
272                                      bcst=packet.data['bf'],
273                                      sid=packet.data['si'],
274                                      tid=packet.data['ti'],
275                                      txData=data,
276                                      rxPacket=packet)
277        if kind == raeting.PcktKind.ack:
278            alivent.alive()
279        else:
280            alivent.nack(kind=kind)
281
282    def aliveBrokenInner(self, stack, uid=None, timeout=None, cascade=False):
283        '''
284        Initiate alive transaction
285        If duid is None then create remote at ha
286        '''
287        remote = stack.retrieveRemote(uid=uid)
288        self.assertIsNotNone(remote)
289        data = odict(hk=stack.Hk, bk=stack.Bk, fk=stack.Fk, ck=stack.Ck)
290        data['ck'] = -1
291        aliver = transacting.Aliver(stack=stack,
292                                    remote=remote,
293                                    timeout=timeout,
294                                    txData=data,
295                                    cascade=cascade)
296        aliver.alive()
297
298    def testJoinNameRoleDiffer(self):
299        '''
300        Test join from other where name and role are different
301        '''
302        console.terse("{0}\n".format(self.testJoinNameRoleDiffer.__doc__))
303
304        mainData = self.createRoadData(name='main_stack',
305                                       role='main',
306                                       base=self.base,
307                                       auto=raeting.AutoMode.once.value)
308        keeping.clearAllKeep(mainData['dirpath'])
309        main = self.createRoadStack(data=mainData,
310                                     main=True,
311                                     auto=mainData['auto'],
312                                     ha=None)
313
314        otherData = self.createRoadData(name='other_stack',
315                                        role='other',
316                                        base=self.base,
317                                        auto=raeting.AutoMode.once.value)
318        keeping.clearAllKeep(otherData['dirpath'])
319        other = self.createRoadStack(data=otherData,
320                                     main=None,
321                                     ha=("", raeting.RAET_TEST_PORT))
322
323        self.assertNotEqual(main.local.name, main.local.role)
324        self.assertNotEqual(other.local.name, other.local.role)
325        self.assertIs(other.main, None)
326        self.assertIs(other.keep.auto, raeting.AutoMode.once.value)
327
328        console.terse("\nJoin Other to Main *********\n")
329        self.join(other, main)
330        for stack in [main, other]:
331            self.assertEqual(len(stack.transactions), 0)
332            self.assertEqual(len(stack.remotes), 1)
333            self.assertEqual(len(stack.nameRemotes), 1)
334            remote = stack.remotes.values()[0]
335            self.assertIs(remote.joined, True)
336            self.assertIs(remote.allowed, None)
337            self.assertIs(remote.alived, None)
338
339        console.terse("\nAllow Other to Main *********\n")
340        self.allow(other, main)
341        for stack in [main, other]:
342            self.assertEqual(len(stack.transactions), 0)
343            self.assertEqual(len(stack.remotes), 1)
344            self.assertEqual(len(stack.nameRemotes), 1)
345            remote = stack.remotes.values()[0]
346            self.assertIs(remote.joined, True)
347            self.assertIs(remote.allowed, True)
348            self.assertIs(remote.alived, True)  # fast alive
349
350        console.terse("\nAlive Other to Main *********\n")
351        self.alive(other, main)
352        for stack in [main, other]:
353            self.assertEqual(len(stack.transactions), 0)
354            self.assertEqual(len(stack.remotes), 1)
355            self.assertEqual(len(stack.nameRemotes), 1)
356            remote = stack.remotes.values()[0]
357            self.assertIs(remote.joined, True)
358            self.assertIs(remote.allowed, True)
359            self.assertIs(remote.alived, True)
360
361        console.terse("\nAlive Main to Other *********\n")
362        self.alive(main, other)
363        for stack in [main, other]:
364            self.assertEqual(len(stack.transactions), 0)
365            self.assertEqual(len(stack.remotes), 1)
366            self.assertEqual(len(stack.nameRemotes), 1)
367            remote = stack.remotes.values()[0]
368            self.assertIs(remote.joined, True)
369            self.assertIs(remote.allowed, True)
370            self.assertIs(remote.alived, True)
371
372        for stack in [main, other]:
373            stack.server.close()
374            stack.clearAllKeeps()
375
376    def testJoinFromMain(self):
377        '''
378        Test join,initiated by main
379        '''
380        console.terse("{0}\n".format(self.testJoinFromMain.__doc__))
381
382        mainData = self.createRoadData(name='main',
383                                       base=self.base,
384                                       auto=raeting.AutoMode.once.value)
385        keeping.clearAllKeep(mainData['dirpath'])
386        main = self.createRoadStack(data=mainData,
387                                     main=True,
388                                     auto=mainData['auto'],
389                                     ha=None)
390
391        otherData = self.createRoadData(name='other',
392                                        base=self.base,
393                                        auto=raeting.AutoMode.never.value)
394        keeping.clearAllKeep(otherData['dirpath'])
395        other = self.createRoadStack(data=otherData,
396                                     main=None,
397                                     ha=("", raeting.RAET_TEST_PORT))
398
399        self.assertIs(other.keep.auto, raeting.AutoMode.never.value)
400        self.assertIs(other.main, None)
401
402        console.terse("\nJoin Main to Other *********\n")
403        self.join(main, other) # vacuous join fails because other not main
404        for stack in [main, other]:
405            self.assertEqual(len(stack.transactions), 0)
406            self.assertEqual(len(stack.remotes), 0)
407            self.assertEqual(len(stack.nameRemotes), 0)
408
409        # now fix it so other can accept vacuous joins
410        other.main = True
411        other.keep.auto = raeting.AutoMode.once.value
412        self.assertIs(other.main, True)
413        self.assertIs(other.keep.auto, raeting.AutoMode.once.value)
414
415        self.join(main, other)
416        for stack in [main, other]:
417            self.assertEqual(len(stack.transactions), 0)
418            self.assertEqual(len(stack.remotes), 1)
419            self.assertEqual(len(stack.nameRemotes), 1)
420            remote = stack.remotes.values()[0]
421            self.assertIs(remote.joined, True)
422            self.assertIs(remote.allowed, None)
423            self.assertIs(remote.alived, None)
424
425        console.terse("\nAllow Main to Other *********\n")
426        self.allow(main, other)
427        for stack in [main, other]:
428            self.assertEqual(len(stack.transactions), 0)
429            self.assertEqual(len(stack.remotes), 1)
430            self.assertEqual(len(stack.nameRemotes), 1)
431            remote = stack.remotes.values()[0]
432            self.assertIs(remote.joined, True)
433            self.assertIs(remote.allowed, True)
434            self.assertIs(remote.alived, True)  # fast alive
435
436        main.remotes.values()[0].alived = None   # reset alived
437        other.remotes.values()[0].alived = None  # reset alived
438
439        console.terse("\nAlive Main to other *********\n")
440        self.alive(main, other)
441        for stack in [main, other]:
442            self.assertEqual(len(stack.transactions), 0)
443            self.assertEqual(len(stack.remotes), 1)
444            self.assertEqual(len(stack.nameRemotes), 1)
445            remote = stack.remotes.values()[0]
446            self.assertIs(remote.joined, True)
447            self.assertIs(remote.allowed, True)
448            self.assertIs(remote.alived, True)
449
450        main.remotes.values()[0].alived = None   # reset alived
451        other.remotes.values()[0].alived = None  # reset alived
452
453        console.terse("\nAlive Other to Main *********\n")
454        self.alive(other, main)
455        for stack in [main, other]:
456            self.assertEqual(len(stack.transactions), 0)
457            self.assertEqual(len(stack.remotes), 1)
458            self.assertEqual(len(stack.nameRemotes), 1)
459            remote = stack.remotes.values()[0]
460            self.assertIs(remote.joined, True)
461            self.assertIs(remote.allowed, True)
462            self.assertIs(remote.alived, True)
463
464
465        for stack in [main, other]:
466            stack.server.close()
467            stack.clearAllKeeps()
468
469    def testJoinFromMainNameRoleDiffer(self):
470        '''
471        Test join from main where name and role are different
472        '''
473        console.terse("{0}\n".format(self.testJoinFromMainNameRoleDiffer.__doc__))
474
475        mainData = self.createRoadData(name='main_stack',
476                                       role='main',
477                                       base=self.base,
478                                       auto=raeting.AutoMode.once.value)
479        keeping.clearAllKeep(mainData['dirpath'])
480        main = self.createRoadStack(data=mainData,
481                                     main=True,
482                                     auto=mainData['auto'],
483                                     ha=None)
484
485        otherData = self.createRoadData(name='other_stack',
486                                        role='other',
487                                        base=self.base,
488                                        auto=raeting.AutoMode.never.value)
489        keeping.clearAllKeep(otherData['dirpath'])
490        other = self.createRoadStack(data=otherData,
491                                     main=None,
492                                     ha=("", raeting.RAET_TEST_PORT))
493
494        self.assertNotEqual(main.local.name, main.local.role)
495        self.assertNotEqual(other.local.name, other.local.role)
496        self.assertIs(other.keep.auto, raeting.AutoMode.never.value)
497        self.assertIs(other.main, None)
498
499        console.terse("\nJoin Main to Other *********\n")
500        self.join(main, other) # vacuous join fails because other not main
501        for stack in [main, other]:
502            self.assertEqual(len(stack.transactions), 0)
503            self.assertEqual(len(stack.remotes), 0)
504            self.assertEqual(len(stack.nameRemotes), 0)
505
506        # now fix it so other can accept vacuous joins
507        other.main = True
508        other.keep.auto = raeting.AutoMode.once.value
509        self.assertIs(other.main, True)
510        self.assertIs(other.keep.auto, raeting.AutoMode.once.value)
511
512
513        self.join(main, other)
514        for stack in [main, other]:
515            self.assertEqual(len(stack.transactions), 0)
516            self.assertEqual(len(stack.remotes), 1)
517            self.assertEqual(len(stack.nameRemotes), 1)
518            remote = stack.remotes.values()[0]
519            self.assertIs(remote.joined, True)
520            self.assertIs(remote.allowed, None)
521            self.assertIs(remote.alived, None)
522
523        console.terse("\nAllow Main to Other *********\n")
524        self.allow(main, other)
525        for stack in [main, other]:
526            self.assertEqual(len(stack.transactions), 0)
527            self.assertEqual(len(stack.remotes), 1)
528            self.assertEqual(len(stack.nameRemotes), 1)
529            remote = stack.remotes.values()[0]
530            self.assertIs(remote.joined, True)
531            self.assertIs(remote.allowed, True)
532            self.assertIs(remote.alived, True)  # fast alive
533
534        main.remotes.values()[0].alived = None   # reset alived
535        other.remotes.values()[0].alived = None  # reset alived
536
537        console.terse("\nAlive Main to other *********\n")
538        self.alive(main, other)
539        for stack in [main, other]:
540            self.assertEqual(len(stack.transactions), 0)
541            self.assertEqual(len(stack.remotes), 1)
542            self.assertEqual(len(stack.nameRemotes), 1)
543            remote = stack.remotes.values()[0]
544            self.assertIs(remote.joined, True)
545            self.assertIs(remote.allowed, True)
546            self.assertIs(remote.alived, True)
547
548        main.remotes.values()[0].alived = None   # reset alived
549        other.remotes.values()[0].alived = None  # reset alived
550
551        console.terse("\nAlive Other to Main *********\n")
552        self.alive(other, main)
553        for stack in [main, other]:
554            self.assertEqual(len(stack.transactions), 0)
555            self.assertEqual(len(stack.remotes), 1)
556            self.assertEqual(len(stack.nameRemotes), 1)
557            remote = stack.remotes.values()[0]
558            self.assertIs(remote.joined, True)
559            self.assertIs(remote.allowed, True)
560            self.assertIs(remote.alived, True)
561
562        for stack in [main, other]:
563            stack.server.close()
564            stack.clearAllKeeps()
565
566    def testJoinFromMainKindChange(self):
567        '''
568        Test allow from main where name changed from join
569        This reproduces what happens if name changed after successful join
570        and reboot
571        so joined is persisted so allow fails
572        '''
573        console.terse("{0}\n".format(self.testJoinFromMainKindChange.__doc__))
574
575        mainData = self.createRoadData(name='main_stack',
576                                       role='main',
577                                       base=self.base,
578                                       auto=raeting.AutoMode.once.value)
579        keeping.clearAllKeep(mainData['dirpath'])
580        main = self.createRoadStack(data=mainData,
581                                     main=True,
582                                     auto=mainData['auto'],
583                                     ha=None)
584
585        otherData = self.createRoadData(name='other_stack',
586                                        role='other',
587                                        base=self.base,
588                                        auto=raeting.AutoMode.once.value)
589        keeping.clearAllKeep(otherData['dirpath'])
590        other = self.createRoadStack(data=otherData,
591                                     main=True,
592                                     ha=("", raeting.RAET_TEST_PORT))
593
594        self.assertNotEqual(main.local.name, main.local.role)
595        self.assertNotEqual(other.local.name, other.local.role)
596        self.assertIs(other.main, True)
597        self.assertIs(other.keep.auto, raeting.AutoMode.once.value)
598
599        console.terse("\nJoin Main to Other *********\n")
600        self.join(main, other)
601        for stack in [main, other]:
602            self.assertEqual(len(stack.transactions), 0)
603            self.assertEqual(len(stack.remotes), 1)
604            self.assertEqual(len(stack.nameRemotes), 1)
605            remote = stack.remotes.values()[0]
606            self.assertIs(remote.joined, True)
607            self.assertIs(remote.allowed, None)
608            self.assertIs(remote.alived, None)
609
610        # Change kind
611        other.kind =  1
612        self.assertNotEqual(other.kind, otherData['kind'])
613        main.kind =  1
614        self.assertNotEqual(main.kind, mainData['kind'])
615
616        self.join(main, other) # fails because not same all and immutable road
617        for stack in [main, other]:
618            self.assertEqual(len(stack.transactions), 0)
619
620        self.assertEqual(len(main.remotes), 0)
621        self.assertEqual(len(main.nameRemotes), 0)
622
623        self.assertEqual(len(other.remotes), 1)
624        self.assertEqual(len(other.nameRemotes), 1)
625        remote = other.remotes.values()[0]
626        self.assertIs(remote.joined, True)
627        self.assertIs(remote.allowed, None)
628        self.assertIs(remote.alived, None)
629
630        other.mutable = True
631        self.assertIs(other.mutable, True)
632        self.join(main, other) # fails because not same all and immutable road
633        for stack in [main, other]:
634            self.assertEqual(len(stack.transactions), 0)
635            self.assertEqual(len(stack.remotes), 1)
636            self.assertEqual(len(stack.nameRemotes), 1)
637            remote = stack.remotes.values()[0]
638            self.assertIs(remote.joined, True)
639            self.assertIs(remote.allowed, None)
640            self.assertIs(remote.alived, None)
641
642        console.terse("\nAllow Main to Other *********\n")
643        self.allow(main, other)
644        for stack in [main, other]:
645            self.assertEqual(len(stack.transactions), 0)
646            self.assertEqual(len(stack.remotes), 1)
647            self.assertEqual(len(stack.nameRemotes), 1)
648            remote = stack.remotes.values()[0]
649            self.assertIs(remote.joined, True)
650            self.assertIs(remote.allowed, True)
651            self.assertIs(remote.alived, True)  # fast alive
652
653        main.remotes.values()[0].alived = None   # reset alived
654        other.remotes.values()[0].alived = None  # reset alived
655
656        console.terse("\nAlive Main to other *********\n")
657        self.alive(main, other)
658        for stack in [main, other]:
659            self.assertEqual(len(stack.transactions), 0)
660            self.assertEqual(len(stack.remotes), 1)
661            self.assertEqual(len(stack.nameRemotes), 1)
662            remote = stack.remotes.values()[0]
663            self.assertIs(remote.joined, True)
664            self.assertIs(remote.allowed, True)
665            self.assertIs(remote.alived, True)
666
667        main.remotes.values()[0].alived = None   # reset alived
668        other.remotes.values()[0].alived = None  # reset alived
669
670        console.terse("\nAlive Other to Main *********\n")
671        self.alive(other, main)
672        for stack in [main, other]:
673            self.assertEqual(len(stack.transactions), 0)
674            self.assertEqual(len(stack.remotes), 1)
675            self.assertEqual(len(stack.nameRemotes), 1)
676            remote = stack.remotes.values()[0]
677            self.assertIs(remote.joined, True)
678            self.assertIs(remote.allowed, True)
679            self.assertIs(remote.alived, True)
680
681
682        for stack in [main, other]:
683            stack.server.close()
684            stack.clearAllKeeps()
685
686    def testAliveDead(self):
687        '''
688        Test basic alive transaction given already joined and allowed
689        '''
690        console.terse("{0}\n".format(self.testAliveDead.__doc__))
691
692        mainData = self.createRoadData(name='main',
693                                       base=self.base,
694                                       auto=raeting.AutoMode.once.value)
695        keeping.clearAllKeep(mainData['dirpath'])
696        main = self.createRoadStack(data=mainData,
697                                     main=True,
698                                     auto=mainData['auto'],
699                                     ha=None)
700
701        otherData = self.createRoadData(name='other', base=self.base)
702        keeping.clearAllKeep(otherData['dirpath'])
703        other = self.createRoadStack(data=otherData,
704                                     main=None,
705                                     auto=raeting.AutoMode.once.value,
706                                     ha=("", raeting.RAET_TEST_PORT))
707
708        self.join(other, main)
709        remotes = []
710        for stack in [main, other]:
711            self.assertEqual(len(stack.transactions), 0)
712            self.assertEqual(len(stack.remotes), 1)
713            remote = stack.remotes.values()[0]
714            remotes.append(remote)
715            self.assertIs(remote.joined, True)
716            self.assertNotEqual(remote.nuid, 0)
717            self.assertNotEqual(remote.fuid, 0)
718
719        self.assertEqual(remotes[0].fuid, remotes[1].nuid)
720        self.assertEqual(remotes[1].fuid, remotes[0].nuid)
721
722        for stack in [main, other]:
723            self.assertIs(stack.remotes.values()[0].allowed, None)
724        self.allow(other, main)
725        for stack in [main, other]:
726            self.assertEqual(len(stack.transactions), 0)
727            self.assertEqual(len(stack.remotes), 1)
728            remote = stack.remotes.values()[0]
729            self.assertIs(remote.joined, True)
730            self.assertIs(remote.allowed, True)
731            self.assertIs(remote.alived, True)
732
733        console.terse("\nAlive Other to Main *********\n")
734        for stack in [main, other]:
735            remote = stack.remotes.values()[0]
736            remote.alived = None
737            self.assertIs(stack.remotes.values()[0].alived, None)
738        self.alive(other, main, duid=other.remotes.values()[0].uid)
739        for stack in [main, other]:
740            self.assertEqual(len(stack.transactions), 0)
741            self.assertEqual(len(stack.remotes), 1)
742            remote = stack.remotes.values()[0]
743            self.assertIs(remote.joined, True)
744            self.assertIs(remote.allowed, True)
745            self.assertIs(remote.alived, True)
746
747        console.terse("\nAlive Main to Other *********\n")
748        for stack in [main, other]:
749            stack.remotes.values()[0].alived = None
750            self.assertIs(stack.remotes.values()[0].alived, None)
751        self.alive(main, other, duid=main.remotes.values()[0].uid)
752        for stack in [main, other]:
753            self.assertEqual(len(stack.transactions), 0)
754            self.assertEqual(len(stack.remotes), 1)
755            remote = stack.remotes.values()[0]
756            self.assertIs(remote.joined, True)
757            self.assertIs(remote.allowed, True)
758            self.assertIs(remote.alived, True)
759
760        console.terse("\nDead Other from Main *********\n")
761        # start alive from main to other but do not service responses from other
762        # so main alive trans times out and other  appears dead to main
763        for stack in [main, other]:
764            self.assertIs(stack.remotes.values()[0].alived, True)
765        main.alive()
766        self.serviceStack(main, duration=3.0) # only service main
767        self.assertEqual(len(main.transactions), 0) # timed out
768        self.assertIs(main.remotes.values()[0].alived,  False)
769        self.serviceStack(other, duration=3.0) # now service other side
770        self.assertEqual(len(other.transactions), 0) # gets queued requests
771        self.assertIs(other.remotes.values()[0].alived,  True)
772        self.assertIs(main.remotes.values()[0].alived,  False)
773
774        console.terse("\nDead Main from Other *********\n")
775        # start alive from other to main but do not service responses from main
776        # so other alive trans times out and main  appears dead to other
777        for stack in [main, other]:
778            self.assertEqual(len(stack.transactions), 0)
779            self.assertEqual(len(stack.remotes), 1)
780        self.assertIs(other.remotes.values()[0].alived, True)
781        other.alive()
782        self.serviceStack(other, duration=3.0) # only service other
783        self.assertEqual(len(other.transactions), 0) # timed out
784        self.assertIs(other.remotes.values()[0].alived, False)
785        self.serviceStack(main, duration=3.0) # now service main side
786        self.assertEqual(len(main.transactions), 0) # gets queued requests
787        self.assertIs(main.remotes.values()[0].alived,  True)
788        self.assertIs(other.remotes.values()[0].alived,  False)
789
790        for stack in [main, other]:
791            stack.server.close()
792            stack.clearAllKeeps()
793
794    def testAliveDeadMultiple(self):
795        '''
796        Test alive transaction with multiple remotes given already joined and allowed
797        '''
798        console.terse("{0}\n".format(self.testAliveDeadMultiple.__doc__))
799
800        mainData = self.createRoadData(name='main',
801                                       base=self.base,
802                                       auto=raeting.AutoMode.once.value)
803        keeping.clearAllKeep(mainData['dirpath'])
804        main = self.createRoadStack(data=mainData,
805                                     main=True,
806                                     auto=mainData['auto'],
807                                     ha=None)
808
809        otherData = self.createRoadData(name='other',
810                                        base=self.base,
811                                        auto=raeting.AutoMode.once.value)
812        keeping.clearAllKeep(otherData['dirpath'])
813        other = self.createRoadStack(data=otherData,
814                                     main=None,
815                                     auto=otherData['auto'],
816                                     ha=("", raeting.RAET_TEST_PORT))
817
818
819        self.join(other, main)
820        for stack in [main, other]:
821            self.assertEqual(len(stack.transactions), 0)
822            self.assertEqual(len(stack.remotes), 1)
823            remote = stack.remotes.values()[0]
824            self.assertIs(remote.joined, True)
825            self.assertIs(remote.allowed, None)
826            self.assertIs(remote.alived, None)
827
828        self.allow(other, main)
829        for stack in [main, other]:
830            self.assertEqual(len(stack.transactions), 0)
831            self.assertEqual(len(stack.remotes), 1)
832            remote = stack.remotes.values()[0]
833            self.assertIs(remote.joined, True)
834            self.assertIs(remote.allowed, True)
835            self.assertIs(remote.alived, True)
836
837        other1Data = self.createRoadData(name='other1',
838                                         base=self.base,
839                                         auto=raeting.AutoMode.once.value)
840        keeping.clearAllKeep(other1Data['dirpath'])
841        other1 = self.createRoadStack(data=other1Data,
842                                     main=None,
843                                     auto=other1Data['auto'],
844                                     ha=("", 7533))
845
846        self.join(other1, main)
847        for stack in [main, other1]:
848            self.assertEqual(len(stack.transactions), 0)
849
850        self.assertEqual(len(main.remotes), 2)
851        self.assertEqual(len(other1.remotes), 1)
852
853        for remote in [main.remotes.values()[1],
854                        other1.remotes.values()[0]]:
855            self.assertIs(remote.joined, True)
856            self.assertIs(remote.allowed, None)
857            self.assertIs(remote.alived, None)
858
859        self.allow(other1, main)
860        for stack in [main, other1]:
861            self.assertEqual(len(stack.transactions), 0)
862        for remote in [main.remotes.values()[1],
863                        other1.remotes.values()[0]]:
864            self.assertIs(remote.joined, True)
865            self.assertIs(remote.allowed, True)
866            self.assertIs(remote.alived, True)
867
868        console.terse("\nAlive Other to Main *********\n")
869        for remote in [main.remotes.values()[0],
870                       other.remotes.values()[0]]:
871            self.assertIs(remote.joined, True)
872            self.assertIs(remote.allowed, True)
873            remote.alived = None  # force not alived
874            self.assertIs(remote.alived, None)
875
876        self.alive(other, main)
877        for stack in [main, other]:
878            self.assertEqual(len(stack.transactions), 0)
879        for remote in [main.remotes.values()[0],
880                       other.remotes.values()[0]]:
881            self.assertIs(remote.joined, True)
882            self.assertIs(remote.allowed, True)
883            self.assertIs(remote.alived, True)
884
885        console.terse("\nAlive Main to Other *********\n")
886        for remote in [main.remotes.values()[0],
887                       other.remotes.values()[0]]:
888            remote.alived =  None
889            self.assertIs(remote.alived, None)
890
891        self.alive(main, other, duid=main.remotes.values()[0].uid)
892        for stack in [main, other]:
893            self.assertEqual(len(stack.transactions), 0)
894        for remote in [main.remotes.values()[0],
895                       other.remotes.values()[0]]:
896            self.assertIs(remote.joined, True)
897            self.assertIs(remote.allowed, True)
898            self.assertIs(remote.alived, True)
899
900        console.terse("\nAlive Other1 to Main *********\n")
901        for remote in [main.remotes.values()[1],
902                       other1.remotes.values()[0]]:
903            remote.alived =  None
904            self.assertIs(remote.alived, None)
905
906        self.alive(other1, main, duid=other1.remotes.values()[0].uid)
907        for stack in [main, other1]:
908            self.assertEqual(len(stack.transactions), 0)
909        for remote in [main.remotes.values()[1],
910                       other1.remotes.values()[0]]:
911            self.assertIs(remote.joined, True)
912            self.assertIs(remote.allowed, True)
913            self.assertIs(remote.alived, True)
914
915        console.terse("\nAlive Main to Other1 *********\n")
916        for remote in [main.remotes.values()[1],
917                       other1.remotes.values()[0]]:
918            remote.alived =  None
919            self.assertIs(remote.alived, None)
920
921        self.alive(main, other1, duid=main.remotes.values()[1].uid)
922        for stack in [main, other1]:
923            self.assertEqual(len(stack.transactions), 0)
924        for remote in [main.remotes.values()[1],
925                       other1.remotes.values()[0]]:
926            self.assertIs(remote.joined, True)
927            self.assertIs(remote.allowed, True)
928            self.assertIs(remote.alived, True)
929
930        console.terse("\nDead Other to Main from Main *********\n")
931        for remote in main.remotes.values():
932            self.assertIs(remote.alived, True)
933            main.alive(uid=remote.nuid)
934        # do not service other stack so other appears dead to main
935        self.serviceStacks([main, other1], duration=4.0)
936        for stack in [main, other, other1]:
937            self.assertEqual(len(stack.transactions), 0)
938        self.assertIs(main.remotes.values()[0].alived, False)
939        self.assertIs(main.remotes.values()[1].alived, True)
940        self.assertIs(other.remotes.values()[0].alived, True)
941        self.assertIs(other1.remotes.values()[0].alived, True)
942
943        console.terse("\nDead Main to Other From Other *********\n")
944        self.flushReceives(other) # flush packets sent by main to other
945        # do not service main stack so main appears dead to other
946        other.alive(uid=other.remotes.values()[0].nuid)
947        self.serviceStacks([other], duration=3.0) # other
948        for stack in [main, other, other1]:
949            self.assertEqual(len(stack.transactions), 0)
950        self.assertIs(main.remotes.values()[0].alived, False)
951        self.assertIs(main.remotes.values()[1].alived, True)
952        self.assertIs(other.remotes.values()[0].alived, False)
953        self.assertIs(other1.remotes.values()[0].alived, True)
954        self.flushReceives(main) # flush buffered packets on main
955
956        #bring other back to life
957        console.terse("\nReliven main to other from other *********\n")
958        for  remote in [main.remotes.values()[0],
959                        other.remotes.values()[0]]:
960            self.assertIs(remote.alived, False)
961        other.alive(cascade=True)
962        self.serviceStacks([other, main], duration=3.0)
963        for  remote in [main.remotes.values()[0],
964                        other.remotes.values()[0]]:
965            self.assertIs(remote.alived, True)
966
967        console.terse("\nAlive Dead Other 1 to Main from Main *********\n")
968        for remote in main.remotes.values():
969            self.assertIs(remote.alived, True)
970            main.alive(uid=remote.nuid)
971        # do not service other1 stack so other appears dead to main
972        self.serviceStacks([main, other], duration=3.0)
973        for stack in [main, other]:
974            self.assertEqual(len(stack.transactions), 0)
975        self.assertIs(main.remotes.values()[0].alived, True)
976        self.assertIs(main.remotes.values()[1].alived, False)
977        self.assertIs(other.remotes.values()[0].alived, True)
978        self.assertIs(other1.remotes.values()[0].alived, True)
979
980        console.terse("\nDead Main to Other1 From Other1 *********\n")
981        # do not service main stack so main appears dead to other1
982        self.flushReceives(other1) # flush packets sent by main to other
983        other1.alive(uid=other.remotes.values()[0].nuid)
984        self.serviceStacks([other1], duration=3.0) # other
985        for stack in [other1]:
986            self.assertEqual(len(stack.transactions), 0)
987        self.assertIs(main.remotes.values()[0].alived, True)
988        self.assertIs(main.remotes.values()[1].alived, False)
989        self.assertIs(other.remotes.values()[0].alived, True)
990        self.assertIs(other1.remotes.values()[0].alived, False)
991        self.flushReceives(main) # flush packets sent other1 to main
992
993
994        console.terse("\nReliven main to other1 from other 1 *********\n")
995        #bring main back to life from other1
996        for  remote in [main.remotes.values()[1],
997                        other1.remotes.values()[0]]:
998            self.assertIs(remote.alived, False)
999        other1.alive(cascade=True)
1000        self.serviceStacks([other1, main], duration=3.0)
1001        for  remote in [main.remotes.values()[1],
1002                        other1.remotes.values()[0]]:
1003            self.assertIs(remote.alived, True)
1004
1005        for stack in [main, other, other1]:
1006            stack.server.close()
1007            stack.clearAllKeeps()
1008        time.sleep(0.1)
1009
1010
1011    def testAliveUnjoinedUnallowedBoth(self):
1012        '''
1013        Test alive transaction for other to main  given
1014        unjoined and/or unallowed on both main and other
1015
1016        Merely set the joined, allowed, and alived status
1017        Do not create fresh stacks.
1018        '''
1019        console.terse("{0}\n".format(self.testAliveUnjoinedUnallowedBoth.__doc__))
1020
1021        mainData = self.createRoadData(name='main',
1022                                       base=self.base,
1023                                       auto=raeting.AutoMode.once.value)
1024        keeping.clearAllKeep(mainData['dirpath'])
1025        main = self.createRoadStack(data=mainData,
1026                                     #uid=1,
1027                                     main=True,
1028                                     auto=mainData['auto'],
1029                                     ha=None)
1030
1031        otherData = self.createRoadData(name='other',
1032                                        base=self.base,
1033                                        auto=raeting.AutoMode.once.value)
1034        keeping.clearAllKeep(otherData['dirpath'])
1035        other = self.createRoadStack(data=otherData,
1036                                     #uid=1,
1037                                     main=None,
1038                                     auto=otherData['auto'],
1039                                     ha=("", raeting.RAET_TEST_PORT))
1040
1041        self.join(other, main) # bootstrap channel
1042        for stack in [main, other]:
1043            self.assertEqual(len(stack.transactions), 0)
1044            self.assertEqual(len(stack.remotes), 1)
1045            remote = stack.remotes.values()[0]
1046            self.assertIs(remote.joined, True)
1047
1048        console.terse("\nAllow Other to Main *********\n")
1049        # force unjoined already unallowed unalived
1050        for stack in [main, other]:
1051            stack.remotes.values()[0].joined = None
1052
1053        for stack in [main, other]:
1054            remote = stack.remotes.values()[0]
1055            self.assertIs(remote.joined, None)
1056            self.assertIs(remote.allowed, None)
1057            self.assertIs(remote.alived, None)
1058        self.allow(other, main) # will join instead since unjoined
1059        for stack in [main, other]:
1060            self.assertEqual(len(stack.transactions), 0)
1061            self.assertEqual(len(stack.remotes), 1)
1062            remote = stack.remotes.values()[0]
1063            self.assertIs(remote.joined, True)
1064            self.assertIs(remote.allowed, None)
1065
1066        self.allow(other, main) # now try to allow again
1067        for stack in [main, other]:
1068            self.assertEqual(len(stack.transactions), 0)
1069            self.assertEqual(len(stack.remotes), 1)
1070            remote = stack.remotes.values()[0]
1071            self.assertIs(remote.joined, True)
1072            self.assertIs(remote.allowed, True)
1073            self.assertIs(remote.alived, True)
1074
1075
1076        console.terse("\nAllow Main to Other *********\n")
1077        # force unjoined already unallowed unalived
1078        for stack in [main, other]:
1079            stack.remotes.values()[0].joined = None
1080            stack.remotes.values()[0].allowed = None
1081            stack.remotes.values()[0].alived = None
1082
1083        for stack in [main, other]:
1084            remote = stack.remotes.values()[0]
1085            self.assertIs(remote.joined, None)
1086            self.assertIs(remote.allowed, None)
1087            self.assertIs(remote.alived, None)
1088        self.allow(main, other) # will join instead since unjoined
1089        for stack in [main, other]:
1090            self.assertEqual(len(stack.transactions), 0)
1091            self.assertEqual(len(stack.remotes), 1)
1092            remote = stack.remotes.values()[0]
1093            self.assertIs(remote.joined, True)
1094            self.assertIs(remote.allowed, None)
1095
1096        self.allow(main, other) # now try to allow again
1097        for stack in [main, other]:
1098            self.assertEqual(len(stack.transactions), 0)
1099            self.assertEqual(len(stack.remotes), 1)
1100            remote = stack.remotes.values()[0]
1101            self.assertIs(remote.joined, True)
1102            self.assertIs(remote.allowed, True)
1103            self.assertIs(remote.alived, True)
1104
1105        console.terse("\nAlive Other to Main *********\n")
1106        # force unjoined unallowed unalived
1107        for stack in [main, other]:
1108            stack.remotes.values()[0].joined = None
1109            stack.remotes.values()[0].allowed = None
1110            stack.remotes.values()[0].alived = None
1111
1112        # alive checks for joined and if not then joins
1113        for stack in [main, other]:
1114            remote = stack.remotes.values()[0]
1115            self.assertIs(remote.joined, None)
1116            self.assertIs(remote.allowed, None)
1117            self.assertIs(remote.alived, None)
1118        self.alive(other, main)
1119        for stack in [main, other]:
1120            self.assertEqual(len(stack.transactions), 0)
1121            self.assertEqual(len(stack.remotes), 1)
1122            remote = stack.remotes.values()[0]
1123            self.assertIs(remote.joined, True)
1124            self.assertIs(remote.allowed, None)
1125            self.assertIs(remote.alived, None)
1126
1127        # Alive checks for allowed and if not allows
1128        self.alive(other, main)
1129        for stack in [main, other]:
1130            self.assertEqual(len(stack.transactions), 0)
1131            self.assertEqual(len(stack.remotes), 1)
1132            remote = stack.remotes.values()[0]
1133            self.assertIs(remote.joined, True)
1134            self.assertIs(remote.allowed, True)
1135            self.assertIs(remote.alived, True)
1136
1137        # Alive should complete now
1138        for stack in [main, other]:
1139            self.assertEqual(len(stack.remotes), 1)
1140            remote = stack.remotes.values()[0]
1141            remote.alived = None
1142            self.assertIs(remote.alived, None)
1143
1144        self.alive(other, main)
1145        for stack in [main, other]:
1146            self.assertEqual(len(stack.transactions), 0)
1147            self.assertEqual(len(stack.remotes), 1)
1148            remote = stack.remotes.values()[0]
1149            self.assertIs(remote.joined, True)
1150            self.assertIs(remote.allowed, True)
1151            self.assertIs(remote.alived, True)
1152
1153        console.terse("\nAlive Main to Other *********\n")
1154        # now do same from main to other
1155        # force unjoined unallowed unalived
1156        for stack in [main, other]:
1157            remote = stack.remotes.values()[0]
1158            remote.joined = None
1159            remote.allowed = None
1160            remote.alived = None
1161
1162        for stack in [main, other]:
1163            remote = stack.remotes.values()[0]
1164            self.assertIs(remote.joined, None)
1165            self.assertIs(remote.allowed, None)
1166            self.assertIs(remote.alived, None)
1167        self.alive(main, other)
1168        for stack in [main, other]:
1169            self.assertEqual(len(stack.transactions), 0)
1170            self.assertEqual(len(stack.remotes), 1)
1171            remote = stack.remotes.values()[0]
1172            self.assertIs(remote.joined, True)
1173            self.assertIs(remote.allowed, None)
1174            self.assertIs(remote.alived, None)
1175
1176        # Alive checks for allowed and if not allows
1177        self.alive(main, other)
1178        for stack in [main, other]:
1179            self.assertEqual(len(stack.transactions), 0)
1180            self.assertEqual(len(stack.remotes), 1)
1181            remote = stack.remotes.values()[0]
1182            self.assertIs(remote.joined, True)
1183            self.assertIs(remote.allowed, True)
1184            self.assertIs(remote.alived, True)
1185
1186        # Alive should complete now
1187        for stack in [main, other]:
1188            self.assertEqual(len(stack.remotes), 1)
1189            remote = stack.remotes.values()[0]
1190            remote.alived = None
1191            self.assertIs(remote.alived, None)
1192
1193        self.alive(main, other)
1194        for stack in [main, other]:
1195            self.assertEqual(len(stack.transactions), 0)
1196            self.assertEqual(len(stack.remotes), 1)
1197            remote = stack.remotes.values()[0]
1198            self.assertIs(remote.joined, True)
1199            self.assertIs(remote.allowed, True)
1200            self.assertIs(remote.alived, True)
1201
1202        for stack in [main, other]:
1203            stack.server.close()
1204            stack.clearAllKeeps()
1205
1206    def testCascadeBoth(self):
1207        '''
1208        Test allow and alive cascades for other to main unjoined and/or unallowed on both
1209        but with cascade = True so it repairs joined and allowed status and redos
1210        alive
1211
1212        create fresh stacks on each cascade.
1213        '''
1214        console.terse("{0}\n".format(self.testCascadeBoth.__doc__))
1215
1216        mainData = self.createRoadData(name='main',
1217                                       base=self.base,
1218                                       auto=raeting.AutoMode.once.value)
1219        keeping.clearAllKeep(mainData['dirpath'])
1220        main = self.createRoadStack(data=mainData,
1221                                     main=True,
1222                                     auto=mainData['auto'],
1223                                     ha=None)
1224
1225        otherData = self.createRoadData(name='other',
1226                                        base=self.base,
1227                                        auto=raeting.AutoMode.once.value)
1228        keeping.clearAllKeep(otherData['dirpath'])
1229        other = self.createRoadStack(data=otherData,
1230                                     main=None,
1231                                     auto=otherData['auto'],
1232                                     ha=("", raeting.RAET_TEST_PORT))
1233
1234        console.terse("\nJoin Other to Main Cascade *********\n")
1235        for stack in [main, other]:
1236            self.assertEqual(len(stack.remotes), 0)
1237
1238        self.join(other, main, cascade=True) # now join cascade w
1239        for stack in [main, other]:
1240            self.assertEqual(len(stack.transactions), 0)
1241            self.assertEqual(len(stack.remotes), 1)
1242            remote = stack.remotes.values()[0]
1243            self.assertIs(remote.joined, True)
1244            self.assertIs(remote.allowed, True) # cascade will include alive
1245            self.assertIs(remote.alived, True)  # cascade will include alive
1246
1247        console.terse("\Join Main to Other Cascade *********\n")
1248        main.server.close()
1249        other.server.close()
1250        keeping.clearAllKeep(mainData['dirpath'])
1251        main = self.createRoadStack(data=mainData,
1252                                     main=True,
1253                                     auto=mainData['auto'],
1254                                     ha=None)
1255
1256        keeping.clearAllKeep(otherData['dirpath'])
1257        other = self.createRoadStack(data=otherData,
1258                                     main=None,
1259                                     auto=raeting.AutoMode.never.value,
1260                                     ha=("", raeting.RAET_TEST_PORT))
1261
1262        for stack in [main, other]:
1263            self.assertEqual(len(stack.remotes), 0)
1264
1265        self.join(main, other) # bootstrap channel fails because other not main
1266        for stack in [main, other]:
1267            self.assertEqual(len(stack.transactions), 0)
1268            self.assertEqual(len(stack.remotes), 0)
1269
1270        # now fix it so other can accept vacuous joins
1271        other.main = True
1272        other.keep.auto = raeting.AutoMode.once.value
1273
1274        self.join(main, other) # bootstrap channel
1275        for stack in [main, other]:
1276            self.assertEqual(len(stack.transactions), 0)
1277            self.assertEqual(len(stack.remotes), 1)
1278            remote = stack.remotes.values()[0]
1279            self.assertIs(remote.joined, True)
1280
1281        # force unjoined already unallowed unalived
1282        for stack in [main, other]:
1283            stack.remotes.values()[0].joined = None
1284
1285        for stack in [main, other]:
1286            remote = stack.remotes.values()[0]
1287            self.assertIs(remote.joined, None)
1288            self.assertIs(remote.allowed, None)
1289            self.assertIs(remote.alived, None)
1290        self.join(main, other, cascade=True) # now alive cascade
1291        for stack in [main, other]:
1292            self.assertEqual(len(stack.transactions), 0)
1293            self.assertEqual(len(stack.remotes), 1)
1294            remote = stack.remotes.values()[0]
1295            self.assertIs(remote.joined, True)
1296            self.assertIs(remote.allowed, True)
1297            self.assertIs(remote.alived, True)  # cascade will include alive
1298
1299        console.terse("\nAllow Other to Main Cascade *********\n")
1300        main.server.close()
1301        other.server.close()
1302        keeping.clearAllKeep(mainData['dirpath'])
1303        main = self.createRoadStack(data=mainData,
1304                                     main=True,
1305                                     auto=mainData['auto'],
1306                                     ha=None)
1307
1308        keeping.clearAllKeep(otherData['dirpath'])
1309        other = self.createRoadStack(data=otherData,
1310                                     main=None,
1311                                     auto=otherData['auto'],
1312                                     ha=("", raeting.RAET_TEST_PORT))
1313
1314        for stack in [main, other]:
1315            self.assertEqual(len(stack.remotes), 0)
1316
1317        self.join(other, main) # bootstrap channel since allow requires remote
1318        for stack in [main, other]:
1319            self.assertEqual(len(stack.transactions), 0)
1320            self.assertEqual(len(stack.remotes), 1)
1321            remote = stack.remotes.values()[0]
1322            self.assertIs(remote.joined, True)
1323
1324        # force unjoined already unallowed unalived
1325        for stack in [main, other]:
1326            stack.remotes.values()[0].joined = None
1327
1328        for stack in [main, other]:
1329            remote = stack.remotes.values()[0]
1330            self.assertIs(remote.joined, None)
1331            self.assertIs(remote.allowed, None)
1332            self.assertIs(remote.alived, None)
1333        self.allow(other, main, cascade=True) # now allow cascade so join then allow
1334        for stack in [main, other]:
1335            self.assertEqual(len(stack.transactions), 0)
1336            self.assertEqual(len(stack.remotes), 1)
1337            remote = stack.remotes.values()[0]
1338            self.assertIs(remote.joined, True)
1339            self.assertIs(remote.allowed, True)
1340            self.assertIs(remote.alived, True)  # cascade will include alive
1341
1342        console.terse("\nAllow Main to Other Cascade *********\n")
1343        main.server.close()
1344        other.server.close()
1345        keeping.clearAllKeep(mainData['dirpath'])
1346        main = self.createRoadStack(data=mainData,
1347                                     main=True,
1348                                     auto=mainData['auto'],
1349                                     ha=None)
1350
1351        keeping.clearAllKeep(otherData['dirpath'])
1352        other = self.createRoadStack(data=otherData,
1353                                     main=None,
1354                                     auto=raeting.AutoMode.never.value,
1355                                     ha=("", raeting.RAET_TEST_PORT))
1356
1357        for stack in [main, other]:
1358            self.assertEqual(len(stack.remotes), 0)
1359
1360        self.join(main, other) # bootstrap channel fails because other not main
1361        for stack in [main, other]:
1362            self.assertEqual(len(stack.transactions), 0)
1363            self.assertEqual(len(stack.remotes), 0)
1364
1365        # now fix it so other can accept vacuous joins
1366        other.main = True
1367        other.keep.auto = raeting.AutoMode.once.value
1368
1369        self.join(main, other) # bootstrap channel since allow requires remote
1370        for stack in [main, other]:
1371            self.assertEqual(len(stack.transactions), 0)
1372            self.assertEqual(len(stack.remotes), 1)
1373            remote = stack.remotes.values()[0]
1374            self.assertIs(remote.joined, True)
1375
1376        # force unjoined already unallowed unalived
1377        for stack in [main, other]:
1378            stack.remotes.values()[0].joined = None
1379
1380        for stack in [main, other]:
1381            remote = stack.remotes.values()[0]
1382            self.assertIs(remote.joined, None)
1383            self.assertIs(remote.allowed, None)
1384            self.assertIs(remote.alived, None)
1385        self.allow(main, other, cascade=True) # now allow cascade so join then allow
1386        for stack in [main, other]:
1387            self.assertEqual(len(stack.transactions), 0)
1388            self.assertEqual(len(stack.remotes), 1)
1389            remote = stack.remotes.values()[0]
1390            self.assertIs(remote.joined, True)
1391            self.assertIs(remote.allowed, True)
1392            self.assertIs(remote.alived, True)  # cascade will include alive
1393
1394        console.terse("\nAlive Other to Main Cascade *********\n")
1395        main.server.close()
1396        other.server.close()
1397        keeping.clearAllKeep(mainData['dirpath'])
1398        main = self.createRoadStack(data=mainData,
1399                                     main=True,
1400                                     auto=mainData['auto'],
1401                                     ha=None)
1402
1403        keeping.clearAllKeep(otherData['dirpath'])
1404        other = self.createRoadStack(data=otherData,
1405                                     main=None,
1406                                     auto=otherData['auto'],
1407                                     ha=("", raeting.RAET_TEST_PORT))
1408
1409        for stack in [main, other]:
1410            self.assertEqual(len(stack.remotes), 0)
1411
1412        self.join(other, main) # bootstrap channel
1413        for stack in [main, other]:
1414            self.assertEqual(len(stack.transactions), 0)
1415            self.assertEqual(len(stack.remotes), 1)
1416            remote = stack.remotes.values()[0]
1417            self.assertIs(remote.joined, True)
1418
1419        # force unjoined already unallowed unalived
1420        for stack in [main, other]:
1421            stack.remotes.values()[0].joined = None
1422
1423        for stack in [main, other]:
1424            remote = stack.remotes.values()[0]
1425            self.assertIs(remote.joined, None)
1426            self.assertIs(remote.allowed, None)
1427            self.assertIs(remote.alived, None)
1428        self.alive(other, main, cascade=True) # now alive cascade
1429        for stack in [main, other]:
1430            self.assertEqual(len(stack.transactions), 0)
1431            self.assertEqual(len(stack.remotes), 1)
1432            remote = stack.remotes.values()[0]
1433            self.assertIs(remote.joined, True)
1434            self.assertIs(remote.allowed, True)
1435            self.assertIs(remote.alived, True)  # cascade will include alive
1436
1437        console.terse("\nAlive Main to Other Cascade *********\n")
1438        main.server.close()
1439        other.server.close()
1440        keeping.clearAllKeep(mainData['dirpath'])
1441        main = self.createRoadStack(data=mainData,
1442                                     main=True,
1443                                     auto=mainData['auto'],
1444                                     ha=None)
1445
1446        keeping.clearAllKeep(otherData['dirpath'])
1447        other = self.createRoadStack(data=otherData,
1448                                     main=None,
1449                                     auto=raeting.AutoMode.never.value,
1450                                     ha=("", raeting.RAET_TEST_PORT))
1451
1452        for stack in [main, other]:
1453            self.assertEqual(len(stack.remotes), 0)
1454
1455        self.join(main, other) # bootstrap channel fails because other not main
1456        for stack in [main, other]:
1457            self.assertEqual(len(stack.transactions), 0)
1458            self.assertEqual(len(stack.remotes), 0)
1459
1460        # now fix it so other can accept vacuous joins
1461        other.main = True
1462        other.keep.auto = raeting.AutoMode.once.value
1463
1464        self.join(main, other) # bootstrap channel
1465        for stack in [main, other]:
1466            self.assertEqual(len(stack.transactions), 0)
1467            self.assertEqual(len(stack.remotes), 1)
1468            remote = stack.remotes.values()[0]
1469            self.assertIs(remote.joined, True)
1470
1471        # force unjoined already unallowed unalived
1472        for stack in [main, other]:
1473            stack.remotes.values()[0].joined = None
1474
1475        for stack in [main, other]:
1476            remote = stack.remotes.values()[0]
1477            self.assertIs(remote.joined, None)
1478            self.assertIs(remote.allowed, None)
1479            self.assertIs(remote.alived, None)
1480        self.alive(main, other, cascade=True) # now alive cascade
1481        for stack in [main, other]:
1482            self.assertEqual(len(stack.transactions), 0)
1483            self.assertEqual(len(stack.remotes), 1)
1484            remote = stack.remotes.values()[0]
1485            self.assertIs(remote.joined, True)
1486            self.assertIs(remote.allowed, True)
1487            self.assertIs(remote.alived, True)  # cascade will include alive
1488
1489        for stack in [main, other]:
1490            stack.server.close()
1491            stack.clearAllKeeps()
1492
1493    def testManageOneSide(self):
1494        '''
1495        Test stack manage remotes
1496        '''
1497        console.terse("{0}\n".format(self.testManageOneSide.__doc__))
1498
1499        mainData = self.createRoadData(name='main',
1500                                       base=self.base,
1501                                       auto=raeting.AutoMode.once.value)
1502        keeping.clearAllKeep(mainData['dirpath'])
1503        main = self.createRoadStack(data=mainData,
1504                                     main=True,
1505                                     auto=mainData['auto'],
1506                                     ha=None)
1507
1508        otherData = self.createRoadData(name='other',
1509                                        base=self.base,
1510                                        auto=raeting.AutoMode.once.value)
1511        keeping.clearAllKeep(otherData['dirpath'])
1512        other = self.createRoadStack(data=otherData,
1513                                     main=None,
1514                                     auto=otherData['auto'],
1515                                     ha=("", raeting.RAET_TEST_PORT))
1516
1517        other1Data = self.createRoadData(name='other1',
1518                                         base=self.base,
1519                                         auto=raeting.AutoMode.once.value,)
1520        keeping.clearAllKeep(other1Data['dirpath'])
1521        other1 = self.createRoadStack(data=other1Data,
1522                                     main=None,
1523                                     auto=other1Data['auto'],
1524                                     ha=("", 7533))
1525
1526        for  stack in [other, other1]:
1527            self.join(stack, main)
1528            self.allow(stack, main)
1529
1530        console.terse("\nTest manage remotes presence *********\n")
1531        console.terse("\nMake all alive *********\n")
1532        stacks = [main, other, other1]
1533        for remote in main.remotes.values(): #make all alive
1534            main.alive(uid=remote.uid)
1535        self.serviceStacks(stacks, duration=3.0)
1536        for remote in main.remotes.values():
1537            self.assertTrue(remote.alived)
1538
1539        main.manage()
1540        for stack in stacks: # no alive transactions started
1541            self.assertEqual(len(stack.transactions), 0)
1542
1543        console.terse("\nMake all expired so send alive *********\n")
1544        # advance clock so remote keep alive timers expire
1545        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
1546        main.manage()
1547        for remote in main.remotes.values(): # should start
1548            self.assertIs(remote.alived, True)
1549
1550        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1551
1552        self.serviceStacks(stacks, duration=3.0)
1553        for stack in stacks:
1554            self.assertEqual(len(stack.transactions), 0)
1555        for remote in main.remotes.values():
1556            self.assertTrue(remote.alived)
1557
1558
1559        for stack in [main, other, other1]:
1560            stack.server.close()
1561            stack.clearAllKeeps()
1562        time.sleep(0.1)
1563
1564    def testManageBothSides(self):
1565        '''
1566        Test stack manage remotes main and others
1567        '''
1568        console.terse("{0}\n".format(self.testManageBothSides.__doc__))
1569
1570        mainData = self.createRoadData(name='main',
1571                                       base=self.base,
1572                                       auto=raeting.AutoMode.once.value)
1573        keeping.clearAllKeep(mainData['dirpath'])
1574        main = self.createRoadStack(data=mainData,
1575                                     main=True,
1576                                     auto=mainData['auto'],
1577                                     ha=None)
1578
1579        otherData = self.createRoadData(name='other',
1580                                        base=self.base,
1581                                        auto=raeting.AutoMode.once.value)
1582        keeping.clearAllKeep(otherData['dirpath'])
1583        other = self.createRoadStack(data=otherData,
1584                                     main=None,
1585                                     auto=otherData['auto'],
1586                                     ha=("", raeting.RAET_TEST_PORT))
1587
1588        other1Data = self.createRoadData(name='other1',
1589                                         base=self.base,
1590                                         auto=raeting.AutoMode.once.value)
1591        keeping.clearAllKeep(other1Data['dirpath'])
1592        other1 = self.createRoadStack(data=other1Data,
1593                                     main=None,
1594                                     auto=other1Data['auto'],
1595                                     ha=("", 7533))
1596
1597        for stack in [other, other1]:
1598            self.join(stack, main)
1599            self.allow(stack, main)
1600
1601        console.terse("\nTest manage remotes presence both ways *********\n")
1602        console.terse("\nMake all alive *********\n")
1603        stacks = [main, other, other1]
1604        for remote in main.remotes.values(): #make all alive
1605            main.alive(uid=remote.uid)
1606        self.serviceStacks(stacks, duration=3.0)
1607        for remote in main.remotes.values():
1608            self.assertTrue(remote.alived)
1609
1610        for stack in stacks:
1611            stack.manage()
1612
1613        for stack in stacks: # no alive transactions started
1614            self.assertEqual(len(stack.transactions), 0)
1615
1616        console.terse("\nMake all expired so send alive *********\n")
1617        # advance clock so remote keep alive timers expire
1618        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
1619        for stack in stacks:
1620            stack.manage()
1621
1622        for stack in stacks:
1623            for remote in stack.remotes.values():
1624                self.assertIs(remote.alived, True)
1625
1626        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1627        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
1628        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
1629
1630        self.serviceManageStacks(stacks, duration=3.0)
1631        for stack in stacks:
1632            self.assertEqual(len(stack.transactions), 0)
1633
1634        for stack in stacks:
1635            for remote in stack.remotes.values():
1636                self.assertIs(remote.alived, True)
1637
1638
1639        for stack in [main, other, other1]:
1640            stack.server.close()
1641            stack.clearAllKeeps()
1642        time.sleep(0.1)
1643
1644    def testManageMainRebootCascade(self):
1645        '''
1646        Test stack manage remotes as if main were rebooted
1647        '''
1648        console.terse("{0}\n".format(self.testManageMainRebootCascade.__doc__))
1649
1650        mainData = self.createRoadData(name='main',
1651                                       base=self.base,
1652                                       auto=raeting.AutoMode.once.value)
1653        mainDirpath = mainData['dirpath']
1654        keeping.clearAllKeep(mainData['dirpath'])
1655        main = self.createRoadStack(data=mainData,
1656                                     main=True,
1657                                     auto=mainData['auto'],
1658                                     ha=None)
1659
1660        otherData = self.createRoadData(name='other',
1661                                        base=self.base,
1662                                        auto=raeting.AutoMode.once.value)
1663        otherDirpath = otherData['dirpath']
1664        keeping.clearAllKeep(otherData['dirpath'])
1665        other = self.createRoadStack(data=otherData,
1666                                     main=None,
1667                                     auto=otherData['auto'],
1668                                     ha=("", raeting.RAET_TEST_PORT))
1669
1670        other1Data = self.createRoadData(name='other1',
1671                                         base=self.base,
1672                                         auto=raeting.AutoMode.once.value)
1673        other1Dirpath = other1Data['dirpath']
1674        keeping.clearAllKeep(other1Data['dirpath'])
1675        other1 = self.createRoadStack(data=other1Data,
1676                                     main=None,
1677                                     auto=other1Data['auto'],
1678                                     ha=("", 7533))
1679
1680        for stack in [other, other1]:
1681            self.join(stack, main)
1682            self.allow(stack, main)
1683
1684        console.terse("\nTest manage remotes presence *********\n")
1685        console.terse("\nMake all alive *********\n")
1686        stacks = [main, other, other1]
1687        for remote in main.remotes.values(): #make all alive
1688            main.alive(uid=remote.uid)
1689        self.serviceStacks(stacks, duration=3.0)
1690        for remote in main.remotes.values():
1691            self.assertTrue(remote.alived)
1692
1693        main.manage(immediate=True)
1694        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1695        for remote in main.remotes.values():
1696            self.assertIs(remote.alived, True)
1697
1698        self.serviceStacks(stacks, duration=3.0)
1699        for stack in stacks:
1700            self.assertEqual(len(stack.transactions), 0)
1701        for remote in main.remotes.values():
1702            self.assertTrue(remote.alived)
1703
1704        # now close down main and reload from saved data and manage
1705        console.terse("\nMake all alive with cascade after main reboots *********\n")
1706        main.server.close()
1707        main = stacking.RoadStack(store=self.store,
1708                                  main=True,
1709                                  dirpath=mainDirpath,
1710                                  )
1711        stacks = [main, other, other1]
1712
1713        for remote in main.remotes.values():
1714            self.assertIs(remote.joined, True) #joined status is persisted
1715            self.assertIs(remote.allowed, None)
1716            self.assertIs(remote.alived, None)
1717
1718        main.manage(immediate=True, cascade=True)
1719        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1720        for remote in main.remotes.values(): # should reset alive to None
1721            self.assertIs(remote.alived, None)
1722        self.serviceStacks(stacks, duration=3.0)
1723        for stack in stacks:
1724            self.assertEqual(len(stack.transactions), 0)
1725        for remote in main.remotes.values():
1726            self.assertIs(remote.joined, True)
1727            self.assertIs(remote.allowed, True)
1728            self.assertIs(remote.alived, True)
1729
1730        # Now test as if others are rebooted
1731        console.terse("\nMake all alive with cascade after others reboot *********\n")
1732        other.server.close()
1733        other1.server.close()
1734        other = stacking.RoadStack(dirpath=otherDirpath,
1735                                   store=self.store)
1736        other1 = stacking.RoadStack(dirpath=other1Dirpath,
1737                                    store=self.store)
1738        stacks = [main, other, other1]
1739
1740        for stack in [other, other1]:
1741            for remote in stack.remotes.values():
1742                self.assertIs(remote.joined, True)
1743                self.assertIs(remote.allowed, None)
1744                self.assertIs(remote.alived, None)
1745
1746        main.manage(immediate=True, cascade=True)
1747        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1748        for remote in main.remotes.values():
1749            self.assertIs(remote.alived, True)
1750        self.serviceStacks(stacks, duration=3.0)
1751        for stack in stacks:
1752            self.assertEqual(len(stack.transactions), 0)
1753        for stack in [other, other1]:
1754            for remote in stack.remotes.values():
1755                self.assertIs(remote.joined, True)
1756                self.assertIs(remote.allowed, True)
1757                self.assertIs(remote.alived, True)
1758
1759        for stack in [main, other, other1]:
1760            stack.server.close()
1761            stack.clearAllKeeps()
1762        time.sleep(0.1)
1763
1764    def testManageRebootCascadeBothSides(self):
1765        '''
1766        Test stack manage remotes as if main were rebooted
1767        '''
1768        console.terse("{0}\n".format(self.testManageRebootCascadeBothSides.__doc__))
1769
1770        mainData = self.createRoadData(name='main',
1771                                       base=self.base,
1772                                       auto=raeting.AutoMode.once.value)
1773        mainDirpath = mainData['dirpath']
1774        keeping.clearAllKeep(mainData['dirpath'])
1775        main = self.createRoadStack(data=mainData,
1776                                     main=True,
1777                                     auto=mainData['auto'],
1778                                     ha=None)
1779
1780        otherData = self.createRoadData(name='other',
1781                                        base=self.base,
1782                                        auto=raeting.AutoMode.once.value)
1783        otherDirpath = otherData['dirpath']
1784        keeping.clearAllKeep(otherData['dirpath'])
1785        other = self.createRoadStack(data=otherData,
1786                                     main=None,
1787                                     auto=otherData['auto'],
1788                                     ha=("", raeting.RAET_TEST_PORT))
1789
1790        other1Data = self.createRoadData(name='other1',
1791                                         base=self.base,
1792                                         auto=raeting.AutoMode.once.value)
1793        other1Dirpath = other1Data['dirpath']
1794        keeping.clearAllKeep(other1Data['dirpath'])
1795        other1 = self.createRoadStack(data=other1Data,
1796                                     main=None,
1797                                     auto=other1Data['auto'],
1798                                     ha=("", 7533))
1799
1800        for stack in (other, other1):
1801            self.join(stack, main)
1802            self.allow(stack, main)
1803
1804        console.terse("\nTest manage remotes presence *********\n")
1805        console.terse("\nMake all alive *********\n")
1806        stacks = [main, other, other1]
1807        for remote in main.remotes.values(): #make all alive
1808            main.alive(uid=remote.uid)
1809        self.serviceStacks(stacks, duration=3.0)
1810        for stack in stacks:
1811            for remote in stack.remotes.values():
1812                self.assertIs(remote.alived, True)
1813
1814        main.manage(immediate=True)
1815        other.manage(immediate=True)
1816        other1.manage(immediate=True)
1817        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1818        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
1819        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
1820
1821        for stack in stacks:
1822            for remote in stack.remotes.values():
1823                self.assertIs(remote.alived, True)
1824
1825        self.serviceManageStacks(stacks, duration=3.0)
1826        for stack in stacks:
1827            self.assertEqual(len(stack.transactions), 0)
1828        for stack in stacks:
1829            for remote in stack.remotes.values():
1830                self.assertIs(remote.alived, True)
1831
1832        # now test if main rebooted
1833        console.terse("\nMake all alive with cascade after main reboots *********\n")
1834        main.server.close()
1835        main = stacking.RoadStack(store=self.store,
1836                                  main=True,
1837                                  dirpath=mainDirpath,)
1838        stacks = [main, other, other1]
1839
1840        for stack in [main]:
1841            for remote in stack.remotes.values():
1842                self.assertIs(remote.joined, True) #joined status is persisted
1843                self.assertIs(remote.allowed, None)
1844                self.assertIs(remote.alived, None)
1845
1846        for stack in [other, other1]:
1847            for remote in stack.remotes.values():
1848                self.assertIs(remote.joined, True) #joined status is persisted
1849                self.assertIs(remote.allowed, True)
1850                self.assertIs(remote.alived, True)
1851
1852        main.manage(immediate=True, cascade=True)
1853        other.manage(cascade=True)
1854        other1.manage(cascade=True)
1855        self.assertEqual(len(main.transactions), 2)
1856        self.assertEqual(len(other.transactions), 0)
1857        self.assertEqual(len(other1.transactions), 0)
1858
1859        for stack in [main]:
1860            for remote in stack.remotes.values():
1861                self.assertIs(remote.alived, None)
1862
1863        for stack in [other, other1]:
1864             for remote in stack.remotes.values():
1865                self.assertIs(remote.alived, True)
1866
1867        self.serviceManageStacks(stacks, duration=3.0, cascade=True)
1868        for stack in stacks:
1869            self.assertEqual(len(stack.transactions), 0)
1870        for stack in stacks:
1871            for remote in stack.remotes.values():
1872                self.assertIs(remote.joined, True) #joined status is persisted
1873                self.assertIs(remote.allowed, True)
1874                self.assertIs(remote.alived, True)
1875
1876        # Now test as if others are rebooted but not main
1877        console.terse("\nMake all alive with cascade after others reboot *********\n")
1878        other.server.close()
1879        other1.server.close()
1880        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
1881        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
1882        stacks = [main, other, other1]
1883
1884        for stack in [main]:
1885            for remote in stack.remotes.values():
1886                self.assertIs(remote.alived, True)
1887
1888        for stack in [other, other1]:
1889             for remote in stack.remotes.values():
1890                self.assertIs(remote.joined, True)
1891                self.assertIs(remote.allowed, None)
1892                self.assertIs(remote.alived, None)
1893
1894        main.manage(cascade=True)
1895        other.manage(immediate=True,cascade=True)
1896        other1.manage(immediate=True,cascade=True)
1897        self.assertEqual(len(main.transactions), 0)
1898        self.assertEqual(len(other.transactions), 1)
1899        self.assertEqual(len(other1.transactions), 1)
1900
1901        for stack in [main]:
1902            for remote in stack.remotes.values():
1903                self.assertIs(remote.alived, True)
1904
1905        for stack in [other, other1]:
1906            for remote in stack.remotes.values():
1907                self.assertIs(remote.joined, True)
1908                self.assertIs(remote.allowed, None)
1909                self.assertIs(remote.alived, None)
1910
1911        self.serviceManageStacks(stacks, duration=3.0, cascade=True)
1912        for stack in stacks:
1913            self.assertEqual(len(stack.transactions), 0)
1914        for stack in stacks:
1915            for remote in stack.remotes.values():
1916                self.assertIs(remote.joined, True)
1917                self.assertIs(remote.allowed, True)
1918                self.assertIs(remote.alived, True)
1919
1920        # now close down all and reload from saved data and manage
1921        console.terse("\nMake all alive with cascade after all reboot *********\n")
1922        main.server.close()
1923        main = stacking.RoadStack(store=self.store,
1924                                  main=True,
1925                                  dirpath=mainDirpath,)
1926        other.server.close()
1927        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
1928        other1.server.close()
1929        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
1930
1931        stacks = [main, other, other1]
1932
1933        for stack in stacks:
1934            for remote in stack.remotes.values():
1935                self.assertIs(remote.joined, True) #joined status is persisted
1936                self.assertIs(remote.allowed, None) # None on reload from file
1937                self.assertIs(remote.alived, None) # None on reload from file
1938
1939        main.manage(immediate=True, cascade=True)
1940        other.manage(immediate=True, cascade=True)
1941        other1.manage(immediate=True, cascade=True)
1942        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1943        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
1944        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
1945        for stack in stacks:
1946            for remote in stack.remotes.values():
1947                self.assertIs(remote.joined, True) #joined status is persisted
1948                self.assertIs(remote.allowed, None) # None on reload from file
1949                self.assertIs(remote.alived, None) # None on reload from file
1950
1951        self.serviceManageStacks(stacks, duration=3.0)
1952        for stack in stacks:
1953            self.assertEqual(len(stack.transactions), 0)
1954        for stack in stacks:
1955            for remote in stack.remotes.values():
1956                self.assertIs(remote.joined, True)
1957                self.assertIs(remote.allowed, True)
1958                self.assertIs(remote.alived, True)
1959
1960        # now close down all and reload from saved data but loose joined status
1961        console.terse("\nMake all alive with cascade after all reboot and lose join *********\n")
1962        main.server.close()
1963        main = stacking.RoadStack(store=self.store,
1964                                  main=True,
1965                                  dirpath=mainDirpath,)
1966        other.server.close()
1967        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
1968        other1.server.close()
1969        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
1970
1971        stacks = [main, other, other1]
1972
1973        for stack in stacks:
1974            for remote in stack.remotes.values():
1975                remote.joined = None
1976                self.assertIs(remote.joined, None) #joined status is persisted
1977                self.assertIs(remote.allowed, None) # None on reload from file
1978                self.assertIs(remote.alived, None) # None on reload from file
1979
1980        main.manage(immediate=True, cascade=True)
1981        other.manage(immediate=True, cascade=True)
1982        other1.manage(immediate=True, cascade=True)
1983        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
1984        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
1985        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
1986        for stack in stacks:
1987            for remote in stack.remotes.values():
1988                self.assertIs(remote.joined, None) #joined status is persisted
1989                self.assertIs(remote.allowed, None) # None on reload from file
1990                self.assertIs(remote.alived, None) # None on reload from file
1991
1992        self.serviceManageStacks(stacks, duration=3.0)
1993        for stack in stacks:
1994            self.assertEqual(len(stack.transactions), 0)
1995        for stack in stacks:
1996            for remote in stack.remotes.values():
1997                self.assertIs(remote.joined, True)
1998                self.assertIs(remote.allowed, True)
1999                self.assertIs(remote.alived, True)
2000
2001        for stack in [main, other, other1]:
2002            stack.server.close()
2003            stack.clearAllKeeps()
2004        time.sleep(0.1)
2005
2006
2007    def testManageRebootCascadeBothSidesAlt(self):
2008        '''
2009        Test stack manage remotes as if main were rebooted and main name
2010        means it loses simultaneous join name resolution
2011        '''
2012        console.terse("{0}\n".format(self.testManageRebootCascadeBothSidesAlt.__doc__))
2013
2014        mainData = self.createRoadData(name='zmain',
2015                                       base=self.base,
2016                                       auto=raeting.AutoMode.once.value)
2017        mainDirpath = mainData['dirpath']
2018        keeping.clearAllKeep(mainData['dirpath'])
2019        main = self.createRoadStack(data=mainData,
2020                                     main=True,
2021                                     auto=mainData['auto'],
2022                                     ha=None)
2023
2024        otherData = self.createRoadData(name='other',
2025                                        base=self.base,
2026                                        auto=raeting.AutoMode.once.value)
2027        otherDirpath = otherData['dirpath']
2028        keeping.clearAllKeep(otherData['dirpath'])
2029        other = self.createRoadStack(data=otherData,
2030                                     main=None,
2031                                     auto=otherData['auto'],
2032                                     ha=("", raeting.RAET_TEST_PORT))
2033
2034        other1Data = self.createRoadData(name='other1',
2035                                         base=self.base,
2036                                         auto=raeting.AutoMode.once.value)
2037        other1Dirpath = other1Data['dirpath']
2038        keeping.clearAllKeep(other1Data['dirpath'])
2039        other1 = self.createRoadStack(data=other1Data,
2040                                     main=None,
2041                                     auto=other1Data['auto'],
2042                                     ha=("", 7533))
2043
2044
2045        for stack in (other, other1):
2046            self.join(stack, main, cascade=True)
2047            self.allow(stack, main, cascade=True)
2048
2049        stacks = [main, other, other1]
2050        self.serviceStacks(stacks, duration=3.0)
2051        for stack in stacks:
2052            for remote in stack.remotes.values():
2053                self.assertIs(remote.joined, True)
2054                self.assertIs(remote.allowed, True)
2055                self.assertIs(remote.alived, True)
2056
2057
2058        console.terse("\nTest manage remotes presence *********\n")
2059        main.manage(immediate=True)
2060        other.manage(immediate=True)
2061        other1.manage(immediate=True)
2062        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
2063        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
2064        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
2065
2066        for stack in stacks:
2067            for remote in stack.remotes.values():
2068                self.assertIs(remote.alived, True)
2069
2070        self.serviceManageStacks(stacks, duration=3.0)
2071        for stack in stacks:
2072            self.assertEqual(len(stack.transactions), 0)
2073        for stack in stacks:
2074            for remote in stack.remotes.values():
2075                self.assertIs(remote.alived, True)
2076
2077        # now test if main rebooted
2078        console.terse("\nMake all alive with cascade after main reboots *********\n")
2079        main.server.close()
2080        main = stacking.RoadStack(store=self.store,
2081                                  main=True,
2082                                  dirpath=mainDirpath,)
2083        stacks = [main, other, other1]
2084
2085        for stack in [main]:
2086            for remote in stack.remotes.values():
2087                self.assertIs(remote.joined, True) #joined status is persisted
2088                self.assertIs(remote.allowed, None)
2089                self.assertIs(remote.alived, None)
2090
2091        for stack in [other, other1]:
2092            for remote in stack.remotes.values():
2093                self.assertIs(remote.joined, True) #joined status is persisted
2094                self.assertIs(remote.allowed, True)
2095                self.assertIs(remote.alived, True)
2096
2097        main.manage(immediate=True, cascade=True)
2098        other.manage(cascade=True)
2099        other1.manage(cascade=True)
2100        self.assertEqual(len(main.transactions), 2)
2101        self.assertEqual(len(other.transactions), 0)
2102        self.assertEqual(len(other1.transactions), 0)
2103
2104        for stack in [main]:
2105            for remote in stack.remotes.values():
2106                self.assertIs(remote.alived, None)
2107
2108        for stack in [other, other1]:
2109             for remote in stack.remotes.values():
2110                self.assertIs(remote.alived, True)
2111
2112        self.serviceManageStacks(stacks, duration=3.0, cascade=True)
2113        for stack in stacks:
2114            self.assertEqual(len(stack.transactions), 0)
2115        for stack in stacks:
2116            for remote in stack.remotes.values():
2117                self.assertIs(remote.joined, True) #joined status is persisted
2118                self.assertIs(remote.allowed, True)
2119                self.assertIs(remote.alived, True)
2120
2121        # Now test as if others are rebooted but not main
2122        console.terse("\nMake all alive with cascade after others reboot *********\n")
2123        other.server.close()
2124        other1.server.close()
2125        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
2126        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
2127        stacks = [main, other, other1]
2128
2129        for stack in [main]:
2130            for remote in stack.remotes.values():
2131                self.assertIs(remote.alived, True)
2132
2133        for stack in [other, other1]:
2134             for remote in stack.remotes.values():
2135                self.assertIs(remote.joined, True)
2136                self.assertIs(remote.allowed, None)
2137                self.assertIs(remote.alived, None)
2138
2139        main.manage(cascade=True)
2140        other.manage(immediate=True,cascade=True)
2141        other1.manage(immediate=True,cascade=True)
2142        self.assertEqual(len(main.transactions), 0)
2143        self.assertEqual(len(other.transactions), 1)
2144        self.assertEqual(len(other1.transactions), 1)
2145
2146        for stack in [main]:
2147            for remote in stack.remotes.values():
2148                self.assertIs(remote.alived, True)
2149
2150        for stack in [other, other1]:
2151            for remote in stack.remotes.values():
2152                self.assertIs(remote.joined, True)
2153                self.assertIs(remote.allowed, None)
2154                self.assertIs(remote.alived, None)
2155
2156        self.serviceManageStacks(stacks, duration=3.0, cascade=True)
2157        for stack in stacks:
2158            self.assertEqual(len(stack.transactions), 0)
2159        for stack in stacks:
2160            for remote in stack.remotes.values():
2161                self.assertIs(remote.joined, True)
2162                self.assertIs(remote.allowed, True)
2163                self.assertIs(remote.alived, True)
2164
2165        # now close down all and reload from saved data and manage
2166        console.terse("\nMake all alive with cascade after all reboot *********\n")
2167        main.server.close()
2168        main = stacking.RoadStack(store=self.store,
2169                                  main=True,
2170                                  dirpath=mainDirpath,)
2171        other.server.close()
2172        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
2173        other1.server.close()
2174        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
2175
2176        stacks = [main, other, other1]
2177
2178        for stack in stacks:
2179            for remote in stack.remotes.values():
2180                self.assertIs(remote.joined, True) #joined status is persisted
2181                self.assertIs(remote.allowed, None) # None on reload from file
2182                self.assertIs(remote.alived, None) # None on reload from file
2183
2184        main.manage(immediate=True, cascade=True)
2185        other.manage(immediate=True, cascade=True)
2186        other1.manage(immediate=True, cascade=True)
2187        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
2188        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
2189        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
2190        for stack in stacks:
2191            for remote in stack.remotes.values():
2192                self.assertIs(remote.joined, True) #joined status is persisted
2193                self.assertIs(remote.allowed, None) # None on reload from file
2194                self.assertIs(remote.alived, None) # None on reload from file
2195
2196        self.serviceManageStacks(stacks, duration=3.0)
2197        for stack in stacks:
2198            self.assertEqual(len(stack.transactions), 0)
2199        for stack in stacks:
2200            for remote in stack.remotes.values():
2201                self.assertIs(remote.joined, True)
2202                self.assertIs(remote.allowed, True)
2203                self.assertIs(remote.alived, True)
2204
2205        # now close down all and reload from saved data but loose joined status
2206        console.terse("\nMake all alive with cascade after all reboot and lose join *********\n")
2207        main.server.close()
2208        main = stacking.RoadStack(store=self.store,
2209                                  main=True,
2210                                  dirpath=mainDirpath,)
2211        other.server.close()
2212        other = stacking.RoadStack(dirpath=otherDirpath, store=self.store)
2213        other1.server.close()
2214        other1 = stacking.RoadStack(dirpath=other1Dirpath, store=self.store)
2215
2216        stacks = [main, other, other1]
2217
2218        for stack in stacks:
2219            for remote in stack.remotes.values():
2220                remote.joined = None
2221                self.assertIs(remote.joined, None) #joined status is persisted
2222                self.assertIs(remote.allowed, None) # None on reload from file
2223                self.assertIs(remote.alived, None) # None on reload from file
2224
2225        main.manage(immediate=True, cascade=True)
2226        other.manage(immediate=True, cascade=True)
2227        other1.manage(immediate=True, cascade=True)
2228        self.assertEqual(len(main.transactions), 2) # started 2 alive transactions
2229        self.assertEqual(len(other.transactions), 1) # started 1 alive transactions
2230        self.assertEqual(len(other1.transactions), 1) # started 1 alive transactions
2231        for stack in stacks:
2232            for remote in stack.remotes.values():
2233                self.assertIs(remote.joined, None) #joined status is persisted
2234                self.assertIs(remote.allowed, None) # None on reload from file
2235                self.assertIs(remote.alived, None) # None on reload from file
2236
2237        self.serviceManageStacks(stacks, duration=3.0)
2238        for stack in stacks:
2239            self.assertEqual(len(stack.transactions), 0)
2240        for stack in stacks:
2241            for remote in stack.remotes.values():
2242                self.assertIs(remote.joined, True)
2243                self.assertIs(remote.allowed, True)
2244                self.assertIs(remote.alived, True)
2245
2246        for stack in [main, other, other1]:
2247            stack.server.close()
2248            stack.clearAllKeeps()
2249        time.sleep(0.1)
2250
2251    def testManageUnjoinedAllTimersExpired(self):
2252        '''
2253        Test stack manage unjoined remotes as if both presence and reap timers expired (A1)
2254        '''
2255        console.terse("{0}\n".format(self.testManageUnjoinedAllTimersExpired.__doc__))
2256
2257        mainData = self.createRoadData(name='main',
2258                                       base=self.base,
2259                                       auto=raeting.AutoMode.once.value)
2260        keeping.clearAllKeep(mainData['dirpath'])
2261        main = self.createRoadStack(data=mainData,
2262                                    main=True,
2263                                    auto=mainData['auto'],
2264                                    ha=None)
2265
2266        otherData = self.createRoadData(name='other',
2267                                        base=self.base,
2268                                        auto=raeting.AutoMode.once.value)
2269        keeping.clearAllKeep(otherData['dirpath'])
2270        other = self.createRoadStack(data=otherData,
2271                                     main=None,
2272                                     auto=otherData['auto'],
2273                                     ha=("", raeting.RAET_TEST_PORT))
2274
2275        stacks = [main, other]
2276
2277        console.terse("\nBootstrap channel *********\n")
2278        self.join(other, main) # bootstrap channel
2279        for stack in [main, other]:
2280            self.assertEqual(len(stack.transactions), 0)
2281            self.assertEqual(len(stack.remotes), 1)
2282            remote = stack.remotes.values()[0]
2283            self.assertTrue(remote.joined)
2284
2285        # force unjoined already unallowed unalived
2286        for stack in [main, other]:
2287            stack.remotes.values()[0].joined = None
2288
2289        for stack in [main, other]:
2290            remote = stack.remotes.values()[0]
2291            self.assertIs(remote.joined, None)
2292            self.assertIs(remote.allowed, None)
2293            self.assertIs(remote.alived, None)
2294            self.assertIs(remote.reaped, None)
2295
2296        console.terse("\nTest manage the unjoined remote *********\n")
2297        console.terse("\nMake all expired so send alive and reap *********\n")
2298        # advance clock so both remote timers expire
2299        self.store.advanceStamp(stacking.RoadStack.Interim)
2300        remote = main.remotes.values()[0]
2301        self.assertTrue(remote.timer.expired)
2302        self.assertTrue(remote.reapTimer.expired)
2303
2304        # Manage
2305        main.clearStats()
2306        main.manage()
2307        remote = main.remotes.values()[0]
2308
2309        self.assertEqual(len(main.transactions), 1) # started 1 join transactions
2310        self.assertIs(remote.joined, None)
2311        self.assertIs(remote.allowed, None)
2312        self.assertIs(remote.alived, None)
2313        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
2314
2315        # Service both sides
2316        self.serviceStacks(stacks)
2317        for stack in stacks:
2318            self.assertEqual(len(stack.transactions), 0) # join is done
2319        remote = main.remotes.values()[0]
2320        self.assertTrue(remote.joined) # join success
2321        self.assertIs(remote.allowed, None)
2322        self.assertIs(remote.alived, None)
2323        self.assertTrue(remote.reaped) # still reaped
2324
2325        for stack in [main, other]:
2326            stack.server.close()
2327            stack.clearAllKeeps()
2328
2329    def testManageUnallowedAllTimersExpired(self):
2330        '''
2331        Test stack manage unallowed remotes as if both presence and reap timers expired (A2)
2332        '''
2333        console.terse("{0}\n".format(self.testManageUnallowedAllTimersExpired.__doc__))
2334
2335        mainData = self.createRoadData(name='main',
2336                                       base=self.base,
2337                                       auto=raeting.AutoMode.once.value)
2338        keeping.clearAllKeep(mainData['dirpath'])
2339        main = self.createRoadStack(data=mainData,
2340                                    main=True,
2341                                    auto=mainData['auto'],
2342                                    ha=None)
2343
2344        otherData = self.createRoadData(name='other',
2345                                        base=self.base,
2346                                        auto=raeting.AutoMode.once.value)
2347        keeping.clearAllKeep(otherData['dirpath'])
2348        other = self.createRoadStack(data=otherData,
2349                                     main=None,
2350                                     auto=otherData['auto'],
2351                                     ha=("", raeting.RAET_TEST_PORT))
2352
2353        stacks = [main, other]
2354
2355        console.terse("\nBootstrap channel *********\n")
2356        self.join(other, main) # bootstrap channel
2357        for stack in [main, other]:
2358            self.assertEqual(len(stack.transactions), 0)
2359            self.assertEqual(len(stack.remotes), 1)
2360            remote = stack.remotes.values()[0]
2361            self.assertTrue(remote.joined)
2362            self.assertIs(remote.allowed, None)
2363            self.assertIs(remote.alived, None)
2364            self.assertIs(remote.reaped, None)
2365
2366        console.terse("\nTest manage the unallowed remote *********\n")
2367        console.terse("\nMake all expired so send alive and reap *********\n")
2368        # advance clock so both remote timers expire
2369        self.store.advanceStamp(stacking.RoadStack.Interim)
2370        remote = main.remotes.values()[0]
2371        self.assertTrue(remote.timer.expired)
2372        self.assertTrue(remote.reapTimer.expired)
2373
2374        # Manage
2375        main.manage()
2376        remote = main.remotes.values()[0]
2377
2378        self.assertEqual(len(main.transactions), 1) # started 1 allow transactions
2379        self.assertTrue(remote.joined)
2380        self.assertIs(remote.allowed, None)
2381        self.assertIs(remote.alived, None)
2382        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
2383
2384        # Service both sides
2385        self.serviceStacks(stacks)
2386        for stack in stacks:
2387            self.assertEqual(len(stack.transactions), 0) # allow is done
2388        remote = main.remotes.values()[0]
2389        self.assertTrue(remote.joined)
2390        self.assertTrue(remote.allowed) # allow success
2391        self.assertTrue(remote.alived) # fast alive
2392        self.assertFalse(remote.reaped) # unreaped by first non-join transaction
2393
2394        for stack in [main, other]:
2395            stack.server.close()
2396            stack.clearAllKeeps()
2397
2398    def testManageCascadeUnjoinedAllTimersExpired(self):
2399        '''
2400        Test stack manage cascade unjoined remotes as if both presence and reap timers expired (A3)
2401        '''
2402        console.terse("{0}\n".format(self.testManageCascadeUnjoinedAllTimersExpired.__doc__))
2403
2404        mainData = self.createRoadData(name='main',
2405                                       base=self.base,
2406                                       auto=raeting.AutoMode.once.value)
2407        keeping.clearAllKeep(mainData['dirpath'])
2408        main = self.createRoadStack(data=mainData,
2409                                    main=True,
2410                                    auto=mainData['auto'],
2411                                    ha=None)
2412
2413        otherData = self.createRoadData(name='other',
2414                                        base=self.base,
2415                                        auto=raeting.AutoMode.once.value)
2416        keeping.clearAllKeep(otherData['dirpath'])
2417        other = self.createRoadStack(data=otherData,
2418                                     main=None,
2419                                     auto=otherData['auto'],
2420                                     ha=("", raeting.RAET_TEST_PORT))
2421
2422        stacks = [main, other]
2423
2424        console.terse("\nBootstrap channel *********\n")
2425        self.join(other, main) # bootstrap channel
2426        for stack in [main, other]:
2427            self.assertEqual(len(stack.transactions), 0)
2428            self.assertEqual(len(stack.remotes), 1)
2429            remote = stack.remotes.values()[0]
2430            self.assertTrue(remote.joined)
2431
2432        # force unjoined already unallowed unalived
2433        for stack in [main, other]:
2434            stack.remotes.values()[0].joined = None
2435
2436        for stack in [main, other]:
2437            remote = stack.remotes.values()[0]
2438            self.assertIs(remote.joined, None)
2439            self.assertIs(remote.allowed, None)
2440            self.assertIs(remote.alived, None)
2441            self.assertIs(remote.reaped, None)
2442
2443        console.terse("\nTest manage cascade the unjoined remote *********\n")
2444        console.terse("\nMake all expired so send alive and reap *********\n")
2445        # advance clock so both remote timers expire
2446        self.store.advanceStamp(stacking.RoadStack.Interim)
2447        remote = main.remotes.values()[0]
2448        self.assertTrue(remote.timer.expired)
2449        self.assertTrue(remote.reapTimer.expired)
2450
2451        # Manage
2452        main.manage(cascade=True)
2453        remote = main.remotes.values()[0]
2454
2455        self.assertEqual(len(main.transactions), 1) # started 1 join transactions
2456        self.assertIs(remote.joined, None)
2457        self.assertIs(remote.allowed, None)
2458        self.assertIs(remote.alived, None)
2459        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
2460
2461        # Service both sides
2462        self.serviceStacks(stacks, duration=3.0)
2463        for stack in stacks:
2464            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2465        remote = main.remotes.values()[0]
2466        self.assertTrue(remote.joined) # join success
2467        self.assertTrue(remote.allowed) # allow success
2468        self.assertTrue(remote.alived) # alive success
2469        self.assertFalse(remote.reaped) # unreaped
2470
2471        for stack in [main, other]:
2472            stack.server.close()
2473            stack.clearAllKeeps()
2474
2475    def testManageCascadeUnallowedAllTimersExpired(self):
2476        '''
2477        Test stack manage cascade unallowed remotes as if both presence and reap timers expired (A4)
2478        '''
2479        console.terse("{0}\n".format(self.testManageCascadeUnallowedAllTimersExpired.__doc__))
2480
2481        mainData = self.createRoadData(name='main',
2482                                       base=self.base,
2483                                       auto=raeting.AutoMode.once.value)
2484        keeping.clearAllKeep(mainData['dirpath'])
2485        main = self.createRoadStack(data=mainData,
2486                                    main=True,
2487                                    auto=mainData['auto'],
2488                                    ha=None)
2489
2490        otherData = self.createRoadData(name='other',
2491                                        base=self.base,
2492                                        auto=raeting.AutoMode.once.value)
2493        keeping.clearAllKeep(otherData['dirpath'])
2494        other = self.createRoadStack(data=otherData,
2495                                     main=None,
2496                                     auto=otherData['auto'],
2497                                     ha=("", raeting.RAET_TEST_PORT))
2498
2499        stacks = [main, other]
2500
2501        console.terse("\nBootstrap channel *********\n")
2502        self.join(other, main) # bootstrap channel
2503        for stack in [main, other]:
2504            self.assertEqual(len(stack.transactions), 0)
2505            self.assertEqual(len(stack.remotes), 1)
2506            remote = stack.remotes.values()[0]
2507            self.assertTrue(remote.joined)
2508            self.assertIs(remote.allowed, None)
2509            self.assertIs(remote.alived, None)
2510            self.assertIs(remote.reaped, None)
2511
2512        console.terse("\nTest manage cascade the unallowed remote *********\n")
2513        console.terse("\nMake all expired so send alive and reap *********\n")
2514        # advance clock so both remote timers expire
2515        self.store.advanceStamp(stacking.RoadStack.Interim)
2516        remote = main.remotes.values()[0]
2517        self.assertTrue(remote.timer.expired)
2518        self.assertTrue(remote.reapTimer.expired)
2519
2520        # Manage
2521        main.manage(cascade=True)
2522        remote = main.remotes.values()[0]
2523
2524        self.assertEqual(len(main.transactions), 1) # started 1 allow transactions
2525        self.assertTrue(remote.joined)
2526        self.assertIs(remote.allowed, None)
2527        self.assertIs(remote.alived, None)
2528        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
2529
2530        # Service both sides
2531        self.serviceStacks(stacks)
2532        for stack in stacks:
2533            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2534        remote = main.remotes.values()[0]
2535        self.assertTrue(remote.joined)
2536        self.assertTrue(remote.allowed) # allow success
2537        self.assertTrue(remote.alived) # alive success
2538        self.assertFalse(remote.reaped) # unreaped
2539
2540        for stack in [main, other]:
2541            stack.server.close()
2542            stack.clearAllKeeps()
2543
2544    def testManageUnalivedAllTimersExpired(self):
2545        '''
2546        Test stack manage not alived remotes as if both presence and reap timers expired (A5)
2547        '''
2548        console.terse("{0}\n".format(self.testManageUnalivedAllTimersExpired.__doc__))
2549
2550        mainData = self.createRoadData(name='main',
2551                                       base=self.base,
2552                                       auto=raeting.AutoMode.once.value)
2553        keeping.clearAllKeep(mainData['dirpath'])
2554        main = self.createRoadStack(data=mainData,
2555                                    main=True,
2556                                    auto=mainData['auto'],
2557                                    ha=None)
2558
2559        otherData = self.createRoadData(name='other',
2560                                        base=self.base,
2561                                        auto=raeting.AutoMode.once.value)
2562        keeping.clearAllKeep(otherData['dirpath'])
2563        other = self.createRoadStack(data=otherData,
2564                                     main=None,
2565                                     auto=otherData['auto'],
2566                                     ha=("", raeting.RAET_TEST_PORT))
2567
2568        stacks = [main, other]
2569
2570        console.terse("\nBootstrap channel *********\n")
2571        self.join(other, main) # bootstrap channel
2572        self.allow(other, main)
2573        for stack in [main, other]:
2574            self.assertEqual(len(stack.transactions), 0)
2575            self.assertEqual(len(stack.remotes), 1)
2576            remote = stack.remotes.values()[0]
2577            self.assertTrue(remote.joined)
2578            self.assertTrue(remote.allowed)
2579            self.assertTrue(remote.alived)  # fast alive
2580            self.assertIs(remote.reaped, None)
2581
2582        console.terse("\nTest manage the joined and allowed remote *********\n")
2583        console.terse("\nMake all expired so send alive and reap *********\n")
2584        # advance clock so both remote timers expire
2585        self.store.advanceStamp(stacking.RoadStack.Interim)
2586        remote = main.remotes.values()[0]
2587        self.assertTrue(remote.timer.expired)
2588        self.assertTrue(remote.reapTimer.expired)
2589        main.remotes.values()[0].alived = None  # reset alived
2590
2591        # Manage
2592        main.manage()
2593        remote = main.remotes.values()[0]
2594
2595        self.assertEqual(len(main.transactions), 1) # started 1 alive transactions
2596        self.assertTrue(remote.joined)
2597        self.assertTrue(remote.allowed)
2598        self.assertIsNone(remote.alived)  # alived status isn't changed
2599        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
2600
2601        # Service both sides
2602        self.serviceStacks(stacks)
2603        for stack in stacks:
2604            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2605        remote = main.remotes.values()[0]
2606        self.assertTrue(remote.joined)
2607        self.assertTrue(remote.allowed)
2608        self.assertTrue(remote.alived) # alive success
2609        self.assertFalse(remote.reaped) # unreaped
2610
2611        for stack in [main, other]:
2612            stack.server.close()
2613            stack.clearAllKeeps()
2614
2615    def testManageUnjoinedPresenceTimerExpired(self):
2616        '''
2617        Test stack manage unjoined remotes as if presence timer expired (B1)
2618        '''
2619        console.terse("{0}\n".format(self.testManageUnjoinedPresenceTimerExpired.__doc__))
2620
2621        mainData = self.createRoadData(name='main',
2622                                       base=self.base,
2623                                       auto=raeting.AutoMode.once.value)
2624        keeping.clearAllKeep(mainData['dirpath'])
2625        main = self.createRoadStack(data=mainData,
2626                                    main=True,
2627                                    auto=mainData['auto'],
2628                                    ha=None)
2629
2630        otherData = self.createRoadData(name='other',
2631                                        base=self.base,
2632                                        auto=raeting.AutoMode.once.value)
2633        keeping.clearAllKeep(otherData['dirpath'])
2634        other = self.createRoadStack(data=otherData,
2635                                     main=None,
2636                                     auto=otherData['auto'],
2637                                     ha=("", raeting.RAET_TEST_PORT))
2638
2639        stacks = [main, other]
2640
2641        console.terse("\nBootstrap channel *********\n")
2642        self.join(other, main) # bootstrap channel
2643        for stack in [main, other]:
2644            self.assertEqual(len(stack.transactions), 0)
2645            self.assertEqual(len(stack.remotes), 1)
2646            remote = stack.remotes.values()[0]
2647            self.assertTrue(remote.joined)
2648
2649        # force unjoined already unallowed unalived
2650        for stack in [main, other]:
2651            stack.remotes.values()[0].joined = None
2652
2653        for stack in [main, other]:
2654            remote = stack.remotes.values()[0]
2655            self.assertIs(remote.joined, None)
2656            self.assertIs(remote.allowed, None)
2657            self.assertIs(remote.alived, None)
2658            self.assertIs(remote.reaped, None)
2659
2660        console.terse("\nTest manage the unjoined remote *********\n")
2661        console.terse("\nMake all expired so send alive and reap *********\n")
2662        # advance clock so remote keep alive timers expire
2663        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
2664        remote = main.remotes.values()[0]
2665        self.assertTrue(remote.timer.expired)
2666        self.assertFalse(remote.reapTimer.expired)
2667
2668        # Manage
2669        main.clearStats()
2670        main.manage()
2671        remote = main.remotes.values()[0]
2672
2673        self.assertEqual(len(main.transactions), 1) # started 1 join transactions
2674        self.assertIs(remote.joined, None)
2675        self.assertIs(remote.allowed, None)
2676        self.assertIs(remote.alived, None)
2677        self.assertIs(remote.reaped, None)
2678
2679        # Service both sides
2680        self.serviceStacks(stacks)
2681        for stack in stacks:
2682            self.assertEqual(len(stack.transactions), 0) # join is done
2683        remote = main.remotes.values()[0]
2684        self.assertTrue(remote.joined) # join success
2685        self.assertIs(remote.allowed, None)
2686        self.assertIs(remote.alived, None)
2687        self.assertIs(remote.reaped, None)
2688
2689        for stack in [main, other]:
2690            stack.server.close()
2691            stack.clearAllKeeps()
2692
2693    def testManageUnallowedPresenceTimerExpired(self):
2694        '''
2695        Test stack manage unallowed remotes as if presence timer expired (B2)
2696        '''
2697        console.terse("{0}\n".format(self.testManageUnallowedPresenceTimerExpired.__doc__))
2698
2699        mainData = self.createRoadData(name='main',
2700                                       base=self.base,
2701                                       auto=raeting.AutoMode.once.value)
2702        keeping.clearAllKeep(mainData['dirpath'])
2703        main = self.createRoadStack(data=mainData,
2704                                    main=True,
2705                                    auto=mainData['auto'],
2706                                    ha=None)
2707
2708        otherData = self.createRoadData(name='other',
2709                                        base=self.base,
2710                                        auto=raeting.AutoMode.once.value)
2711        keeping.clearAllKeep(otherData['dirpath'])
2712        other = self.createRoadStack(data=otherData,
2713                                     main=None,
2714                                     auto=otherData['auto'],
2715                                     ha=("", raeting.RAET_TEST_PORT))
2716
2717        stacks = [main, other]
2718
2719        console.terse("\nBootstrap channel *********\n")
2720        self.join(other, main) # bootstrap channel
2721        for stack in [main, other]:
2722            self.assertEqual(len(stack.transactions), 0)
2723            self.assertEqual(len(stack.remotes), 1)
2724            remote = stack.remotes.values()[0]
2725            self.assertTrue(remote.joined)
2726            self.assertIs(remote.allowed, None)
2727            self.assertIs(remote.alived, None)
2728            self.assertIs(remote.reaped, None)
2729
2730        console.terse("\nTest manage the unallowed remote *********\n")
2731        console.terse("\nMake all expired so send alive and reap *********\n")
2732        # advance clock so remote keep alive timers expire
2733        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
2734        remote = main.remotes.values()[0]
2735        self.assertTrue(remote.timer.expired)
2736        self.assertFalse(remote.reapTimer.expired)
2737
2738        # Manage
2739        main.manage()
2740        remote = main.remotes.values()[0]
2741
2742        self.assertEqual(len(main.transactions), 1) # started 1 allow transactions
2743        self.assertTrue(remote.joined)
2744        self.assertIs(remote.allowed, None)
2745        self.assertIs(remote.alived, None)
2746        self.assertIs(remote.reaped, None)
2747
2748        # Service both sides
2749        self.serviceStacks(stacks)
2750        for stack in stacks:
2751            self.assertEqual(len(stack.transactions), 0) # allow is done
2752        remote = main.remotes.values()[0]
2753        self.assertTrue(remote.joined)
2754        self.assertTrue(remote.allowed) # allow success
2755        self.assertTrue(remote.alived) # fast alive
2756        self.assertIs(remote.reaped, None)
2757
2758        for stack in [main, other]:
2759            stack.server.close()
2760            stack.clearAllKeeps()
2761
2762    def testManageCascadeUnjoinedPresenceTimerExpired(self):
2763        '''
2764        Test stack manage cascade unjoined remotes as if presence timer expired (B3)
2765        '''
2766        console.terse("{0}\n".format(self.testManageCascadeUnjoinedPresenceTimerExpired.__doc__))
2767
2768        mainData = self.createRoadData(name='main',
2769                                       base=self.base,
2770                                       auto=raeting.AutoMode.once.value)
2771        keeping.clearAllKeep(mainData['dirpath'])
2772        main = self.createRoadStack(data=mainData,
2773                                    main=True,
2774                                    auto=mainData['auto'],
2775                                    ha=None)
2776
2777        otherData = self.createRoadData(name='other',
2778                                        base=self.base,
2779                                        auto=raeting.AutoMode.once.value)
2780        keeping.clearAllKeep(otherData['dirpath'])
2781        other = self.createRoadStack(data=otherData,
2782                                     main=None,
2783                                     auto=otherData['auto'],
2784                                     ha=("", raeting.RAET_TEST_PORT))
2785
2786        stacks = [main, other]
2787
2788        console.terse("\nBootstrap channel *********\n")
2789        self.join(other, main) # bootstrap channel
2790        for stack in [main, other]:
2791            self.assertEqual(len(stack.transactions), 0)
2792            self.assertEqual(len(stack.remotes), 1)
2793            remote = stack.remotes.values()[0]
2794            self.assertTrue(remote.joined)
2795
2796        # force unjoined already unallowed unalived
2797        for stack in [main, other]:
2798            stack.remotes.values()[0].joined = None
2799
2800        for stack in [main, other]:
2801            remote = stack.remotes.values()[0]
2802            self.assertIs(remote.joined, None)
2803            self.assertIs(remote.allowed, None)
2804            self.assertIs(remote.alived, None)
2805            self.assertIs(remote.reaped, None)
2806
2807        console.terse("\nTest manage cascade the unjoined remote *********\n")
2808        console.terse("\nMake all expired so send alive and reap *********\n")
2809        # advance clock so remote keep alive timers expire
2810        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
2811        remote = main.remotes.values()[0]
2812        self.assertTrue(remote.timer.expired)
2813        self.assertFalse(remote.reapTimer.expired)
2814
2815        # Manage
2816        main.manage(cascade=True)
2817        remote = main.remotes.values()[0]
2818
2819        self.assertEqual(len(main.transactions), 1) # started 1 join transactions
2820        self.assertIs(remote.joined, None)
2821        self.assertIs(remote.allowed, None)
2822        self.assertIs(remote.alived, None)
2823        self.assertIs(remote.reaped, None)
2824
2825        # Service both sides
2826        self.serviceStacks(stacks, duration=3.0)
2827        for stack in stacks:
2828            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2829        remote = main.remotes.values()[0]
2830        self.assertTrue(remote.joined) # join success
2831        self.assertTrue(remote.allowed) # allow success
2832        self.assertTrue(remote.alived) # alive success
2833        self.assertIs(remote.reaped, None)
2834
2835        for stack in [main, other]:
2836            stack.server.close()
2837            stack.clearAllKeeps()
2838
2839    def testManageCascadeUnallowedPresenceTimerExpired(self):
2840        '''
2841        Test stack manage cascade unallowed remotes as if presence timer expired (B4)
2842        '''
2843        console.terse("{0}\n".format(self.testManageCascadeUnallowedPresenceTimerExpired.__doc__))
2844
2845        mainData = self.createRoadData(name='main',
2846                                       base=self.base,
2847                                       auto=raeting.AutoMode.once.value)
2848        keeping.clearAllKeep(mainData['dirpath'])
2849        main = self.createRoadStack(data=mainData,
2850                                    main=True,
2851                                    auto=mainData['auto'],
2852                                    ha=None)
2853
2854        otherData = self.createRoadData(name='other',
2855                                        base=self.base,
2856                                        auto=raeting.AutoMode.once.value)
2857        keeping.clearAllKeep(otherData['dirpath'])
2858        other = self.createRoadStack(data=otherData,
2859                                     main=None,
2860                                     auto=otherData['auto'],
2861                                     ha=("", raeting.RAET_TEST_PORT))
2862
2863        stacks = [main, other]
2864
2865        console.terse("\nBootstrap channel *********\n")
2866        self.join(other, main) # bootstrap channel
2867        for stack in [main, other]:
2868            self.assertEqual(len(stack.transactions), 0)
2869            self.assertEqual(len(stack.remotes), 1)
2870            remote = stack.remotes.values()[0]
2871            self.assertTrue(remote.joined)
2872            self.assertIs(remote.allowed, None)
2873            self.assertIs(remote.alived, None)
2874            self.assertIs(remote.reaped, None)
2875
2876        console.terse("\nTest manage cascade the unallowed remote *********\n")
2877        console.terse("\nMake all expired so send alive and reap *********\n")
2878        # advance clock so remote keep alive timers expire
2879        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
2880        remote = main.remotes.values()[0]
2881        self.assertTrue(remote.timer.expired)
2882        self.assertFalse(remote.reapTimer.expired)
2883
2884        # Manage
2885        main.manage(cascade=True)
2886        remote = main.remotes.values()[0]
2887
2888        self.assertEqual(len(main.transactions), 1) # started 1 allow transactions
2889        self.assertTrue(remote.joined)
2890        self.assertIs(remote.allowed, None)
2891        self.assertIs(remote.alived, None)
2892        self.assertIs(remote.reaped, None)
2893
2894        # Service both sides
2895        self.serviceStacks(stacks)
2896        for stack in stacks:
2897            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2898        remote = main.remotes.values()[0]
2899        self.assertTrue(remote.joined)
2900        self.assertTrue(remote.allowed) # allow success
2901        self.assertTrue(remote.alived) # alive success
2902        self.assertIs(remote.reaped, None)
2903
2904        for stack in [main, other]:
2905            stack.server.close()
2906            stack.clearAllKeeps()
2907
2908    def testManageUnalivedPresenceTimerExpired(self):
2909        '''
2910        Test stack manage not alived remotes as if presence timer expired (B5)
2911        '''
2912        console.terse("{0}\n".format(self.testManageUnalivedPresenceTimerExpired.__doc__))
2913
2914        mainData = self.createRoadData(name='main',
2915                                       base=self.base,
2916                                       auto=raeting.AutoMode.once.value)
2917        keeping.clearAllKeep(mainData['dirpath'])
2918        main = self.createRoadStack(data=mainData,
2919                                    main=True,
2920                                    auto=mainData['auto'],
2921                                    ha=None)
2922
2923        otherData = self.createRoadData(name='other',
2924                                        base=self.base,
2925                                        auto=raeting.AutoMode.once.value)
2926        keeping.clearAllKeep(otherData['dirpath'])
2927        other = self.createRoadStack(data=otherData,
2928                                     main=None,
2929                                     auto=otherData['auto'],
2930                                     ha=("", raeting.RAET_TEST_PORT))
2931
2932        stacks = [main, other]
2933
2934        console.terse("\nBootstrap channel *********\n")
2935        self.join(other, main) # bootstrap channel
2936        self.allow(other, main)
2937        for stack in [main, other]:
2938            self.assertEqual(len(stack.transactions), 0)
2939            self.assertEqual(len(stack.remotes), 1)
2940            remote = stack.remotes.values()[0]
2941            self.assertTrue(remote.joined)
2942            self.assertTrue(remote.allowed)
2943            self.assertTrue(remote.alived)  # fast alive
2944            self.assertIs(remote.reaped, None)
2945
2946        console.terse("\nTest manage the joined and allowed remote *********\n")
2947        console.terse("\nMake timer expired so send alive *********\n")
2948        # advance clock so remote keep alive timers expire
2949        self.store.advanceStamp(stacking.RoadStack.Period + stacking.RoadStack.Offset)
2950        remote = main.remotes.values()[0]
2951        self.assertTrue(remote.timer.expired)
2952        self.assertFalse(remote.reapTimer.expired)
2953        main.remotes.values()[0].alived = None  # reset alived
2954
2955        # Manage
2956        main.manage()
2957        remote = main.remotes.values()[0]
2958
2959        self.assertEqual(len(main.transactions), 1) # started 1 alive transactions
2960        self.assertTrue(remote.joined)
2961        self.assertTrue(remote.allowed)
2962        self.assertIs(remote.alived, None)
2963        self.assertIs(remote.reaped, None)
2964
2965        # Service both sides
2966        self.serviceStacks(stacks)
2967        for stack in stacks:
2968            self.assertEqual(len(stack.transactions), 0) # all transactions finished
2969        remote = main.remotes.values()[0]
2970        self.assertTrue(remote.joined)
2971        self.assertTrue(remote.allowed)
2972        self.assertTrue(remote.alived) # alive success
2973        self.assertIs(remote.reaped, None)
2974
2975        for stack in [main, other]:
2976            stack.server.close()
2977            stack.clearAllKeeps()
2978
2979    def testManageReapTimerExpired(self):
2980        '''
2981        Test stack manage remotes as if reap timer expired (C1)
2982        '''
2983        console.terse("{0}\n".format(self.testManageReapTimerExpired.__doc__))
2984
2985        mainData = self.createRoadData(name='main',
2986                                       base=self.base,
2987                                       auto=raeting.AutoMode.once.value)
2988        keeping.clearAllKeep(mainData['dirpath'])
2989        main = self.createRoadStack(data=mainData,
2990                                    main=True,
2991                                    auto=mainData['auto'],
2992                                    ha=None,
2993                                    period=stacking.RoadStack.Interim*2) # set presence timeout longer than reap
2994
2995        otherData = self.createRoadData(name='other',
2996                                        base=self.base,
2997                                        auto=raeting.AutoMode.once.value)
2998        keeping.clearAllKeep(otherData['dirpath'])
2999        other = self.createRoadStack(data=otherData,
3000                                     main=None,
3001                                     auto=otherData['auto'],
3002                                     ha=("", raeting.RAET_TEST_PORT))
3003
3004        stacks = [main, other]
3005
3006        console.terse("\nBootstrap channel *********\n")
3007        self.join(other, main) # bootstrap channel
3008        for stack in [main, other]:
3009            self.assertEqual(len(stack.transactions), 0)
3010            self.assertEqual(len(stack.remotes), 1)
3011            remote = stack.remotes.values()[0]
3012            self.assertTrue(remote.joined)
3013            self.assertIs(remote.allowed, None)
3014            self.assertIs(remote.alived, None)
3015            self.assertIs(remote.reaped, None)
3016
3017        console.terse("\nTest manage remote *********\n")
3018        console.terse("\nMake reap timer expired so send nothing just reap *********\n")
3019        # advance clock so reap timer expire
3020        self.store.advanceStamp(stacking.RoadStack.Interim)
3021        remote = main.remotes.values()[0]
3022        self.assertFalse(remote.timer.expired)
3023        self.assertTrue(remote.reapTimer.expired)
3024
3025        # Manage
3026        main.clearStats()
3027        main.manage()
3028        remote = main.remotes.values()[0]
3029
3030        self.assertEqual(len(main.transactions), 0) # started no transactions because presence timer isn't expired
3031        self.assertTrue(remote.joined)
3032        self.assertIs(remote.allowed, None)
3033        self.assertIs(remote.alived, None)
3034        self.assertTrue(remote.reaped) # reaped the remote because reap timer expired
3035
3036        # Service both sides
3037        self.serviceStacks(stacks)
3038        for stack in stacks:
3039            self.assertEqual(len(stack.transactions), 0)
3040        remote = main.remotes.values()[0]
3041        self.assertTrue(remote.joined)
3042        self.assertIs(remote.allowed, None)
3043        self.assertIs(remote.alived, None)
3044        self.assertTrue(remote.reaped) # still reaped since there was no transactions
3045
3046        for stack in [main, other]:
3047            stack.server.close()
3048            stack.clearAllKeeps()
3049
3050    def testManageNoTimerExpired(self):
3051        '''
3052        Test stack manage remotes as if no timer expired (C2)
3053        '''
3054        console.terse("{0}\n".format(self.testManageNoTimerExpired.__doc__))
3055
3056        mainData = self.createRoadData(name='main',
3057                                       base=self.base,
3058                                       auto=raeting.AutoMode.once.value)
3059        keeping.clearAllKeep(mainData['dirpath'])
3060        main = self.createRoadStack(data=mainData,
3061                                    main=True,
3062                                    auto=mainData['auto'],
3063                                    ha=None)
3064
3065        otherData = self.createRoadData(name='other',
3066                                        base=self.base,
3067                                        auto=raeting.AutoMode.once.value)
3068        keeping.clearAllKeep(otherData['dirpath'])
3069        other = self.createRoadStack(data=otherData,
3070                                     main=None,
3071                                     auto=otherData['auto'],
3072                                     ha=("", raeting.RAET_TEST_PORT))
3073
3074        stacks = [main, other]
3075
3076        console.terse("\nBootstrap channel *********\n")
3077        self.join(other, main) # bootstrap channel
3078        for stack in [main, other]:
3079            self.assertEqual(len(stack.transactions), 0)
3080            self.assertEqual(len(stack.remotes), 1)
3081            remote = stack.remotes.values()[0]
3082            self.assertTrue(remote.joined)
3083            self.assertIs(remote.allowed, None)
3084            self.assertIs(remote.alived, None)
3085            self.assertIs(remote.reaped, None)
3086
3087        console.terse("\nTest manage remote *********\n")
3088        # ensure no timer expired
3089        remote = main.remotes.values()[0]
3090        self.assertFalse(remote.timer.expired)
3091        self.assertFalse(remote.reapTimer.expired)
3092
3093        # Manage
3094        main.clearStats()
3095        main.manage()
3096        remote = main.remotes.values()[0]
3097
3098        self.assertEqual(len(main.transactions), 0) # started no transactions because presence timer isn't expired
3099        self.assertTrue(remote.joined)
3100        self.assertIs(remote.allowed, None)
3101        self.assertIs(remote.alived, None)
3102        self.assertIs(remote.reaped, None)
3103
3104        # Service both sides
3105        self.serviceStacks(stacks)
3106        for stack in stacks:
3107            self.assertEqual(len(stack.transactions), 0)
3108        remote = main.remotes.values()[0]
3109        self.assertTrue(remote.joined)
3110        self.assertIs(remote.allowed, None)
3111        self.assertIs(remote.alived, None)
3112        self.assertIs(remote.reaped, None)
3113
3114        for stack in [main, other]:
3115            stack.server.close()
3116            stack.clearAllKeeps()
3117
3118    def testManageReaped(self):
3119        '''
3120        Test stack manage reaped remotes (D1)
3121        '''
3122        console.terse("{0}\n".format(self.testManageReaped.__doc__))
3123
3124        mainData = self.createRoadData(name='main',
3125                                       base=self.base,
3126                                       auto=raeting.AutoMode.once.value)
3127        keeping.clearAllKeep(mainData['dirpath'])
3128        main = self.createRoadStack(data=mainData,
3129                                    main=True,
3130                                    auto=mainData['auto'],
3131                                    ha=None)
3132
3133        otherData = self.createRoadData(name='other',
3134                                        base=self.base,
3135                                        auto=raeting.AutoMode.once.value)
3136        keeping.clearAllKeep(otherData['dirpath'])
3137        other = self.createRoadStack(data=otherData,
3138                                     main=None,
3139                                     auto=otherData['auto'],
3140                                     ha=("", raeting.RAET_TEST_PORT))
3141
3142        stacks = [main, other]
3143
3144        console.terse("\nBootstrap channel *********\n")
3145        self.join(other, main) # bootstrap channel
3146        for stack in [main, other]:
3147            self.assertEqual(len(stack.transactions), 0)
3148            self.assertEqual(len(stack.remotes), 1)
3149            remote = stack.remotes.values()[0]
3150            self.assertTrue(remote.joined)
3151            self.assertIs(remote.allowed, None)
3152            self.assertIs(remote.alived, None)
3153            self.assertIs(remote.reaped, None)
3154
3155        console.terse("\nTest manage the unallowed remote *********\n")
3156        console.terse("\nMake all expired so send alive and reap *********\n")
3157        # advance clock so both remote timers expire
3158        self.store.advanceStamp(stacking.RoadStack.Interim)
3159        remote = main.remotes.values()[0]
3160        self.assertTrue(remote.timer.expired)
3161        self.assertTrue(remote.reapTimer.expired)
3162        # force reap the remote
3163        remote.reaped = True
3164
3165        # Manage
3166        main.manage()
3167        remote = main.remotes.values()[0]
3168
3169        # check nothing has changed
3170        self.assertEqual(len(main.transactions), 0) # started no transactions
3171        self.assertTrue(remote.joined)
3172        self.assertIs(remote.allowed, None)
3173        self.assertIs(remote.alived, None)
3174        self.assertTrue(remote.reaped)
3175
3176        # Service both sides
3177        self.serviceStacks(stacks)
3178        # check nothing has changed
3179        for stack in stacks:
3180            self.assertEqual(len(stack.transactions), 0)
3181        remote = main.remotes.values()[0]
3182        self.assertTrue(remote.joined)
3183        self.assertIs(remote.allowed, None)
3184        self.assertIs(remote.alived, None)
3185        self.assertTrue(remote.reaped)
3186
3187        for stack in [main, other]:
3188            stack.server.close()
3189            stack.clearAllKeeps()
3190
3191    def testAliventUnjoined(self):
3192        '''
3193        Test unjoined alivent receives alive (A1)
3194        Refresh no change alive, nack unjoined, remove
3195        '''
3196        console.terse("{0}\n".format(self.testAliventUnjoined.__doc__))
3197
3198        mainData = self.createRoadData(name='main',
3199                                       base=self.base,
3200                                       auto=raeting.AutoMode.once.value)
3201        keeping.clearAllKeep(mainData['dirpath'])
3202        main = self.createRoadStack(data=mainData,
3203                                    main=True,
3204                                    auto=mainData['auto'],
3205                                    ha=None)
3206
3207        otherData = self.createRoadData(name='other',
3208                                        base=self.base,
3209                                        auto=raeting.AutoMode.once.value)
3210        keeping.clearAllKeep(otherData['dirpath'])
3211        other = self.createRoadStack(data=otherData,
3212                                     main=None,
3213                                     auto=otherData['auto'],
3214                                     ha=("", raeting.RAET_TEST_PORT))
3215
3216        stacks = [main, other]
3217
3218        console.terse("\nBootstrap channel *********\n")
3219        # remote on joiner side have to be joined and allowed
3220        self.join(other, main) # bootstrap channel
3221        self.allow(other, main)
3222        for stack in [main, other]:
3223            self.assertEqual(len(stack.transactions), 0)
3224            self.assertEqual(len(stack.remotes), 1)
3225            remote = stack.remotes.values()[0]
3226            self.assertTrue(remote.joined)
3227            self.assertTrue(remote.allowed)
3228            self.assertTrue(remote.alived)  # fast alive
3229
3230        # force unjoin, unallow and unalive other side
3231        other.remotes.values()[0].joined = None
3232        other.remotes.values()[0].allowed = None
3233        other.remotes.values()[0].alived = None
3234
3235        self.assertTrue(main.remotes.values()[0].joined)
3236        self.assertTrue(main.remotes.values()[0].allowed)
3237        self.assertTrue(main.remotes.values()[0].alived)  # fast alive
3238        self.assertIs(other.remotes.values()[0].joined, None)
3239        self.assertIs(other.remotes.values()[0].allowed, None)
3240        self.assertIs(other.remotes.values()[0].alived, None)
3241        for stack in [main, other]:
3242            remote = stack.remotes.values()[0]
3243            self.assertIs(remote.reaped, None)
3244
3245        console.terse("\nTest unjoined alivent *********\n")
3246        main.alive() # no cascade alive, unjoined so would be joined as a final result
3247        self.serviceStack(main, duration=0.1)
3248        self.serviceStack(other, duration=0.1)
3249        self.assertEqual(len(other.transactions), 0) # transaction was removed
3250        self.assertIn('unjoined_alive_attempt', other.stats) # the reason is 'unjoined'
3251        self.assertEqual(other.stats['unjoined_alive_attempt'], 1)
3252        self.assertEqual(len(main.transactions), 1)
3253        self.assertEqual(len(other.remotes), 1)
3254        remote = other.remotes.values()[0]
3255        self.assertIs(remote.joined, None)
3256        self.assertIs(remote.allowed, None)
3257        self.assertIs(remote.alived, None) # not changed
3258        self.assertIs(remote.reaped, None)
3259
3260        # alive nacked as 'unjoined', the result would be join attempt from aliver side
3261        self.serviceStacks(stacks)
3262        for stack in stacks:
3263            self.assertEqual(len(stack.transactions), 0)
3264            self.assertEqual(len(stack.remotes), 1)
3265            remote = other.remotes.values()[0]
3266            self.assertTrue(remote.joined)
3267            self.assertIs(remote.allowed, None)
3268            self.assertIs(remote.alived, None)
3269            self.assertIs(remote.reaped, None)
3270
3271        for stack in [main, other]:
3272            stack.server.close()
3273            stack.clearAllKeeps()
3274
3275    def testAliventUnallowed(self):
3276        '''
3277        Test unallowed alivent receives alive (A2)
3278        Refresh no change alive, nack unallowed, remove
3279        '''
3280        console.terse("{0}\n".format(self.testAliventUnallowed.__doc__))
3281
3282        mainData = self.createRoadData(name='main',
3283                                       base=self.base,
3284                                       auto=raeting.AutoMode.once.value)
3285        keeping.clearAllKeep(mainData['dirpath'])
3286        main = self.createRoadStack(data=mainData,
3287                                    main=True,
3288                                    auto=mainData['auto'],
3289                                    ha=None)
3290
3291        otherData = self.createRoadData(name='other',
3292                                        base=self.base,
3293                                        auto=raeting.AutoMode.once.value)
3294        keeping.clearAllKeep(otherData['dirpath'])
3295        other = self.createRoadStack(data=otherData,
3296                                     main=None,
3297                                     auto=otherData['auto'],
3298                                     ha=("", raeting.RAET_TEST_PORT))
3299
3300        stacks = [main, other]
3301
3302        console.terse("\nBootstrap channel *********\n")
3303        # remote on joiner side have to be joined and allowed
3304        self.join(other, main) # bootstrap channel
3305        self.allow(other, main)
3306        for stack in [main, other]:
3307            self.assertEqual(len(stack.transactions), 0)
3308            self.assertEqual(len(stack.remotes), 1)
3309            remote = stack.remotes.values()[0]
3310            self.assertTrue(remote.joined)
3311            self.assertTrue(remote.allowed)
3312            self.assertTrue(remote.alived)  # fast alive
3313
3314        # force unallow and unalive other side
3315        other.remotes.values()[0].allowed = None
3316        other.remotes.values()[0].alived = None
3317
3318        self.assertTrue(main.remotes.values()[0].allowed)
3319        self.assertTrue(main.remotes.values()[0].alived)
3320        self.assertIs(other.remotes.values()[0].allowed, None)
3321        self.assertIs(other.remotes.values()[0].alived, None)
3322        for stack in [main, other]:
3323            remote = stack.remotes.values()[0]
3324            self.assertTrue(remote.joined)
3325            self.assertIs(remote.reaped, None)
3326
3327        console.terse("\nTest unallowed alivent *********\n")
3328        main.alive() # no cascade alive, unallowed so would be allowed as a final result
3329        self.serviceStack(main, duration=0.1)
3330        self.serviceStack(other, duration=0.1)
3331        self.assertEqual(len(other.transactions), 0) # transaction was removed
3332        self.assertIn('unallowed_alive_attempt', other.stats) # the reason is 'unallowed'
3333        self.assertEqual(other.stats['unallowed_alive_attempt'], 1)
3334        self.assertEqual(len(main.transactions), 1)
3335        self.assertEqual(len(other.remotes), 1)
3336        remote = other.remotes.values()[0]
3337        self.assertTrue(remote.joined)
3338        self.assertIs(remote.allowed, None)
3339        self.assertIs(remote.alived, None) # not changed
3340        self.assertIs(remote.reaped, None)
3341
3342        # alive nacked as 'unallowed', the result would be allow attempt from aliver side
3343        self.serviceStacks(stacks)
3344        for stack in stacks:
3345            self.assertEqual(len(stack.transactions), 0)
3346            self.assertEqual(len(stack.remotes), 1)
3347            remote = other.remotes.values()[0]
3348            self.assertTrue(remote.joined)
3349            self.assertTrue(remote.allowed)
3350            self.assertTrue(remote.alived)  # fast alive
3351            self.assertIs(remote.reaped, None)
3352
3353        for stack in [main, other]:
3354            stack.server.close()
3355            stack.clearAllKeeps()
3356
3357    def testAliventJoinedAllowed(self):
3358        '''
3359        Test joined and allowed alivent receives alive (C1)
3360        Ack, refresh alive, unreap
3361        '''
3362        console.terse("{0}\n".format(self.testAliventUnallowed.__doc__))
3363
3364        mainData = self.createRoadData(name='main',
3365                                       base=self.base,
3366                                       auto=raeting.AutoMode.once.value)
3367        keeping.clearAllKeep(mainData['dirpath'])
3368        main = self.createRoadStack(data=mainData,
3369                                    main=True,
3370                                    auto=mainData['auto'],
3371                                    ha=None)
3372
3373        otherData = self.createRoadData(name='other',
3374                                        base=self.base,
3375                                        auto=raeting.AutoMode.once.value)
3376        keeping.clearAllKeep(otherData['dirpath'])
3377        other = self.createRoadStack(data=otherData,
3378                                     main=True, # only main can reap and unreap
3379                                     auto=otherData['auto'],
3380                                     ha=("", raeting.RAET_TEST_PORT))
3381
3382        stacks = [main, other]
3383
3384        console.terse("\nBootstrap channel *********\n")
3385        # remote on joiner side have to be joined and allowed
3386        self.join(other, main) # bootstrap channel
3387        self.allow(other, main)
3388        for stack in [main, other]:
3389            self.assertEqual(len(stack.transactions), 0)
3390            self.assertEqual(len(stack.remotes), 1)
3391            remote = stack.remotes.values()[0]
3392            self.assertTrue(remote.joined)
3393            self.assertTrue(remote.allowed)
3394            self.assertTrue(remote.alived) # fast alive
3395            self.assertIs(remote.reaped, None)
3396
3397        # force unalive and reap to check it would be unreaped as result
3398        other.remotes.values()[0].alived = None
3399        other.remotes.values()[0].reaped = True
3400
3401        console.terse("\nTest joined allowed alivent *********\n")
3402        main.alive() # no cascade alive, joined and allowed so ack, refresh alive and unreap
3403        self.serviceStack(main, duration=0.1)
3404        self.serviceStack(other,duration=0.1)
3405        self.assertEqual(len(other.transactions), 0) # transaction finished
3406        self.assertIn('alive_complete', other.stats) # the reason is 'unallowed'
3407        self.assertEqual(other.stats['alive_complete'], 1)
3408        self.assertEqual(len(main.transactions), 1)
3409        self.assertEqual(len(other.remotes), 1)
3410        remote = other.remotes.values()[0]
3411        self.assertTrue(remote.joined)
3412        self.assertTrue(remote.allowed)
3413        self.assertTrue(remote.alived) # alive updated
3414        self.assertFalse(remote.reaped) # unreaped
3415
3416        # alive acked
3417        self.serviceStacks(stacks)
3418        for stack in stacks:
3419            self.assertEqual(len(stack.transactions), 0)
3420            self.assertEqual(len(stack.remotes), 1)
3421            remote = other.remotes.values()[0]
3422            self.assertTrue(remote.joined)
3423            self.assertTrue(remote.allowed)
3424            self.assertTrue(remote.alived)
3425        self.assertIs(main.remotes.values()[0].reaped, None)
3426        self.assertFalse(other.remotes.values()[0].reaped)
3427
3428        for stack in [main, other]:
3429            stack.server.close()
3430            stack.clearAllKeeps()
3431
3432    def testAliverReceiveNack(self):
3433        '''
3434        Test aliver receive nack: nack, refuse (A2) and reject (A5)
3435        Other cases which are ack (A1), unjoined (A3) and unallowed (A4) are covered by other tests
3436        '''
3437        console.terse("{0}\n".format(self.testAliverReceiveNack.__doc__))
3438
3439        mainData = self.createRoadData(name='main',
3440                                       base=self.base,
3441                                       auto=raeting.AutoMode.once.value)
3442        keeping.clearAllKeep(mainData['dirpath'])
3443        main = self.createRoadStack(data=mainData,
3444                                    main=True,
3445                                    auto=mainData['auto'],
3446                                    ha=None)
3447
3448        otherData = self.createRoadData(name='other',
3449                                        base=self.base,
3450                                        auto=raeting.AutoMode.once.value)
3451        keeping.clearAllKeep(otherData['dirpath'])
3452        other = self.createRoadStack(data=otherData,
3453                                     main=None,
3454                                     auto=otherData['auto'],
3455                                     ha=("", raeting.RAET_TEST_PORT))
3456
3457        stacks = [main, other]
3458
3459        console.terse("\nBootstrap channel *********\n")
3460        # remote on joiner side have to be joined and allowed
3461        self.join(other, main) # bootstrap channel
3462        self.allow(other, main)
3463        self.alive(main, other)
3464        for stack in [main, other]:
3465            self.assertEqual(len(stack.transactions), 0)
3466            self.assertEqual(len(stack.remotes), 1)
3467            remote = stack.remotes.values()[0]
3468            self.assertTrue(remote.joined)
3469            self.assertTrue(remote.allowed)
3470            self.assertTrue(remote.alived)
3471            self.assertIs(remote.reaped, None)
3472
3473        console.terse("\nTest aliver receive nack *********\n")
3474        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3475        self.serviceStack(main, duration=0.1)
3476        self.answerAlive(other, kind=raeting.PcktKind.nack.value)
3477
3478        self.serviceStacks(stacks)
3479        for stack in stacks:
3480            self.assertEqual(len(stack.transactions), 0)
3481            self.assertEqual(len(stack.remotes), 1)
3482            remote = other.remotes.values()[0]
3483            self.assertTrue(remote.joined)
3484            self.assertTrue(remote.allowed)
3485            self.assertTrue(remote.alived) # alived status isn't changed
3486            self.assertIs(remote.reaped, None)
3487        self.assertIn('aliver_transaction_failure', main.stats) # transaction failed
3488        self.assertEqual(main.stats['aliver_transaction_failure'], 1)
3489
3490        main.clearStats()
3491        console.terse("\nTest aliver receive refuse *********\n")
3492        main.alive() # no cascade alive, aliver will set alive to False and remove transaction
3493        self.serviceStack(main, duration=0.1)
3494        self.answerAlive(other, kind=raeting.PcktKind.refuse.value)
3495
3496        self.serviceStacks(stacks)
3497        for stack in stacks:
3498            self.assertEqual(len(stack.transactions), 0)
3499            self.assertEqual(len(stack.remotes), 1)
3500            remote = other.remotes.values()[0]
3501            self.assertTrue(remote.joined)
3502            self.assertTrue(remote.allowed)
3503            self.assertTrue(remote.alived) # alived status isn't changed
3504            self.assertIs(remote.reaped, None)
3505        self.assertIn('aliver_transaction_failure', main.stats) # transaction failed
3506        self.assertEqual(main.stats['aliver_transaction_failure'], 1)
3507
3508        main.clearStats()
3509        console.terse("\nTest aliver receive reject *********\n")
3510        main.alive() # no cascade alive, aliver will set alive to False and remove transaction
3511        self.serviceStack(main, duration=0.1)
3512        self.answerAlive(other, kind=raeting.PcktKind.reject.value)
3513
3514        self.serviceStacks(stacks)
3515        for stack in stacks:
3516            self.assertEqual(len(stack.transactions), 0)
3517            self.assertEqual(len(stack.remotes), 1)
3518            remote = other.remotes.values()[0]
3519            self.assertTrue(remote.joined)
3520            self.assertTrue(remote.allowed)
3521            self.assertIs(remote.reaped, None)
3522        self.assertIs(main.remotes.values()[0].alived, False) # alive is set to false
3523        self.assertIn('aliver_transaction_failure', main.stats) # transaction failed
3524        self.assertEqual(main.stats['aliver_transaction_failure'], 1)
3525
3526        main.clearStats()
3527        console.terse("\nTest invalid (unexpected) nack kind *********\n")
3528        main.alive() # no cascade alive, aliver will set alive to False and remove transaction
3529        self.serviceStack(main, duration=0.1)
3530        self.answerAlive(other, kind=raeting.PcktKind.unknown.value) # unknown is invalid kind for nack. it will be set to nack
3531        self.serviceStack(other) # send nack
3532        self.serviceStack(main) # receive nack
3533
3534        self.assertEqual(len(main.transactions), 1) # transaction wasn't dropped, will attempt to redo
3535        self.assertEqual(len(other.transactions), 0)
3536
3537        self.serviceStacks(stacks)
3538        for stack in stacks:
3539            self.assertEqual(len(stack.transactions), 0)
3540            self.assertEqual(len(stack.remotes), 1)
3541            remote = other.remotes.values()[0]
3542            self.assertTrue(remote.joined)
3543            self.assertTrue(remote.allowed)
3544            self.assertIs(remote.reaped, None)
3545        self.assertTrue(main.remotes.values()[0].alived) # alived after redo
3546        for stack in [main, other]:
3547            stack.server.close()
3548            stack.clearAllKeeps()
3549
3550    def testAliverAlivePackError(self):
3551        '''
3552        Test packet.pack() throws error in aliver.alive() (coverage)
3553        '''
3554        console.terse("{0}\n".format(self.testAliverAlivePackError.__doc__))
3555
3556        mainData = self.createRoadData(name='main',
3557                                       base=self.base,
3558                                       auto=raeting.AutoMode.once.value)
3559        keeping.clearAllKeep(mainData['dirpath'])
3560        main = self.createRoadStack(data=mainData,
3561                                    main=True,
3562                                    auto=mainData['auto'],
3563                                    ha=None)
3564
3565        otherData = self.createRoadData(name='other',
3566                                        base=self.base,
3567                                        auto=raeting.AutoMode.once.value)
3568        keeping.clearAllKeep(otherData['dirpath'])
3569        other = self.createRoadStack(data=otherData,
3570                                     main=None,
3571                                     auto=otherData['auto'],
3572                                     ha=("", raeting.RAET_TEST_PORT))
3573
3574        stacks = [main, other]
3575
3576        console.terse("\nBootstrap channel *********\n")
3577        # remote on joiner side have to be joined and allowed
3578        self.join(other, main) # bootstrap channel
3579        self.allow(other, main)
3580        self.alive(main, other)
3581        for stack in [main, other]:
3582            self.assertEqual(len(stack.transactions), 0)
3583            self.assertEqual(len(stack.remotes), 1)
3584            remote = stack.remotes.values()[0]
3585            self.assertTrue(remote.joined)
3586            self.assertTrue(remote.allowed)
3587            self.assertTrue(remote.alived)
3588            self.assertIs(remote.reaped, None)
3589
3590        main.clearStats()
3591        console.terse("\nTest aliver packet.pack error *********\n")
3592        default_size = raeting.UDP_MAX_PACKET_SIZE
3593        raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will throw PacketError
3594        main.alive() # will fail with packing error
3595        raeting.UDP_MAX_PACKET_SIZE = default_size
3596
3597        self.assertEqual(len(main.transactions), 0)
3598        self.assertEqual(len(main.remotes), 1)
3599        remote = other.remotes.values()[0]
3600        self.assertTrue(remote.joined)
3601        self.assertTrue(remote.allowed)
3602        self.assertTrue(remote.alived) # alived status isn't changed
3603        self.assertIs(remote.reaped, None)
3604        self.assertIn('packing_error', main.stats) # transaction failed
3605        self.assertEqual(main.stats['packing_error'], 1)
3606
3607        for stack in [main, other]:
3608            stack.server.close()
3609            stack.clearAllKeeps()
3610
3611    def testAliverErrorParseInner(self):
3612        '''
3613        Test aliver behavior if packet.parseInner() fail (coverage)
3614        '''
3615        console.terse("{0}\n".format(self.testAliverErrorParseInner.__doc__))
3616
3617        mainData = self.createRoadData(name='main',
3618                                       base=self.base,
3619                                       auto=raeting.AutoMode.once.value)
3620        keeping.clearAllKeep(mainData['dirpath'])
3621        main = self.createRoadStack(data=mainData,
3622                                    main=True,
3623                                    auto=mainData['auto'],
3624                                    ha=None)
3625
3626        otherData = self.createRoadData(name='other',
3627                                        base=self.base,
3628                                        auto=raeting.AutoMode.once.value)
3629        keeping.clearAllKeep(otherData['dirpath'])
3630        other = self.createRoadStack(data=otherData,
3631                                     main=None,
3632                                     auto=otherData['auto'],
3633                                     ha=("", raeting.RAET_TEST_PORT))
3634
3635        stacks = [main, other]
3636
3637        console.terse("\nBootstrap channel *********\n")
3638        # remote on joiner side have to be joined and allowed
3639        self.join(other, main) # bootstrap channel
3640        self.allow(other, main)
3641        self.alive(main, other)
3642        for stack in [main, other]:
3643            self.assertEqual(len(stack.transactions), 0)
3644            self.assertEqual(len(stack.remotes), 1)
3645            remote = stack.remotes.values()[0]
3646            self.assertTrue(remote.joined)
3647            self.assertTrue(remote.allowed)
3648            self.assertTrue(remote.alived)
3649            self.assertIs(remote.reaped, None)
3650
3651        main.clearStats()
3652        console.terse("\nTest aliver receive broken ack *********\n")
3653        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3654        self.serviceStack(main, duration=0.125) # main send alive to other
3655        self.answerAlive(other, kind=raeting.PcktKind.ack.value, dataMod={'ck': -1}) # other receive and answer
3656        self.serviceStack(other, duration=0.125) # other send the answer to main
3657        self.serviceStack(main, duration=0.125) # main handle the answer
3658
3659        self.assertEqual(len(main.transactions), 1) # transaction wasn't handled
3660        self.assertEqual(len(main.remotes), 1)
3661        self.assertIn('parsing_inner_error', main.stats) # transaction failed
3662        self.assertEqual(main.stats['parsing_inner_error'], 1)
3663
3664        self.serviceStacks(stacks) # retry and finish transaction
3665        for stack in stacks:
3666            self.assertEqual(len(stack.transactions), 0)
3667            self.assertEqual(len(stack.remotes), 1)
3668            remote = other.remotes.values()[0]
3669            self.assertTrue(remote.joined)
3670            self.assertTrue(remote.allowed)
3671            self.assertTrue(remote.alived) # alived status isn't changed
3672            self.assertIs(remote.reaped, None)
3673        self.assertIn('alive_complete', main.stats) # transaction failed
3674        self.assertEqual(main.stats['alive_complete'], 1)
3675
3676        main.clearStats()
3677        console.terse("\nTest aliver receive broken refuse *********\n")
3678        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3679        self.serviceStack(main, duration=0.125) # main send alive to other
3680        self.answerAlive(other, kind=raeting.PcktKind.refuse.value, dataMod={'ck': -1}) # other receive and answer
3681        self.serviceStack(other, duration=0.125) # other send the answer to main
3682        self.serviceStack(main, duration=0.125) # main handle the answer
3683
3684        self.assertEqual(len(main.transactions), 1) # transaction wasn't handled
3685        self.assertEqual(len(main.remotes), 1)
3686        self.assertIn('parsing_inner_error', main.stats) # transaction failed
3687        self.assertEqual(main.stats['parsing_inner_error'], 1)
3688
3689        self.serviceStacks(stacks) # retry and finish transaction
3690        for stack in stacks:
3691            self.assertEqual(len(stack.transactions), 0)
3692            self.assertEqual(len(stack.remotes), 1)
3693            remote = other.remotes.values()[0]
3694            self.assertTrue(remote.joined)
3695            self.assertTrue(remote.allowed)
3696            self.assertTrue(remote.alived) # alived status isn't changed
3697            self.assertIs(remote.reaped, None)
3698        self.assertIn('alive_complete', main.stats) # transaction failed
3699        self.assertEqual(main.stats['alive_complete'], 1)
3700
3701        main.clearStats()
3702        console.terse("\nTest aliver receive broken reject *********\n")
3703        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3704        self.serviceStack(main, duration=0.125) # main send alive to other
3705        self.answerAlive(other, kind=raeting.PcktKind.reject.value, dataMod={'ck': -1}) # other receive and answer
3706        self.serviceStack(other, duration=0.125) # other send the answer to main
3707        self.serviceStack(main, duration=0.125) # main handle the answer
3708
3709        self.assertEqual(len(main.transactions), 1) # transaction wasn't handled
3710        self.assertEqual(len(main.remotes), 1)
3711        self.assertIn('parsing_inner_error', main.stats) # transaction failed
3712        self.assertEqual(main.stats['parsing_inner_error'], 1)
3713
3714        self.serviceStacks(stacks) # retry and finish transaction
3715        for stack in stacks:
3716            self.assertEqual(len(stack.transactions), 0)
3717            self.assertEqual(len(stack.remotes), 1)
3718            remote = other.remotes.values()[0]
3719            self.assertTrue(remote.joined)
3720            self.assertTrue(remote.allowed)
3721            self.assertTrue(remote.alived) # alived status isn't changed
3722            self.assertIs(remote.reaped, None)
3723        self.assertIn('alive_complete', main.stats) # transaction failed
3724        self.assertEqual(main.stats['alive_complete'], 1)
3725
3726        main.clearStats()
3727        console.terse("\nTest aliver receive broken unjoined *********\n")
3728        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3729        self.serviceStack(main, duration=0.125) # main send alive to other
3730        self.answerAlive(other, kind=raeting.PcktKind.unjoined.value, dataMod={'ck': -1}) # other receive and answer
3731        self.serviceStack(other, duration=0.125) # other send the answer to main
3732        self.serviceStack(main, duration=0.125) # main handle the answer
3733
3734        self.assertEqual(len(main.transactions), 1) # transaction wasn't handled
3735        self.assertEqual(len(main.remotes), 1)
3736        self.assertIn('parsing_inner_error', main.stats) # transaction failed
3737        self.assertEqual(main.stats['parsing_inner_error'], 1)
3738
3739        self.serviceStacks(stacks) # retry and finish transaction
3740        for stack in stacks:
3741            self.assertEqual(len(stack.transactions), 0)
3742            self.assertEqual(len(stack.remotes), 1)
3743            remote = other.remotes.values()[0]
3744            self.assertTrue(remote.joined)
3745            self.assertTrue(remote.allowed)
3746            self.assertTrue(remote.alived) # alived status isn't changed
3747            self.assertIs(remote.reaped, None)
3748        self.assertIn('alive_complete', main.stats) # transaction failed
3749        self.assertEqual(main.stats['alive_complete'], 1)
3750
3751        main.clearStats()
3752        console.terse("\nTest aliver receive broken unallowed *********\n")
3753        main.alive() # no cascade alive, answer nack, aliver will not change alive status and remove transaction
3754        self.serviceStack(main, duration=0.125) # main send alive to other
3755        self.answerAlive(other, kind=raeting.PcktKind.unallowed.value, dataMod={'ck': -1}) # other receive and answer
3756        self.serviceStack(other, duration=0.125) # other send the answer to main
3757        self.serviceStack(main, duration=0.125) # main handle the answer
3758
3759        self.assertEqual(len(main.transactions), 1) # transaction wasn't handled
3760        self.assertEqual(len(main.remotes), 1)
3761        self.assertIn('parsing_inner_error', main.stats) # transaction failed
3762        self.assertEqual(main.stats['parsing_inner_error'], 1)
3763
3764        self.serviceStacks(stacks) # retry and finish transaction
3765        for stack in stacks:
3766            self.assertEqual(len(stack.transactions), 0)
3767            self.assertEqual(len(stack.remotes), 1)
3768            remote = other.remotes.values()[0]
3769            self.assertTrue(remote.joined)
3770            self.assertTrue(remote.allowed)
3771            self.assertTrue(remote.alived) # alived status isn't changed
3772            self.assertIs(remote.reaped, None)
3773        self.assertIn('alive_complete', main.stats) # transaction failed
3774        self.assertEqual(main.stats['alive_complete'], 1)
3775
3776        for stack in [main, other]:
3777            stack.server.close()
3778            stack.clearAllKeeps()
3779
3780    def testAliventAliveErrorParseInner(self):
3781        '''
3782        Test for Alivent.alive() error on parse packet inner. (coverage)
3783        '''
3784        console.terse("{0}\n".format(self.testAliventAliveErrorParseInner.__doc__))
3785
3786        mainData = self.createRoadData(name='main',
3787                                       base=self.base,
3788                                       auto=raeting.AutoMode.once.value)
3789        keeping.clearAllKeep(mainData['dirpath'])
3790        main = self.createRoadStack(data=mainData,
3791                                    main=True,
3792                                    auto=mainData['auto'],
3793                                    ha=None)
3794
3795        otherData = self.createRoadData(name='other',
3796                                        base=self.base,
3797                                        auto=raeting.AutoMode.once.value)
3798        keeping.clearAllKeep(otherData['dirpath'])
3799        other = self.createRoadStack(data=otherData,
3800                                     main=True, # only main can reap and unreap
3801                                     auto=otherData['auto'],
3802                                     ha=("", raeting.RAET_TEST_PORT))
3803
3804        stacks = [main, other]
3805
3806        console.terse("\nBootstrap channel *********\n")
3807        # remote on joiner side have to be joined and allowed
3808        self.join(other, main) # bootstrap channel
3809        self.allow(other, main)
3810        for stack in [main, other]:
3811            self.assertEqual(len(stack.transactions), 0)
3812            self.assertEqual(len(stack.remotes), 1)
3813            remote = stack.remotes.values()[0]
3814            self.assertTrue(remote.joined)
3815            self.assertTrue(remote.allowed)
3816            self.assertTrue(remote.alived)  # fast alive
3817            self.assertIs(remote.reaped, None)
3818
3819        other.remotes.values()[0].alived = None  # reset alived
3820        other.clearStats()
3821        console.terse("\nTest alive with broken coat kind *********\n")
3822        self.aliveBrokenInner(main) # alive with broken coat kind
3823        self.serviceStack(main, duration=0.1)
3824        self.serviceStack(other,duration=0.1)
3825        self.assertEqual(len(other.transactions), 0) # transaction finished
3826        self.assertIn('parsing_inner_error', other.stats) # Error ocured
3827        self.assertEqual(other.stats['parsing_inner_error'], 1)
3828        self.assertEqual(len(main.transactions), 1)
3829        self.assertEqual(len(other.remotes), 1)
3830        remote = other.remotes.values()[0]
3831        self.assertTrue(remote.joined)
3832        self.assertTrue(remote.allowed)
3833        self.assertIsNone(remote.alived)  # alived status isn't changed
3834        self.assertIsNone(remote.reaped)
3835
3836        # redo the broken packet then drop it
3837        self.serviceStacks(stacks, duration=10.0)
3838        for stack in stacks:
3839            self.assertEqual(len(stack.transactions), 0)
3840            self.assertEqual(len(stack.remotes), 1)
3841            remote = other.remotes.values()[0]
3842            self.assertTrue(remote.joined)
3843            self.assertTrue(remote.allowed)
3844            self.assertFalse(remote.alived) # dead
3845        self.assertIs(main.remotes.values()[0].reaped, None)
3846        self.assertFalse(other.remotes.values()[0].reaped)
3847
3848        for stack in [main, other]:
3849            stack.server.close()
3850            stack.clearAllKeeps()
3851
3852    def testAliventAliveErrorPacketPack(self):
3853        '''
3854        Test for Alivent.alive() error on packing a packet. (coverage)
3855        '''
3856        console.terse("{0}\n".format(self.testAliventAliveErrorPacketPack.__doc__))
3857
3858        mainData = self.createRoadData(name='main',
3859                                       base=self.base,
3860                                       auto=raeting.AutoMode.once.value)
3861        keeping.clearAllKeep(mainData['dirpath'])
3862        main = self.createRoadStack(data=mainData,
3863                                    main=True,
3864                                    auto=mainData['auto'],
3865                                    ha=None)
3866
3867        otherData = self.createRoadData(name='other',
3868                                        base=self.base,
3869                                        auto=raeting.AutoMode.once.value)
3870        keeping.clearAllKeep(otherData['dirpath'])
3871        other = self.createRoadStack(data=otherData,
3872                                     main=True, # only main can reap and unreap
3873                                     auto=otherData['auto'],
3874                                     ha=("", raeting.RAET_TEST_PORT))
3875
3876        stacks = [main, other]
3877
3878        console.terse("\nBootstrap channel *********\n")
3879        # remote on joiner side have to be joined and allowed
3880        self.join(other, main) # bootstrap channel
3881        self.allow(other, main)
3882        for stack in [main, other]:
3883            self.assertEqual(len(stack.transactions), 0)
3884            self.assertEqual(len(stack.remotes), 1)
3885            remote = stack.remotes.values()[0]
3886            self.assertTrue(remote.joined)
3887            self.assertTrue(remote.allowed)
3888            self.assertTrue(remote.alived)
3889            self.assertIs(remote.reaped, None)
3890
3891        other.remotes.values()[0].alived = None
3892        other.clearStats()
3893        console.terse("\nTest alivent alive with packet pack error *********\n")
3894        main.alive() # alive
3895        self.serviceStack(main, duration=0.1) # aliver send
3896        default_size = raeting.UDP_MAX_PACKET_SIZE # backup
3897        raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will fail
3898        self.serviceStack(other,duration=0.1) # alivent receive
3899        raeting.UDP_MAX_PACKET_SIZE = default_size
3900
3901        self.assertEqual(len(other.transactions), 0) # transaction removed
3902        self.assertIn('packing_error', other.stats) # Error ocured
3903        self.assertEqual(other.stats['packing_error'], 1)
3904        self.assertEqual(len(main.transactions), 1)
3905        self.assertEqual(len(other.remotes), 1)
3906        remote = other.remotes.values()[0]
3907        self.assertTrue(remote.joined)
3908        self.assertTrue(remote.allowed)
3909        self.assertIsNone(remote.alived)
3910        self.assertIsNone(remote.reaped)
3911
3912        # redo with normal udp packet limit
3913        self.serviceStacks(stacks)
3914        for stack in stacks:
3915            self.assertEqual(len(stack.transactions), 0)
3916            self.assertEqual(len(stack.remotes), 1)
3917            remote = other.remotes.values()[0]
3918            self.assertTrue(remote.joined)
3919            self.assertTrue(remote.allowed)
3920            self.assertTrue(remote.alived) # redo
3921        self.assertIs(main.remotes.values()[0].reaped, None)
3922        self.assertFalse(other.remotes.values()[0].reaped)
3923
3924        for stack in [main, other]:
3925            stack.server.close()
3926            stack.clearAllKeeps()
3927
3928    def testAliventNackErrorPacketPack(self):
3929        '''
3930        Test for Alivent.nack() error on packing a packet. (coverage)
3931        '''
3932        console.terse("{0}\n".format(self.testAliventNackErrorPacketPack.__doc__))
3933
3934        mainData = self.createRoadData(name='main',
3935                                       base=self.base,
3936                                       auto=raeting.AutoMode.once.value)
3937        keeping.clearAllKeep(mainData['dirpath'])
3938        main = self.createRoadStack(data=mainData,
3939                                    main=True,
3940                                    auto=mainData['auto'],
3941                                    ha=None)
3942
3943        otherData = self.createRoadData(name='other',
3944                                        base=self.base,
3945                                        auto=raeting.AutoMode.once.value)
3946        keeping.clearAllKeep(otherData['dirpath'])
3947        other = self.createRoadStack(data=otherData,
3948                                     main=True, # only main can reap and unreap
3949                                     auto=otherData['auto'],
3950                                     ha=("", raeting.RAET_TEST_PORT))
3951
3952        stacks = [main, other]
3953
3954        console.terse("\nBootstrap channel *********\n")
3955        # remote on joiner side have to be joined and allowed
3956        self.join(other, main) # bootstrap channel
3957        self.allow(other, main)
3958        for stack in [main, other]:
3959            self.assertEqual(len(stack.transactions), 0)
3960            self.assertEqual(len(stack.remotes), 1)
3961            remote = stack.remotes.values()[0]
3962            self.assertTrue(remote.joined)
3963            self.assertTrue(remote.allowed)
3964            self.assertTrue(remote.alived) # fast alive
3965            self.assertIs(remote.reaped, None)
3966
3967        # force unallow and unalive joinent
3968        other.remotes.values()[0].allowed = None
3969        other.remotes.values()[0].alived = None
3970        self.assertIsNone(other.remotes.values()[0].allowed)
3971        self.assertIsNone(other.remotes.values()[0].alived)
3972
3973        other.clearStats()
3974        console.terse("\nTest alivent nack with packet pack error *********\n")
3975        main.alive() # alive
3976        self.serviceStack(main, duration=0.1) # aliver send
3977        default_size = raeting.UDP_MAX_PACKET_SIZE # backup
3978        raeting.UDP_MAX_PACKET_SIZE = 10 # packet.pack() will fail
3979        self.serviceStack(other,duration=0.1) # alivent receive
3980        raeting.UDP_MAX_PACKET_SIZE = default_size
3981
3982        self.assertEqual(len(other.transactions), 0) # transaction removed
3983        self.assertIn('packing_error', other.stats) # Error ocured
3984        self.assertEqual(other.stats['packing_error'], 1)
3985        self.assertEqual(len(main.transactions), 1)
3986        self.assertEqual(len(other.remotes), 1)
3987        remote = other.remotes.values()[0]
3988        self.assertTrue(remote.joined)
3989        self.assertIsNone(remote.allowed)
3990        self.assertIsNone(remote.alived)
3991        self.assertIsNone(remote.reaped)
3992
3993        # redo with normal udp packet limit
3994        self.serviceStacks(stacks)
3995        for stack in stacks:
3996            self.assertEqual(len(stack.transactions), 0)
3997            self.assertEqual(len(stack.remotes), 1)
3998            remote = other.remotes.values()[0]
3999            self.assertTrue(remote.joined)
4000            self.assertTrue(remote.allowed)
4001            self.assertTrue(remote.alived) # fast alive
4002        self.assertIs(main.remotes.values()[0].reaped, None)
4003        self.assertFalse(other.remotes.values()[0].reaped)
4004
4005        for stack in [main, other]:
4006            stack.server.close()
4007            stack.clearAllKeeps()
4008
4009    def testAliventReceiveRequest(self):
4010        '''
4011        Test for Alivent.receive() (coverage)
4012        '''
4013        console.terse("{0}\n".format(self.testAliventReceiveRequest.__doc__))
4014
4015        mainData = self.createRoadData(name='main',
4016                                       base=self.base,
4017                                       auto=raeting.AutoMode.once.value)
4018        keeping.clearAllKeep(mainData['dirpath'])
4019        main = self.createRoadStack(data=mainData,
4020                                    main=True,
4021                                    auto=mainData['auto'],
4022                                    ha=None)
4023
4024        otherData = self.createRoadData(name='other',
4025                                        base=self.base,
4026                                        auto=raeting.AutoMode.once.value)
4027        keeping.clearAllKeep(otherData['dirpath'])
4028        other = self.createRoadStack(data=otherData,
4029                                     main=True, # only main can reap and unreap
4030                                     auto=otherData['auto'],
4031                                     ha=("", raeting.RAET_TEST_PORT))
4032
4033        stacks = [main, other]
4034
4035        console.terse("\nBootstrap channel *********\n")
4036        # remote on joiner side have to be joined and allowed
4037        self.join(other, main) # bootstrap channel
4038        self.allow(other, main)
4039        for stack in [main, other]:
4040            self.assertEqual(len(stack.transactions), 0)
4041            self.assertEqual(len(stack.remotes), 1)
4042            remote = stack.remotes.values()[0]
4043            self.assertTrue(remote.joined)
4044            self.assertTrue(remote.allowed)
4045            self.assertTrue(remote.alived)  # fast alive
4046            self.assertIs(remote.reaped, None)
4047
4048        other.remotes.values()[0].alived = None  # reset alived
4049        other.clearStats()
4050        console.terse("\nTest alivent receive request *********\n")
4051        main.alive() # alive from main to other
4052        self.serviceStack(main, duration=0.1) # Send alive
4053        # receive alive on other in custom way: use transaction receive instead of stack.correspond()
4054        other.serviceReceives()
4055        # handle rx
4056        raw, sa = other.rxes.popleft()
4057        console.verbose("{0} received packet\n{1}\n".format(other.name, raw))
4058        packet = packeting.RxPacket(stack=other, packed=raw)
4059        packet.parseOuter()
4060        sh, sp = sa
4061        packet.data.update(sh=sh, sp=sp)
4062        # process rx
4063        nuid = packet.data['de']
4064        self.assertNotEqual(nuid, 0)
4065        remote = other.remotes.get(nuid, None)
4066        self.assertIsNotNone(remote)
4067        rsid = packet.data['si']
4068        self.assertNotEqual(rsid, 0)
4069        remote.rsid = rsid
4070        # create transaction
4071        data = odict(hk=other.Hk, bk=other.Bk, fk=other.Fk, ck=other.Ck)
4072        alivent = transacting.Alivent(stack=other,
4073                                      remote=remote,
4074                                      bcst=packet.data['bf'],
4075                                      sid=packet.data['si'],
4076                                      tid=packet.data['ti'],
4077                                      txData=data,
4078                                      rxPacket=packet)
4079        # receive
4080        alivent.receive(packet)
4081        self.serviceStacks(stacks) # Process alive
4082
4083        for stack in stacks:
4084            self.assertEqual(len(stack.transactions), 0)
4085            self.assertEqual(len(stack.remotes), 1)
4086            remote = other.remotes.values()[0]
4087            self.assertTrue(remote.joined)
4088            self.assertTrue(remote.allowed)
4089            self.assertTrue(remote.alived)
4090
4091        for stack in [main, other]:
4092            stack.server.close()
4093            stack.clearAllKeeps()
4094
4095    def testAliventProcessTimeout(self):
4096        '''
4097        Test for Alivent.process() (coverage)
4098        '''
4099        console.terse("{0}\n".format(self.testAliventProcessTimeout.__doc__))
4100
4101        mainData = self.createRoadData(name='main',
4102                                       base=self.base,
4103                                       auto=raeting.AutoMode.once.value)
4104        keeping.clearAllKeep(mainData['dirpath'])
4105        main = self.createRoadStack(data=mainData,
4106                                    main=True,
4107                                    auto=mainData['auto'],
4108                                    ha=None)
4109
4110        otherData = self.createRoadData(name='other',
4111                                        base=self.base,
4112                                        auto=raeting.AutoMode.once.value)
4113        keeping.clearAllKeep(otherData['dirpath'])
4114        other = self.createRoadStack(data=otherData,
4115                                     main=True, # only main can reap and unreap
4116                                     auto=otherData['auto'],
4117                                     ha=("", raeting.RAET_TEST_PORT))
4118
4119        stacks = [main, other]
4120
4121        console.terse("\nBootstrap channel *********\n")
4122        # remote on joiner side have to be joined and allowed
4123        self.join(other, main) # bootstrap channel
4124        self.allow(other, main)
4125        for stack in [main, other]:
4126            self.assertEqual(len(stack.transactions), 0)
4127            self.assertEqual(len(stack.remotes), 1)
4128            remote = stack.remotes.values()[0]
4129            self.assertTrue(remote.joined)
4130            self.assertTrue(remote.allowed)
4131            self.assertTrue(remote.alived)  # fast alive
4132            self.assertIs(remote.reaped, None)
4133
4134        other.remotes.values()[0].alived = None  # reset alived
4135        other.clearStats()
4136        console.terse("\nTest alivent process alive transaction timeout *********\n")
4137        main.alive() # alive from main to other
4138        self.serviceStack(main, duration=0.1) # Send alive
4139        # receive alive on other in custom way: use transaction receive instead of stack.correspond()
4140        other.serviceReceives()
4141        # handle rx
4142        raw, sa = other.rxes.popleft()
4143        console.verbose("{0} received packet\n{1}\n".format(other.name, raw))
4144        packet = packeting.RxPacket(stack=other, packed=raw)
4145        packet.parseOuter()
4146        sh, sp = sa
4147        packet.data.update(sh=sh, sp=sp)
4148        # process rx
4149        nuid = packet.data['de']
4150        self.assertNotEqual(nuid, 0)
4151        remote = other.remotes.get(nuid, None)
4152        self.assertIsNotNone(remote)
4153        rsid = packet.data['si']
4154        self.assertNotEqual(rsid, 0)
4155        remote.rsid = rsid
4156        # create transaction
4157        data = odict(hk=other.Hk, bk=other.Bk, fk=other.Fk, ck=other.Ck)
4158        alivent = transacting.Alivent(stack=other,
4159                                      remote=remote,
4160                                      bcst=packet.data['bf'],
4161                                      sid=packet.data['si'],
4162                                      tid=packet.data['ti'],
4163                                      txData=data,
4164                                      rxPacket=packet)
4165        # timeout
4166        self.assertTrue(alivent.timeout > 0.0)
4167        self.store.advanceStamp(alivent.timeout)
4168        alivent.process()
4169        self.serviceStacks(stacks) # Process alive
4170
4171        self.assertIn('alivent_transaction_failure', stack.stats)
4172        self.assertEqual(stack.stats['alivent_transaction_failure'], 1)
4173
4174        for stack in stacks:
4175            self.assertEqual(len(stack.transactions), 0)
4176            self.assertEqual(len(stack.remotes), 1)
4177            remote = other.remotes.values()[0]
4178            self.assertTrue(remote.joined)
4179            self.assertTrue(remote.allowed)
4180            self.assertIsNone(remote.alived)
4181
4182        for stack in [main, other]:
4183            stack.server.close()
4184            stack.clearAllKeeps()
4185
4186    def testFirstAliveRequestDropped(self):
4187        '''
4188        Test network dropped first alive request (redo timeout)
4189        '''
4190        console.terse("{0}\n".format(self.testFirstAliveRequestDropped.__doc__))
4191
4192        mainData = self.createRoadData(name='main',
4193                                       base=self.base,
4194                                       auto=raeting.AutoMode.once.value)
4195        keeping.clearAllKeep(mainData['dirpath'])
4196        main = self.createRoadStack(data=mainData,
4197                                    main=True,
4198                                    auto=mainData['auto'],
4199                                    ha=None)
4200
4201        otherData = self.createRoadData(name='other',
4202                                        base=self.base,
4203                                        auto=raeting.AutoMode.once.value)
4204        keeping.clearAllKeep(otherData['dirpath'])
4205        other = self.createRoadStack(data=otherData,
4206                                     main=True, # only main can reap and unreap
4207                                     auto=otherData['auto'],
4208                                     ha=("", raeting.RAET_TEST_PORT))
4209
4210        stacks = [main, other]
4211
4212        console.terse("\nBootstrap channel *********\n")
4213        # remote on joiner side have to be joined and allowed
4214        self.join(other, main) # bootstrap channel
4215        self.allow(other, main)
4216        for stack in [main, other]:
4217            self.assertEqual(len(stack.transactions), 0)
4218            self.assertEqual(len(stack.remotes), 1)
4219            remote = stack.remotes.values()[0]
4220            self.assertTrue(remote.joined)
4221            self.assertTrue(remote.allowed)
4222            self.assertTrue(remote.alived)  # fast alive
4223            self.assertIs(remote.reaped, None)
4224
4225        main.remotes.values()[0].alived = None  # reset alived
4226        main.clearStats()
4227        console.terse("\nTest aliver didn't received response, redo *********\n")
4228        main.alive() # alive from main to other
4229        self.serviceStack(main, duration=0.1) # Send alive
4230        self.flushReceives(other)
4231        self.serviceStacks(stacks) # timeout, redo, alive
4232        self.serviceStacks(stacks) # fix race condition
4233
4234        self.assertIn('redo_alive', main.stats)
4235        self.assertEqual(main.stats['redo_alive'], 1) # 1 redo
4236        for stack in stacks:
4237            self.assertEqual(len(stack.transactions), 0)
4238            self.assertEqual(len(stack.remotes), 1)
4239            remote = stack.remotes.values()[0]
4240            self.assertTrue(remote.joined)
4241            self.assertTrue(remote.allowed)
4242            self.assertTrue(remote.alived) # Alive complete
4243
4244        for stack in [main, other]:
4245            stack.server.close()
4246            stack.clearAllKeeps()
4247
4248    def testAllAliveRequestsDropped(self):
4249        '''
4250        Test network dropped all alive requests (transaction timeout)
4251        '''
4252        console.terse("{0}\n".format(self.testAllAliveRequestsDropped.__doc__))
4253
4254        mainData = self.createRoadData(name='main',
4255                                       base=self.base,
4256                                       auto=raeting.AutoMode.once.value)
4257        keeping.clearAllKeep(mainData['dirpath'])
4258        main = self.createRoadStack(data=mainData,
4259                                    main=True,
4260                                    auto=mainData['auto'],
4261                                    ha=None)
4262
4263        otherData = self.createRoadData(name='other',
4264                                        base=self.base,
4265                                        auto=raeting.AutoMode.once.value)
4266        keeping.clearAllKeep(otherData['dirpath'])
4267        other = self.createRoadStack(data=otherData,
4268                                     main=True, # only main can reap and unreap
4269                                     auto=otherData['auto'],
4270                                     ha=("", raeting.RAET_TEST_PORT))
4271
4272        stacks = [main, other]
4273
4274        console.terse("\nBootstrap channel *********\n")
4275        # remote on joiner side have to be joined and allowed
4276        self.join(other, main) # bootstrap channel
4277        self.allow(other, main)
4278        for stack in [main, other]:
4279            self.assertEqual(len(stack.transactions), 0)
4280            self.assertEqual(len(stack.remotes), 1)
4281            remote = stack.remotes.values()[0]
4282            self.assertTrue(remote.joined)
4283            self.assertTrue(remote.allowed)
4284            self.assertTrue(remote.alived)  # fast alive
4285            self.assertIs(remote.reaped, None)
4286
4287        other.remotes.values()[0].alived = None  # reset alived
4288        main.clearStats()
4289        console.terse("\nTest aliver received no response, transaction timeout *********\n")
4290        main.alive() # alive from main to other
4291        self.serviceStacksDropRx(stacks, duration=3.0)  # redo timeout, packet timeout, drop
4292
4293        self.assertIn('redo_alive', main.stats)
4294        self.assertEqual(main.stats['redo_alive'], 3) # 3 redo
4295        for stack in stacks:
4296            self.assertEqual(len(stack.transactions), 0)
4297            self.assertEqual(len(stack.remotes), 1)
4298            remote = stack.remotes.values()[0]
4299            self.assertTrue(remote.joined)
4300            self.assertTrue(remote.allowed)
4301            self.assertFalse(remote.alived)
4302
4303        for stack in [main, other]:
4304            stack.server.close()
4305            stack.clearAllKeeps()
4306
4307    def testFirstAliveRequestDelayed(self):
4308        '''
4309        Test network delayed response so it has been received after redo.
4310        '''
4311        console.terse("{0}\n".format(self.testFirstAliveRequestDelayed.__doc__))
4312
4313        mainData = self.createRoadData(name='main',
4314                                       base=self.base,
4315                                       auto=raeting.AutoMode.once.value)
4316        keeping.clearAllKeep(mainData['dirpath'])
4317        main = self.createRoadStack(data=mainData,
4318                                    main=True,
4319                                    auto=mainData['auto'],
4320                                    ha=None)
4321
4322        otherData = self.createRoadData(name='other',
4323                                        base=self.base,
4324                                        auto=raeting.AutoMode.once.value)
4325        keeping.clearAllKeep(otherData['dirpath'])
4326        other = self.createRoadStack(data=otherData,
4327                                     main=True, # only main can reap and unreap
4328                                     auto=otherData['auto'],
4329                                     ha=("", raeting.RAET_TEST_PORT))
4330
4331        stacks = [main, other]
4332
4333        console.terse("\nBootstrap channel *********\n")
4334        # remote on joiner side have to be joined and allowed
4335        self.join(other, main) # bootstrap channel
4336        self.allow(other, main)
4337        for stack in [main, other]:
4338            self.assertEqual(len(stack.transactions), 0)
4339            self.assertEqual(len(stack.remotes), 1)
4340            remote = stack.remotes.values()[0]
4341            self.assertTrue(remote.joined)
4342            self.assertTrue(remote.allowed)
4343            self.assertTrue(remote.alived)  # fast alive
4344            self.assertIs(remote.reaped, None)
4345
4346        main.remotes.values()[0].alived = None   # reset alived
4347        other.remotes.values()[0].alived = None  # reset alived
4348        main.clearStats()
4349        other.clearStats()
4350        console.terse("\nTest alivent received both request and redo *********\n")
4351        main.alive() # alive from main to other
4352        self.serviceStack(main, duration=0.5) # Send alive and redo
4353        self.serviceStacks(stacks) # service delayed messages
4354        self.serviceStacks(stacks) # fix race condition
4355
4356        self.assertIn('redo_alive', main.stats)
4357        self.assertEqual(main.stats['redo_alive'], 1) # 1 redo
4358        self.assertIn('stale_correspondent_attempt', main.stats)
4359        self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt
4360        self.assertIn('stale_correspondent_nack', main.stats)
4361        self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer
4362        self.assertIn('alive_complete', main.stats)
4363        self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main
4364        self.assertIn('stale_packet', other.stats)
4365        self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other
4366        self.assertIn('alive_complete', other.stats)
4367        self.assertEqual(other.stats['alive_complete'], 2) # 2 complete on other
4368        for stack in stacks:
4369            self.assertEqual(len(stack.transactions), 0)
4370            self.assertEqual(len(stack.remotes), 1)
4371            remote = stack.remotes.values()[0]
4372            self.assertTrue(remote.joined)
4373            self.assertTrue(remote.allowed)
4374            self.assertTrue(remote.alived) # Alive complete
4375
4376        for stack in [main, other]:
4377            stack.server.close()
4378            stack.clearAllKeeps()
4379
4380    def testAllAliveRequestsDelayed(self):
4381        '''
4382        Test network delayed all alive requests (aliver receive response after transaction dropped)
4383        '''
4384        console.terse("{0}\n".format(self.testAllAliveRequestsDelayed.__doc__))
4385
4386        mainData = self.createRoadData(name='main',
4387                                       base=self.base,
4388                                       auto=raeting.AutoMode.once.value)
4389        keeping.clearAllKeep(mainData['dirpath'])
4390        main = self.createRoadStack(data=mainData,
4391                                    main=True,
4392                                    auto=mainData['auto'],
4393                                    ha=None)
4394
4395        otherData = self.createRoadData(name='other',
4396                                        base=self.base,
4397                                        auto=raeting.AutoMode.once.value)
4398        keeping.clearAllKeep(otherData['dirpath'])
4399        other = self.createRoadStack(data=otherData,
4400                                     main=True, # only main can reap and unreap
4401                                     auto=otherData['auto'],
4402                                     ha=("", raeting.RAET_TEST_PORT))
4403
4404        stacks = [main, other]
4405
4406        console.terse("\nBootstrap channel *********\n")
4407        # remote on joiner side have to be joined and allowed
4408        self.join(other, main) # bootstrap channel
4409        self.allow(other, main)
4410        for stack in [main, other]:
4411            self.assertEqual(len(stack.transactions), 0)
4412            self.assertEqual(len(stack.remotes), 1)
4413            remote = stack.remotes.values()[0]
4414            self.assertTrue(remote.joined)
4415            self.assertTrue(remote.allowed)
4416            self.assertTrue(remote.alived)  # fast alive
4417            self.assertIs(remote.reaped, None)
4418
4419        main.remotes.values()[0].alived = None   # reset alived
4420        other.remotes.values()[0].alived = None  # reset alived
4421        main.clearStats()
4422        other.clearStats()
4423        console.terse("\nTest alivent received alive and redo after aliver transaction timeout *********\n")
4424        main.alive() # alive from main to other
4425        self.serviceStack(main, duration=3.0)  # redo timeout, packet timeout, drop
4426        self.serviceStacks(stacks) # ack 4 alives on other
4427        self.serviceStacks(stacks) # stale nack 4 alives on main
4428        for stack in stacks:
4429            self.assertEqual(len(stack.txes), 0) # ensure both stacks done
4430
4431        self.assertIn('redo_alive', main.stats)
4432        self.assertEqual(main.stats['redo_alive'], 3) # 3 redo
4433        self.assertIn('stale_correspondent_attempt', main.stats)
4434        self.assertEqual(main.stats['stale_correspondent_attempt'], 4) # 4 stale attempt
4435        self.assertIn('stale_correspondent_nack', main.stats)
4436        self.assertEqual(main.stats['stale_correspondent_nack'], 4) # 4 stale nack answer
4437        self.assertIn('stale_packet', other.stats)
4438        self.assertEqual(other.stats['stale_packet'], 4) # 4 stale nack on other
4439        self.assertIn('alive_complete', other.stats)
4440        self.assertEqual(other.stats['alive_complete'], 4) # 2 complete on other
4441        for stack in stacks:
4442            self.assertEqual(len(stack.transactions), 0)
4443            self.assertEqual(len(stack.remotes), 1)
4444            remote = stack.remotes.values()[0]
4445            self.assertTrue(remote.joined)
4446            self.assertTrue(remote.allowed)
4447        self.assertFalse(main.remotes.values()[0].alived) # main didn't received correct alive ack
4448        self.assertTrue(other.remotes.values()[0].alived) # other received correct alive request
4449
4450        for stack in [main, other]:
4451            stack.server.close()
4452            stack.clearAllKeeps()
4453
4454    def testFirstAliveRequestDuplicated(self):
4455        '''
4456        Test network duplicated first alive request (aliver ack both, alivent stale nack second)
4457        '''
4458        console.terse("{0}\n".format(self.testFirstAliveRequestDuplicated.__doc__))
4459
4460        mainData = self.createRoadData(name='main',
4461                                       base=self.base,
4462                                       auto=raeting.AutoMode.once.value)
4463        keeping.clearAllKeep(mainData['dirpath'])
4464        main = self.createRoadStack(data=mainData,
4465                                    main=True,
4466                                    auto=mainData['auto'],
4467                                    ha=None)
4468
4469        otherData = self.createRoadData(name='other',
4470                                        base=self.base,
4471                                        auto=raeting.AutoMode.once.value)
4472        keeping.clearAllKeep(otherData['dirpath'])
4473        other = self.createRoadStack(data=otherData,
4474                                     main=True, # only main can reap and unreap
4475                                     auto=otherData['auto'],
4476                                     ha=("", raeting.RAET_TEST_PORT))
4477
4478        stacks = [main, other]
4479
4480        console.terse("\nBootstrap channel *********\n")
4481        # remote on joiner side have to be joined and allowed
4482        self.join(other, main) # bootstrap channel
4483        self.allow(other, main)
4484        for stack in [main, other]:
4485            self.assertEqual(len(stack.transactions), 0)
4486            self.assertEqual(len(stack.remotes), 1)
4487            remote = stack.remotes.values()[0]
4488            self.assertTrue(remote.joined)
4489            self.assertTrue(remote.allowed)
4490            self.assertTrue(remote.alived)  # fast alive
4491            self.assertIs(remote.reaped, None)
4492
4493        main.remotes.values()[0].alived = None  # reset alived
4494        main.clearStats()
4495        console.terse("\nTest alivent received duplicated request *********\n")
4496        main.alive() # alive from main to other
4497        self.serviceStack(main, duration=0.1) # Send alive
4498        self.dupReceives(other)
4499        self.serviceStacks(stacks) # 1 req, 2 ack, 1st accept, 2nd nack
4500        self.serviceStacks(stacks) # fix race condition
4501
4502        self.assertIn('stale_correspondent_attempt', main.stats)
4503        self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt (dup)
4504        self.assertIn('stale_correspondent_nack', main.stats)
4505        self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer (dup)
4506        self.assertIn('alive_complete', main.stats)
4507        self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main
4508        self.assertIn('stale_packet', other.stats)
4509        self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other (dup)
4510        self.assertIn('alive_complete', other.stats)
4511        self.assertEqual(other.stats['alive_complete'], 2) # 2 complete on other
4512        for stack in stacks:
4513            self.assertEqual(len(stack.transactions), 0)
4514            self.assertEqual(len(stack.remotes), 1)
4515            remote = stack.remotes.values()[0]
4516            self.assertTrue(remote.joined)
4517            self.assertTrue(remote.allowed)
4518            self.assertTrue(remote.alived) # Alive complete
4519
4520        for stack in [main, other]:
4521            stack.server.close()
4522            stack.clearAllKeeps()
4523
4524    def testAliveAckDuplicated(self):
4525        '''
4526        Test network duplicated alive ack response (stale nack the second one)
4527        '''
4528        console.terse("{0}\n".format(self.testAliveAckDuplicated.__doc__))
4529
4530        mainData = self.createRoadData(name='main',
4531                                       base=self.base,
4532                                       auto=raeting.AutoMode.once.value)
4533        keeping.clearAllKeep(mainData['dirpath'])
4534        main = self.createRoadStack(data=mainData,
4535                                    main=True,
4536                                    auto=mainData['auto'],
4537                                    ha=None)
4538
4539        otherData = self.createRoadData(name='other',
4540                                        base=self.base,
4541                                        auto=raeting.AutoMode.once.value)
4542        keeping.clearAllKeep(otherData['dirpath'])
4543        other = self.createRoadStack(data=otherData,
4544                                     main=True, # only main can reap and unreap
4545                                     auto=otherData['auto'],
4546                                     ha=("", raeting.RAET_TEST_PORT))
4547
4548        stacks = [main, other]
4549
4550        console.terse("\nBootstrap channel *********\n")
4551        # remote on joiner side have to be joined and allowed
4552        self.join(other, main) # bootstrap channel
4553        self.allow(other, main)
4554        for stack in [main, other]:
4555            self.assertEqual(len(stack.transactions), 0)
4556            self.assertEqual(len(stack.remotes), 1)
4557            remote = stack.remotes.values()[0]
4558            self.assertTrue(remote.joined)
4559            self.assertTrue(remote.allowed)
4560            self.assertIs(remote.alived, True)
4561            self.assertIs(remote.reaped, None)
4562
4563        main.clearStats()
4564        other.clearStats()
4565        console.terse("\nTest aliver received response twice *********\n")
4566        for stack in stacks:
4567            self.assertEqual(len(stack.remotes), 1)
4568            remote = stack.remotes.values()[0]
4569            remote.alived = None
4570            self.assertIs(remote.alived,  None) #  Reset alived
4571
4572        main.alive() # alive from main to other
4573        self.serviceStack(main, duration=0.125) # Send alive
4574        self.serviceStack(other, duration=0.125) # Send ack
4575        time.sleep(0.3)
4576        self.dupReceives(main) # duplicate response
4577        self.serviceStacks(stacks, duration=0.25) # 1st accept, 2nd stale nack
4578        self.serviceStacks(stacks, duration=1.0) # fix race condition
4579
4580        self.assertIn('stale_correspondent_attempt', main.stats)
4581        self.assertEqual(main.stats['stale_correspondent_attempt'], 1) # 1 stale attempt (dup)
4582        self.assertIn('stale_correspondent_nack', main.stats)
4583        self.assertEqual(main.stats['stale_correspondent_nack'], 1) # 1 stale nack answer (dup)
4584        self.assertIn('alive_complete', main.stats)
4585        self.assertEqual(main.stats['alive_complete'], 1) # 1 complete on main
4586        self.assertIn('stale_packet', other.stats)
4587        self.assertEqual(other.stats['stale_packet'], 1) # 1 stale nack on other (dup)
4588        self.assertIn('alive_complete', other.stats)
4589        self.assertEqual(other.stats['alive_complete'], 1) # 1 complete on other
4590        for stack in stacks:
4591            self.assertEqual(len(stack.transactions), 0)
4592            self.assertEqual(len(stack.remotes), 1)
4593            remote = stack.remotes.values()[0]
4594            self.assertTrue(remote.joined)
4595            self.assertTrue(remote.allowed)
4596            self.assertTrue(remote.alived) # Alive complete
4597
4598        for stack in [main, other]:
4599            stack.server.close()
4600            stack.clearAllKeeps()
4601
4602def runOne(test):
4603    '''
4604    Unittest Runner
4605    '''
4606    test = BasicTestCase(test)
4607    suite = unittest.TestSuite([test])
4608    unittest.TextTestRunner(verbosity=2).run(suite)
4609
4610def runSome():
4611    '''
4612    Unittest runner
4613    '''
4614    tests =  []
4615    names = [
4616                'testJoinNameRoleDiffer',
4617                'testJoinFromMain',
4618                'testJoinFromMainNameRoleDiffer',
4619                'testJoinFromMainKindChange',
4620                'testAliveDead',
4621                'testAliveDeadMultiple',
4622                'testAliveUnjoinedUnallowedBoth',
4623                'testCascadeBoth',
4624                'testManageOneSide',
4625                'testManageBothSides',
4626                'testManageMainRebootCascade',
4627                'testManageRebootCascadeBothSides',
4628                'testManageRebootCascadeBothSidesAlt',
4629                'testManageUnjoinedAllTimersExpired',
4630                'testManageUnallowedAllTimersExpired',
4631                'testManageCascadeUnjoinedAllTimersExpired',
4632                'testManageCascadeUnallowedAllTimersExpired',
4633                'testManageUnalivedAllTimersExpired',
4634                'testManageUnjoinedPresenceTimerExpired',
4635                'testManageUnallowedPresenceTimerExpired',
4636                'testManageCascadeUnjoinedPresenceTimerExpired',
4637                'testManageCascadeUnallowedPresenceTimerExpired',
4638                'testManageUnalivedPresenceTimerExpired',
4639                'testManageReapTimerExpired',
4640                'testManageNoTimerExpired',
4641                'testManageReaped',
4642                'testAliventUnjoined',
4643                'testAliventUnallowed',
4644                'testAliventJoinedAllowed',
4645                'testAliverReceiveNack',
4646                'testAliverAlivePackError',
4647                'testAliverErrorParseInner',
4648                'testAliventAliveErrorParseInner',
4649                'testAliventAliveErrorPacketPack',
4650                'testAliventNackErrorPacketPack',
4651                'testAliventReceiveRequest',
4652                'testAliventProcessTimeout',
4653                'testFirstAliveRequestDropped',
4654                'testAllAliveRequestsDropped',
4655                'testFirstAliveRequestDelayed',
4656                'testAllAliveRequestsDelayed',
4657                'testFirstAliveRequestDuplicated',
4658                'testAliveAckDuplicated',
4659
4660            ]
4661
4662    tests.extend(map(BasicTestCase, names))
4663
4664    suite = unittest.TestSuite(tests)
4665    unittest.TextTestRunner(verbosity=2).run(suite)
4666
4667def runAll():
4668    '''
4669    Unittest runner
4670    '''
4671    suite = unittest.TestSuite()
4672    suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BasicTestCase))
4673
4674    unittest.TextTestRunner(verbosity=2).run(suite)
4675
4676if __name__ == '__main__' and __package__ is None:
4677
4678    #console.reinit(verbosity=console.Wordage.concise)
4679
4680    runAll() #run all unittests
4681
4682    #runSome()#only run some
4683
4684    #runOne('testAliveAckDuplicated')
4685