1# Copyright 2010 United States Government as represented by the 2# Administrator of the National Aeronautics and Space Administration. 3# All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may 6# not use this file except in compliance with the License. You may obtain 7# a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations 15# under the License. 16 17"""Unit Tests for service class""" 18 19import logging 20import multiprocessing 21import os 22import signal 23import socket 24import time 25import traceback 26from unittest import mock 27 28import eventlet 29from eventlet import event 30from oslotest import base as test_base 31 32from oslo_service import service 33from oslo_service.tests import base 34from oslo_service.tests import eventlet_service 35 36 37LOG = logging.getLogger(__name__) 38 39 40class ExtendedService(service.Service): 41 def test_method(self): 42 return 'service' 43 44 45class ServiceManagerTestCase(test_base.BaseTestCase): 46 """Test cases for Services.""" 47 def test_override_manager_method(self): 48 serv = ExtendedService() 49 serv.start() 50 self.assertEqual('service', serv.test_method()) 51 52 53class ServiceWithTimer(service.Service): 54 def __init__(self, ready_event=None): 55 super(ServiceWithTimer, self).__init__() 56 self.ready_event = ready_event 57 58 def start(self): 59 super(ServiceWithTimer, self).start() 60 self.timer_fired = 0 61 self.tg.add_timer(1, self.timer_expired) 62 63 def wait(self): 64 if self.ready_event: 65 self.ready_event.set() 66 super(ServiceWithTimer, self).wait() 67 68 def timer_expired(self): 69 self.timer_fired = self.timer_fired + 1 70 71 72class ServiceCrashOnStart(ServiceWithTimer): 73 def start(self): 74 super(ServiceCrashOnStart, self).start() 75 raise ValueError 76 77 78class ServiceTestBase(base.ServiceBaseTestCase): 79 """A base class for ServiceLauncherTest and ServiceRestartTest.""" 80 81 def _spawn_service(self, 82 workers=1, 83 service_maker=None, 84 launcher_maker=None): 85 self.workers = workers 86 pid = os.fork() 87 if pid == 0: 88 os.setsid() 89 # NOTE(johannes): We can't let the child processes exit back 90 # into the unit test framework since then we'll have multiple 91 # processes running the same tests (and possibly forking more 92 # processes that end up in the same situation). So we need 93 # to catch all exceptions and make sure nothing leaks out, in 94 # particular SystemExit, which is raised by sys.exit(). We use 95 # os._exit() which doesn't have this problem. 96 status = 0 97 try: 98 serv = service_maker() if service_maker else ServiceWithTimer() 99 if launcher_maker: 100 launcher = launcher_maker() 101 launcher.launch_service(serv, workers=workers) 102 else: 103 launcher = service.launch(self.conf, serv, workers=workers) 104 status = launcher.wait() 105 except SystemExit as exc: 106 status = exc.code 107 except BaseException: 108 # We need to be defensive here too 109 try: 110 traceback.print_exc() 111 except BaseException: 112 print("Couldn't print traceback") 113 status = 2 114 # Really exit 115 os._exit(status or 0) 116 return pid 117 118 def _wait(self, cond, timeout): 119 start = time.time() 120 while not cond(): 121 if time.time() - start > timeout: 122 break 123 time.sleep(.1) 124 125 def setUp(self): 126 super(ServiceTestBase, self).setUp() 127 # NOTE(markmc): ConfigOpts.log_opt_values() uses CONF.config-file 128 self.conf(args=[], default_config_files=[]) 129 self.addCleanup(self.conf.reset) 130 self.addCleanup(self._reap_pid) 131 self.pid = 0 132 133 def _reap_pid(self): 134 if self.pid: 135 # Make sure all processes are stopped 136 os.kill(self.pid, signal.SIGTERM) 137 138 # Make sure we reap our test process 139 self._reap_test() 140 141 def _reap_test(self): 142 pid, status = os.waitpid(self.pid, 0) 143 self.pid = None 144 return status 145 146 147class ServiceLauncherTest(ServiceTestBase): 148 """Originally from nova/tests/integrated/test_multiprocess_api.py.""" 149 150 def _spawn(self): 151 self.pid = self._spawn_service(workers=2) 152 153 # Wait at most 10 seconds to spawn workers 154 cond = lambda: self.workers == len(self._get_workers()) 155 timeout = 10 156 self._wait(cond, timeout) 157 158 workers = self._get_workers() 159 self.assertEqual(len(workers), self.workers) 160 return workers 161 162 def _get_workers(self): 163 f = os.popen('ps ax -o pid,ppid,command') 164 # Skip ps header 165 f.readline() 166 167 processes = [tuple(int(p) for p in line.strip().split()[:2]) 168 for line in f] 169 return [p for p, pp in processes if pp == self.pid] 170 171 def test_killed_worker_recover(self): 172 start_workers = self._spawn() 173 174 # kill one worker and check if new worker can come up 175 LOG.info('pid of first child is %s' % start_workers[0]) 176 os.kill(start_workers[0], signal.SIGTERM) 177 178 # Wait at most 5 seconds to respawn a worker 179 cond = lambda: start_workers != self._get_workers() 180 timeout = 5 181 self._wait(cond, timeout) 182 183 # Make sure worker pids don't match 184 end_workers = self._get_workers() 185 LOG.info('workers: %r' % end_workers) 186 self.assertNotEqual(start_workers, end_workers) 187 188 def _terminate_with_signal(self, sig): 189 self._spawn() 190 191 os.kill(self.pid, sig) 192 193 # Wait at most 5 seconds to kill all workers 194 cond = lambda: not self._get_workers() 195 timeout = 5 196 self._wait(cond, timeout) 197 198 workers = self._get_workers() 199 LOG.info('workers: %r' % workers) 200 self.assertFalse(workers, 'No OS processes left.') 201 202 def test_terminate_sigkill(self): 203 self._terminate_with_signal(signal.SIGKILL) 204 status = self._reap_test() 205 self.assertTrue(os.WIFSIGNALED(status)) 206 self.assertEqual(signal.SIGKILL, os.WTERMSIG(status)) 207 208 def test_terminate_sigterm(self): 209 self._terminate_with_signal(signal.SIGTERM) 210 status = self._reap_test() 211 self.assertTrue(os.WIFEXITED(status)) 212 self.assertEqual(0, os.WEXITSTATUS(status)) 213 214 def test_crashed_service(self): 215 service_maker = lambda: ServiceCrashOnStart() 216 self.pid = self._spawn_service(service_maker=service_maker) 217 status = self._reap_test() 218 self.assertTrue(os.WIFEXITED(status)) 219 self.assertEqual(1, os.WEXITSTATUS(status)) 220 221 def test_child_signal_sighup(self): 222 start_workers = self._spawn() 223 224 os.kill(start_workers[0], signal.SIGHUP) 225 # Wait at most 5 seconds to respawn a worker 226 cond = lambda: start_workers != self._get_workers() 227 timeout = 5 228 self._wait(cond, timeout) 229 230 # Make sure worker pids match 231 end_workers = self._get_workers() 232 LOG.info('workers: %r' % end_workers) 233 self.assertEqual(start_workers, end_workers) 234 235 def test_parent_signal_sighup(self): 236 start_workers = self._spawn() 237 238 os.kill(self.pid, signal.SIGHUP) 239 240 def cond(): 241 workers = self._get_workers() 242 return (len(workers) == len(start_workers) and 243 not set(start_workers).intersection(workers)) 244 245 # Wait at most 5 seconds to respawn a worker 246 timeout = 10 247 self._wait(cond, timeout) 248 self.assertTrue(cond()) 249 250 251class ServiceRestartTest(ServiceTestBase): 252 253 def _spawn(self): 254 ready_event = multiprocessing.Event() 255 service_maker = lambda: ServiceWithTimer(ready_event=ready_event) 256 self.pid = self._spawn_service(service_maker=service_maker) 257 return ready_event 258 259 def test_service_restart(self): 260 ready = self._spawn() 261 262 timeout = 5 263 ready.wait(timeout) 264 self.assertTrue(ready.is_set(), 'Service never became ready') 265 ready.clear() 266 267 os.kill(self.pid, signal.SIGHUP) 268 ready.wait(timeout) 269 self.assertTrue(ready.is_set(), 'Service never back after SIGHUP') 270 271 def test_terminate_sigterm(self): 272 ready = self._spawn() 273 timeout = 5 274 ready.wait(timeout) 275 self.assertTrue(ready.is_set(), 'Service never became ready') 276 277 os.kill(self.pid, signal.SIGTERM) 278 279 status = self._reap_test() 280 self.assertTrue(os.WIFEXITED(status)) 281 self.assertEqual(0, os.WEXITSTATUS(status)) 282 283 def test_mutate_hook_service_launcher(self): 284 """Test mutate_config_files is called by ServiceLauncher on SIGHUP. 285 286 Not using _spawn_service because ServiceLauncher doesn't fork and it's 287 simplest to stay all in one process. 288 """ 289 mutate = multiprocessing.Event() 290 self.conf.register_mutate_hook(lambda c, f: mutate.set()) 291 launcher = service.launch( 292 self.conf, ServiceWithTimer(), restart_method='mutate') 293 294 self.assertFalse(mutate.is_set(), "Hook was called too early") 295 launcher.restart() 296 self.assertTrue(mutate.is_set(), "Hook wasn't called") 297 298 def test_mutate_hook_process_launcher(self): 299 """Test mutate_config_files is called by ProcessLauncher on SIGHUP. 300 301 Forks happen in _spawn_service and ProcessLauncher. So we get three 302 tiers of processes, the top tier being the test process. self.pid 303 refers to the middle tier, which represents our application. Both 304 service_maker and launcher_maker execute in the middle tier. The bottom 305 tier is the workers. 306 307 The behavior we want is that when the application (middle tier) 308 receives a SIGHUP, it catches that, calls mutate_config_files and 309 relaunches all the workers. This causes them to inherit the mutated 310 config. 311 """ 312 mutate = multiprocessing.Event() 313 ready = multiprocessing.Event() 314 315 def service_maker(): 316 self.conf.register_mutate_hook(lambda c, f: mutate.set()) 317 return ServiceWithTimer(ready) 318 319 def launcher_maker(): 320 return service.ProcessLauncher(self.conf, restart_method='mutate') 321 322 self.pid = self._spawn_service(1, service_maker, launcher_maker) 323 324 timeout = 5 325 ready.wait(timeout) 326 self.assertTrue(ready.is_set(), 'Service never became ready') 327 ready.clear() 328 329 self.assertFalse(mutate.is_set(), "Hook was called too early") 330 os.kill(self.pid, signal.SIGHUP) 331 ready.wait(timeout) 332 self.assertTrue(ready.is_set(), 'Service never back after SIGHUP') 333 self.assertTrue(mutate.is_set(), "Hook wasn't called") 334 335 336class _Service(service.Service): 337 def __init__(self): 338 super(_Service, self).__init__() 339 self.init = event.Event() 340 self.cleaned_up = False 341 342 def start(self): 343 self.init.send() 344 345 def stop(self): 346 self.cleaned_up = True 347 super(_Service, self).stop() 348 349 350class LauncherTest(base.ServiceBaseTestCase): 351 352 def test_graceful_shutdown(self): 353 # test that services are given a chance to clean up: 354 svc = _Service() 355 356 launcher = service.launch(self.conf, svc) 357 # wait on 'init' so we know the service had time to start: 358 svc.init.wait() 359 360 launcher.stop() 361 self.assertTrue(svc.cleaned_up) 362 363 # make sure stop can be called more than once. (i.e. play nice with 364 # unit test fixtures in nova bug #1199315) 365 launcher.stop() 366 367 @mock.patch('oslo_service.service.ServiceLauncher.launch_service') 368 def _test_launch_single(self, workers, mock_launch): 369 svc = service.Service() 370 service.launch(self.conf, svc, workers=workers) 371 mock_launch.assert_called_with(svc, workers=workers) 372 373 def test_launch_none(self): 374 self._test_launch_single(None) 375 376 def test_launch_one_worker(self): 377 self._test_launch_single(1) 378 379 def test_launch_invalid_workers_number(self): 380 svc = service.Service() 381 for num_workers in [0, -1]: 382 self.assertRaises(ValueError, service.launch, self.conf, 383 svc, num_workers) 384 for num_workers in ["0", "a", "1"]: 385 self.assertRaises(TypeError, service.launch, self.conf, 386 svc, num_workers) 387 388 @mock.patch('signal.alarm') 389 @mock.patch('oslo_service.service.ProcessLauncher.launch_service') 390 def test_multiple_worker(self, mock_launch, alarm_mock): 391 svc = service.Service() 392 service.launch(self.conf, svc, workers=3) 393 mock_launch.assert_called_with(svc, workers=3) 394 395 def test_launch_wrong_service_base_class(self): 396 # check that services that do not subclass service.ServiceBase 397 # can not be launched. 398 svc = mock.Mock() 399 self.assertRaises(TypeError, service.launch, self.conf, svc) 400 401 @mock.patch('signal.alarm') 402 @mock.patch("oslo_service.service.Services.add") 403 @mock.patch("oslo_service.eventlet_backdoor.initialize_if_enabled") 404 def test_check_service_base(self, initialize_if_enabled_mock, 405 services_mock, 406 alarm_mock): 407 initialize_if_enabled_mock.return_value = None 408 launcher = service.Launcher(self.conf) 409 serv = _Service() 410 launcher.launch_service(serv) 411 412 @mock.patch('signal.alarm') 413 @mock.patch("oslo_service.service.Services.add") 414 @mock.patch("oslo_service.eventlet_backdoor.initialize_if_enabled") 415 def test_check_service_base_fails(self, initialize_if_enabled_mock, 416 services_mock, 417 alarm_mock): 418 initialize_if_enabled_mock.return_value = None 419 launcher = service.Launcher(self.conf) 420 421 class FooService(object): 422 def __init__(self): 423 pass 424 serv = FooService() 425 self.assertRaises(TypeError, launcher.launch_service, serv) 426 427 428class ProcessLauncherTest(base.ServiceBaseTestCase): 429 430 @mock.patch('signal.alarm') 431 @mock.patch("signal.signal") 432 def test_stop(self, signal_mock, alarm_mock): 433 signal_mock.SIGTERM = 15 434 launcher = service.ProcessLauncher(self.conf) 435 self.assertTrue(launcher.running) 436 437 pid_nums = [22, 222] 438 fakeServiceWrapper = service.ServiceWrapper(service.Service(), 1) 439 launcher.children = {pid_nums[0]: fakeServiceWrapper, 440 pid_nums[1]: fakeServiceWrapper} 441 with mock.patch('oslo_service.service.os.kill') as mock_kill: 442 with mock.patch.object(launcher, '_wait_child') as _wait_child: 443 444 def fake_wait_child(): 445 pid = pid_nums.pop() 446 return launcher.children.pop(pid) 447 448 _wait_child.side_effect = fake_wait_child 449 with mock.patch('oslo_service.service.Service.stop') as \ 450 mock_service_stop: 451 mock_service_stop.side_effect = lambda: None 452 launcher.stop() 453 454 self.assertFalse(launcher.running) 455 self.assertFalse(launcher.children) 456 mock_kill.assert_has_calls([mock.call(222, signal_mock.SIGTERM), 457 mock.call(22, signal_mock.SIGTERM)], 458 any_order=True) 459 self.assertEqual(2, mock_kill.call_count) 460 mock_service_stop.assert_called_once_with() 461 462 def test__handle_signal(self): 463 signal_handler = service.SignalHandler() 464 signal_handler.clear() 465 self.assertEqual(0, 466 len(signal_handler._signal_handlers[signal.SIGTERM])) 467 call_1, call_2 = mock.Mock(), mock.Mock() 468 signal_handler.add_handler('SIGTERM', call_1) 469 signal_handler.add_handler('SIGTERM', call_2) 470 self.assertEqual(2, 471 len(signal_handler._signal_handlers[signal.SIGTERM])) 472 signal_handler._handle_signal(signal.SIGTERM, 'test') 473 # execute pending eventlet callbacks 474 time.sleep(0) 475 for m in signal_handler._signal_handlers[signal.SIGTERM]: 476 m.assert_called_once_with(signal.SIGTERM, 'test') 477 signal_handler.clear() 478 479 def test_setup_signal_interruption_no_select_poll(self): 480 # NOTE(claudiub): SignalHandler is a singleton, which means that it 481 # might already be initialized. We need to clear to clear the cache 482 # in order to prevent race conditions between tests. 483 service.SignalHandler.__class__._instances.clear() 484 with mock.patch('eventlet.patcher.original', 485 return_value=object()) as get_original: 486 signal_handler = service.SignalHandler() 487 get_original.assert_called_with('select') 488 self.addCleanup(service.SignalHandler.__class__._instances.clear) 489 self.assertFalse( 490 signal_handler._SignalHandler__force_interrupt_on_signal) 491 492 def test_setup_signal_interruption_select_poll(self): 493 # NOTE(claudiub): SignalHandler is a singleton, which means that it 494 # might already be initialized. We need to clear to clear the cache 495 # in order to prevent race conditions between tests. 496 service.SignalHandler.__class__._instances.clear() 497 signal_handler = service.SignalHandler() 498 self.addCleanup(service.SignalHandler.__class__._instances.clear) 499 self.assertTrue( 500 signal_handler._SignalHandler__force_interrupt_on_signal) 501 502 @mock.patch('signal.alarm') 503 @mock.patch("os.kill") 504 @mock.patch("oslo_service.service.ProcessLauncher.stop") 505 @mock.patch("oslo_service.service.ProcessLauncher._respawn_children") 506 @mock.patch("oslo_service.service.ProcessLauncher.handle_signal") 507 @mock.patch("oslo_config.cfg.CONF.log_opt_values") 508 @mock.patch("oslo_service.systemd.notify_once") 509 @mock.patch("oslo_config.cfg.CONF.reload_config_files") 510 @mock.patch("oslo_service.service._is_sighup_and_daemon") 511 def test_parent_process_reload_config(self, 512 is_sighup_and_daemon_mock, 513 reload_config_files_mock, 514 notify_once_mock, 515 log_opt_values_mock, 516 handle_signal_mock, 517 respawn_children_mock, 518 stop_mock, 519 kill_mock, 520 alarm_mock): 521 is_sighup_and_daemon_mock.return_value = True 522 respawn_children_mock.side_effect = [None, 523 eventlet.greenlet.GreenletExit()] 524 launcher = service.ProcessLauncher(self.conf) 525 launcher.sigcaught = 1 526 launcher.children = {} 527 528 wrap_mock = mock.Mock() 529 launcher.children[222] = wrap_mock 530 launcher.wait() 531 532 reload_config_files_mock.assert_called_once_with() 533 wrap_mock.service.reset.assert_called_once_with() 534 535 @mock.patch("oslo_service.service.ProcessLauncher._start_child") 536 @mock.patch("oslo_service.service.ProcessLauncher.handle_signal") 537 @mock.patch("eventlet.greenio.GreenPipe") 538 @mock.patch("os.pipe") 539 def test_check_service_base(self, pipe_mock, green_pipe_mock, 540 handle_signal_mock, start_child_mock): 541 pipe_mock.return_value = [None, None] 542 launcher = service.ProcessLauncher(self.conf) 543 serv = _Service() 544 launcher.launch_service(serv, workers=0) 545 546 @mock.patch("oslo_service.service.ProcessLauncher._start_child") 547 @mock.patch("oslo_service.service.ProcessLauncher.handle_signal") 548 @mock.patch("eventlet.greenio.GreenPipe") 549 @mock.patch("os.pipe") 550 def test_check_service_base_fails(self, pipe_mock, green_pipe_mock, 551 handle_signal_mock, start_child_mock): 552 pipe_mock.return_value = [None, None] 553 launcher = service.ProcessLauncher(self.conf) 554 555 class FooService(object): 556 def __init__(self): 557 pass 558 serv = FooService() 559 self.assertRaises(TypeError, launcher.launch_service, serv, 0) 560 561 @mock.patch("oslo_service.service.ProcessLauncher._start_child") 562 @mock.patch("oslo_service.service.ProcessLauncher.handle_signal") 563 @mock.patch("eventlet.greenio.GreenPipe") 564 @mock.patch("os.pipe") 565 def test_double_sighup(self, pipe_mock, green_pipe_mock, 566 handle_signal_mock, start_child_mock): 567 # Test that issuing two SIGHUPs in a row does not exit; then send a 568 # TERM that does cause an exit. 569 pipe_mock.return_value = [None, None] 570 launcher = service.ProcessLauncher(self.conf) 571 serv = _Service() 572 launcher.launch_service(serv, workers=0) 573 574 def stager(): 575 # -1: start state 576 # 0: post-init 577 # 1: first HUP sent 578 # 2: second HUP sent 579 # 3: TERM sent 580 stager.stage += 1 581 if stager.stage < 3: 582 launcher._handle_hup(1, mock.sentinel.frame) 583 elif stager.stage == 3: 584 launcher._handle_term(15, mock.sentinel.frame) 585 else: 586 self.fail("TERM did not kill launcher") 587 stager.stage = -1 588 handle_signal_mock.side_effect = stager 589 590 launcher.wait() 591 self.assertEqual(3, stager.stage) 592 593 594class GracefulShutdownTestService(service.Service): 595 def __init__(self): 596 super(GracefulShutdownTestService, self).__init__() 597 self.finished_task = event.Event() 598 599 def start(self, sleep_amount): 600 def sleep_and_send(finish_event): 601 time.sleep(sleep_amount) 602 finish_event.send() 603 self.tg.add_thread(sleep_and_send, self.finished_task) 604 605 606def exercise_graceful_test_service(sleep_amount, time_to_wait, graceful): 607 svc = GracefulShutdownTestService() 608 svc.start(sleep_amount) 609 svc.stop(graceful) 610 611 def wait_for_task(svc): 612 svc.finished_task.wait() 613 614 return eventlet.timeout.with_timeout(time_to_wait, wait_for_task, 615 svc=svc, timeout_value="Timeout!") 616 617 618class ServiceTest(test_base.BaseTestCase): 619 def test_graceful_stop(self): 620 # Here we wait long enough for the task to gracefully finish. 621 self.assertIsNone(exercise_graceful_test_service(1, 2, True)) 622 623 def test_ungraceful_stop(self): 624 # Here we stop ungracefully, and will never see the task finish. 625 self.assertEqual("Timeout!", 626 exercise_graceful_test_service(1, 2, False)) 627 628 629class EventletServerProcessLauncherTest(base.ServiceBaseTestCase): 630 def setUp(self): 631 super(EventletServerProcessLauncherTest, self).setUp() 632 self.conf(args=[], default_config_files=[]) 633 self.addCleanup(self.conf.reset) 634 self.workers = 3 635 636 def run_server(self): 637 queue = multiprocessing.Queue() 638 # NOTE(bnemec): process_time of 5 needs to be longer than the graceful 639 # shutdown timeout in the "exceeded" test below, but also needs to be 640 # shorter than the timeout in the regular graceful shutdown test. 641 proc = multiprocessing.Process(target=eventlet_service.run, 642 args=(queue,), 643 kwargs={'workers': self.workers, 644 'process_time': 5}) 645 proc.start() 646 647 port = queue.get() 648 conn = socket.create_connection(('127.0.0.1', port)) 649 # Send request to make the connection active. 650 conn.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') 651 652 # NOTE(blk-u): The sleep shouldn't be necessary. There must be a bug in 653 # the server implementation where it takes some time to set up the 654 # server or signal handlers. 655 time.sleep(1) 656 657 return (proc, conn) 658 659 def test_shuts_down_on_sigint_when_client_connected(self): 660 proc, conn = self.run_server() 661 662 # check that server is live 663 self.assertTrue(proc.is_alive()) 664 665 # send SIGINT to the server and wait for it to exit while client still 666 # connected. 667 os.kill(proc.pid, signal.SIGINT) 668 proc.join() 669 conn.close() 670 671 def test_graceful_shuts_down_on_sigterm_when_client_connected(self): 672 self.config(graceful_shutdown_timeout=7) 673 proc, conn = self.run_server() 674 675 # send SIGTERM to the server and wait for it to exit while client still 676 # connected. 677 os.kill(proc.pid, signal.SIGTERM) 678 679 # server with graceful shutdown must wait forever if 680 # option graceful_shutdown_timeout is not specified. 681 # we can not wait forever ... so 1 second is enough. 682 # NOTE(bnemec): In newer versions of eventlet that drop idle 683 # connections, this needs to be long enough to allow the signal 684 # handler to fire but short enough that our request doesn't complete 685 # or the connection will be closed and the server will stop. 686 time.sleep(1) 687 688 self.assertTrue(proc.is_alive()) 689 690 conn.close() 691 proc.join() 692 693 def test_graceful_stop_with_exceeded_graceful_shutdown_timeout(self): 694 # Server must exit if graceful_shutdown_timeout exceeded 695 graceful_shutdown_timeout = 4 696 self.config(graceful_shutdown_timeout=graceful_shutdown_timeout) 697 proc, conn = self.run_server() 698 699 time_before = time.time() 700 os.kill(proc.pid, signal.SIGTERM) 701 self.assertTrue(proc.is_alive()) 702 proc.join() 703 self.assertFalse(proc.is_alive()) 704 time_after = time.time() 705 706 self.assertTrue(time_after - time_before > graceful_shutdown_timeout) 707 708 709class EventletServerServiceLauncherTest(EventletServerProcessLauncherTest): 710 def setUp(self): 711 super(EventletServerServiceLauncherTest, self).setUp() 712 self.workers = 1 713