1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "chrome/browser/metrics/thread_watcher.h"
6
7 #include <math.h>
8 #include <stdint.h>
9
10 #include <memory>
11
12 #include "base/bind.h"
13 #include "base/cancelable_callback.h"
14 #include "base/check.h"
15 #include "base/location.h"
16 #include "base/run_loop.h"
17 #include "base/single_thread_task_runner.h"
18 #include "base/stl_util.h"
19 #include "base/strings/string_number_conversions.h"
20 #include "base/strings/string_split.h"
21 #include "base/strings/string_tokenizer.h"
22 #include "base/synchronization/condition_variable.h"
23 #include "base/synchronization/lock.h"
24 #include "base/synchronization/waitable_event.h"
25 #include "base/test/bind.h"
26 #include "base/threading/platform_thread.h"
27 #include "base/threading/thread_task_runner_handle.h"
28 #include "base/time/time.h"
29 #include "build/build_config.h"
30 #include "chrome/common/chrome_switches.h"
31 #include "components/crash/core/common/crash_key.h"
32 #include "content/public/browser/browser_task_traits.h"
33 #include "content/public/browser/browser_thread.h"
34 #include "content/public/test/browser_task_environment.h"
35 #include "testing/gtest/include/gtest/gtest.h"
36 #include "testing/platform_test.h"
37
38 using base::TimeDelta;
39 using base::TimeTicks;
40 using content::BrowserThread;
41
42 enum State {
43 INITIALIZED, // Created ThreadWatch object.
44 ACTIVATED, // Thread watching activated.
45 SENT_PING, // Sent ping message to watched thread.
46 RECEIVED_PONG, // Received Pong message.
47 DEACTIVATED, // Thread watching de-activated.
48 };
49
50 enum WaitState {
51 UNINITIALIZED,
52 STARTED_WAITING, // Start waiting for state_ to change to expected_state.
53 STOPPED_WAITING, // Done with the waiting.
54 ALL_DONE, // Done with waiting for STOPPED_WAITING.
55 };
56
57 enum CheckResponseState {
58 UNKNOWN,
59 SUCCESSFUL, // CheckResponse was successful.
60 FAILED, // CheckResponse has failed.
61 };
62
63 // This class helps to track and manipulate thread state during tests. This
64 // class also has utility method to simulate hanging of watched thread by making
65 // the watched thread wait for a very long time by posting a task on watched
66 // thread that keeps it busy. It also has an utility method to block running of
67 // tests until ThreadWatcher object's post-condition state changes to an
68 // expected state.
69 class CustomThreadWatcher : public ThreadWatcher {
70 public:
71 State thread_watcher_state_;
72 // Wait state may be accessed from VeryLongMethod on another thread.
73 base::Lock wait_state_lock_;
74 base::ConditionVariable wait_state_changed_;
75 WaitState wait_state_;
76 CheckResponseState check_response_state_;
77 uint64_t ping_sent_;
78 uint64_t pong_received_;
79 int32_t success_response_;
80 int32_t failed_response_;
81 base::TimeTicks saved_ping_time_;
82 uint64_t saved_ping_sequence_number_;
83 base::RepeatingClosure on_state_changed_;
84
CustomThreadWatcher(const BrowserThread::ID thread_id,const std::string thread_name,const TimeDelta & sleep_time,const TimeDelta & unresponsive_time)85 CustomThreadWatcher(const BrowserThread::ID thread_id,
86 const std::string thread_name,
87 const TimeDelta& sleep_time,
88 const TimeDelta& unresponsive_time)
89 : ThreadWatcher(WatchingParams(thread_id,
90 thread_name,
91 sleep_time,
92 unresponsive_time,
93 ThreadWatcherList::kUnresponsiveCount,
94 true)),
95 thread_watcher_state_(INITIALIZED),
96 wait_state_changed_(&wait_state_lock_),
97 wait_state_(UNINITIALIZED),
98 check_response_state_(UNKNOWN),
99 ping_sent_(0),
100 pong_received_(0),
101 success_response_(0),
102 failed_response_(0),
103 saved_ping_time_(base::TimeTicks::Now()),
104 saved_ping_sequence_number_(0) {}
105
UpdateState(State new_state)106 State UpdateState(State new_state) {
107 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
108 State old_state = thread_watcher_state_;
109 if (old_state != DEACTIVATED)
110 thread_watcher_state_ = new_state;
111 if (new_state == SENT_PING)
112 ++ping_sent_;
113 if (new_state == RECEIVED_PONG)
114 ++pong_received_;
115 saved_ping_time_ = ping_time();
116 saved_ping_sequence_number_ = ping_sequence_number();
117 OnStateChanged();
118 return old_state;
119 }
120
UpdateWaitState(WaitState new_state)121 void UpdateWaitState(WaitState new_state) {
122 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
123 {
124 base::AutoLock auto_lock(wait_state_lock_);
125 wait_state_ = new_state;
126 }
127 wait_state_changed_.Broadcast();
128 OnStateChanged();
129 }
130
ActivateThreadWatching()131 void ActivateThreadWatching() override {
132 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
133 State old_state = UpdateState(ACTIVATED);
134 EXPECT_EQ(old_state, INITIALIZED);
135 ThreadWatcher::ActivateThreadWatching();
136 }
137
DeActivateThreadWatching()138 void DeActivateThreadWatching() override {
139 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
140 State old_state = UpdateState(DEACTIVATED);
141 EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
142 old_state == RECEIVED_PONG);
143 ThreadWatcher::DeActivateThreadWatching();
144 }
145
PostPingMessage()146 void PostPingMessage() override {
147 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
148 State old_state = UpdateState(SENT_PING);
149 EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
150 ThreadWatcher::PostPingMessage();
151 }
152
OnPongMessage(uint64_t ping_sequence_number)153 void OnPongMessage(uint64_t ping_sequence_number) override {
154 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
155 State old_state = UpdateState(RECEIVED_PONG);
156 EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
157 ThreadWatcher::OnPongMessage(ping_sequence_number);
158 }
159
OnCheckResponsiveness(uint64_t ping_sequence_number)160 void OnCheckResponsiveness(uint64_t ping_sequence_number) override {
161 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
162 ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
163 if (responsive_) {
164 ++success_response_;
165 check_response_state_ = SUCCESSFUL;
166 } else {
167 ++failed_response_;
168 check_response_state_ = FAILED;
169 }
170 OnStateChanged();
171 }
172
WaitForWaitStateChange(TimeDelta wait_time,WaitState expected_state)173 void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
174 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
175 base::RunLoop run_loop(base::RunLoop::Type::kNestableTasksAllowed);
176 base::RepeatingClosure quit_closure = run_loop.QuitClosure();
177 on_state_changed_ = base::BindRepeating(
178 [](CustomThreadWatcher* watcher, base::RepeatingClosure quit_closure,
179 WaitState expected_state) {
180 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
181 // No need to acquire wait_state_lock_ since we're on the same thread
182 // that modifies wait_state_.
183 if (watcher->wait_state_ == expected_state)
184 quit_closure.Run();
185 },
186 base::Unretained(this), quit_closure, expected_state);
187 base::CancelableClosure timeout_closure(base::BindRepeating(
188 [](base::RepeatingClosure quit_closure) {
189 ADD_FAILURE() << "WaitForWaitStateChange timed out";
190 quit_closure.Run();
191 },
192 quit_closure));
193 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
194 FROM_HERE, timeout_closure.callback(), wait_time);
195 run_loop.Run();
196 on_state_changed_.Reset();
197 }
198
199 // May be called on any thread other than the WatchDogThread.
BusyWaitForWaitStateChange(TimeDelta wait_time,WaitState expected_state)200 void BusyWaitForWaitStateChange(TimeDelta wait_time,
201 WaitState expected_state) {
202 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
203 TimeTicks end_time = TimeTicks::Now() + wait_time;
204 {
205 base::AutoLock auto_lock(wait_state_lock_);
206 while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
207 wait_state_changed_.TimedWait(end_time - TimeTicks::Now());
208 }
209 }
210
VeryLongMethod(TimeDelta wait_time)211 void VeryLongMethod(TimeDelta wait_time) {
212 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
213 // ThreadWatcher tasks should not be allowed to execute while we're waiting,
214 // so hog the thread until the state changes.
215 BusyWaitForWaitStateChange(wait_time, STOPPED_WAITING);
216 WatchDogThread::PostTask(
217 FROM_HERE, base::BindRepeating(&CustomThreadWatcher::UpdateWaitState,
218 base::Unretained(this), ALL_DONE));
219 }
220
WaitForStateChange(const TimeDelta & wait_time,State expected_state)221 State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
222 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
223 UpdateWaitState(STARTED_WAITING);
224
225 // Keep the watch dog thread looping until the state changes to the
226 // expected_state or until wait_time elapses enough times for the
227 // unresponsive threshold to be reached.
228 base::RunLoop run_loop(base::RunLoop::Type::kNestableTasksAllowed);
229 base::RepeatingClosure quit_closure = run_loop.QuitClosure();
230 State exit_state = INITIALIZED;
231 on_state_changed_ = base::BindRepeating(
232 [](CustomThreadWatcher* watcher, base::RepeatingClosure quit_closure,
233 State expected_state, State* exit_state) {
234 *exit_state = watcher->thread_watcher_state_;
235 if (watcher->thread_watcher_state_ == expected_state)
236 quit_closure.Run();
237 },
238 base::Unretained(this), quit_closure, expected_state,
239 base::Unretained(&exit_state));
240 base::CancelableClosure timeout_closure(base::BindRepeating(
241 [](base::RepeatingClosure quit_closure) { quit_closure.Run(); },
242 quit_closure));
243 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
244 FROM_HERE, timeout_closure.callback(),
245 wait_time * unresponsive_threshold_);
246 run_loop.Run();
247 on_state_changed_.Reset();
248
249 UpdateWaitState(STOPPED_WAITING);
250 return exit_state;
251 }
252
WaitForCheckResponse(const TimeDelta & wait_time,CheckResponseState expected_state)253 CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
254 CheckResponseState expected_state) {
255 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
256 UpdateWaitState(STARTED_WAITING);
257
258 // Keep the watch dog thread looping until the state changes to the
259 // expected_state or until wait_time elapses enough times for the
260 // unresponsive threshold to be reached.
261 base::RunLoop run_loop(base::RunLoop::Type::kNestableTasksAllowed);
262 base::RepeatingClosure quit_closure = run_loop.QuitClosure();
263 CheckResponseState exit_state = UNKNOWN;
264 on_state_changed_ = base::BindRepeating(
265 [](CustomThreadWatcher* watcher, base::RepeatingClosure quit_closure,
266 CheckResponseState expected_state, CheckResponseState* exit_state) {
267 *exit_state = watcher->check_response_state_;
268 if (watcher->check_response_state_ == expected_state)
269 quit_closure.Run();
270 },
271 base::Unretained(this), quit_closure, expected_state,
272 base::Unretained(&exit_state));
273 base::CancelableClosure timeout_closure(base::BindRepeating(
274 [](base::RepeatingClosure quit_closure) { quit_closure.Run(); },
275 quit_closure));
276 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
277 FROM_HERE, timeout_closure.callback(),
278 wait_time * unresponsive_threshold_);
279 run_loop.Run();
280 on_state_changed_.Reset();
281
282 UpdateWaitState(STOPPED_WAITING);
283 return exit_state;
284 }
285
OnStateChanged()286 void OnStateChanged() {
287 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
288 if (on_state_changed_)
289 on_state_changed_.Run();
290 }
291 };
292
293 class ThreadWatcherTest : public ::testing::Test {
294 public:
295 static constexpr TimeDelta kSleepTime = TimeDelta::FromMilliseconds(50);
296 static constexpr TimeDelta kUnresponsiveTime =
297 TimeDelta::FromMilliseconds(500);
298 static constexpr char kIOThreadName[] = "IO";
299 static constexpr char kUIThreadName[] = "UI";
300 static constexpr char kCrashOnHangThreadNames[] = "UI,IO";
301 static constexpr char kCrashOnHangThreadData[] = "UI:12,IO:12";
302
303 CustomThreadWatcher* io_watcher_;
304 CustomThreadWatcher* ui_watcher_;
305 ThreadWatcherList* thread_watcher_list_;
306
307 template <typename... TaskEnvironmentTraits>
ThreadWatcherTest(base::test::TaskEnvironment::TimeSource time_source=base::test::TaskEnvironment::TimeSource::SYSTEM_TIME)308 ThreadWatcherTest(base::test::TaskEnvironment::TimeSource time_source =
309 base::test::TaskEnvironment::TimeSource::SYSTEM_TIME)
310 : task_environment_(content::BrowserTaskEnvironment::REAL_IO_THREAD,
311 time_source),
312 setup_complete_(&lock_),
313 initialized_(false) {
314 crash_reporter::InitializeCrashKeysForTesting();
315
316 // Make sure UI and IO threads are started and ready.
317 task_environment_.RunIOThreadUntilIdle();
318
319 watchdog_thread_.reset(new WatchDogThread());
320 watchdog_thread_->StartAndWaitForTesting();
321
322 WatchDogThread::PostTask(
323 FROM_HERE, base::BindRepeating(&ThreadWatcherTest::SetUpObjects,
324 base::Unretained(this)));
325
326 WaitForSetUp(TimeDelta::FromMinutes(1));
327 }
328
SetUpObjects()329 void SetUpObjects() {
330 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
331
332 // Setup the registry for thread watchers.
333 thread_watcher_list_ = new ThreadWatcherList();
334
335 // Create thread watcher object for the IO thread.
336 std::unique_ptr<CustomThreadWatcher> io_watcher(new CustomThreadWatcher(
337 BrowserThread::IO, kIOThreadName, kSleepTime, kUnresponsiveTime));
338 io_watcher_ = io_watcher.get();
339 ThreadWatcher* registered_io_watcher =
340 ThreadWatcherList::Register(std::move(io_watcher));
341 EXPECT_EQ(io_watcher_, registered_io_watcher);
342 EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(BrowserThread::IO));
343
344 // Create thread watcher object for the UI thread.
345 std::unique_ptr<CustomThreadWatcher> ui_watcher(new CustomThreadWatcher(
346 BrowserThread::UI, kUIThreadName, kSleepTime, kUnresponsiveTime));
347 ui_watcher_ = ui_watcher.get();
348 ThreadWatcher* registered_ui_watcher =
349 ThreadWatcherList::Register(std::move(ui_watcher));
350 EXPECT_EQ(ui_watcher_, registered_ui_watcher);
351 EXPECT_EQ(ui_watcher_, thread_watcher_list_->Find(BrowserThread::UI));
352
353 {
354 base::AutoLock lock(lock_);
355 initialized_ = true;
356 }
357 setup_complete_.Signal();
358 }
359
WaitForSetUp(TimeDelta wait_time)360 void WaitForSetUp(TimeDelta wait_time) {
361 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
362 TimeTicks end_time = TimeTicks::Now() + wait_time;
363 {
364 base::AutoLock auto_lock(lock_);
365 while (!initialized_ && TimeTicks::Now() < end_time)
366 setup_complete_.TimedWait(end_time - TimeTicks::Now());
367 }
368 }
369
~ThreadWatcherTest()370 ~ThreadWatcherTest() override {
371 ThreadWatcherList::DeleteAll();
372 io_watcher_ = nullptr;
373 ui_watcher_ = nullptr;
374 watchdog_thread_.reset();
375 thread_watcher_list_ = nullptr;
376
377 crash_reporter::ResetCrashKeysForTesting();
378 }
379
380 protected:
381 content::BrowserTaskEnvironment task_environment_;
382 base::Lock lock_;
383 base::ConditionVariable setup_complete_;
384 bool initialized_;
385 std::unique_ptr<WatchDogThread> watchdog_thread_;
386
387 DISALLOW_COPY_AND_ASSIGN(ThreadWatcherTest);
388 };
389
390 class ThreadWatcherTestWithMockTime : public ThreadWatcherTest {
391 public:
ThreadWatcherTestWithMockTime()392 ThreadWatcherTestWithMockTime()
393 : ThreadWatcherTest(base::test::TaskEnvironment::TimeSource::MOCK_TIME) {}
394 };
395
396 // Verify that the "seconds-since-last-memory-pressure" crash key is written
397 // correctly.
398 //
399 // Note: It is not possible to split this test in 3 smaller tests, because
400 // reusing the same crash key in multiple unit tests is broken with breakpad.
401 // https://crbug.com/1041106.
TEST_F(ThreadWatcherTestWithMockTime,MemoryPressureCrashKey)402 TEST_F(ThreadWatcherTestWithMockTime, MemoryPressureCrashKey) {
403 // The "seconds-since-last-memory-pressure" crash key should hold "No memory
404 // pressure" when there has never been any memory pressure signal.
405 watchdog_thread_->PostTask(
406 FROM_HERE, base::BindLambdaForTesting([&]() {
407 ui_watcher_->SetTimeSinceLastCriticalMemoryPressureCrashKey();
408 EXPECT_EQ("No memory pressure",
409 crash_reporter::GetCrashKeyValue(
410 "seconds-since-last-memory-pressure"));
411 }));
412
413 watchdog_thread_->FlushForTesting();
414
415 // The "seconds-since-last-memory-pressure" crash key should hold "No memory
416 // pressure" when there has been a MODERATE memory pressure signal, but no
417 // CRITICAL memory pressure signal.
418 base::MemoryPressureListener::SimulatePressureNotification(
419 base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_MODERATE);
420 watchdog_thread_->FlushForTesting();
421
422 watchdog_thread_->PostTask(
423 FROM_HERE, base::BindLambdaForTesting([&]() {
424 ui_watcher_->SetTimeSinceLastCriticalMemoryPressureCrashKey();
425 EXPECT_EQ("No memory pressure",
426 crash_reporter::GetCrashKeyValue(
427 "seconds-since-last-memory-pressure"));
428 }));
429
430 watchdog_thread_->FlushForTesting();
431
432 // The "seconds-since-last-memory-pressure" crash key should hold "4" when set
433 // 4 seconds after a CRITICAL memory pressure signal.
434 base::MemoryPressureListener::SimulatePressureNotification(
435 base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_CRITICAL);
436 watchdog_thread_->FlushForTesting();
437
438 task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(4));
439 watchdog_thread_->PostTask(
440 FROM_HERE, base::BindLambdaForTesting([&]() {
441 ui_watcher_->SetTimeSinceLastCriticalMemoryPressureCrashKey();
442 EXPECT_EQ("4", crash_reporter::GetCrashKeyValue(
443 "seconds-since-last-memory-pressure"));
444 }));
445
446 watchdog_thread_->FlushForTesting();
447 }
448
449 // Test fixture that runs a test body on the WatchDogThread. Subclasses override
450 // TestBodyOnWatchDogThread() and should call RunTestOnWatchDogThread() in their
451 // TEST_F() declaration.
452 class ThreadWatcherTestOnWatchDogThread : public ThreadWatcherTest {
453 public:
ThreadWatcherTestOnWatchDogThread()454 ThreadWatcherTestOnWatchDogThread()
455 : test_body_run_loop_(base::RunLoop::Type::kNestableTasksAllowed) {}
456
457 protected:
RunTestOnWatchDogThread()458 void RunTestOnWatchDogThread() {
459 WatchDogThread::PostTask(FROM_HERE,
460 base::BindRepeating(
461 [](ThreadWatcherTestOnWatchDogThread* test) {
462 test->TestBodyOnWatchDogThread();
463 test->test_body_run_loop_.Quit();
464 },
465 base::Unretained(this)));
466 test_body_run_loop_.Run();
467 }
468
469 virtual void TestBodyOnWatchDogThread() = 0;
470
471 protected:
472 base::RunLoop test_body_run_loop_;
473 };
474
475 // Declare storage for ThreadWatcherTest's static constants.
476 constexpr TimeDelta ThreadWatcherTest::kSleepTime;
477 constexpr TimeDelta ThreadWatcherTest::kUnresponsiveTime;
478 constexpr char ThreadWatcherTest::kIOThreadName[];
479 constexpr char ThreadWatcherTest::kUIThreadName[];
480 constexpr char ThreadWatcherTest::kCrashOnHangThreadNames[];
481 constexpr char ThreadWatcherTest::kCrashOnHangThreadData[];
482
TEST_F(ThreadWatcherTest,ThreadNamesOnlyArgs)483 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
484 // Setup command_line arguments.
485 base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
486 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
487 kCrashOnHangThreadNames);
488
489 // Parse command_line arguments.
490 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
491 uint32_t unresponsive_threshold;
492 ThreadWatcherList::ParseCommandLine(command_line,
493 &unresponsive_threshold,
494 &crash_on_hang_threads);
495
496 // Verify the data.
497 base::CStringTokenizer tokens(
498 kCrashOnHangThreadNames,
499 kCrashOnHangThreadNames + (base::size(kCrashOnHangThreadNames) - 1), ",");
500 while (tokens.GetNext()) {
501 std::vector<base::StringPiece> values = base::SplitStringPiece(
502 tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
503 std::string thread_name = values[0].as_string();
504
505 auto it = crash_on_hang_threads.find(thread_name);
506 bool crash_on_hang = (it != crash_on_hang_threads.end());
507 EXPECT_TRUE(crash_on_hang);
508 EXPECT_LT(0u, it->second);
509 }
510 }
511
TEST_F(ThreadWatcherTest,CrashOnHangThreadsAllArgs)512 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
513 // Setup command_line arguments.
514 base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
515 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
516 kCrashOnHangThreadData);
517
518 // Parse command_line arguments.
519 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
520 uint32_t unresponsive_threshold;
521 ThreadWatcherList::ParseCommandLine(command_line,
522 &unresponsive_threshold,
523 &crash_on_hang_threads);
524
525 // Verify the data.
526 base::CStringTokenizer tokens(
527 kCrashOnHangThreadData,
528 kCrashOnHangThreadData + (base::size(kCrashOnHangThreadData) - 1), ",");
529 while (tokens.GetNext()) {
530 std::vector<base::StringPiece> values = base::SplitStringPiece(
531 tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
532 std::string thread_name = values[0].as_string();
533
534 auto it = crash_on_hang_threads.find(thread_name);
535
536 bool crash_on_hang = (it != crash_on_hang_threads.end());
537 EXPECT_TRUE(crash_on_hang);
538
539 uint32_t crash_unresponsive_threshold = it->second;
540 uint32_t crash_on_unresponsive_seconds =
541 ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
542 EXPECT_EQ(12u, crash_on_unresponsive_seconds);
543 }
544 }
545
546 // Test registration. When thread_watcher_list_ goes out of scope after
547 // TearDown, all thread watcher objects will be deleted.
548 class ThreadWatcherTestRegistration : public ThreadWatcherTestOnWatchDogThread {
549 protected:
TestBodyOnWatchDogThread()550 void TestBodyOnWatchDogThread() override {
551 // Check ThreadWatcher object has all correct parameters.
552 EXPECT_EQ(BrowserThread::IO, io_watcher_->thread_id());
553 EXPECT_EQ(kIOThreadName, io_watcher_->thread_name());
554 EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
555 EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
556 EXPECT_FALSE(io_watcher_->active());
557
558 // Check ThreadWatcher object of watched UI thread has correct data.
559 EXPECT_EQ(BrowserThread::UI, ui_watcher_->thread_id());
560 EXPECT_EQ(kUIThreadName, ui_watcher_->thread_name());
561 EXPECT_EQ(kSleepTime, ui_watcher_->sleep_time());
562 EXPECT_EQ(kUnresponsiveTime, ui_watcher_->unresponsive_time());
563 EXPECT_FALSE(ui_watcher_->active());
564 }
565 };
566
TEST_F(ThreadWatcherTestRegistration,RunTest)567 TEST_F(ThreadWatcherTestRegistration, RunTest) {
568 RunTestOnWatchDogThread();
569 }
570
571 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
572 // method also checks that pong message was sent by the watched thread and pong
573 // message was received by the WatchDogThread. It also checks that
574 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched
575 // thread is not hung.
576 class ThreadWatcherTestThreadResponding
577 : public ThreadWatcherTestOnWatchDogThread {
578 protected:
TestBodyOnWatchDogThread()579 void TestBodyOnWatchDogThread() override {
580 TimeTicks time_before_ping = TimeTicks::Now();
581 // Activate watching IO thread.
582 io_watcher_->ActivateThreadWatching();
583
584 // Activate would have started ping/pong messaging. Expect at least one
585 // ping/pong messaging sequence to happen.
586 io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
587 RECEIVED_PONG);
588 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64_t>(0));
589 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64_t>(0));
590 EXPECT_TRUE(io_watcher_->active());
591 EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
592 EXPECT_GE(io_watcher_->saved_ping_sequence_number_,
593 static_cast<uint64_t>(0));
594
595 // Verify watched thread is responding with ping/pong messaging.
596 io_watcher_->WaitForCheckResponse(
597 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
598 EXPECT_GT(io_watcher_->success_response_, 0);
599 EXPECT_EQ(io_watcher_->failed_response_, 0);
600
601 // DeActivate thread watching for shutdown.
602 io_watcher_->DeActivateThreadWatching();
603 }
604 };
605
TEST_F(ThreadWatcherTestThreadResponding,RunTest)606 TEST_F(ThreadWatcherTestThreadResponding, RunTest) {
607 RunTestOnWatchDogThread();
608 }
609
610 // This test posts a task on watched thread that takes very long time (this is
611 // to simulate hanging of watched thread). It then checks for
612 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
613 // if the watched thread is not responding).
614 class ThreadWatcherTestThreadNotResponding
615 : public ThreadWatcherTestOnWatchDogThread {
616 protected:
TestBodyOnWatchDogThread()617 void TestBodyOnWatchDogThread() override {
618 // Simulate hanging of watched thread by making the watched thread wait for
619 // a very long time by posting a task on watched thread that keeps it busy.
620 // It is safe to use base::Unretained because test is waiting for the method
621 // to finish.
622 content::GetIOThreadTaskRunner({})->PostTask(
623 FROM_HERE,
624 base::BindOnce(&CustomThreadWatcher::VeryLongMethod,
625 base::Unretained(io_watcher_), kUnresponsiveTime * 10));
626
627 // Activate thread watching.
628 io_watcher_->ActivateThreadWatching();
629
630 // Verify watched thread is not responding for ping messages.
631 io_watcher_->WaitForCheckResponse(
632 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
633 EXPECT_EQ(io_watcher_->success_response_, 0);
634 EXPECT_GT(io_watcher_->failed_response_, 0);
635
636 // DeActivate thread watching for shutdown.
637 io_watcher_->DeActivateThreadWatching();
638
639 // Wait for the io_watcher_'s VeryLongMethod to finish.
640 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
641 }
642 };
643
TEST_F(ThreadWatcherTestThreadNotResponding,RunTest)644 TEST_F(ThreadWatcherTestThreadNotResponding, RunTest) {
645 RunTestOnWatchDogThread();
646 }
647
648 // Test watching of multiple threads with all threads not responding.
649 class ThreadWatcherTestMultipleThreadsResponding
650 : public ThreadWatcherTestOnWatchDogThread {
651 protected:
TestBodyOnWatchDogThread()652 void TestBodyOnWatchDogThread() override {
653 // Check for UI thread to perform ping/pong messaging.
654 ui_watcher_->ActivateThreadWatching();
655
656 // Check for IO thread to perform ping/pong messaging.
657 io_watcher_->ActivateThreadWatching();
658
659 // Verify UI thread is responding with ping/pong messaging.
660 ui_watcher_->WaitForCheckResponse(
661 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
662 EXPECT_GT(ui_watcher_->ping_sent_, static_cast<uint64_t>(0));
663 EXPECT_GT(ui_watcher_->pong_received_, static_cast<uint64_t>(0));
664 EXPECT_GE(ui_watcher_->ping_sequence_number(), static_cast<uint64_t>(0));
665 EXPECT_GT(ui_watcher_->success_response_, 0);
666 EXPECT_EQ(ui_watcher_->failed_response_, 0);
667
668 // Verify IO thread is responding with ping/pong messaging.
669 io_watcher_->WaitForCheckResponse(
670 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
671 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64_t>(0));
672 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64_t>(0));
673 EXPECT_GE(io_watcher_->ping_sequence_number(), static_cast<uint64_t>(0));
674 EXPECT_GT(io_watcher_->success_response_, 0);
675 EXPECT_EQ(io_watcher_->failed_response_, 0);
676
677 // DeActivate thread watching for shutdown.
678 io_watcher_->DeActivateThreadWatching();
679 ui_watcher_->DeActivateThreadWatching();
680 }
681 };
682
TEST_F(ThreadWatcherTestMultipleThreadsResponding,RunTest)683 TEST_F(ThreadWatcherTestMultipleThreadsResponding, RunTest) {
684 RunTestOnWatchDogThread();
685 }
686
687 // Test watching of multiple threads with one of the threads not responding.
688 class ThreadWatcherTestMultipleThreadsNotResponding
689 : public ThreadWatcherTestOnWatchDogThread {
690 protected:
TestBodyOnWatchDogThread()691 void TestBodyOnWatchDogThread() override {
692 // Simulate hanging of watched thread by making the watched thread wait for
693 // a very long time by posting a task on watched thread that keeps it busy.
694 // It is safe to use base::Unretained because test is waiting for the method
695 // to finish.
696 content::GetIOThreadTaskRunner({})->PostTask(
697 FROM_HERE,
698 base::BindOnce(&CustomThreadWatcher::VeryLongMethod,
699 base::Unretained(io_watcher_), kUnresponsiveTime * 10));
700
701 // Activate watching of UI thread.
702 ui_watcher_->ActivateThreadWatching();
703
704 // Activate watching of IO thread.
705 io_watcher_->ActivateThreadWatching();
706
707 // Verify UI thread is responding with ping/pong messaging.
708 ui_watcher_->WaitForCheckResponse(
709 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
710 EXPECT_GT(ui_watcher_->success_response_, 0);
711 EXPECT_EQ(ui_watcher_->failed_response_, 0);
712
713 // Verify IO thread is not responding for ping messages.
714 io_watcher_->WaitForCheckResponse(
715 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
716 EXPECT_EQ(io_watcher_->success_response_, 0);
717 EXPECT_GT(io_watcher_->failed_response_, 0);
718
719 // DeActivate thread watching for shutdown.
720 io_watcher_->DeActivateThreadWatching();
721 ui_watcher_->DeActivateThreadWatching();
722
723 // Wait for the io_watcher_'s VeryLongMethod to finish.
724 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
725 }
726 };
727
TEST_F(ThreadWatcherTestMultipleThreadsNotResponding,RunTest)728 TEST_F(ThreadWatcherTestMultipleThreadsNotResponding, RunTest) {
729 RunTestOnWatchDogThread();
730 }
731
732 class ThreadWatcherListTest : public ::testing::Test {
733 protected:
ThreadWatcherListTest()734 ThreadWatcherListTest()
735 : done_(&lock_),
736 state_available_(false),
737 has_thread_watcher_list_(false),
738 stopped_(false) {}
739
ReadStateOnWatchDogThread()740 void ReadStateOnWatchDogThread() {
741 CHECK(WatchDogThread::CurrentlyOnWatchDogThread());
742 {
743 base::AutoLock auto_lock(lock_);
744 has_thread_watcher_list_ =
745 ThreadWatcherList::g_thread_watcher_list_ != nullptr;
746 stopped_ = ThreadWatcherList::g_stopped_;
747 state_available_ = true;
748 }
749 done_.Signal();
750 }
751
CheckState(bool has_thread_watcher_list,bool stopped,const char * const msg)752 void CheckState(bool has_thread_watcher_list,
753 bool stopped,
754 const char* const msg) {
755 CHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
756 {
757 base::AutoLock auto_lock(lock_);
758 state_available_ = false;
759 }
760
761 WatchDogThread::PostTask(
762 FROM_HERE,
763 base::BindRepeating(&ThreadWatcherListTest::ReadStateOnWatchDogThread,
764 base::Unretained(this)));
765 {
766 base::AutoLock auto_lock(lock_);
767 while (!state_available_)
768 done_.Wait();
769
770 EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg;
771 EXPECT_EQ(stopped, stopped_) << msg;
772 }
773 }
774
775 content::BrowserTaskEnvironment task_environment_;
776 base::Lock lock_;
777 base::ConditionVariable done_;
778
779 bool state_available_;
780 bool has_thread_watcher_list_;
781 bool stopped_;
782 };
783
TEST_F(ThreadWatcherListTest,Restart)784 TEST_F(ThreadWatcherListTest, Restart) {
785 ThreadWatcherList::g_initialize_delay_seconds = 1;
786
787 std::unique_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread());
788 watchdog_thread_->StartAndWaitForTesting();
789
790 // See http://crbug.com/347887.
791 // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_,
792 // whilst StopWatchingAll() will just PostTask to destroy it.
793 // Ensure that when Stop is called, Start will NOT create
794 // g_thread_watcher_list_ later on.
795 ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
796 ThreadWatcherList::StopWatchingAll();
797 {
798 base::RunLoop run_loop;
799 content::GetUIThreadTaskRunner({})->PostDelayedTask(
800 FROM_HERE, run_loop.QuitWhenIdleClosure(),
801 base::TimeDelta::FromSeconds(
802 ThreadWatcherList::g_initialize_delay_seconds));
803 run_loop.Run();
804 }
805
806 CheckState(false /* has_thread_watcher_list */, true /* stopped */,
807 "Start / Stopped");
808
809 // Proceed with just |StartWatchingAll| and ensure it'll be started.
810 ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
811 {
812 base::RunLoop run_loop;
813 content::GetUIThreadTaskRunner({})->PostDelayedTask(
814 FROM_HERE, run_loop.QuitWhenIdleClosure(),
815 base::TimeDelta::FromSeconds(
816 ThreadWatcherList::g_initialize_delay_seconds + 1));
817 run_loop.Run();
818 }
819
820 CheckState(true /* has_thread_watcher_list */, false /* stopped */,
821 "Started");
822
823 // Finally, StopWatchingAll() must stop.
824 ThreadWatcherList::StopWatchingAll();
825 {
826 base::RunLoop run_loop;
827 content::GetUIThreadTaskRunner({})->PostDelayedTask(
828 FROM_HERE, run_loop.QuitWhenIdleClosure(),
829 base::TimeDelta::FromSeconds(
830 ThreadWatcherList::g_initialize_delay_seconds));
831 run_loop.Run();
832 }
833
834 CheckState(false /* has_thread_watcher_list */, true /* stopped */,
835 "Stopped");
836 }
837