| // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "chrome/browser/metrics/thread_watcher.h" |
| |
| #include <math.h> |
| #include <stdint.h> |
| |
| #include <memory> |
| |
| #include "base/bind.h" |
| #include "base/location.h" |
| #include "base/logging.h" |
| #include "base/macros.h" |
| #include "base/message_loop/message_loop.h" |
| #include "base/run_loop.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/strings/string_number_conversions.h" |
| #include "base/strings/string_split.h" |
| #include "base/strings/string_tokenizer.h" |
| #include "base/synchronization/condition_variable.h" |
| #include "base/synchronization/lock.h" |
| #include "base/synchronization/spin_wait.h" |
| #include "base/synchronization/waitable_event.h" |
| #include "base/threading/platform_thread.h" |
| #include "base/time/time.h" |
| #include "build/build_config.h" |
| #include "chrome/common/chrome_switches.h" |
| #include "content/public/test/test_browser_thread.h" |
| #include "testing/gtest/include/gtest/gtest.h" |
| #include "testing/platform_test.h" |
| |
| using base::TimeDelta; |
| using base::TimeTicks; |
| using content::BrowserThread; |
| |
| enum State { |
| INITIALIZED, // Created ThreadWatch object. |
| ACTIVATED, // Thread watching activated. |
| SENT_PING, // Sent ping message to watched thread. |
| RECEIVED_PONG, // Received Pong message. |
| DEACTIVATED, // Thread watching de-activated. |
| }; |
| |
| enum WaitState { |
| UNINITIALIZED, |
| STARTED_WAITING, // Start waiting for state_ to change to expected_state. |
| STOPPED_WAITING, // Done with the waiting. |
| ALL_DONE, // Done with waiting for STOPPED_WAITING. |
| }; |
| |
| enum CheckResponseState { |
| UNKNOWN, |
| SUCCESSFUL, // CheckResponse was successful. |
| FAILED, // CheckResponse has failed. |
| }; |
| |
| // This class helps to track and manipulate thread state during tests. This |
| // class also has utility method to simulate hanging of watched thread by making |
| // the watched thread wait for a very long time by posting a task on watched |
| // thread that keeps it busy. It also has an utility method to block running of |
| // tests until ThreadWatcher object's post-condition state changes to an |
| // expected state. |
| class CustomThreadWatcher : public ThreadWatcher { |
| public: |
| base::Lock custom_lock_; |
| base::ConditionVariable state_changed_; |
| State thread_watcher_state_; |
| WaitState wait_state_; |
| CheckResponseState check_response_state_; |
| uint64_t ping_sent_; |
| uint64_t pong_received_; |
| base::subtle::Atomic32 success_response_; |
| base::subtle::Atomic32 failed_response_; |
| base::TimeTicks saved_ping_time_; |
| uint64_t saved_ping_sequence_number_; |
| |
| CustomThreadWatcher(const BrowserThread::ID thread_id, |
| const std::string thread_name, |
| const TimeDelta& sleep_time, |
| const TimeDelta& unresponsive_time) |
| : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time, |
| unresponsive_time, ThreadWatcherList::kUnresponsiveCount, |
| true, ThreadWatcherList::kLiveThreadsThreshold)), |
| state_changed_(&custom_lock_), |
| thread_watcher_state_(INITIALIZED), |
| wait_state_(UNINITIALIZED), |
| check_response_state_(UNKNOWN), |
| ping_sent_(0), |
| pong_received_(0), |
| success_response_(0), |
| failed_response_(0), |
| saved_ping_time_(base::TimeTicks::Now()), |
| saved_ping_sequence_number_(0) { |
| } |
| |
| State UpdateState(State new_state) { |
| State old_state; |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| old_state = thread_watcher_state_; |
| if (old_state != DEACTIVATED) |
| thread_watcher_state_ = new_state; |
| if (new_state == SENT_PING) |
| ++ping_sent_; |
| if (new_state == RECEIVED_PONG) |
| ++pong_received_; |
| saved_ping_time_ = ping_time(); |
| saved_ping_sequence_number_ = ping_sequence_number(); |
| } |
| state_changed_.Broadcast(); |
| return old_state; |
| } |
| |
| WaitState UpdateWaitState(WaitState new_state) { |
| WaitState old_state; |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| old_state = wait_state_; |
| wait_state_ = new_state; |
| } |
| state_changed_.Broadcast(); |
| return old_state; |
| } |
| |
| void ActivateThreadWatching() override { |
| State old_state = UpdateState(ACTIVATED); |
| EXPECT_EQ(old_state, INITIALIZED); |
| ThreadWatcher::ActivateThreadWatching(); |
| } |
| |
| void DeActivateThreadWatching() override { |
| State old_state = UpdateState(DEACTIVATED); |
| EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING || |
| old_state == RECEIVED_PONG); |
| ThreadWatcher::DeActivateThreadWatching(); |
| } |
| |
| void PostPingMessage() override { |
| State old_state = UpdateState(SENT_PING); |
| EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG); |
| ThreadWatcher::PostPingMessage(); |
| } |
| |
| void OnPongMessage(uint64_t ping_sequence_number) override { |
| State old_state = UpdateState(RECEIVED_PONG); |
| EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED); |
| ThreadWatcher::OnPongMessage(ping_sequence_number); |
| } |
| |
| void OnCheckResponsiveness(uint64_t ping_sequence_number) override { |
| ThreadWatcher::OnCheckResponsiveness(ping_sequence_number); |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| if (responsive_) { |
| base::subtle::Release_Store(&success_response_, |
| base::subtle::Acquire_Load(&success_response_) + 1); |
| check_response_state_ = SUCCESSFUL; |
| } else { |
| base::subtle::Release_Store(&failed_response_, |
| base::subtle::Acquire_Load(&failed_response_) + 1); |
| check_response_state_ = FAILED; |
| } |
| } |
| // Broadcast to indicate we have checked responsiveness of the thread that |
| // is watched. |
| state_changed_.Broadcast(); |
| } |
| |
| void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) { |
| DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| TimeTicks end_time = TimeTicks::Now() + wait_time; |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| while (wait_state_ != expected_state && TimeTicks::Now() < end_time) |
| state_changed_.TimedWait(end_time - TimeTicks::Now()); |
| } |
| } |
| |
| void VeryLongMethod(TimeDelta wait_time) { |
| DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| WaitForWaitStateChange(wait_time, STOPPED_WAITING); |
| UpdateWaitState(ALL_DONE); |
| } |
| |
| State WaitForStateChange(const TimeDelta& wait_time, State expected_state) { |
| DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| UpdateWaitState(STARTED_WAITING); |
| |
| State exit_state = INITIALIZED; |
| // Keep the thread that is running the tests waiting until ThreadWatcher |
| // object's state changes to the expected_state or until wait_time elapses. |
| for (uint32_t i = 0; i < unresponsive_threshold_; ++i) { |
| TimeTicks end_time = TimeTicks::Now() + wait_time; |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| while (thread_watcher_state_ != expected_state && |
| TimeTicks::Now() < end_time) { |
| TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); |
| state_changed_.TimedWait(state_change_wait_time); |
| } |
| // Capture the thread_watcher_state_ before it changes and return it |
| // to the caller. |
| exit_state = thread_watcher_state_; |
| if (exit_state == expected_state) |
| break; |
| } |
| } |
| UpdateWaitState(STOPPED_WAITING); |
| return exit_state; |
| } |
| |
| CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time, |
| CheckResponseState expected_state) { |
| DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| UpdateWaitState(STARTED_WAITING); |
| |
| CheckResponseState exit_state = UNKNOWN; |
| // Keep the thread that is running the tests waiting until ThreadWatcher |
| // object's check_response_state_ changes to the expected_state or until |
| // wait_time elapses. |
| for (uint32_t i = 0; i < unresponsive_threshold_; ++i) { |
| TimeTicks end_time = TimeTicks::Now() + wait_time; |
| { |
| base::AutoLock auto_lock(custom_lock_); |
| while (check_response_state_ != expected_state && |
| TimeTicks::Now() < end_time) { |
| TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); |
| state_changed_.TimedWait(state_change_wait_time); |
| } |
| // Capture the check_response_state_ before it changes and return it |
| // to the caller. |
| exit_state = check_response_state_; |
| if (exit_state == expected_state) |
| break; |
| } |
| } |
| UpdateWaitState(STOPPED_WAITING); |
| return exit_state; |
| } |
| }; |
| |
| class ThreadWatcherTest : public ::testing::Test { |
| public: |
| static const TimeDelta kSleepTime; |
| static const TimeDelta kUnresponsiveTime; |
| static const char kIOThreadName[]; |
| static const char kUIThreadName[]; |
| static const char kCrashOnHangThreadNames[]; |
| static const char kThreadNamesAndLiveThreshold[]; |
| static const char kCrashOnHangThreadData[]; |
| |
| CustomThreadWatcher* io_watcher_; |
| CustomThreadWatcher* ui_watcher_; |
| ThreadWatcherList* thread_watcher_list_; |
| |
| ThreadWatcherTest() |
| : setup_complete_(&lock_), |
| initialized_(false) { |
| ui_thread_.reset(new content::TestBrowserThread(BrowserThread::UI)); |
| io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO)); |
| watchdog_thread_.reset(new WatchDogThread()); |
| ui_thread_->StartAndWaitForTesting(); |
| io_thread_->StartAndWaitForTesting(); |
| watchdog_thread_->StartAndWaitForTesting(); |
| |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this))); |
| |
| WaitForSetUp(TimeDelta::FromMinutes(1)); |
| } |
| |
| void SetUpObjects() { |
| DCHECK(WatchDogThread::CurrentlyOnWatchDogThread()); |
| |
| // Setup the registry for thread watchers. |
| thread_watcher_list_ = new ThreadWatcherList(); |
| |
| // Create thread watcher object for the IO thread. |
| std::unique_ptr<CustomThreadWatcher> io_watcher( |
| new CustomThreadWatcher(BrowserThread::IO, kIOThreadName, |
| kSleepTime, kUnresponsiveTime)); |
| io_watcher_ = io_watcher.get(); |
| ThreadWatcher* registered_io_watcher = |
| ThreadWatcherList::Register(std::move(io_watcher)); |
| EXPECT_EQ(io_watcher_, registered_io_watcher); |
| EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(BrowserThread::IO)); |
| |
| // Create thread watcher object for the UI thread. |
| std::unique_ptr<CustomThreadWatcher> ui_watcher(new CustomThreadWatcher( |
| BrowserThread::UI, kUIThreadName, kSleepTime, kUnresponsiveTime)); |
| ui_watcher_ = ui_watcher.get(); |
| ThreadWatcher* registered_ui_watcher = |
| ThreadWatcherList::Register(std::move(ui_watcher)); |
| EXPECT_EQ(ui_watcher_, registered_ui_watcher); |
| EXPECT_EQ(ui_watcher_, thread_watcher_list_->Find(BrowserThread::UI)); |
| |
| { |
| base::AutoLock lock(lock_); |
| initialized_ = true; |
| } |
| setup_complete_.Signal(); |
| } |
| |
| void WaitForSetUp(TimeDelta wait_time) { |
| DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| TimeTicks end_time = TimeTicks::Now() + wait_time; |
| { |
| base::AutoLock auto_lock(lock_); |
| while (!initialized_ && TimeTicks::Now() < end_time) |
| setup_complete_.TimedWait(end_time - TimeTicks::Now()); |
| } |
| } |
| |
| ~ThreadWatcherTest() override { |
| ThreadWatcherList::DeleteAll(); |
| io_watcher_ = nullptr; |
| ui_watcher_ = nullptr; |
| io_thread_.reset(); |
| ui_thread_.reset(); |
| watchdog_thread_.reset(); |
| thread_watcher_list_ = nullptr; |
| } |
| |
| private: |
| base::MessageLoop message_loop_; |
| base::Lock lock_; |
| base::ConditionVariable setup_complete_; |
| bool initialized_; |
| std::unique_ptr<content::TestBrowserThread> ui_thread_; |
| std::unique_ptr<content::TestBrowserThread> io_thread_; |
| std::unique_ptr<WatchDogThread> watchdog_thread_; |
| |
| DISALLOW_COPY_AND_ASSIGN(ThreadWatcherTest); |
| }; |
| |
| // Define static constants. |
| const TimeDelta ThreadWatcherTest::kSleepTime = TimeDelta::FromMilliseconds(50); |
| const TimeDelta ThreadWatcherTest::kUnresponsiveTime = |
| TimeDelta::FromMilliseconds(500); |
| const char ThreadWatcherTest::kIOThreadName[] = "IO"; |
| const char ThreadWatcherTest::kUIThreadName[] = "UI"; |
| const char ThreadWatcherTest::kCrashOnHangThreadNames[] = "UI,IO"; |
| const char ThreadWatcherTest::kThreadNamesAndLiveThreshold[] = "UI:4,IO:4"; |
| const char ThreadWatcherTest::kCrashOnHangThreadData[] = |
| "UI:5:12,IO:5:12,FILE:5:12"; |
| |
| TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) { |
| // Setup command_line arguments. |
| base::CommandLine command_line(base::CommandLine::NO_PROGRAM); |
| command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, |
| kCrashOnHangThreadNames); |
| |
| // Parse command_line arguments. |
| ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; |
| uint32_t unresponsive_threshold; |
| ThreadWatcherList::ParseCommandLine(command_line, |
| &unresponsive_threshold, |
| &crash_on_hang_threads); |
| |
| // Verify the data. |
| base::CStringTokenizer tokens( |
| kCrashOnHangThreadNames, |
| kCrashOnHangThreadNames + (arraysize(kCrashOnHangThreadNames) - 1), ","); |
| while (tokens.GetNext()) { |
| std::vector<base::StringPiece> values = base::SplitStringPiece( |
| tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); |
| std::string thread_name = values[0].as_string(); |
| |
| ThreadWatcherList::CrashOnHangThreadMap::iterator it = |
| crash_on_hang_threads.find(thread_name); |
| bool crash_on_hang = (it != crash_on_hang_threads.end()); |
| EXPECT_TRUE(crash_on_hang); |
| EXPECT_LT(0u, it->second.live_threads_threshold); |
| EXPECT_LT(0u, it->second.unresponsive_threshold); |
| } |
| } |
| |
| TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) { |
| // Setup command_line arguments. |
| base::CommandLine command_line(base::CommandLine::NO_PROGRAM); |
| command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, |
| kThreadNamesAndLiveThreshold); |
| |
| // Parse command_line arguments. |
| ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; |
| uint32_t unresponsive_threshold; |
| ThreadWatcherList::ParseCommandLine(command_line, |
| &unresponsive_threshold, |
| &crash_on_hang_threads); |
| |
| // Verify the data. |
| base::CStringTokenizer tokens( |
| kThreadNamesAndLiveThreshold, |
| kThreadNamesAndLiveThreshold + |
| (arraysize(kThreadNamesAndLiveThreshold) - 1), |
| ","); |
| while (tokens.GetNext()) { |
| std::vector<base::StringPiece> values = base::SplitStringPiece( |
| tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); |
| std::string thread_name = values[0].as_string(); |
| |
| ThreadWatcherList::CrashOnHangThreadMap::iterator it = |
| crash_on_hang_threads.find(thread_name); |
| bool crash_on_hang = (it != crash_on_hang_threads.end()); |
| EXPECT_TRUE(crash_on_hang); |
| EXPECT_EQ(4u, it->second.live_threads_threshold); |
| EXPECT_LT(0u, it->second.unresponsive_threshold); |
| } |
| } |
| |
| TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) { |
| // Setup command_line arguments. |
| base::CommandLine command_line(base::CommandLine::NO_PROGRAM); |
| command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, |
| kCrashOnHangThreadData); |
| |
| // Parse command_line arguments. |
| ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; |
| uint32_t unresponsive_threshold; |
| ThreadWatcherList::ParseCommandLine(command_line, |
| &unresponsive_threshold, |
| &crash_on_hang_threads); |
| |
| // Verify the data. |
| base::CStringTokenizer tokens( |
| kCrashOnHangThreadData, |
| kCrashOnHangThreadData + (arraysize(kCrashOnHangThreadData) - 1), ","); |
| while (tokens.GetNext()) { |
| std::vector<base::StringPiece> values = base::SplitStringPiece( |
| tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); |
| std::string thread_name = values[0].as_string(); |
| |
| ThreadWatcherList::CrashOnHangThreadMap::iterator it = |
| crash_on_hang_threads.find(thread_name); |
| |
| bool crash_on_hang = (it != crash_on_hang_threads.end()); |
| EXPECT_TRUE(crash_on_hang); |
| |
| uint32_t crash_live_threads_threshold = it->second.live_threads_threshold; |
| EXPECT_EQ(5u, crash_live_threads_threshold); |
| |
| uint32_t crash_unresponsive_threshold = it->second.unresponsive_threshold; |
| uint32_t crash_on_unresponsive_seconds = |
| ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold; |
| EXPECT_EQ(12u, crash_on_unresponsive_seconds); |
| } |
| } |
| |
| // Test registration. When thread_watcher_list_ goes out of scope after |
| // TearDown, all thread watcher objects will be deleted. |
| TEST_F(ThreadWatcherTest, Registration) { |
| // Check ThreadWatcher object has all correct parameters. |
| EXPECT_EQ(BrowserThread::IO, io_watcher_->thread_id()); |
| EXPECT_EQ(kIOThreadName, io_watcher_->thread_name()); |
| EXPECT_EQ(kSleepTime, io_watcher_->sleep_time()); |
| EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time()); |
| EXPECT_FALSE(io_watcher_->active()); |
| |
| // Check ThreadWatcher object of watched UI thread has correct data. |
| EXPECT_EQ(BrowserThread::UI, ui_watcher_->thread_id()); |
| EXPECT_EQ(kUIThreadName, ui_watcher_->thread_name()); |
| EXPECT_EQ(kSleepTime, ui_watcher_->sleep_time()); |
| EXPECT_EQ(kUnresponsiveTime, ui_watcher_->unresponsive_time()); |
| EXPECT_FALSE(ui_watcher_->active()); |
| } |
| |
| // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This |
| // method also checks that pong message was sent by the watched thread and pong |
| // message was received by the WatchDogThread. It also checks that |
| // OnCheckResponsiveness has verified the ping-pong mechanism and the watched |
| // thread is not hung. |
| TEST_F(ThreadWatcherTest, ThreadResponding) { |
| TimeTicks time_before_ping = TimeTicks::Now(); |
| // Activate watching IO thread. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| // Activate would have started ping/pong messaging. Expect atleast one |
| // ping/pong messaging sequence to happen. |
| io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1), |
| RECEIVED_PONG); |
| EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64_t>(0)); |
| EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64_t>(0)); |
| EXPECT_TRUE(io_watcher_->active()); |
| EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping); |
| EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64_t>(0)); |
| |
| // Verify watched thread is responding with ping/pong messaging. |
| io_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // DeActivate thread watching for shutdown. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| } |
| |
| // This test posts a task on watched thread that takes very long time (this is |
| // to simulate hanging of watched thread). It then checks for |
| // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false |
| // if the watched thread is not responding). |
| TEST_F(ThreadWatcherTest, ThreadNotResponding) { |
| // Simulate hanging of watched thread by making the watched thread wait for a |
| // very long time by posting a task on watched thread that keeps it busy. |
| // It is safe to use base::Unretained because test is waiting for the method |
| // to finish. |
| BrowserThread::PostTask( |
| BrowserThread::IO, FROM_HERE, |
| base::BindOnce(&CustomThreadWatcher::VeryLongMethod, |
| base::Unretained(io_watcher_), kUnresponsiveTime * 10)); |
| |
| // Activate thread watching. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| // Verify watched thread is not responding for ping messages. |
| io_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // DeActivate thread watching for shutdown. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| // Wait for the io_watcher_'s VeryLongMethod to finish. |
| io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); |
| } |
| |
| // Test watching of multiple threads with all threads not responding. |
| TEST_F(ThreadWatcherTest, MultipleThreadsResponding) { |
| // Check for UI thread to perform ping/pong messaging. |
| WatchDogThread::PostTask(FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(ui_watcher_))); |
| |
| // Check for IO thread to perform ping/pong messaging. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| // Verify UI thread is responding with ping/pong messaging. |
| ui_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); |
| EXPECT_GT(ui_watcher_->ping_sent_, static_cast<uint64_t>(0)); |
| EXPECT_GT(ui_watcher_->pong_received_, static_cast<uint64_t>(0)); |
| EXPECT_GE(ui_watcher_->ping_sequence_number_, static_cast<uint64_t>(0)); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(ui_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(ui_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // Verify IO thread is responding with ping/pong messaging. |
| io_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); |
| EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64_t>(0)); |
| EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64_t>(0)); |
| EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64_t>(0)); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // DeActivate thread watching for shutdown. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| WatchDogThread::PostTask(FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(ui_watcher_))); |
| } |
| |
| // Test watching of multiple threads with one of the threads not responding. |
| TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) { |
| // Simulate hanging of watched thread by making the watched thread wait for a |
| // very long time by posting a task on watched thread that keeps it busy. |
| // It is safe ot use base::Unretained because test is waiting for the method |
| // to finish. |
| BrowserThread::PostTask( |
| BrowserThread::IO, FROM_HERE, |
| base::BindOnce(&CustomThreadWatcher::VeryLongMethod, |
| base::Unretained(io_watcher_), kUnresponsiveTime * 10)); |
| |
| // Activate watching of UI thread. |
| WatchDogThread::PostTask(FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(ui_watcher_))); |
| |
| // Activate watching of IO thread. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::ActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| |
| // Verify UI thread is responding with ping/pong messaging. |
| ui_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(ui_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(ui_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // Verify IO thread is not responding for ping messages. |
| io_watcher_->WaitForCheckResponse( |
| kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); |
| EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), |
| static_cast<base::subtle::Atomic32>(0)); |
| |
| // DeActivate thread watching for shutdown. |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(io_watcher_))); |
| WatchDogThread::PostTask(FROM_HERE, |
| base::Bind(&ThreadWatcher::DeActivateThreadWatching, |
| base::Unretained(ui_watcher_))); |
| |
| // Wait for the io_watcher_'s VeryLongMethod to finish. |
| io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); |
| } |
| |
| class ThreadWatcherListTest : public ::testing::Test { |
| protected: |
| ThreadWatcherListTest() |
| : done_(&lock_), |
| state_available_(false), |
| has_thread_watcher_list_(false), |
| stopped_(false) { |
| } |
| |
| void ReadStateOnWatchDogThread() { |
| CHECK(WatchDogThread::CurrentlyOnWatchDogThread()); |
| { |
| base::AutoLock auto_lock(lock_); |
| has_thread_watcher_list_ = |
| ThreadWatcherList::g_thread_watcher_list_ != nullptr; |
| stopped_ = ThreadWatcherList::g_stopped_; |
| state_available_ = true; |
| } |
| done_.Signal(); |
| } |
| |
| void CheckState(bool has_thread_watcher_list, |
| bool stopped, |
| const char* const msg) { |
| CHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); |
| { |
| base::AutoLock auto_lock(lock_); |
| state_available_ = false; |
| } |
| |
| WatchDogThread::PostTask( |
| FROM_HERE, |
| base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread, |
| base::Unretained(this))); |
| { |
| base::AutoLock auto_lock(lock_); |
| while (!state_available_) |
| done_.Wait(); |
| |
| EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg; |
| EXPECT_EQ(stopped, stopped_) << msg; |
| } |
| } |
| |
| base::Lock lock_; |
| base::ConditionVariable done_; |
| |
| bool state_available_; |
| bool has_thread_watcher_list_; |
| bool stopped_; |
| }; |
| |
| TEST_F(ThreadWatcherListTest, Restart) { |
| ThreadWatcherList::g_initialize_delay_seconds = 1; |
| |
| base::MessageLoopForUI message_loop_for_ui; |
| content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui); |
| |
| std::unique_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread()); |
| watchdog_thread_->StartAndWaitForTesting(); |
| |
| // See http://crbug.com/347887. |
| // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_, |
| // whilst StopWatchingAll() will just PostTask to destroy it. |
| // Ensure that when Stop is called, Start will NOT create |
| // g_thread_watcher_list_ later on. |
| ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess()); |
| ThreadWatcherList::StopWatchingAll(); |
| message_loop_for_ui.task_runner()->PostDelayedTask( |
| FROM_HERE, message_loop_for_ui.QuitWhenIdleClosure(), |
| base::TimeDelta::FromSeconds( |
| ThreadWatcherList::g_initialize_delay_seconds)); |
| base::RunLoop().Run(); |
| |
| CheckState(false /* has_thread_watcher_list */, |
| true /* stopped */, |
| "Start / Stopped"); |
| |
| // Proceed with just |StartWatchingAll| and ensure it'll be started. |
| ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess()); |
| message_loop_for_ui.task_runner()->PostDelayedTask( |
| FROM_HERE, message_loop_for_ui.QuitWhenIdleClosure(), |
| base::TimeDelta::FromSeconds( |
| ThreadWatcherList::g_initialize_delay_seconds + 1)); |
| base::RunLoop().Run(); |
| |
| CheckState(true /* has_thread_watcher_list */, |
| false /* stopped */, |
| "Started"); |
| |
| // Finally, StopWatchingAll() must stop. |
| ThreadWatcherList::StopWatchingAll(); |
| message_loop_for_ui.task_runner()->PostDelayedTask( |
| FROM_HERE, message_loop_for_ui.QuitWhenIdleClosure(), |
| base::TimeDelta::FromSeconds( |
| ThreadWatcherList::g_initialize_delay_seconds)); |
| base::RunLoop().Run(); |
| |
| CheckState(false /* has_thread_watcher_list */, |
| true /* stopped */, |
| "Stopped"); |
| } |
