Added command line switches "crash-on-hang-threads" and"crash-on-hang-seconds" for crashing the browser whenthreads specified in "crash-on-hang-threads" don't respondfor "crash-on-hang-seconds".TEST=thread watcher unit testsBUG=85282R=jar
Review URL: http://codereview.chromium.org/7134007
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@89630 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/chrome/browser/metrics/thread_watcher.cc b/chrome/browser/metrics/thread_watcher.cc
index 2b2ab3b..f9cd49f 100644
--- a/chrome/browser/metrics/thread_watcher.cc
+++ b/chrome/browser/metrics/thread_watcher.cc
@@ -2,10 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include <math.h> // ceil
+
+#include "base/string_tokenizer.h"
#include "base/threading/thread_restrictions.h"
#include "build/build_config.h"
#include "chrome/browser/metrics/metrics_service.h"
#include "chrome/browser/metrics/thread_watcher.h"
+#include "chrome/common/chrome_switches.h"
#include "content/common/notification_service.h"
#if defined(OS_WIN)
@@ -15,14 +19,14 @@
// static
const int ThreadWatcher::kPingCount = 6;
-// static
-const int ThreadWatcher::kUnresponsiveCount = 6;
-
// ThreadWatcher methods and members.
ThreadWatcher::ThreadWatcher(const BrowserThread::ID& thread_id,
const std::string& thread_name,
const base::TimeDelta& sleep_time,
- const base::TimeDelta& unresponsive_time)
+ const base::TimeDelta& unresponsive_time,
+ uint32 unresponsive_threshold,
+ bool crash_on_hang,
+ uint32 live_threads_threshold)
: thread_id_(thread_id),
thread_name_(thread_name),
sleep_time_(sleep_time),
@@ -36,7 +40,11 @@
unresponsive_time_histogram_(NULL),
unresponsive_count_(0),
hung_processing_complete_(false),
+ unresponsive_threshold_(unresponsive_threshold),
+ crash_on_hang_(crash_on_hang),
+ live_threads_threshold_(live_threads_threshold),
ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
Initialize();
}
@@ -46,7 +54,10 @@
void ThreadWatcher::StartWatching(const BrowserThread::ID& thread_id,
const std::string& thread_name,
const base::TimeDelta& sleep_time,
- const base::TimeDelta& unresponsive_time) {
+ const base::TimeDelta& unresponsive_time,
+ uint32 unresponsive_threshold,
+ bool crash_on_hang,
+ uint32 live_threads_threshold) {
DCHECK_GE(sleep_time.InMilliseconds(), 0);
DCHECK_GE(unresponsive_time.InMilliseconds(), sleep_time.InMilliseconds());
@@ -55,17 +66,27 @@
if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
WatchDogThread::PostTask(
FROM_HERE,
- NewRunnableFunction(
- &ThreadWatcher::StartWatching,
- thread_id, thread_name, sleep_time, unresponsive_time));
+ NewRunnableFunction(&ThreadWatcher::StartWatching,
+ thread_id,
+ thread_name,
+ sleep_time,
+ unresponsive_time,
+ unresponsive_threshold,
+ crash_on_hang,
+ live_threads_threshold));
return;
}
DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
// Create a new thread watcher object for the given thread and activate it.
- ThreadWatcher* watcher =
- new ThreadWatcher(thread_id, thread_name, sleep_time, unresponsive_time);
+ ThreadWatcher* watcher = new ThreadWatcher(thread_id,
+ thread_name,
+ sleep_time,
+ unresponsive_time,
+ unresponsive_threshold,
+ crash_on_hang,
+ live_threads_threshold);
DCHECK(watcher);
// If we couldn't register the thread watcher object, we are shutting down,
// then don't activate thread watching.
@@ -196,6 +217,7 @@
}
void ThreadWatcher::Initialize() {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
ThreadWatcherList::Register(this);
const std::string response_time_histogram_name =
@@ -244,9 +266,8 @@
void ThreadWatcher::GotNoResponse() {
DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
- // Record how other threads are responding when we don't get a response for
- // ping message atleast kUnresponsiveCount times.
- if (++unresponsive_count_ < kUnresponsiveCount)
+ ++unresponsive_count_;
+ if (!IsVeryUnresponsive())
return;
// Record total unresponsive_time since last pong message.
@@ -257,210 +278,313 @@
if (hung_processing_complete_)
return;
- int no_of_responding_threads = 0;
- int no_of_unresponding_threads = 0;
- ThreadWatcherList::GetStatusOfThreads(&no_of_responding_threads,
- &no_of_unresponding_threads);
+ // Record how other threads are responding.
+ uint32 responding_thread_count = 0;
+ uint32 unresponding_thread_count = 0;
+ ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
+ &unresponding_thread_count);
// Record how many watched threads are responding.
- responsive_count_histogram_->Add(no_of_responding_threads);
+ responsive_count_histogram_->Add(responding_thread_count);
// Record how many watched threads are not responding.
- unresponsive_count_histogram_->Add(no_of_unresponding_threads);
+ unresponsive_count_histogram_->Add(unresponding_thread_count);
- // Crash the browser if IO thread hasn't responded atleast kUnresponsiveCount
- // times and if the number of other threads is equal to 1. We picked 1 to
- // reduce the number of crashes and to get some sample data.
- if (thread_id_ == BrowserThread::IO && no_of_responding_threads == 1) {
+ // Crash the browser if the watched thread is to be crashed on hang and if the
+ // number of other threads responding is equal to live_threads_threshold_.
+ if (crash_on_hang_ && responding_thread_count == live_threads_threshold_) {
int* crash = NULL;
- CHECK(crash++);
+ CHECK(crash + thread_id_);
}
hung_processing_complete_ = true;
}
+bool ThreadWatcher::IsVeryUnresponsive() {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ return unresponsive_count_ >= unresponsive_threshold_;
+}
+
// ThreadWatcherList methods and members.
//
// static
-ThreadWatcherList* ThreadWatcherList::global_ = NULL;
+ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
// static
const int ThreadWatcherList::kSleepSeconds = 1;
// static
const int ThreadWatcherList::kUnresponsiveSeconds = 2;
-
-ThreadWatcherList::ThreadWatcherList()
- : last_wakeup_time_(base::TimeTicks::Now()) {
- // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
- // are on UI thread, but Unit tests are not running on UI thread.
- DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
- CHECK(!global_);
- global_ = this;
- // Register Notifications observer.
- MetricsService::SetUpNotifications(®istrar_, this);
-}
-
-ThreadWatcherList::~ThreadWatcherList() {
- base::AutoLock auto_lock(lock_);
- DCHECK(this == global_);
- global_ = NULL;
-}
+// static
+const int ThreadWatcherList::kUnresponsiveCount = 6;
+// static
+const int ThreadWatcherList::kLiveThreadsThreshold = 1;
// static
-void ThreadWatcherList::Register(ThreadWatcher* watcher) {
- if (!global_)
- return;
- base::AutoLock auto_lock(global_->lock_);
- DCHECK(!global_->PreLockedFind(watcher->thread_id()));
- global_->registered_[watcher->thread_id()] = watcher;
-}
+void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
+ ThreadWatcherObserver::SetupNotifications(
+ base::TimeDelta::FromSeconds(kSleepSeconds * ThreadWatcher::kPingCount));
-// static
-bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
- return NULL != ThreadWatcherList::Find(thread_id);
-}
+ uint32 unresponsive_threshold;
+ std::set<std::string> crash_on_hang_thread_names;
+ uint32 live_threads_threshold;
+ ParseCommandLine(command_line,
+ &unresponsive_threshold,
+ &crash_on_hang_thread_names,
+ &live_threads_threshold);
-// static
-void ThreadWatcherList::StartWatchingAll() {
- if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
- WatchDogThread::PostDelayedTask(
- FROM_HERE,
- NewRunnableFunction(&ThreadWatcherList::StartWatchingAll),
- base::TimeDelta::FromSeconds(120).InMilliseconds());
- return;
- }
- DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
- const base::TimeDelta kSleepTime =
- base::TimeDelta::FromSeconds(kSleepSeconds);
- const base::TimeDelta kUnresponsiveTime =
- base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
- if (BrowserThread::IsMessageLoopValid(BrowserThread::UI)) {
- ThreadWatcher::StartWatching(BrowserThread::UI, "UI", kSleepTime,
- kUnresponsiveTime);
- }
- if (BrowserThread::IsMessageLoopValid(BrowserThread::IO)) {
- ThreadWatcher::StartWatching(BrowserThread::IO, "IO", kSleepTime,
- kUnresponsiveTime);
- }
- if (BrowserThread::IsMessageLoopValid(BrowserThread::DB)) {
- ThreadWatcher::StartWatching(BrowserThread::DB, "DB", kSleepTime,
- kUnresponsiveTime);
- }
- if (BrowserThread::IsMessageLoopValid(BrowserThread::FILE)) {
- ThreadWatcher::StartWatching(BrowserThread::FILE, "FILE", kSleepTime,
- kUnresponsiveTime);
- }
- if (BrowserThread::IsMessageLoopValid(BrowserThread::CACHE)) {
- ThreadWatcher::StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime,
- kUnresponsiveTime);
- }
+ WatchDogThread::PostDelayedTask(
+ FROM_HERE,
+ NewRunnableFunction(&ThreadWatcherList::InitializeAndStartWatching,
+ unresponsive_threshold,
+ crash_on_hang_thread_names,
+ live_threads_threshold),
+ base::TimeDelta::FromSeconds(120).InMilliseconds());
}
// static
void ThreadWatcherList::StopWatchingAll() {
- // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
- // are on UI thread, but Unit tests are not running on UI thread.
- DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
- if (!global_)
- return;
-
- // Remove all notifications for all watched threads.
- RemoveNotifications();
-
- // Delete all thread watcher objects on WatchDogThread.
- WatchDogThread::PostTask(
- FROM_HERE,
- NewRunnableMethod(global_, &ThreadWatcherList::DeleteAll));
+ ThreadWatcherObserver::RemoveNotifications();
+ DeleteAll();
}
// static
-void ThreadWatcherList::RemoveNotifications() {
- // Assert we are not running on WATCHDOG thread. Would be ideal to assert we
- // are on UI thread, but Unit tests are not running on UI thread.
- DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
- if (!global_)
- return;
- base::AutoLock auto_lock(global_->lock_);
- global_->registrar_.RemoveAll();
-}
-
-// static
-void ThreadWatcherList::GetStatusOfThreads(int* no_of_responding_threads,
- int* no_of_unresponding_threads) {
+void ThreadWatcherList::Register(ThreadWatcher* watcher) {
DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
- *no_of_responding_threads = 0;
- *no_of_unresponding_threads = 0;
- if (!global_)
+ if (!g_thread_watcher_list_)
+ return;
+ DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
+ g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
+}
+
+// static
+bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ return NULL != ThreadWatcherList::Find(thread_id);
+}
+
+// static
+void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
+ uint32* unresponding_thread_count) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ *responding_thread_count = 0;
+ *unresponding_thread_count = 0;
+ if (!g_thread_watcher_list_)
return;
- base::AutoLock auto_lock(global_->lock_);
- for (RegistrationList::iterator it = global_->registered_.begin();
- global_->registered_.end() != it;
+ for (RegistrationList::iterator it =
+ g_thread_watcher_list_->registered_.begin();
+ g_thread_watcher_list_->registered_.end() != it;
++it) {
- if (it->second->unresponsive_count_ < ThreadWatcher::kUnresponsiveCount)
- ++(*no_of_responding_threads);
+ if (it->second->IsVeryUnresponsive())
+ ++(*unresponding_thread_count);
else
- ++(*no_of_unresponding_threads);
+ ++(*responding_thread_count);
}
}
-void ThreadWatcherList::DeleteAll() {
- DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
- base::AutoLock auto_lock(lock_);
- while (!registered_.empty()) {
- RegistrationList::iterator it = registered_.begin();
- delete it->second;
- registered_.erase(it->first);
- }
-}
-
-void ThreadWatcherList::Observe(NotificationType type,
- const NotificationSource& source,
- const NotificationDetails& details) {
- DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
- // There is some user activity, see if thread watchers are to be awakened.
- bool need_to_awaken = false;
- base::TimeTicks now = base::TimeTicks::Now();
- {
- base::AutoLock lock(lock_);
- if (now - last_wakeup_time_ > base::TimeDelta::FromSeconds(kSleepSeconds)) {
- need_to_awaken = true;
- last_wakeup_time_ = now;
- }
- }
- if (need_to_awaken) {
- WatchDogThread::PostTask(
- FROM_HERE,
- NewRunnableMethod(this, &ThreadWatcherList::WakeUpAll));
- }
-}
-
+// static
void ThreadWatcherList::WakeUpAll() {
DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
- if (!global_)
+ if (!g_thread_watcher_list_)
return;
- base::AutoLock auto_lock(lock_);
- for (RegistrationList::iterator it = global_->registered_.begin();
- global_->registered_.end() != it;
+
+ for (RegistrationList::iterator it =
+ g_thread_watcher_list_->registered_.begin();
+ g_thread_watcher_list_->registered_.end() != it;
++it)
it->second->WakeUp();
}
-// static
-ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
- if (!global_)
- return NULL;
- base::AutoLock auto_lock(global_->lock_);
- return global_->PreLockedFind(thread_id);
+ThreadWatcherList::ThreadWatcherList() {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ CHECK(!g_thread_watcher_list_);
+ g_thread_watcher_list_ = this;
}
-ThreadWatcher* ThreadWatcherList::PreLockedFind(
- const BrowserThread::ID& thread_id) {
- RegistrationList::iterator it = registered_.find(thread_id);
- if (registered_.end() == it)
+ThreadWatcherList::~ThreadWatcherList() {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ DCHECK(this == g_thread_watcher_list_);
+ g_thread_watcher_list_ = NULL;
+}
+
+// static
+void ThreadWatcherList::ParseCommandLine(
+ const CommandLine& command_line,
+ uint32* unresponsive_threshold,
+ std::set<std::string>* crash_on_hang_thread_names,
+ uint32* live_threads_threshold) {
+ // Determine |unresponsive_threshold| based on switches::kCrashOnHangSeconds.
+ *unresponsive_threshold = kUnresponsiveCount;
+ std::string crash_on_hang_seconds =
+ command_line.GetSwitchValueASCII(switches::kCrashOnHangSeconds);
+ if (!crash_on_hang_seconds.empty()) {
+ int crash_seconds = atoi(crash_on_hang_seconds.c_str());
+ if (crash_seconds > 0) {
+ *unresponsive_threshold = static_cast<uint32>(
+ ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
+ }
+ }
+
+ // Default to crashing the browser if UI or IO threads are not responsive.
+ std::string crash_on_hang_threads = "UI,IO";
+ if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
+ crash_on_hang_threads =
+ command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
+ }
+ StringTokenizer tokens(crash_on_hang_threads, ",");
+ while (tokens.GetNext())
+ crash_on_hang_thread_names->insert(tokens.token());
+
+ // Determine |live_threads_threshold| based on switches::kCrashOnLive.
+ *live_threads_threshold = kLiveThreadsThreshold;
+ if (command_line.HasSwitch(switches::kCrashOnLive)) {
+ std::string live_threads =
+ command_line.GetSwitchValueASCII(switches::kCrashOnLive);
+ *live_threads_threshold = static_cast<uint32>(atoi(live_threads.c_str()));
+ }
+}
+
+// static
+void ThreadWatcherList::InitializeAndStartWatching(
+ uint32 unresponsive_threshold,
+ const std::set<std::string>& crash_on_hang_thread_names,
+ uint32 live_threads_threshold) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+
+ ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
+ CHECK(thread_watcher_list);
+
+ const base::TimeDelta kSleepTime =
+ base::TimeDelta::FromSeconds(kSleepSeconds);
+ const base::TimeDelta kUnresponsiveTime =
+ base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
+
+ StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
+ unresponsive_threshold, crash_on_hang_thread_names,
+ live_threads_threshold);
+ StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
+ unresponsive_threshold, crash_on_hang_thread_names,
+ live_threads_threshold);
+ StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
+ unresponsive_threshold, crash_on_hang_thread_names,
+ live_threads_threshold);
+ StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
+ unresponsive_threshold, crash_on_hang_thread_names,
+ live_threads_threshold);
+ StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
+ unresponsive_threshold, crash_on_hang_thread_names,
+ live_threads_threshold);
+}
+
+// static
+void ThreadWatcherList::StartWatching(
+ const BrowserThread::ID& thread_id,
+ const std::string& thread_name,
+ const base::TimeDelta& sleep_time,
+ const base::TimeDelta& unresponsive_time,
+ uint32 unresponsive_threshold,
+ const std::set<std::string>& crash_on_hang_thread_names,
+ uint32 live_threads_threshold) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+
+ if (!BrowserThread::IsMessageLoopValid(thread_id))
+ return;
+
+ std::set<std::string>::const_iterator it =
+ crash_on_hang_thread_names.find(thread_name);
+ bool crash_on_hang = (it != crash_on_hang_thread_names.end());
+
+ ThreadWatcher::StartWatching(thread_id,
+ thread_name,
+ sleep_time,
+ unresponsive_time,
+ unresponsive_threshold,
+ crash_on_hang,
+ live_threads_threshold);
+}
+
+// static
+void ThreadWatcherList::DeleteAll() {
+ if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
+ WatchDogThread::PostTask(
+ FROM_HERE,
+ NewRunnableFunction(&ThreadWatcherList::DeleteAll));
+ return;
+ }
+
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ if (!g_thread_watcher_list_)
+ return;
+
+ // Delete all thread watcher objects.
+ while (!g_thread_watcher_list_->registered_.empty()) {
+ RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
+ delete it->second;
+ g_thread_watcher_list_->registered_.erase(it);
+ }
+
+ delete g_thread_watcher_list_;
+}
+
+// static
+ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
+ DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
+ if (!g_thread_watcher_list_)
+ return NULL;
+ RegistrationList::iterator it =
+ g_thread_watcher_list_->registered_.find(thread_id);
+ if (g_thread_watcher_list_->registered_.end() == it)
return NULL;
return it->second;
}
+// ThreadWatcherObserver methods and members.
+//
+// static
+ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
+
+ThreadWatcherObserver::ThreadWatcherObserver(
+ const base::TimeDelta& wakeup_interval)
+ : last_wakeup_time_(base::TimeTicks::Now()),
+ wakeup_interval_(wakeup_interval) {
+ CHECK(!g_thread_watcher_observer_);
+ g_thread_watcher_observer_ = this;
+}
+
+ThreadWatcherObserver::~ThreadWatcherObserver() {
+ DCHECK(this == g_thread_watcher_observer_);
+ g_thread_watcher_observer_ = NULL;
+}
+
+// static
+void ThreadWatcherObserver::SetupNotifications(
+ const base::TimeDelta& wakeup_interval) {
+ DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+ ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
+ MetricsService::SetUpNotifications(&observer->registrar_, observer);
+}
+
+// static
+void ThreadWatcherObserver::RemoveNotifications() {
+ DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+ if (!g_thread_watcher_observer_)
+ return;
+ g_thread_watcher_observer_->registrar_.RemoveAll();
+ delete g_thread_watcher_observer_;
+}
+
+void ThreadWatcherObserver::Observe(NotificationType type,
+ const NotificationSource& source,
+ const NotificationDetails& details) {
+ // There is some user activity, see if thread watchers are to be awakened.
+ base::TimeTicks now = base::TimeTicks::Now();
+ if ((now - last_wakeup_time_) < wakeup_interval_)
+ return;
+ last_wakeup_time_ = now;
+ WatchDogThread::PostTask(
+ FROM_HERE,
+ NewRunnableFunction(&ThreadWatcherList::WakeUpAll));
+}
+
// WatchDogThread methods and members.
//
// static
@@ -543,8 +667,8 @@
void WatchDogThread::CleanUpAfterMessageLoopDestruction() {
#if defined(OS_WIN)
- // Closes the COM library on the current thread. CoInitialize must
- // be balanced by a corresponding call to CoUninitialize.
+ // Closes the COM library on the current thread. CoInitialize must be balanced
+ // by a corresponding call to CoUninitialize.
CoUninitialize();
#endif
}
