gpu/tools/check_gpu_bots.py

#!/usr/bin/env python

# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import datetime
import getpass
import json
import os
import smtplib
import sys
import time
import urllib
import urllib2

class Emailer:
  DEFAULT_EMAIL_PASSWORD_FILE = '.email_password'
  GMAIL_SMTP_SERVER = 'smtp.gmail.com:587'
  SUBJECT = 'Chrome GPU Bots Notification'

  def __init__(self, email_from, email_to, email_password_file):
    self.email_from = email_from
    self.email_to = email_to
    self.email_password = Emailer._getEmailPassword(email_password_file)

  @staticmethod
  def format_email_body(time_str, offline_str, failed_str, noteworthy_str):
    return '%s%s%s%s' % (time_str, offline_str, failed_str, noteworthy_str)

  def send_email(self, body):
    message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (self.email_from,
            ','.join(self.email_to), Emailer.SUBJECT, body)

    try:
      server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
      server.starttls()
      server.login(self.email_from, self.email_password)
      server.sendmail(self.email_from, self.email_to, message)
      server.quit()
    except Exception as e:
      print 'Error sending email: %s' % str(e)

  def testEmailLogin(self):
    server = smtplib.SMTP(Emailer.GMAIL_SMTP_SERVER)
    server.starttls()
    server.login(self.email_from, self.email_password)
    server.quit()

  @staticmethod
  def _getEmailPassword(email_password_file):
    password = ''

    password_file = (email_password_file if email_password_file is not None
            else Emailer.DEFAULT_EMAIL_PASSWORD_FILE)

    if os.path.isfile(password_file):
      with open(password_file, 'r') as f:
        password = f.read().strip()
    else:
      password = getpass.getpass(
              'Please enter email password for source email account: ')

    return password

class GpuBot:
  def __init__(self, waterfall_name, bot_name, bot_data):
    self.waterfall_name = waterfall_name
    self.bot_name = bot_name
    self.bot_data = bot_data
    self._end_time = None
    self._hours_since_last_run = None
    self.failure_string = None
    self.bot_url = None
    self.build_url = None

  def getEndTime(self):
    return self._end_time

  def setEndTime(self, end_time):
    self._end_time = end_time
    self._hours_since_last_run = \
            roughTimeDiffInHours(end_time, time.localtime())

  def getHoursSinceLastRun(self):
    return self._hours_since_last_run

  def toDict(self):
    dict = {'waterfall_name': self.waterfall_name, 'bot_name': self.bot_name}

    if self._end_time is not None:
      dict['end_time'] = serialTime(self._end_time)
      dict['hours_since_last_run'] = self._hours_since_last_run

    if self.failure_string is not None:
      dict['failure_string'] = self.failure_string

    if self.bot_url is not None:
      dict['bot_url'] = self.bot_url

    if self.build_url is not None:
      dict['build_url'] = self.build_url

    return dict

  @staticmethod
  def fromDict(dict):
    gpu_bot = GpuBot(dict['waterfall_name'], dict['bot_name'], None)

    if 'end_time' in dict:
      gpu_bot._end_time = unserializeTime(dict['end_time'])

    if 'hours_since_last_run' in dict:
      gpu_bot._hours_since_last_run = dict['hours_since_last_run']

    if 'failure_string' in dict:
      gpu_bot.failure_string = dict['failure_string']

    if 'bot_url' in dict:
      gpu_bot.bot_url = dict['bot_url']

    if 'build_url' in dict:
      gpu_bot.build_url = dict['build_url']

    return gpu_bot

def errorNoMostRecentBuild(waterfall_name, bot_name):
  print 'No most recent build available: %s::%s' % (waterfall_name, bot_name)

class Waterfall:
  BASE_URL = 'http://build.chromium.org/p/'
  BASE_BUILD_URL = BASE_URL + '%s/builders/%s'
  SPECIFIC_BUILD_URL = BASE_URL + '%s/builders/%s/builds/%s'
  BASE_JSON_BUILDERS_URL = BASE_URL + '%s/json/builders'
  BASE_JSON_BUILDS_URL = BASE_URL + '%s/json/builders/%s/builds'
  REGULAR_WATERFALLS = ['chromium.gpu', 'chromium.gpu.fyi']
  WEBKIT_GPU_BOTS = ['GPU Win Builder',
          'GPU Win Builder (dbg)',
          'GPU Win7 (NVIDIA)',
          'GPU Win7 (dbg) (NVIDIA)',
          'GPU Mac Builder',
          'GPU Mac Builder (dbg)',
          'GPU Mac10.7',
          'GPU Mac10.7 (dbg)',
          'GPU Linux Builder',
          'GPU Linux Builder (dbg)',
          'GPU Linux (NVIDIA)',
          'GPU Linux (dbg) (NVIDIA)']
  FILTERED_WATERFALLS = [('chromium.webkit', WEBKIT_GPU_BOTS)]

  @staticmethod
  def getJsonFromUrl(url):
    conn = urllib2.urlopen(url)
    result = conn.read()
    conn.close()
    return json.loads(result)

  @staticmethod
  def getBuildersJsonForWaterfall(waterfall):
    querystring = '?filter'
    return (Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
        % (waterfall, querystring)))

  @staticmethod
  def getLastNBuildsForBuilder(n, waterfall, builder):
    if n <= 0:
      return {}

    querystring = '?'

    for i in range(n):
      querystring += 'select=-%d&' % (i + 1)

    querystring += 'filter'

    return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDS_URL + '%s') %
            (waterfall, urllib.quote(builder), querystring))

  @staticmethod
  def getFilteredBuildersJsonForWaterfall(waterfall, filter):
    querystring = '?'

    for bot_name in filter:
      querystring += 'select=%s&' % urllib.quote(bot_name)

    querystring += 'filter'

    return Waterfall.getJsonFromUrl((Waterfall.BASE_JSON_BUILDERS_URL + '%s')
            % (waterfall, querystring))

  @staticmethod
  def getAllGpuBots():
    allbots = {k: Waterfall.getBuildersJsonForWaterfall(k)
            for k in Waterfall.REGULAR_WATERFALLS}

    filteredbots = {k[0]:
            Waterfall.getFilteredBuildersJsonForWaterfall(k[0], k[1])
            for k in Waterfall.FILTERED_WATERFALLS}

    allbots.update(filteredbots)

    return allbots

  @staticmethod
  def getOfflineBots(bots):
    offline_bots = []

    for waterfall_name in bots:
      waterfall = bots[waterfall_name]

      for bot_name in waterfall:
        bot = waterfall[bot_name]

        if bot['state'] != 'offline':
          continue

        gpu_bot = GpuBot(waterfall_name, bot_name, bot)
        gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
                urllib.quote(bot_name))

        most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
                gpu_bot)

        if (most_recent_build and 'times' in most_recent_build and
                most_recent_build['times']):
          gpu_bot.setEndTime(time.localtime(most_recent_build['times'][1]))
        else:
          errorNoMostRecentBuild(waterfall_name, bot_name)

        offline_bots.append(gpu_bot)

    return offline_bots

  @staticmethod
  def getMostRecentlyCompletedBuildForBot(bot):
    if bot.bot_data is not None and 'most_recent_build' in bot.bot_data:
      return bot.bot_data['most_recent_build']

    # Unfortunately, the JSON API doesn't provide a "most recent completed
    # build" call. We just have to get some number of the most recent (including
    # current, in-progress builds) and give up if that's not enough.
    NUM_BUILDS = 10
    builds = Waterfall.getLastNBuildsForBuilder(NUM_BUILDS, bot.waterfall_name,
            bot.bot_name)

    for i in range(NUM_BUILDS):
      current_build_name = '-%d' % (i + 1)
      current_build = builds[current_build_name]

      if 'results' in current_build and current_build['results'] is not None:
        if bot.bot_data is not None:
          bot.bot_data['most_recent_build'] = current_build

        return current_build

    return None

  @staticmethod
  def getFailedBots(bots):
    failed_bots = []

    for waterfall_name in bots:
      waterfall = bots[waterfall_name]

      for bot_name in waterfall:
        bot = waterfall[bot_name]
        gpu_bot = GpuBot(waterfall_name, bot_name, bot)
        gpu_bot.bot_url = Waterfall.BASE_BUILD_URL % (waterfall_name,
                urllib.quote(bot_name))

        most_recent_build = Waterfall.getMostRecentlyCompletedBuildForBot(
                gpu_bot)

        if (most_recent_build and 'text' in most_recent_build and
                'failed' in most_recent_build['text']):
          gpu_bot.failure_string = ' '.join(most_recent_build['text'])
          gpu_bot.build_url = Waterfall.SPECIFIC_BUILD_URL % (waterfall_name,
                  urllib.quote(bot_name), most_recent_build['number'])
          failed_bots.append(gpu_bot)
        elif not most_recent_build:
          errorNoMostRecentBuild(waterfall_name, bot_name)

    return failed_bots

def formatTime(t):
  return time.strftime("%a, %d %b %Y %H:%M:%S", t)

def roughTimeDiffInHours(t1, t2):
  datetimes = []

  for t in [t1, t2]:
    datetimes.append(datetime.datetime(t.tm_year, t.tm_mon, t.tm_mday,
        t.tm_hour, t.tm_min, t.tm_sec))

  datetime_diff = datetimes[0] - datetimes[1]

  hours = float(datetime_diff.total_seconds()) / 3600.0

  return abs(hours)

def getBotStr(bot):
  s = '  %s::%s\n' % (bot.waterfall_name, bot.bot_name)

  if bot.failure_string is not None:
    s += '  failure: %s\n' % bot.failure_string

  if bot.getEndTime() is not None:
    s += ('  last build end time: %s (roughly %f hours ago)\n' %
    (formatTime(bot.getEndTime()), bot.getHoursSinceLastRun()))

  if bot.bot_url is not None:
    s += '  bot url: %s\n' % bot.bot_url

  if bot.build_url is not None:
    s += '  build url: %s\n' % bot.build_url

  s += '\n'
  return s

def getBotsStr(bots):
  s = ''

  for bot in bots:
    s += getBotStr(bot)

  s += '\n'
  return s

def getOfflineBotsStr(offline_bots):
  return 'Offline bots:\n%s' % getBotsStr(offline_bots)

def getFailedBotsStr(failed_bots):
  return 'Failed bots:\n%s' % getBotsStr(failed_bots)

def getBotDicts(bots):
  dicts = []

  for bot in bots:
    dicts.append(bot.toDict())

  return dicts

def unserializeTime(t):
  return time.struct_time((t['year'], t['mon'], t['day'], t['hour'], t['min'],
      t['sec'], 0, 0, 0))

def serialTime(t):
  return {'year': t.tm_year, 'mon': t.tm_mon, 'day': t.tm_mday,
          'hour': t.tm_hour, 'min': t.tm_min, 'sec': t.tm_sec}

def getSummary(offline_bots, failed_bots):
  offline_bot_dict = getBotDicts(offline_bots)
  failed_bot_dict = getBotDicts(failed_bots)
  return {'offline': offline_bot_dict, 'failed': failed_bot_dict}

def findBot(name, lst):
  for bot in lst:
    if bot.bot_name == name:
      return bot

  return None

def getNoteworthyEvents(offline_bots, failed_bots, previous_results):
  CRITICAL_NUM_HOURS = 1.0

  previous_offline = (previous_results['offline'] if 'offline'
          in previous_results else [])

  previous_failures = (previous_results['failed'] if 'failed'
          in previous_results else [])

  noteworthy_offline = []
  for bot in offline_bots:
    if bot.getHoursSinceLastRun() >= CRITICAL_NUM_HOURS:
      previous_bot = findBot(bot.bot_name, previous_offline)

      if (previous_bot is None or
              previous_bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS):
        noteworthy_offline.append(bot)

  noteworthy_new_failures = []
  for bot in failed_bots:
    previous_bot = findBot(bot.bot_name, previous_failures)

    if previous_bot is None:
      noteworthy_new_failures.append(bot)

  noteworthy_new_offline_recoveries = []
  for bot in previous_offline:
    if bot.getHoursSinceLastRun() < CRITICAL_NUM_HOURS:
      continue

    current_bot = findBot(bot.bot_name, offline_bots)
    if current_bot is None:
      noteworthy_new_offline_recoveries.append(bot)

  noteworthy_new_failure_recoveries = []
  for bot in previous_failures:
    current_bot = findBot(bot.bot_name, failed_bots)

    if current_bot is None:
      noteworthy_new_failure_recoveries.append(bot)

  return {'offline': noteworthy_offline, 'failed': noteworthy_new_failures,
          'recovered_failures': noteworthy_new_failure_recoveries,
          'recovered_offline': noteworthy_new_offline_recoveries}

def getNoteworthyStr(noteworthy_events):
  s = ''

  if noteworthy_events['offline']:
    s += 'IMPORTANT bots newly offline for over an hour:\n'

    for bot in noteworthy_events['offline']:
      s += getBotStr(bot)

    s += '\n'

  if noteworthy_events['failed']:
    s += 'IMPORTANT new failing bots:\n'

    for bot in noteworthy_events['failed']:
      s += getBotStr(bot)

    s += '\n'

  if noteworthy_events['recovered_offline']:
    s += 'IMPORTANT newly recovered previously offline bots:\n'

    for bot in noteworthy_events['recovered_offline']:
      s += getBotStr(bot)

    s += '\n'

  if noteworthy_events['recovered_failures']:
    s += 'IMPORTANT newly recovered failing bots:\n'

    for bot in noteworthy_events['recovered_failures']:
      s += getBotStr(bot)

    s += '\n'

  return s

def dictsToBots(bots):
  offline_bots = []
  for bot in bots['offline']:
    offline_bots.append(GpuBot.fromDict(bot))

  failed_bots = []
  for bot in bots['failed']:
    failed_bots.append(GpuBot.fromDict(bot))

  return {'offline': offline_bots, 'failed': failed_bots}

class GpuBotPoller:
  DEFAULT_PREVIOUS_RESULTS_FILE = '.check_gpu_bots_previous_results'

  def __init__(self, emailer, send_email_for_recovered_offline_bots,
          send_email_for_recovered_failing_bots, send_email_on_error,
          previous_results_file):
    self.emailer = emailer

    self.send_email_for_recovered_offline_bots = \
            send_email_for_recovered_offline_bots

    self.send_email_for_recovered_failing_bots = \
            send_email_for_recovered_failing_bots

    self.send_email_on_error = send_email_on_error
    self.previous_results_file = previous_results_file

  def shouldEmail(self, noteworthy_events):
    if noteworthy_events['offline'] or noteworthy_events['failed']:
      return True

    if (self.send_email_for_recovered_offline_bots and
            noteworthy_events['recovered_offline']):
      return True

    if (self.send_email_for_recovered_failing_bots and
          noteworthy_events['recovered_failures']):
      return True

    return False

  def writeResults(self, summary):
    results_file = (self.previous_results_file
            if self.previous_results_file is not None
            else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

    with open(results_file, 'w') as f:
      f.write(json.dumps(summary))

  def getPreviousResults(self):
    previous_results_file = (self.previous_results_file
            if self.previous_results_file is not None
            else GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE)

    previous_results = {}
    if os.path.isfile(previous_results_file):
      with open(previous_results_file, 'r') as f:
        previous_results = dictsToBots(json.loads(f.read()))

    return previous_results

  def checkBots(self):
    time_str = 'Current time: %s\n\n' % (formatTime(time.localtime()))
    print time_str

    try:
      bots = Waterfall.getAllGpuBots()

      offline_bots = Waterfall.getOfflineBots(bots)
      offline_str = getOfflineBotsStr(offline_bots)
      print offline_str

      failed_bots = Waterfall.getFailedBots(bots)
      failed_str = getFailedBotsStr(failed_bots)
      print failed_str

      previous_results = self.getPreviousResults()
      noteworthy_events = getNoteworthyEvents(offline_bots, failed_bots,
              previous_results)

      noteworthy_str = getNoteworthyStr(noteworthy_events)
      print noteworthy_str

      summary = getSummary(offline_bots, failed_bots)
      self.writeResults(summary)

      if (self.emailer is not None and self.shouldEmail(noteworthy_events)):
        self.emailer.send_email(Emailer.format_email_body(time_str, offline_str,
            failed_str, noteworthy_str))
    except Exception as e:
      error_str = 'Error: %s' % str(e)
      print error_str

      if self.send_email_on_error:
        self.emailer.send_email(error_str)

def parseArgs(sys_args):
  parser = argparse.ArgumentParser(prog=sys_args[0],
          description='Query the Chromium GPU Bots Waterfall, output ' +
          'potential problems, and optionally repeat automatically and/or ' +
          'email notifications of results.')

  parser.add_argument('--repeat-delay', type=int, dest='repeat_delay',
          required=False,
          help='How often to automatically re-run the script, in minutes.')

  parser.add_argument('--email-from', type=str, dest='email_from',
          required=False,
          help='Email address to send from. Requires also specifying ' +
          '\'--email-to\'.')

  parser.add_argument('--email-to', type=str, dest='email_to', required=False,
          nargs='+',
          help='Email address(es) to send to. Requires also specifying ' +
          '\'--email-from\'')

  parser.add_argument('--send-email-for-recovered-offline-bots',
          dest='send_email_for_recovered_offline_bots', action='store_true',
          default=False,
          help='Send an email out when a bot which has been offline for more ' +
          'than 1 hour goes back online.')

  parser.add_argument('--send-email-for-recovered-failing-bots',
          dest='send_email_for_recovered_failing_bots',
          action='store_true', default=False,
          help='Send an email when a failing bot recovers.')

  parser.add_argument('--send-email-on-error',
          dest='send_email_on_error',
          action='store_true', default=False,
          help='Send an email when the script has an error. For example, if ' +
          'the server is unreachable.')

  parser.add_argument('--email-password-file',
          dest='email_password_file',
          required=False,
          help=(('File containing the plaintext password of the source email ' +
          'account. By default, \'%s\' will be tried. If it does not exist, ' +
          'you will be prompted. If you opt to store your password on disk ' +
          'in plaintext, use of a dummy account is strongly recommended.')
          % Emailer.DEFAULT_EMAIL_PASSWORD_FILE))

  parser.add_argument('--previous-results-file',
          dest='previous_results_file',
          required=False,
          help=(('File to store the results of the previous invocation of ' +
              'this script. By default, \'%s\' will be used.')
              % GpuBotPoller.DEFAULT_PREVIOUS_RESULTS_FILE))

  args = parser.parse_args(sys_args[1:])

  if args.email_from is not None and args.email_to is None:
    parser.error('--email-from requires --email-to.')
  elif args.email_to is not None and args.email_from is None:
    parser.error('--email-to requires --email-from.')
  elif args.email_from is None and args.send_email_for_recovered_offline_bots:
    parser.error('--send-email-for-recovered-offline-bots requires ' +
            '--email-to and --email-from.')
  elif (args.email_from is None and args.send_email_for_recovered_failing_bots):
    parser.error('--send-email-for-recovered-failing-bots ' +
            'requires --email-to and --email-from.')
  elif (args.email_from is None and args.send_email_on_error):
    parser.error('--send-email-on-error ' +
            'requires --email-to and --email-from.')
  elif (args.email_password_file and
          not os.path.isfile(args.email_password_file)):
    parser.error('File does not exist: %s' % args.email_password_file)

  return args

def main(sys_args):
  args = parseArgs(sys_args)

  emailer = None
  if args.email_from is not None and args.email_to is not None:
    emailer = Emailer(args.email_from, args.email_to, args.email_password_file)

    try:
      emailer.testEmailLogin()
    except Exception as e:
      print 'Error logging into email account: %s' % str(e)
      return 1

  poller = GpuBotPoller(emailer,
          args.send_email_for_recovered_offline_bots,
          args.send_email_for_recovered_failing_bots,
          args.send_email_on_error,
          args.previous_results_file)

  while True:
    poller.checkBots()

    if args.repeat_delay is None:
      break

    print 'Will run again in %d minutes...\n' % args.repeat_delay
    time.sleep(args.repeat_delay * 60)

  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv))