tools/perf_expectations/update_perf_expectations.py

#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Prepare tests that require re-baselining for input to make_expectations.py.

The regularly running perf-AV tests require re-baselineing of expectations
about once a week. The steps involved in rebaselining are:

1.) Identify the tests to update, based off reported e-mail results.
2.) Figure out reva and revb values, which is the starting and ending revision
 numbers for the range that we should use to obtain new thresholds.
3.) Modify lines in perf_expectations.json referring to the tests to be updated,
 so that they may be used as input to make_expectations.py.

This script automates the last step above.

Here's a sample line from perf_expectations.json:

"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, \
"revb": 164141, "type": "absolute", "better": "higher", "improve": 0, \
"regress": 0, "sha1": "54d94538"},

To get the above test ready for input to make_expectations.py, it should become:

"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": <new reva>, \
"revb": <new revb>, "type": "absolute", "better": "higher", "improve": 0, \
"regress": 0},

Examples:

1.) To update the test specified above and get baseline
values using the revision range 12345 and 23456, run this script with a command
line like this:
  python update_perf_expectations.py -f \
  win-release/media_tests_av_perf/fps/tulip2.m4a --reva 12345 --revb 23456
Or, using an input file,
where the input file contains a single line with text
  win-release/media_tests_av_perf/fps/tulip2.m4a
run with this command line:
  python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456

2.) Let's say you want to update all seek tests on windows, and get baseline
values using the revision range 12345 and 23456.
Run this script with this command line:
  python update_perf_expectations.py -f win-release/media_tests_av_perf/seek/ \
   --reva 12345 --revb 23456
Or:
  python update_perf_expectations.py -f win-release/.*/seek/ --reva 12345 \
  --revb 23456

Or, using an input file,
where the input file contains a single line with text win-release/.*/seek/:
  python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456

3.) Similarly, if you want to update seek tests on all platforms
  python update_perf_expectations.py -f .*-release/.*/seek/ --reva 12345 \
  --revb 23456

"""

import logging
from optparse import OptionParser
import os
import re

import make_expectations as perf_ex_lib

# Default logging is INFO. Use --verbose to enable DEBUG logging.
_DEFAULT_LOG_LEVEL = logging.INFO


def GetTestsToUpdate(contents, all_test_keys):
  """Parses input contents and obtains tests to be re-baselined.

  Args:
    contents: string containing contents of input file.
    all_test_keys: list of keys of test dictionary.
  Returns:
    A list of keys for tests that should be updated.
  """
  # Each line of the input file specifies a test case to update.
  tests_list = []
  for test_case_filter in contents.splitlines():
    # Skip any empty lines.
    if test_case_filter:
      # Sample expected line:
      # win-release/media_tests_av_perf/seek/\
      # CACHED_BUFFERED_SEEK_NoConstraints_crowd1080.ogv
      # Or, if reg-ex, then sample line:
      # win-release/media-tests_av_perf/seek*
      # Skip any leading spaces if they exist in the input file.
      logging.debug('Trying to match %s', test_case_filter)
      tests_list.extend(GetMatchingTests(test_case_filter.strip(),
                                         all_test_keys))
  return tests_list


def GetMatchingTests(tests_to_update, all_test_keys):
  """Parses input reg-ex filter and obtains tests to be re-baselined.

  Args:
    tests_to_update: reg-ex string specifying tests to be updated.
    all_test_keys: list of keys of tests dictionary.
  Returns:
    A list of keys for tests that should be updated.
  """
  tests_list = []
  search_string = re.compile(tests_to_update)
  # Get matching tests from the dictionary of tests
  for test_key in all_test_keys:
    if search_string.match(test_key):
      tests_list.append(test_key)
      logging.debug('%s will be updated', test_key)
  logging.info('%s tests found matching reg-ex: %s', len(tests_list),
               tests_to_update)
  return tests_list


def PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb):
  """Modifies value of tests that are to re-baselined:
     Set reva and revb values to specified new values. Remove sha1.

  Args:
    tests_to_update: list of tests to be updated.
    all_tests: dictionary of all tests.
    reva: oldest revision in range to use for new values.
    revb: newest revision in range to use for new values.
  Raises:
    ValueError: If reva or revb are not valid ints, or if either
    of them are negative.
  """
  reva = int(reva)
  revb = int(revb)

  if reva < 0 or revb < 0:
    raise ValueError('Revision values should be positive.')
  # Ensure reva is less than revb.
  # (this is similar to the check done in make_expectations.py)
  if revb < reva:
    temp = revb
    revb = reva
    reva = temp
  for test_key in tests_to_update:
    # Get original test from the dictionary of tests
    test_value = all_tests[test_key]
    if test_value:
      # Sample line in perf_expectations.json:
      #  "linux-release/media_tests _av_perf/dropped_frames/crowd360.webm":\
      # {"reva": 155180, "revb": 155280, "type": "absolute", \
      # "better": "lower", "improve": 0, "regress": 3, "sha1": "276ba29c"},
      # Set new revision range
      test_value['reva'] = reva
      test_value['revb'] = revb
      # Remove sha1 to indicate this test requires an update
      # Check first to make sure it exist.
      if 'sha1' in test_value:
        del test_value['sha1']
    else:
      logging.warning('%s does not exist.', test_key)
  logging.info('Done preparing tests for update.')


def GetCommandLineOptions():
  """Parse command line arguments.

  Returns:
    An options object containing command line arguments and their values.
  """
  parser = OptionParser()

  parser.add_option('--reva', dest='reva', type='int',
                    help='Starting revision of new range.',
                    metavar='START_REVISION')
  parser.add_option('--revb', dest='revb', type='int',
                    help='Ending revision of new range.',
                    metavar='END_REVISION')
  parser.add_option('-f', dest='tests_filter',
                    help='Regex to use for filtering tests to be updated. '
                    'At least one of -filter or -input_file must be provided. '
                    'If both are provided, then input-file is used.',
                    metavar='FILTER', default='')
  parser.add_option('-i', dest='input_file',
                    help='Optional path to file with reg-exes for tests to'
                    ' update. If provided, it overrides the filter argument.',
                    metavar='INPUT_FILE', default='')
  parser.add_option('--config', dest='config_file',
                    default=perf_ex_lib.DEFAULT_CONFIG_FILE,
                    help='Set the config file to FILE.', metavar='FILE')
  parser.add_option('-v', dest='verbose', action='store_true', default=False,
                    help='Enable verbose output.')
  options = parser.parse_args()[0]
  return options


def Main():
  """Main driver function."""
  options = GetCommandLineOptions()

  _SetLogger(options.verbose)
  # Do some command-line validation
  if not options.input_file and not options.tests_filter:
    logging.error('At least one of input-file or test-filter must be provided.')
    exit(1)
  if options.input_file and options.tests_filter:
    logging.error('Specify only one of input file or test-filter.')
    exit(1)
  if not options.reva or not options.revb:
    logging.error('Start and end revision of range must be specified.')
    exit(1)

  # Load config.
  config = perf_ex_lib.ConvertJsonIntoDict(
      perf_ex_lib.ReadFile(options.config_file))

  # Obtain the perf expectations file from the config file.
  perf_file = os.path.join(
      os.path.dirname(options.config_file), config['perf_file'])

  # We should have all the information we require now.
  # On to the real thang.
  # First, get all the existing tests from the original perf_expectations file.
  all_tests = perf_ex_lib.ConvertJsonIntoDict(
      perf_ex_lib.ReadFile(perf_file))
  all_test_keys = all_tests.keys()
  # Remove the load key, because we don't want to modify it.
  all_test_keys.remove('load')
  # Keep tests sorted, like in the original file.
  all_test_keys.sort()

  # Next, get all tests that have been identified for an update.
  tests_to_update = []
  if options.input_file:
    # Tests to update have been specified in an input_file.
    # Get contents of file.
    tests_filter = perf_ex_lib.ReadFile(options.input_file)
  elif options.tests_filter:
    # Tests to update have been specified as a reg-ex filter.
    tests_filter = options.tests_filter

  # Get tests to update based on filter specified.
  tests_to_update = GetTestsToUpdate(tests_filter, all_test_keys)
  logging.info('Done obtaining matching tests.')

  # Now, prepare tests for update.
  PrepareTestsForUpdate(tests_to_update, all_tests, options.reva, options.revb)

  # Finally, write modified tests back to perf_expectations file.
  perf_ex_lib.WriteJson(perf_file, all_tests, all_test_keys,
                        calculate_sha1=False)
  logging.info('Done writing tests for update to %s.', perf_file)


def _SetLogger(verbose):
  log_level = _DEFAULT_LOG_LEVEL
  if verbose:
    log_level = logging.DEBUG
  logging.basicConfig(level=log_level, format='%(message)s')


if __name__ == '__main__':
  Main()