[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Run Performance Test Bisect Tool |
| 7 | |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 8 | This script is used by a try bot to run the bisect script with the parameters |
| 9 | specified in the bisect config file. It checks out a copy of the depot in |
| 10 | a subdirectory 'bisect' of the working directory provided, annd runs the |
| 11 | bisect scrip there. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 12 | """ |
| 13 | |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 14 | import json |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 15 | import optparse |
| 16 | import os |
[email protected] | 92a0dddb | 2014-04-28 06:41:22 | [diff] [blame] | 17 | import platform |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 18 | import re |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 19 | import shlex |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 20 | import subprocess |
| 21 | import sys |
[email protected] | d6f9b9ef | 2013-05-29 17:13:01 | [diff] [blame] | 22 | import traceback |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 23 | |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 24 | from auto_bisect import bisect_perf_regression |
[email protected] | d4aad20 | 2014-07-10 01:27:09 | [diff] [blame] | 25 | from auto_bisect import bisect_utils |
[email protected] | 15186ef | 2014-08-06 19:02:45 | [diff] [blame] | 26 | from auto_bisect import math_utils |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 27 | from auto_bisect import source_control |
[email protected] | d4aad20 | 2014-07-10 01:27:09 | [diff] [blame] | 28 | |
[email protected] | 2a75ef7 | 2013-06-06 17:39:03 | [diff] [blame] | 29 | CROS_BOARD_ENV = 'BISECT_CROS_BOARD' |
| 30 | CROS_IP_ENV = 'BISECT_CROS_IP' |
qyearsley | 915e727 | 2014-10-02 21:58:27 | [diff] [blame] | 31 | SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 32 | SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir) |
| 33 | BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg') |
| 34 | RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg') |
| 35 | WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join( |
| 36 | SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') |
| 37 | BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect') |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 38 | |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 39 | PERF_BENCHMARKS_PATH = 'tools/perf/benchmarks' |
Bartosz Fabianowski | 85a82381 | 2015-04-16 10:27:51 | [diff] [blame] | 40 | PERF_MEASUREMENTS_PATH = 'tools/perf/measurements' |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 41 | BUILDBOT_BUILDERNAME = 'BUILDBOT_BUILDERNAME' |
| 42 | BENCHMARKS_JSON_FILE = 'benchmarks.json' |
qyearsley | 5867c6e | 2014-08-28 19:04:16 | [diff] [blame] | 43 | |
akuegel | 8ce00b7 | 2015-05-27 08:03:53 | [diff] [blame] | 44 | # This is used to identify tryjobs triggered by the commit queue. |
prasadv | 21b44a7 | 2015-05-29 16:21:41 | [diff] [blame] | 45 | _COMMIT_QUEUE_USERS = [ |
| 46 | '5071639625-1lppvbtck1morgivc6sq4dul7klu27sd@developer.gserviceaccount.com', |
| 47 | '[email protected]'] |
akuegel | 8ce00b7 | 2015-05-27 08:03:53 | [diff] [blame] | 48 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 49 | class Goma(object): |
| 50 | |
| 51 | def __init__(self, path_to_goma): |
| 52 | self._abs_path_to_goma = None |
| 53 | self._abs_path_to_goma_file = None |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 54 | if not path_to_goma: |
| 55 | return |
| 56 | self._abs_path_to_goma = os.path.abspath(path_to_goma) |
| 57 | filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh' |
| 58 | self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 59 | |
| 60 | def __enter__(self): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 61 | if self._HasGomaPath(): |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 62 | self._SetupAndStart() |
| 63 | return self |
| 64 | |
| 65 | def __exit__(self, *_): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 66 | if self._HasGomaPath(): |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 67 | self._Stop() |
| 68 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 69 | def _HasGomaPath(self): |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 70 | return bool(self._abs_path_to_goma) |
| 71 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 72 | def _SetupEnvVars(self): |
| 73 | if os.name == 'nt': |
| 74 | os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + |
| 75 | ' cl.exe') |
| 76 | os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') + |
| 77 | ' cl.exe') |
| 78 | else: |
| 79 | os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma, |
| 80 | os.environ['PATH']]) |
| 81 | |
| 82 | def _SetupAndStart(self): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 83 | """Sets up goma and launches it. |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 84 | |
| 85 | Args: |
| 86 | path_to_goma: Path to goma directory. |
| 87 | |
| 88 | Returns: |
| 89 | True if successful.""" |
| 90 | self._SetupEnvVars() |
| 91 | |
| 92 | # Sometimes goma is lingering around if something went bad on a previous |
| 93 | # run. Stop it before starting a new process. Can ignore the return code |
| 94 | # since it will return an error if it wasn't running. |
| 95 | self._Stop() |
| 96 | |
| 97 | if subprocess.call([self._abs_path_to_goma_file, 'start']): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 98 | raise RuntimeError('Goma failed to start.') |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 99 | |
| 100 | def _Stop(self): |
| 101 | subprocess.call([self._abs_path_to_goma_file, 'stop']) |
| 102 | |
| 103 | |
[email protected] | f7a2a6e | 2014-07-30 21:04:31 | [diff] [blame] | 104 | def _LoadConfigFile(config_file_path): |
[email protected] | 5dcad27 | 2014-07-30 08:00:42 | [diff] [blame] | 105 | """Attempts to load the specified config file as a module |
| 106 | and grab the global config dict. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 107 | |
[email protected] | cada4c2 | 2013-02-26 00:27:46 | [diff] [blame] | 108 | Args: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 109 | config_file_path: Path to the config file. |
[email protected] | cada4c2 | 2013-02-26 00:27:46 | [diff] [blame] | 110 | |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 111 | Returns: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 112 | If successful, returns the config dict loaded from the file. If no |
| 113 | such dictionary could be loaded, returns the empty dictionary. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 114 | """ |
| 115 | try: |
| 116 | local_vars = {} |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 117 | execfile(config_file_path, local_vars) |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 118 | return local_vars['config'] |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 119 | except Exception: |
[email protected] | d6f9b9ef | 2013-05-29 17:13:01 | [diff] [blame] | 120 | print |
| 121 | traceback.print_exc() |
| 122 | print |
[email protected] | b3c0ee7 | 2013-10-22 21:58:38 | [diff] [blame] | 123 | return {} |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 124 | |
| 125 | |
robertocn | b20bc1f | 2014-11-03 19:50:29 | [diff] [blame] | 126 | def _ValidateConfigFile(config_contents, required_parameters): |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 127 | """Validates the config file contents, checking whether all values are |
| 128 | non-empty. |
| 129 | |
| 130 | Args: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 131 | config_contents: A config dictionary. |
robertocn | b20bc1f | 2014-11-03 19:50:29 | [diff] [blame] | 132 | required_parameters: A list of parameters to check for. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 133 | |
| 134 | Returns: |
| 135 | True if valid. |
| 136 | """ |
robertocn | b20bc1f | 2014-11-03 19:50:29 | [diff] [blame] | 137 | for parameter in required_parameters: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 138 | if parameter not in config_contents: |
| 139 | return False |
| 140 | value = config_contents[parameter] |
| 141 | if not value or type(value) is not str: |
| 142 | return False |
| 143 | return True |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 144 | |
| 145 | |
| 146 | def _ValidatePerfConfigFile(config_contents): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 147 | """Validates the perf config file contents. |
| 148 | |
| 149 | This is used when we're doing a perf try job, rather than a bisect. |
| 150 | The config file is called run-perf-test.cfg by default. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 151 | |
| 152 | The parameters checked are the required parameters; any additional optional |
| 153 | parameters won't be checked and validation will still pass. |
| 154 | |
| 155 | Args: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 156 | config_contents: A config dictionary. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 157 | |
| 158 | Returns: |
| 159 | True if valid. |
| 160 | """ |
qyearsley | bd0f0208 | 2015-01-28 00:27:02 | [diff] [blame] | 161 | return _ValidateConfigFile(config_contents, required_parameters=['command']) |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 162 | |
| 163 | |
| 164 | def _ValidateBisectConfigFile(config_contents): |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 165 | """Validates the bisect config file contents. |
| 166 | |
| 167 | The parameters checked are the required parameters; any additional optional |
| 168 | parameters won't be checked and validation will still pass. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 169 | |
| 170 | Args: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 171 | config_contents: A config dictionary. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 172 | |
| 173 | Returns: |
| 174 | True if valid. |
| 175 | """ |
qyearsley | bd0f0208 | 2015-01-28 00:27:02 | [diff] [blame] | 176 | return _ValidateConfigFile( |
| 177 | config_contents, |
| 178 | required_parameters=['command', 'good_revision', 'bad_revision']) |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 179 | |
| 180 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 181 | def _OutputFailedResults(text_to_print): |
| 182 | bisect_utils.OutputAnnotationStepStart('Results - Failed') |
| 183 | print |
| 184 | print text_to_print |
| 185 | print |
| 186 | bisect_utils.OutputAnnotationStepClosed() |
| 187 | |
| 188 | |
| 189 | def _CreateBisectOptionsFromConfig(config): |
[email protected] | 6e6200c8 | 2014-07-16 23:16:30 | [diff] [blame] | 190 | print config['command'] |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 191 | opts_dict = {} |
| 192 | opts_dict['command'] = config['command'] |
[email protected] | 7b9ef3c | 2014-08-15 09:33:19 | [diff] [blame] | 193 | opts_dict['metric'] = config.get('metric') |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 194 | |
| 195 | if config['repeat_count']: |
| 196 | opts_dict['repeat_test_count'] = int(config['repeat_count']) |
| 197 | |
| 198 | if config['truncate_percent']: |
| 199 | opts_dict['truncate_percent'] = int(config['truncate_percent']) |
| 200 | |
| 201 | if config['max_time_minutes']: |
| 202 | opts_dict['max_time_minutes'] = int(config['max_time_minutes']) |
| 203 | |
| 204 | if config.has_key('use_goma'): |
| 205 | opts_dict['use_goma'] = config['use_goma'] |
[email protected] | 7a960ad | 2014-07-08 00:49:57 | [diff] [blame] | 206 | if config.has_key('goma_dir'): |
| 207 | opts_dict['goma_dir'] = config['goma_dir'] |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 208 | |
robertocn | fdd5b1c0 | 2014-10-17 21:55:29 | [diff] [blame] | 209 | if config.has_key('improvement_direction'): |
| 210 | opts_dict['improvement_direction'] = int(config['improvement_direction']) |
| 211 | |
prasadv | 5557c67 | 2015-02-10 20:07:12 | [diff] [blame] | 212 | if config.has_key('target_arch'): |
| 213 | opts_dict['target_arch'] = config['target_arch'] |
| 214 | |
robertocn | b20bc1f | 2014-11-03 19:50:29 | [diff] [blame] | 215 | if config.has_key('bug_id') and str(config['bug_id']).isdigit(): |
| 216 | opts_dict['bug_id'] = config['bug_id'] |
| 217 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 218 | opts_dict['build_preference'] = 'ninja' |
| 219 | opts_dict['output_buildbot_annotations'] = True |
| 220 | |
| 221 | if '--browser=cros' in config['command']: |
| 222 | opts_dict['target_platform'] = 'cros' |
| 223 | |
| 224 | if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: |
| 225 | opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] |
| 226 | opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] |
| 227 | else: |
qyearsley | 8b8d567 | 2014-08-23 21:19:14 | [diff] [blame] | 228 | raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 229 | 'BISECT_CROS_BOARD undefined.') |
| 230 | elif 'android' in config['command']: |
prasadv | 1d2dd72 | 2015-07-17 00:45:22 | [diff] [blame] | 231 | # TODO (prasadv): Remove android-chrome-shell check once we confirm that |
| 232 | # there are no pending bisect jobs with this in command. |
| 233 | if any(item in config['command'] |
| 234 | for item in ['android-chrome-shell', 'android-chromium']): |
[email protected] | 6e6200c8 | 2014-07-16 23:16:30 | [diff] [blame] | 235 | opts_dict['target_platform'] = 'android' |
| 236 | elif 'android-chrome' in config['command']: |
[email protected] | c4d2f1d9 | 2013-12-04 06:35:15 | [diff] [blame] | 237 | opts_dict['target_platform'] = 'android-chrome' |
| 238 | else: |
| 239 | opts_dict['target_platform'] = 'android' |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 240 | |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 241 | return bisect_perf_regression.BisectOptions.FromDict(opts_dict) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 242 | |
| 243 | |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 244 | def _ParseCloudLinksFromOutput(output): |
| 245 | html_results_pattern = re.compile( |
| 246 | r'\s(?P<VALUES>http://storage.googleapis.com/' + |
| 247 | 'chromium-telemetry/html-results/results-[a-z0-9-_]+)\s', |
| 248 | re.MULTILINE) |
| 249 | profiler_pattern = re.compile( |
| 250 | r'\s(?P<VALUES>https://console.developers.google.com/' + |
| 251 | 'm/cloudstorage/b/[a-z-]+/o/profiler-[a-z0-9-_.]+)\s', |
| 252 | re.MULTILINE) |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 253 | |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 254 | results = { |
| 255 | 'html-results': html_results_pattern.findall(output), |
| 256 | 'profiler': profiler_pattern.findall(output), |
| 257 | } |
| 258 | |
| 259 | return results |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 260 | |
| 261 | |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 262 | def _ParseAndOutputCloudLinks( |
| 263 | results_without_patch, results_with_patch, annotations_dict): |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 264 | cloud_links_without_patch = _ParseCloudLinksFromOutput( |
| 265 | results_without_patch[2]) |
| 266 | cloud_links_with_patch = _ParseCloudLinksFromOutput( |
| 267 | results_with_patch[2]) |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 268 | |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 269 | cloud_file_link = (cloud_links_without_patch['html-results'][0] |
| 270 | if cloud_links_without_patch['html-results'] else '') |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 271 | |
simonhatch | 87be98a | 2014-12-01 20:47:11 | [diff] [blame] | 272 | profiler_file_links_with_patch = cloud_links_with_patch['profiler'] |
| 273 | profiler_file_links_without_patch = cloud_links_without_patch['profiler'] |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 274 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 275 | # Calculate the % difference in the means of the 2 runs. |
[email protected] | 7b9ef3c | 2014-08-15 09:33:19 | [diff] [blame] | 276 | percent_diff_in_means = None |
| 277 | std_err = None |
| 278 | if (results_with_patch[0].has_key('mean') and |
| 279 | results_with_patch[0].has_key('values')): |
| 280 | percent_diff_in_means = (results_with_patch[0]['mean'] / |
| 281 | max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0 |
| 282 | std_err = math_utils.PooledStandardError( |
| 283 | [results_with_patch[0]['values'], results_without_patch[0]['values']]) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 284 | |
[email protected] | 7b9ef3c | 2014-08-15 09:33:19 | [diff] [blame] | 285 | if percent_diff_in_means is not None and std_err is not None: |
| 286 | bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' % |
| 287 | (percent_diff_in_means, std_err)) |
| 288 | print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '), |
| 289 | 'Std. Error'.center(20, ' ')) |
| 290 | print ' %s %s %s' % ('Patch'.center(10, ' '), |
| 291 | ('%.02f' % results_with_patch[0]['mean']).center(20, ' '), |
| 292 | ('%.02f' % results_with_patch[0]['std_err']).center(20, ' ')) |
| 293 | print ' %s %s %s' % ('No Patch'.center(10, ' '), |
| 294 | ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), |
| 295 | ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) |
| 296 | if cloud_file_link: |
| 297 | bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
| 298 | bisect_utils.OutputAnnotationStepClosed() |
| 299 | elif cloud_file_link: |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 300 | bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 301 | |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 302 | if profiler_file_links_with_patch and profiler_file_links_without_patch: |
| 303 | for i in xrange(len(profiler_file_links_with_patch)): |
| 304 | bisect_utils.OutputAnnotationStepLink( |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 305 | '%s[%d]' % (annotations_dict.get('profiler_link1'), i), |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 306 | profiler_file_links_with_patch[i]) |
| 307 | for i in xrange(len(profiler_file_links_without_patch)): |
| 308 | bisect_utils.OutputAnnotationStepLink( |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 309 | '%s[%d]' % (annotations_dict.get('profiler_link2'), i), |
simonhatch | a9b78ba | 2014-11-14 15:00:00 | [diff] [blame] | 310 | profiler_file_links_without_patch[i]) |
| 311 | |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 312 | |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 313 | def _ResolveRevisionsFromConfig(config): |
| 314 | if not 'good_revision' in config and not 'bad_revision' in config: |
| 315 | return (None, None) |
| 316 | |
| 317 | bad_revision = source_control.ResolveToRevision( |
| 318 | config['bad_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100) |
| 319 | if not bad_revision: |
| 320 | raise RuntimeError('Failed to resolve [%s] to git hash.', |
| 321 | config['bad_revision']) |
| 322 | good_revision = source_control.ResolveToRevision( |
| 323 | config['good_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, -100) |
| 324 | if not good_revision: |
| 325 | raise RuntimeError('Failed to resolve [%s] to git hash.', |
| 326 | config['good_revision']) |
| 327 | |
| 328 | return (good_revision, bad_revision) |
| 329 | |
| 330 | |
| 331 | def _GetStepAnnotationStringsDict(config): |
| 332 | if 'good_revision' in config and 'bad_revision' in config: |
| 333 | return { |
| 334 | 'build1': 'Building [%s]' % config['good_revision'], |
| 335 | 'build2': 'Building [%s]' % config['bad_revision'], |
| 336 | 'run1': 'Running [%s]' % config['good_revision'], |
| 337 | 'run2': 'Running [%s]' % config['bad_revision'], |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 338 | 'sync1': 'Syncing [%s]' % config['good_revision'], |
| 339 | 'sync2': 'Syncing [%s]' % config['bad_revision'], |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 340 | 'results_label1': config['good_revision'], |
| 341 | 'results_label2': config['bad_revision'], |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 342 | 'profiler_link1': 'Profiler Data - %s' % config['good_revision'], |
| 343 | 'profiler_link2': 'Profiler Data - %s' % config['bad_revision'], |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 344 | } |
| 345 | else: |
| 346 | return { |
| 347 | 'build1': 'Building With Patch', |
| 348 | 'build2': 'Building Without Patch', |
| 349 | 'run1': 'Running With Patch', |
| 350 | 'run2': 'Running Without Patch', |
| 351 | 'results_label1': 'Patch', |
| 352 | 'results_label2': 'ToT', |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 353 | 'profiler_link1': 'With Patch - Profiler Data', |
| 354 | 'profiler_link2': 'Without Patch - Profiler Data', |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 355 | } |
| 356 | |
| 357 | |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 358 | def _RunBuildStepForPerformanceTest(bisect_instance, |
| 359 | build_string, |
| 360 | sync_string, |
| 361 | revision): |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 362 | if revision: |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 363 | bisect_utils.OutputAnnotationStepStart(sync_string) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 364 | if not source_control.SyncToRevision(revision, 'gclient'): |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 365 | raise RuntimeError('Failed [%s].' % sync_string) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 366 | bisect_utils.OutputAnnotationStepClosed() |
| 367 | |
| 368 | bisect_utils.OutputAnnotationStepStart(build_string) |
| 369 | |
| 370 | if bisect_utils.RunGClient(['runhooks']): |
| 371 | raise RuntimeError('Failed to run gclient runhooks') |
| 372 | |
| 373 | if not bisect_instance.ObtainBuild('chromium'): |
| 374 | raise RuntimeError('Patched version failed to build.') |
| 375 | |
| 376 | bisect_utils.OutputAnnotationStepClosed() |
| 377 | |
| 378 | |
| 379 | def _RunCommandStepForPerformanceTest(bisect_instance, |
| 380 | opts, |
| 381 | reset_on_first_run, |
| 382 | upload_on_last_run, |
| 383 | results_label, |
| 384 | run_string): |
| 385 | bisect_utils.OutputAnnotationStepStart(run_string) |
| 386 | |
| 387 | results = bisect_instance.RunPerformanceTestAndParseResults( |
| 388 | opts.command, |
| 389 | opts.metric, |
| 390 | reset_on_first_run=reset_on_first_run, |
| 391 | upload_on_last_run=upload_on_last_run, |
simonhatch | bc1d384 | 2015-03-13 19:18:24 | [diff] [blame] | 392 | results_label=results_label, |
| 393 | allow_flakes=False) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 394 | |
| 395 | if results[1]: |
| 396 | raise RuntimeError('Patched version failed to run performance test.') |
| 397 | |
| 398 | bisect_utils.OutputAnnotationStepClosed() |
| 399 | |
| 400 | return results |
| 401 | |
| 402 | |
| 403 | def _RunPerformanceTest(config): |
| 404 | """Runs a performance test with and without the current patch. |
| 405 | |
| 406 | Args: |
| 407 | config: Contents of the config file, a dictionary. |
| 408 | |
| 409 | Attempts to build and run the current revision with and without the |
| 410 | current patch, with the parameters passed in. |
| 411 | """ |
| 412 | # Bisect script expects to be run from the src directory |
| 413 | os.chdir(SRC_DIR) |
| 414 | |
| 415 | opts = _CreateBisectOptionsFromConfig(config) |
| 416 | revisions = _ResolveRevisionsFromConfig(config) |
| 417 | annotations_dict = _GetStepAnnotationStringsDict(config) |
| 418 | b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) |
| 419 | |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 420 | _RunBuildStepForPerformanceTest(b, |
| 421 | annotations_dict.get('build1'), |
| 422 | annotations_dict.get('sync1'), |
| 423 | revisions[0]) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 424 | |
| 425 | results_with_patch = _RunCommandStepForPerformanceTest( |
| 426 | b, opts, True, True, annotations_dict['results_label1'], |
| 427 | annotations_dict['run1']) |
| 428 | |
| 429 | bisect_utils.OutputAnnotationStepStart('Reverting Patch') |
| 430 | # TODO: When this is re-written to recipes, this should use bot_update's |
| 431 | # revert mechanism to fully revert the client. But for now, since we know that |
| 432 | # the perf try bot currently only supports src/ and src/third_party/WebKit, we |
| 433 | # simply reset those two directories. |
| 434 | bisect_utils.CheckRunGit(['reset', '--hard']) |
| 435 | bisect_utils.CheckRunGit(['reset', '--hard'], |
| 436 | os.path.join('third_party', 'WebKit')) |
| 437 | bisect_utils.OutputAnnotationStepClosed() |
| 438 | |
simonhatch | d674bae | 2014-12-08 21:00:35 | [diff] [blame] | 439 | _RunBuildStepForPerformanceTest(b, |
| 440 | annotations_dict.get('build2'), |
| 441 | annotations_dict.get('sync2'), |
| 442 | revisions[1]) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 443 | |
| 444 | results_without_patch = _RunCommandStepForPerformanceTest( |
| 445 | b, opts, False, True, annotations_dict['results_label2'], |
sullivan | dbb5e44 | 2014-12-05 15:07:48 | [diff] [blame] | 446 | annotations_dict['run2']) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 447 | |
| 448 | # Find the link to the cloud stored results file. |
simonhatch | 3754043 | 2014-12-11 15:25:47 | [diff] [blame] | 449 | _ParseAndOutputCloudLinks( |
| 450 | results_without_patch, results_with_patch, annotations_dict) |
simonhatch | 88ed7eb5 | 2014-12-02 19:01:25 | [diff] [blame] | 451 | |
| 452 | |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 453 | def _SetupAndRunPerformanceTest(config, path_to_goma, is_cq_tryjob=False): |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 454 | """Attempts to build and run the current revision with and without the |
| 455 | current patch, with the parameters passed in. |
| 456 | |
| 457 | Args: |
| 458 | config: The config read from run-perf-test.cfg. |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 459 | path_to_goma: Path to goma directory. |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 460 | is_cq_tryjob: Whether or not the try job was initiated by commit queue. |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 461 | |
| 462 | Returns: |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 463 | An exit code: 0 on success, otherwise 1. |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 464 | """ |
sullivan | 318e47b7e | 2014-09-19 17:16:22 | [diff] [blame] | 465 | if platform.release() == 'XP': |
| 466 | print 'Windows XP is not supported for perf try jobs because it lacks ' |
| 467 | print 'goma support. Please refer to crbug.com/330900.' |
| 468 | return 1 |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 469 | try: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 470 | with Goma(path_to_goma) as _: |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 471 | config['use_goma'] = bool(path_to_goma) |
prasadv | 43d1aaa | 2014-08-28 00:39:14 | [diff] [blame] | 472 | if config['use_goma']: |
| 473 | config['goma_dir'] = os.path.abspath(path_to_goma) |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 474 | if not is_cq_tryjob: |
| 475 | _RunPerformanceTest(config) |
| 476 | else: |
| 477 | return _RunBenchmarksForCommitQueue(config) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 478 | return 0 |
| 479 | except RuntimeError, e: |
simonhatch | bc1d384 | 2015-03-13 19:18:24 | [diff] [blame] | 480 | bisect_utils.OutputAnnotationStepFailure() |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 481 | bisect_utils.OutputAnnotationStepClosed() |
| 482 | _OutputFailedResults('Error: %s' % e.message) |
| 483 | return 1 |
| 484 | |
| 485 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 486 | def _RunBisectionScript( |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 487 | config, working_directory, path_to_goma, path_to_extra_src, dry_run): |
| 488 | """Attempts to execute the bisect script with the given parameters. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 489 | |
| 490 | Args: |
| 491 | config: A dict containing the parameters to pass to the script. |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 492 | working_directory: A working directory to provide to the bisect script, |
| 493 | where it will store it's own copy of the depot. |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 494 | path_to_goma: Path to goma directory. |
[email protected] | c4d2f1d9 | 2013-12-04 06:35:15 | [diff] [blame] | 495 | path_to_extra_src: Path to extra source file. |
[email protected] | 63c7f73e | 2013-10-28 20:02:08 | [diff] [blame] | 496 | dry_run: Do a dry run, skipping sync, build, and performance testing steps. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 497 | |
| 498 | Returns: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 499 | An exit status code: 0 on success, otherwise 1. |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 500 | """ |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 501 | _PrintConfigStep(config) |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 502 | |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 503 | # Construct the basic command with all necessary arguments. |
| 504 | cmd = [ |
| 505 | 'python', |
| 506 | os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'), |
| 507 | '--command', config['command'], |
| 508 | '--good_revision', config['good_revision'], |
| 509 | '--bad_revision', config['bad_revision'], |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 510 | '--working_directory', working_directory, |
| 511 | '--output_buildbot_annotations' |
| 512 | ] |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 513 | |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 514 | # Add flags for any optional config parameters if given in the config. |
| 515 | options = [ |
qyearsley | bd0f0208 | 2015-01-28 00:27:02 | [diff] [blame] | 516 | ('metric', '--metric'), |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 517 | ('repeat_count', '--repeat_test_count'), |
| 518 | ('truncate_percent', '--truncate_percent'), |
| 519 | ('max_time_minutes', '--max_time_minutes'), |
| 520 | ('bisect_mode', '--bisect_mode'), |
| 521 | ('improvement_direction', '--improvement_direction'), |
| 522 | ('bug_id', '--bug_id'), |
qyearsley | 38a6e5a | 2015-01-12 17:59:24 | [diff] [blame] | 523 | ('builder_type', '--builder_type'), |
prasadv | 5557c67 | 2015-02-10 20:07:12 | [diff] [blame] | 524 | ('target_arch', '--target_arch'), |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 525 | ] |
| 526 | for config_key, flag in options: |
| 527 | if config.has_key(config_key): |
| 528 | cmd.extend([flag, config[config_key]]) |
robertocn | b20bc1f | 2014-11-03 19:50:29 | [diff] [blame] | 529 | |
[email protected] | bda7af4 | 2013-06-10 23:49:42 | [diff] [blame] | 530 | cmd.extend(['--build_preference', 'ninja']) |
[email protected] | 2deb10d | 2013-03-15 19:29:50 | [diff] [blame] | 531 | |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 532 | # Possibly set the target platform name based on the browser name in a |
| 533 | # Telemetry command. |
prasadv | 1d2dd72 | 2015-07-17 00:45:22 | [diff] [blame] | 534 | # TODO (prasadv): Remove android-chrome-shell check once we confirm there are |
| 535 | # no pending bisect jobs with this in command. |
| 536 | if any(item in config['command'] |
| 537 | for item in ['android-chrome-shell', 'android-chromium']): |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 538 | cmd.extend(['--target_platform', 'android']) |
| 539 | elif 'android-chrome' in config['command']: |
| 540 | cmd.extend(['--target_platform', 'android-chrome']) |
| 541 | elif 'android' in config['command']: |
| 542 | cmd.extend(['--target_platform', 'android']) |
[email protected] | bda7af4 | 2013-06-10 23:49:42 | [diff] [blame] | 543 | |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 544 | if path_to_goma: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 545 | # For Windows XP platforms, goma service is not supported. |
[email protected] | 92a0dddb | 2014-04-28 06:41:22 | [diff] [blame] | 546 | # Moreover we don't compile chrome when gs_bucket flag is set instead |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 547 | # use builds archives, therefore ignore goma service for Windows XP. |
| 548 | # See http://crbug.com/330900. |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 549 | if platform.release() == 'XP': |
[email protected] | 92a0dddb | 2014-04-28 06:41:22 | [diff] [blame] | 550 | print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' |
| 551 | 'on Windows XP platform. Please refer to crbug.com/330900.') |
| 552 | path_to_goma = None |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 553 | cmd.append('--use_goma') |
simonhatch | 88572ab | 2014-12-23 18:35:06 | [diff] [blame] | 554 | cmd.append('--goma_dir') |
| 555 | cmd.append(os.path.abspath(path_to_goma)) |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 556 | |
[email protected] | c4d2f1d9 | 2013-12-04 06:35:15 | [diff] [blame] | 557 | if path_to_extra_src: |
| 558 | cmd.extend(['--extra_src', path_to_extra_src]) |
| 559 | |
[email protected] | 63c7f73e | 2013-10-28 20:02:08 | [diff] [blame] | 560 | if dry_run: |
qyearsley | 3571227 | 2014-11-22 01:11:37 | [diff] [blame] | 561 | cmd.extend([ |
| 562 | '--debug_ignore_build', |
| 563 | '--debug_ignore_sync', |
| 564 | '--debug_ignore_perf_test' |
| 565 | ]) |
| 566 | |
[email protected] | 2deb10d | 2013-03-15 19:29:50 | [diff] [blame] | 567 | cmd = [str(c) for c in cmd] |
| 568 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 569 | with Goma(path_to_goma) as _: |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 570 | return_code = subprocess.call(cmd) |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 571 | |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 572 | if return_code: |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 573 | print ('Error: bisect_perf_regression.py returned with error %d\n' |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 574 | % return_code) |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 575 | |
| 576 | return return_code |
| 577 | |
| 578 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 579 | def _PrintConfigStep(config): |
| 580 | """Prints out the given config, along with Buildbot annotations.""" |
| 581 | bisect_utils.OutputAnnotationStepStart('Config') |
| 582 | print |
| 583 | for k, v in config.iteritems(): |
| 584 | print ' %s : %s' % (k, v) |
| 585 | print |
| 586 | bisect_utils.OutputAnnotationStepClosed() |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 587 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 588 | |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 589 | def _GetBrowserType(bot_platform): |
| 590 | """Gets the browser type to be used in the run benchmark command.""" |
| 591 | if bot_platform == 'android': |
prasadv | 1d2dd72 | 2015-07-17 00:45:22 | [diff] [blame] | 592 | return 'android-chromium' |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 593 | elif 'x64' in bot_platform: |
| 594 | return 'release_x64' |
| 595 | |
| 596 | return 'release' |
| 597 | |
| 598 | |
| 599 | |
| 600 | def _GuessTelemetryTestCommand(bot_platform, test_name=None): |
| 601 | """Creates a Telemetry benchmark command based on bot and test name.""" |
| 602 | command = [] |
| 603 | # On Windows, Python scripts should be prefixed with the python command. |
| 604 | if bot_platform == 'win': |
| 605 | command.append('python') |
| 606 | command.append('tools/perf/run_benchmark') |
| 607 | command.append('-v') |
| 608 | command.append('--browser=%s' % _GetBrowserType(bot_platform)) |
| 609 | if test_name: |
| 610 | command.append(test_name) |
| 611 | |
| 612 | return ' '.join(command) |
| 613 | |
| 614 | |
| 615 | def _GetConfigBasedOnPlatform(config, bot_name, test_name): |
| 616 | """Generates required options to create BisectPerformanceMetrics instance.""" |
| 617 | opts_dict = { |
| 618 | 'command': _GuessTelemetryTestCommand(bot_name, test_name), |
| 619 | 'target_arch': 'x64' if 'x64' in bot_name else 'ia32', |
| 620 | 'build_preference': 'ninja', |
| 621 | 'output_buildbot_annotations': True, |
| 622 | 'repeat_test_count': 1, |
| 623 | 'bisect_mode': bisect_utils.BISECT_MODE_RETURN_CODE, |
| 624 | } |
| 625 | |
| 626 | if 'use_goma' in config: |
| 627 | opts_dict['use_goma'] = config['use_goma'] |
| 628 | if 'goma_dir' in config: |
| 629 | opts_dict['goma_dir'] = config['goma_dir'] |
prasadv | 1d2dd72 | 2015-07-17 00:45:22 | [diff] [blame] | 630 | # TODO (prasadv): Remove android-chrome-shell check once we confirm there are |
| 631 | # no pending bisect jobs with this in command. |
| 632 | if any(item in opts_dict['command'] |
| 633 | for item in ['android-chrome-shell', 'android-chromium']): |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 634 | opts_dict['target_platform'] = 'android' |
| 635 | |
| 636 | return bisect_perf_regression.BisectOptions.FromDict(opts_dict) |
| 637 | |
| 638 | |
| 639 | def _GetModifiedFilesFromPatch(cwd=None): |
| 640 | """Gets list of files modified in the current patch.""" |
| 641 | log_output = bisect_utils.CheckRunGit( |
| 642 | ['diff', '--no-ext-diff', '--name-only', 'HEAD~1'], cwd=cwd) |
| 643 | modified_files = log_output.split() |
| 644 | return modified_files |
| 645 | |
| 646 | |
| 647 | def _GetAffectedBenchmarkModuleNames(): |
| 648 | """Gets list of modified benchmark files under tools/perf/benchmarks.""" |
| 649 | all_affected_files = _GetModifiedFilesFromPatch() |
| 650 | modified_benchmarks = [] |
| 651 | for affected_file in all_affected_files: |
Bartosz Fabianowski | 85a82381 | 2015-04-16 10:27:51 | [diff] [blame] | 652 | if (affected_file.startswith(PERF_BENCHMARKS_PATH) or |
| 653 | affected_file.startswith(PERF_MEASUREMENTS_PATH)): |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 654 | benchmark = os.path.basename(os.path.splitext(affected_file)[0]) |
| 655 | modified_benchmarks.append(benchmark) |
| 656 | return modified_benchmarks |
| 657 | |
| 658 | |
| 659 | def _ListAvailableBenchmarks(bot_platform): |
| 660 | """Gets all available benchmarks names as a list.""" |
| 661 | browser_type = _GetBrowserType(bot_platform) |
| 662 | if os.path.exists(BENCHMARKS_JSON_FILE): |
| 663 | os.remove(BENCHMARKS_JSON_FILE) |
| 664 | command = [] |
| 665 | if 'win' in bot_platform: |
| 666 | command.append('python') |
| 667 | command.append('tools/perf/run_benchmark') |
| 668 | command.extend([ |
| 669 | 'list', |
| 670 | '--browser', |
| 671 | browser_type, |
| 672 | '--json-output', |
| 673 | BENCHMARKS_JSON_FILE]) |
| 674 | try: |
| 675 | output, return_code = bisect_utils.RunProcessAndRetrieveOutput( |
| 676 | command=command, cwd=SRC_DIR) |
| 677 | if return_code: |
| 678 | raise RuntimeError('Something went wrong while listing benchmarks. ' |
| 679 | 'Please review the command line: %s.\nERROR: [%s]' % |
| 680 | (' '.join(command), output)) |
| 681 | with open(BENCHMARKS_JSON_FILE) as tests_json: |
| 682 | tests_data = json.load(tests_json) |
| 683 | if tests_data.get('steps'): |
| 684 | return tests_data.get('steps').keys() |
| 685 | finally: |
| 686 | try: |
| 687 | if os.path.exists(BENCHMARKS_JSON_FILE): |
| 688 | os.remove(BENCHMARKS_JSON_FILE) |
| 689 | except OSError as e: |
| 690 | if e.errno != errno.ENOENT: |
| 691 | raise |
| 692 | return None |
| 693 | |
| 694 | |
| 695 | def _OutputOverallResults(results): |
| 696 | """Creates results step and prints results on buildbot job.""" |
| 697 | test_status = all(current_value == True for current_value in results.values()) |
| 698 | bisect_utils.OutputAnnotationStepStart( |
| 699 | 'Results - %s' % ('Passed' if test_status else 'Failed')) |
| 700 | print |
| 701 | print 'Results of benchmarks:' |
| 702 | print |
| 703 | for benchmark, result in results.iteritems(): |
| 704 | print '%s: %s' % (benchmark, 'Passed' if result else 'Failed') |
| 705 | if not test_status: |
| 706 | bisect_utils.OutputAnnotationStepFailure() |
| 707 | bisect_utils.OutputAnnotationStepClosed() |
| 708 | # Returns 0 for success and 1 for failure. |
| 709 | return 0 if test_status else 1 |
| 710 | |
| 711 | |
| 712 | def _RunBenchmark(bisect_instance, opts, bot_name, benchmark_name): |
| 713 | """Runs a Telemetry benchmark.""" |
| 714 | bisect_utils.OutputAnnotationStepStart(benchmark_name) |
| 715 | command_to_run = _GuessTelemetryTestCommand(bot_name, benchmark_name) |
| 716 | args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost()) |
| 717 | output, return_code = bisect_utils.RunProcessAndRetrieveOutput(args, SRC_DIR) |
| 718 | # A value other than 0 indicates that the test couldn't be run, and results |
| 719 | # should also include an error message. |
| 720 | if return_code: |
| 721 | print ('Error: Something went wrong running the benchmark: %s.' |
| 722 | 'Please review the command line:%s\n\n%s' % |
| 723 | (benchmark_name, command_to_run, output)) |
| 724 | bisect_utils.OutputAnnotationStepFailure() |
| 725 | print output |
| 726 | bisect_utils.OutputAnnotationStepClosed() |
| 727 | # results[1] contains the return code from subprocess that executes test |
| 728 | # command, On successful test run it contains 0 otherwise any non-zero value. |
| 729 | return return_code == 0 |
| 730 | |
| 731 | |
| 732 | def _RunBenchmarksForCommitQueue(config): |
| 733 | """Runs Telemetry benchmark for the commit queue.""" |
| 734 | os.chdir(SRC_DIR) |
| 735 | # To determine the bot platform by reading buildbot name from environment |
| 736 | # variable. |
| 737 | bot_name = os.environ.get(BUILDBOT_BUILDERNAME) |
| 738 | if not bot_name: |
| 739 | bot_name = sys.platform |
| 740 | bot_name = bot_name.split('_')[0] |
| 741 | |
| 742 | affected_benchmarks = _GetAffectedBenchmarkModuleNames() |
| 743 | # Abort if there are no changes to benchmark any existing benchmark files. |
| 744 | if not affected_benchmarks: |
| 745 | bisect_utils.OutputAnnotationStepStart('Results') |
| 746 | print |
| 747 | print ('There are no modification to Telemetry benchmarks,' |
| 748 | ' aborting the try job.') |
| 749 | bisect_utils.OutputAnnotationStepClosed() |
| 750 | return 0 |
| 751 | |
| 752 | # Bisect script expects to be run from the src directory |
| 753 | # Gets required options inorder to create BisectPerformanceMetrics instance. |
| 754 | # Since command is a required arg in BisectPerformanceMetrics, we just create |
| 755 | # a dummy command for now. |
| 756 | opts = _GetConfigBasedOnPlatform(config, bot_name, test_name='') |
| 757 | annotations_dict = _GetStepAnnotationStringsDict(config) |
| 758 | b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) |
| 759 | _RunBuildStepForPerformanceTest(b, |
| 760 | annotations_dict.get('build1'), |
| 761 | annotations_dict.get('sync1'), |
| 762 | None) |
| 763 | available_benchmarks = _ListAvailableBenchmarks(bot_name) |
| 764 | overall_results = {} |
| 765 | for affected_benchmark in affected_benchmarks: |
| 766 | for benchmark in available_benchmarks: |
| 767 | if (benchmark.startswith(affected_benchmark) and |
| 768 | not benchmark.endswith('reference')): |
| 769 | overall_results[benchmark] = _RunBenchmark(b, opts, bot_name, benchmark) |
| 770 | |
| 771 | return _OutputOverallResults(overall_results) |
| 772 | |
| 773 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 774 | def _OptionParser(): |
| 775 | """Returns the options parser for run-bisect-perf-regression.py.""" |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 776 | |
| 777 | def ConvertJson(option, _, value, parser): |
| 778 | """Provides an OptionParser callback to unmarshal a JSON string.""" |
| 779 | setattr(parser.values, option.dest, json.loads(value)) |
| 780 | |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 781 | usage = ('%prog [options] [-- chromium-options]\n' |
qyearsley | 8b8d567 | 2014-08-23 21:19:14 | [diff] [blame] | 782 | 'Used by a try bot to run the bisection script using the parameters' |
qyearsley | 5867c6e | 2014-08-28 19:04:16 | [diff] [blame] | 783 | ' provided in the auto_bisect/bisect.cfg file.') |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 784 | parser = optparse.OptionParser(usage=usage) |
| 785 | parser.add_option('-w', '--working_directory', |
| 786 | type='str', |
| 787 | help='A working directory to supply to the bisection ' |
| 788 | 'script, which will use it as the location to checkout ' |
| 789 | 'a copy of the chromium depot.') |
[email protected] | e69075f | 2013-03-01 02:21:45 | [diff] [blame] | 790 | parser.add_option('-p', '--path_to_goma', |
| 791 | type='str', |
| 792 | help='Path to goma directory. If this is supplied, goma ' |
| 793 | 'builds will be enabled.') |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 794 | parser.add_option('--path_to_config', |
| 795 | type='str', |
| 796 | help='Path to the config file to use. If this is supplied, ' |
| 797 | 'the bisect script will use this to override the default ' |
| 798 | 'config file path. The script will attempt to load it ' |
| 799 | 'as a bisect config first, then a perf config.') |
[email protected] | c4d2f1d9 | 2013-12-04 06:35:15 | [diff] [blame] | 800 | parser.add_option('--extra_src', |
| 801 | type='str', |
| 802 | help='Path to extra source file. If this is supplied, ' |
| 803 | 'bisect script will use this to override default behavior.') |
[email protected] | 63c7f73e | 2013-10-28 20:02:08 | [diff] [blame] | 804 | parser.add_option('--dry_run', |
| 805 | action="store_true", |
| 806 | help='The script will perform the full bisect, but ' |
| 807 | 'without syncing, building, or running the performance ' |
| 808 | 'tests.') |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 809 | # This argument is passed by buildbot to supply build properties to the bisect |
| 810 | # script. Note: Don't change "--build-properties" property name. |
| 811 | parser.add_option('--build-properties', action='callback', |
| 812 | dest='build_properties', |
| 813 | callback=ConvertJson, type='string', |
| 814 | nargs=1, default={}, |
| 815 | help='build properties in JSON format') |
| 816 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 817 | return parser |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 818 | |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 819 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 820 | def main(): |
| 821 | """Entry point for run-bisect-perf-regression.py. |
| 822 | |
| 823 | Reads the config file, and then tries to either bisect a regression or |
| 824 | just run a performance test, depending on the particular config parameters |
| 825 | specified in the config file. |
| 826 | """ |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 827 | parser = _OptionParser() |
| 828 | opts, _ = parser.parse_args() |
| 829 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 830 | # Use the default config file path unless one was specified. |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 831 | config_path = BISECT_CONFIG_PATH |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 832 | if opts.path_to_config: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 833 | config_path = opts.path_to_config |
| 834 | config = _LoadConfigFile(config_path) |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 835 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 836 | # Check if the config is valid for running bisect job. |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 837 | config_is_valid = _ValidateBisectConfigFile(config) |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 838 | |
[email protected] | b5827602 | 2014-04-10 23:59:19 | [diff] [blame] | 839 | if config and config_is_valid: |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 840 | if not opts.working_directory: |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 841 | print 'Error: missing required parameter: --working_directory\n' |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 842 | parser.print_help() |
| 843 | return 1 |
| 844 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 845 | return _RunBisectionScript( |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 846 | config, opts.working_directory, opts.path_to_goma, opts.extra_src, |
| 847 | opts.dry_run) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 848 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 849 | # If it wasn't valid for running a bisect, then maybe the user wanted |
| 850 | # to run a perf test instead of a bisect job. Try reading any possible |
| 851 | # perf test config files. |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 852 | perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH] |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 853 | for current_perf_cfg_file in perf_cfg_files: |
| 854 | if opts.path_to_config: |
| 855 | path_to_perf_cfg = opts.path_to_config |
| 856 | else: |
| 857 | path_to_perf_cfg = os.path.join( |
| 858 | os.path.abspath(os.path.dirname(sys.argv[0])), |
| 859 | current_perf_cfg_file) |
[email protected] | 774a58c | 2013-10-22 00:10:24 | [diff] [blame] | 860 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 861 | config = _LoadConfigFile(path_to_perf_cfg) |
| 862 | config_is_valid = _ValidatePerfConfigFile(config) |
[email protected] | b3c0ee7 | 2013-10-22 21:58:38 | [diff] [blame] | 863 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 864 | if config and config_is_valid: |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 865 | return _SetupAndRunPerformanceTest(config, opts.path_to_goma) |
[email protected] | b3c0ee7 | 2013-10-22 21:58:38 | [diff] [blame] | 866 | |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 867 | # If there are no changes to config file, then check if the request is |
akuegel | 8ce00b7 | 2015-05-27 08:03:53 | [diff] [blame] | 868 | # from the commit queue, if so then run the modified Telemetry benchmarks for |
| 869 | # the patch. |
prasadv | 21b44a7 | 2015-05-29 16:21:41 | [diff] [blame] | 870 | if opts.build_properties.get('requester') in _COMMIT_QUEUE_USERS: |
prasadv | 155d7e5 | 2015-04-07 17:06:30 | [diff] [blame] | 871 | return _SetupAndRunPerformanceTest( |
| 872 | config={}, path_to_goma=opts.path_to_goma, is_cq_tryjob=True) |
| 873 | |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 874 | print ('Error: Could not load config file. Double check your changes to ' |
qyearsley | 62edbcf | 2014-09-26 01:19:34 | [diff] [blame] | 875 | 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n') |
[email protected] | 401c9dc | 2014-07-25 21:20:42 | [diff] [blame] | 876 | return 1 |
[email protected] | 8997afd | 2013-02-21 17:20:04 | [diff] [blame] | 877 | |
| 878 | |
| 879 | if __name__ == '__main__': |
| 880 | sys.exit(main()) |