blob: 27c0c7755d369556b5e2c33853ca9c9d5fc49b80 [file] [log] [blame]
[email protected]8997afd2013-02-21 17:20:041#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Run Performance Test Bisect Tool
7
qyearsley62edbcf2014-09-26 01:19:348This script is used by a try bot to run the bisect script with the parameters
9specified in the bisect config file. It checks out a copy of the depot in
10a subdirectory 'bisect' of the working directory provided, annd runs the
11bisect scrip there.
[email protected]8997afd2013-02-21 17:20:0412"""
13
prasadv155d7e52015-04-07 17:06:3014import json
[email protected]8997afd2013-02-21 17:20:0415import optparse
16import os
[email protected]92a0dddb2014-04-28 06:41:2217import platform
simonhatch87be98a2014-12-01 20:47:1118import re
prasadv155d7e52015-04-07 17:06:3019import shlex
[email protected]8997afd2013-02-21 17:20:0420import subprocess
21import sys
[email protected]d6f9b9ef2013-05-29 17:13:0122import traceback
[email protected]8997afd2013-02-21 17:20:0423
qyearsley62edbcf2014-09-26 01:19:3424from auto_bisect import bisect_perf_regression
[email protected]d4aad202014-07-10 01:27:0925from auto_bisect import bisect_utils
[email protected]15186ef2014-08-06 19:02:4526from auto_bisect import math_utils
simonhatch88ed7eb52014-12-02 19:01:2527from auto_bisect import source_control
[email protected]d4aad202014-07-10 01:27:0928
[email protected]2a75ef72013-06-06 17:39:0329CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
30CROS_IP_ENV = 'BISECT_CROS_IP'
qyearsley915e7272014-10-02 21:58:2731SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
qyearsley62edbcf2014-09-26 01:19:3432SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
33BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
34RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
35WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
36 SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
37BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
[email protected]774a58c2013-10-22 00:10:2438
prasadv155d7e52015-04-07 17:06:3039PERF_BENCHMARKS_PATH = 'tools/perf/benchmarks'
Bartosz Fabianowski85a823812015-04-16 10:27:5140PERF_MEASUREMENTS_PATH = 'tools/perf/measurements'
prasadv155d7e52015-04-07 17:06:3041BUILDBOT_BUILDERNAME = 'BUILDBOT_BUILDERNAME'
42BENCHMARKS_JSON_FILE = 'benchmarks.json'
qyearsley5867c6e2014-08-28 19:04:1643
akuegel8ce00b72015-05-27 08:03:5344# This is used to identify tryjobs triggered by the commit queue.
prasadv21b44a72015-05-29 16:21:4145_COMMIT_QUEUE_USERS = [
46 '5071639625-1lppvbtck1morgivc6sq4dul7klu27sd@developer.gserviceaccount.com',
47 '[email protected]']
akuegel8ce00b72015-05-27 08:03:5348
[email protected]774a58c2013-10-22 00:10:2449class Goma(object):
50
51 def __init__(self, path_to_goma):
52 self._abs_path_to_goma = None
53 self._abs_path_to_goma_file = None
[email protected]401c9dc2014-07-25 21:20:4254 if not path_to_goma:
55 return
56 self._abs_path_to_goma = os.path.abspath(path_to_goma)
57 filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
58 self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
[email protected]774a58c2013-10-22 00:10:2459
60 def __enter__(self):
[email protected]401c9dc2014-07-25 21:20:4261 if self._HasGomaPath():
[email protected]774a58c2013-10-22 00:10:2462 self._SetupAndStart()
63 return self
64
65 def __exit__(self, *_):
[email protected]401c9dc2014-07-25 21:20:4266 if self._HasGomaPath():
[email protected]774a58c2013-10-22 00:10:2467 self._Stop()
68
[email protected]401c9dc2014-07-25 21:20:4269 def _HasGomaPath(self):
[email protected]774a58c2013-10-22 00:10:2470 return bool(self._abs_path_to_goma)
71
[email protected]774a58c2013-10-22 00:10:2472 def _SetupEnvVars(self):
73 if os.name == 'nt':
74 os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
75 ' cl.exe')
76 os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
77 ' cl.exe')
78 else:
79 os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
80 os.environ['PATH']])
81
82 def _SetupAndStart(self):
[email protected]401c9dc2014-07-25 21:20:4283 """Sets up goma and launches it.
[email protected]774a58c2013-10-22 00:10:2484
85 Args:
86 path_to_goma: Path to goma directory.
87
88 Returns:
89 True if successful."""
90 self._SetupEnvVars()
91
92 # Sometimes goma is lingering around if something went bad on a previous
93 # run. Stop it before starting a new process. Can ignore the return code
94 # since it will return an error if it wasn't running.
95 self._Stop()
96
97 if subprocess.call([self._abs_path_to_goma_file, 'start']):
[email protected]401c9dc2014-07-25 21:20:4298 raise RuntimeError('Goma failed to start.')
[email protected]774a58c2013-10-22 00:10:2499
100 def _Stop(self):
101 subprocess.call([self._abs_path_to_goma_file, 'stop'])
102
103
[email protected]f7a2a6e2014-07-30 21:04:31104def _LoadConfigFile(config_file_path):
[email protected]5dcad272014-07-30 08:00:42105 """Attempts to load the specified config file as a module
106 and grab the global config dict.
[email protected]8997afd2013-02-21 17:20:04107
[email protected]cada4c22013-02-26 00:27:46108 Args:
[email protected]401c9dc2014-07-25 21:20:42109 config_file_path: Path to the config file.
[email protected]cada4c22013-02-26 00:27:46110
[email protected]8997afd2013-02-21 17:20:04111 Returns:
[email protected]401c9dc2014-07-25 21:20:42112 If successful, returns the config dict loaded from the file. If no
113 such dictionary could be loaded, returns the empty dictionary.
[email protected]8997afd2013-02-21 17:20:04114 """
115 try:
116 local_vars = {}
[email protected]401c9dc2014-07-25 21:20:42117 execfile(config_file_path, local_vars)
[email protected]8997afd2013-02-21 17:20:04118 return local_vars['config']
[email protected]401c9dc2014-07-25 21:20:42119 except Exception:
[email protected]d6f9b9ef2013-05-29 17:13:01120 print
121 traceback.print_exc()
122 print
[email protected]b3c0ee72013-10-22 21:58:38123 return {}
[email protected]8997afd2013-02-21 17:20:04124
125
robertocnb20bc1f2014-11-03 19:50:29126def _ValidateConfigFile(config_contents, required_parameters):
[email protected]b58276022014-04-10 23:59:19127 """Validates the config file contents, checking whether all values are
128 non-empty.
129
130 Args:
[email protected]401c9dc2014-07-25 21:20:42131 config_contents: A config dictionary.
robertocnb20bc1f2014-11-03 19:50:29132 required_parameters: A list of parameters to check for.
[email protected]b58276022014-04-10 23:59:19133
134 Returns:
135 True if valid.
136 """
robertocnb20bc1f2014-11-03 19:50:29137 for parameter in required_parameters:
[email protected]401c9dc2014-07-25 21:20:42138 if parameter not in config_contents:
139 return False
140 value = config_contents[parameter]
141 if not value or type(value) is not str:
142 return False
143 return True
[email protected]b58276022014-04-10 23:59:19144
145
146def _ValidatePerfConfigFile(config_contents):
[email protected]401c9dc2014-07-25 21:20:42147 """Validates the perf config file contents.
148
149 This is used when we're doing a perf try job, rather than a bisect.
150 The config file is called run-perf-test.cfg by default.
[email protected]b58276022014-04-10 23:59:19151
152 The parameters checked are the required parameters; any additional optional
153 parameters won't be checked and validation will still pass.
154
155 Args:
[email protected]401c9dc2014-07-25 21:20:42156 config_contents: A config dictionary.
[email protected]b58276022014-04-10 23:59:19157
158 Returns:
159 True if valid.
160 """
qyearsleybd0f02082015-01-28 00:27:02161 return _ValidateConfigFile(config_contents, required_parameters=['command'])
[email protected]b58276022014-04-10 23:59:19162
163
164def _ValidateBisectConfigFile(config_contents):
[email protected]401c9dc2014-07-25 21:20:42165 """Validates the bisect config file contents.
166
167 The parameters checked are the required parameters; any additional optional
168 parameters won't be checked and validation will still pass.
[email protected]b58276022014-04-10 23:59:19169
170 Args:
[email protected]401c9dc2014-07-25 21:20:42171 config_contents: A config dictionary.
[email protected]b58276022014-04-10 23:59:19172
173 Returns:
174 True if valid.
175 """
qyearsleybd0f02082015-01-28 00:27:02176 return _ValidateConfigFile(
177 config_contents,
178 required_parameters=['command', 'good_revision', 'bad_revision'])
[email protected]b58276022014-04-10 23:59:19179
180
[email protected]774a58c2013-10-22 00:10:24181def _OutputFailedResults(text_to_print):
182 bisect_utils.OutputAnnotationStepStart('Results - Failed')
183 print
184 print text_to_print
185 print
186 bisect_utils.OutputAnnotationStepClosed()
187
188
189def _CreateBisectOptionsFromConfig(config):
[email protected]6e6200c82014-07-16 23:16:30190 print config['command']
[email protected]774a58c2013-10-22 00:10:24191 opts_dict = {}
192 opts_dict['command'] = config['command']
[email protected]7b9ef3c2014-08-15 09:33:19193 opts_dict['metric'] = config.get('metric')
[email protected]774a58c2013-10-22 00:10:24194
195 if config['repeat_count']:
196 opts_dict['repeat_test_count'] = int(config['repeat_count'])
197
198 if config['truncate_percent']:
199 opts_dict['truncate_percent'] = int(config['truncate_percent'])
200
201 if config['max_time_minutes']:
202 opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
203
204 if config.has_key('use_goma'):
205 opts_dict['use_goma'] = config['use_goma']
[email protected]7a960ad2014-07-08 00:49:57206 if config.has_key('goma_dir'):
207 opts_dict['goma_dir'] = config['goma_dir']
[email protected]774a58c2013-10-22 00:10:24208
robertocnfdd5b1c02014-10-17 21:55:29209 if config.has_key('improvement_direction'):
210 opts_dict['improvement_direction'] = int(config['improvement_direction'])
211
prasadv5557c672015-02-10 20:07:12212 if config.has_key('target_arch'):
213 opts_dict['target_arch'] = config['target_arch']
214
robertocnb20bc1f2014-11-03 19:50:29215 if config.has_key('bug_id') and str(config['bug_id']).isdigit():
216 opts_dict['bug_id'] = config['bug_id']
217
[email protected]774a58c2013-10-22 00:10:24218 opts_dict['build_preference'] = 'ninja'
219 opts_dict['output_buildbot_annotations'] = True
220
221 if '--browser=cros' in config['command']:
222 opts_dict['target_platform'] = 'cros'
223
224 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
225 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
226 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
227 else:
qyearsley8b8d5672014-08-23 21:19:14228 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
[email protected]774a58c2013-10-22 00:10:24229 'BISECT_CROS_BOARD undefined.')
230 elif 'android' in config['command']:
prasadv1d2dd722015-07-17 00:45:22231 # TODO (prasadv): Remove android-chrome-shell check once we confirm that
232 # there are no pending bisect jobs with this in command.
233 if any(item in config['command']
234 for item in ['android-chrome-shell', 'android-chromium']):
[email protected]6e6200c82014-07-16 23:16:30235 opts_dict['target_platform'] = 'android'
236 elif 'android-chrome' in config['command']:
[email protected]c4d2f1d92013-12-04 06:35:15237 opts_dict['target_platform'] = 'android-chrome'
238 else:
239 opts_dict['target_platform'] = 'android'
[email protected]774a58c2013-10-22 00:10:24240
qyearsley62edbcf2014-09-26 01:19:34241 return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
[email protected]774a58c2013-10-22 00:10:24242
243
simonhatch87be98a2014-12-01 20:47:11244def _ParseCloudLinksFromOutput(output):
245 html_results_pattern = re.compile(
246 r'\s(?P<VALUES>http://storage.googleapis.com/' +
247 'chromium-telemetry/html-results/results-[a-z0-9-_]+)\s',
248 re.MULTILINE)
249 profiler_pattern = re.compile(
250 r'\s(?P<VALUES>https://console.developers.google.com/' +
251 'm/cloudstorage/b/[a-z-]+/o/profiler-[a-z0-9-_.]+)\s',
252 re.MULTILINE)
simonhatcha9b78ba2014-11-14 15:00:00253
simonhatch87be98a2014-12-01 20:47:11254 results = {
255 'html-results': html_results_pattern.findall(output),
256 'profiler': profiler_pattern.findall(output),
257 }
258
259 return results
simonhatcha9b78ba2014-11-14 15:00:00260
261
simonhatch37540432014-12-11 15:25:47262def _ParseAndOutputCloudLinks(
263 results_without_patch, results_with_patch, annotations_dict):
simonhatch87be98a2014-12-01 20:47:11264 cloud_links_without_patch = _ParseCloudLinksFromOutput(
265 results_without_patch[2])
266 cloud_links_with_patch = _ParseCloudLinksFromOutput(
267 results_with_patch[2])
qyearsley35712272014-11-22 01:11:37268
simonhatch87be98a2014-12-01 20:47:11269 cloud_file_link = (cloud_links_without_patch['html-results'][0]
270 if cloud_links_without_patch['html-results'] else '')
[email protected]774a58c2013-10-22 00:10:24271
simonhatch87be98a2014-12-01 20:47:11272 profiler_file_links_with_patch = cloud_links_with_patch['profiler']
273 profiler_file_links_without_patch = cloud_links_without_patch['profiler']
simonhatcha9b78ba2014-11-14 15:00:00274
[email protected]774a58c2013-10-22 00:10:24275 # Calculate the % difference in the means of the 2 runs.
[email protected]7b9ef3c2014-08-15 09:33:19276 percent_diff_in_means = None
277 std_err = None
278 if (results_with_patch[0].has_key('mean') and
279 results_with_patch[0].has_key('values')):
280 percent_diff_in_means = (results_with_patch[0]['mean'] /
281 max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
282 std_err = math_utils.PooledStandardError(
283 [results_with_patch[0]['values'], results_without_patch[0]['values']])
[email protected]774a58c2013-10-22 00:10:24284
[email protected]7b9ef3c2014-08-15 09:33:19285 if percent_diff_in_means is not None and std_err is not None:
286 bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
287 (percent_diff_in_means, std_err))
288 print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
289 'Std. Error'.center(20, ' '))
290 print ' %s %s %s' % ('Patch'.center(10, ' '),
291 ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
292 ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
293 print ' %s %s %s' % ('No Patch'.center(10, ' '),
294 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
295 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
296 if cloud_file_link:
297 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
298 bisect_utils.OutputAnnotationStepClosed()
299 elif cloud_file_link:
[email protected]774a58c2013-10-22 00:10:24300 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
[email protected]774a58c2013-10-22 00:10:24301
simonhatcha9b78ba2014-11-14 15:00:00302 if profiler_file_links_with_patch and profiler_file_links_without_patch:
303 for i in xrange(len(profiler_file_links_with_patch)):
304 bisect_utils.OutputAnnotationStepLink(
simonhatch37540432014-12-11 15:25:47305 '%s[%d]' % (annotations_dict.get('profiler_link1'), i),
simonhatcha9b78ba2014-11-14 15:00:00306 profiler_file_links_with_patch[i])
307 for i in xrange(len(profiler_file_links_without_patch)):
308 bisect_utils.OutputAnnotationStepLink(
simonhatch37540432014-12-11 15:25:47309 '%s[%d]' % (annotations_dict.get('profiler_link2'), i),
simonhatcha9b78ba2014-11-14 15:00:00310 profiler_file_links_without_patch[i])
311
[email protected]774a58c2013-10-22 00:10:24312
simonhatch88ed7eb52014-12-02 19:01:25313def _ResolveRevisionsFromConfig(config):
314 if not 'good_revision' in config and not 'bad_revision' in config:
315 return (None, None)
316
317 bad_revision = source_control.ResolveToRevision(
318 config['bad_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, 100)
319 if not bad_revision:
320 raise RuntimeError('Failed to resolve [%s] to git hash.',
321 config['bad_revision'])
322 good_revision = source_control.ResolveToRevision(
323 config['good_revision'], 'chromium', bisect_utils.DEPOT_DEPS_NAME, -100)
324 if not good_revision:
325 raise RuntimeError('Failed to resolve [%s] to git hash.',
326 config['good_revision'])
327
328 return (good_revision, bad_revision)
329
330
331def _GetStepAnnotationStringsDict(config):
332 if 'good_revision' in config and 'bad_revision' in config:
333 return {
334 'build1': 'Building [%s]' % config['good_revision'],
335 'build2': 'Building [%s]' % config['bad_revision'],
336 'run1': 'Running [%s]' % config['good_revision'],
337 'run2': 'Running [%s]' % config['bad_revision'],
simonhatchd674bae2014-12-08 21:00:35338 'sync1': 'Syncing [%s]' % config['good_revision'],
339 'sync2': 'Syncing [%s]' % config['bad_revision'],
simonhatch88ed7eb52014-12-02 19:01:25340 'results_label1': config['good_revision'],
341 'results_label2': config['bad_revision'],
simonhatch37540432014-12-11 15:25:47342 'profiler_link1': 'Profiler Data - %s' % config['good_revision'],
343 'profiler_link2': 'Profiler Data - %s' % config['bad_revision'],
simonhatch88ed7eb52014-12-02 19:01:25344 }
345 else:
346 return {
347 'build1': 'Building With Patch',
348 'build2': 'Building Without Patch',
349 'run1': 'Running With Patch',
350 'run2': 'Running Without Patch',
351 'results_label1': 'Patch',
352 'results_label2': 'ToT',
simonhatch37540432014-12-11 15:25:47353 'profiler_link1': 'With Patch - Profiler Data',
354 'profiler_link2': 'Without Patch - Profiler Data',
simonhatch88ed7eb52014-12-02 19:01:25355 }
356
357
simonhatchd674bae2014-12-08 21:00:35358def _RunBuildStepForPerformanceTest(bisect_instance,
359 build_string,
360 sync_string,
361 revision):
simonhatch88ed7eb52014-12-02 19:01:25362 if revision:
simonhatchd674bae2014-12-08 21:00:35363 bisect_utils.OutputAnnotationStepStart(sync_string)
simonhatch88ed7eb52014-12-02 19:01:25364 if not source_control.SyncToRevision(revision, 'gclient'):
simonhatchd674bae2014-12-08 21:00:35365 raise RuntimeError('Failed [%s].' % sync_string)
simonhatch88ed7eb52014-12-02 19:01:25366 bisect_utils.OutputAnnotationStepClosed()
367
368 bisect_utils.OutputAnnotationStepStart(build_string)
369
370 if bisect_utils.RunGClient(['runhooks']):
371 raise RuntimeError('Failed to run gclient runhooks')
372
373 if not bisect_instance.ObtainBuild('chromium'):
374 raise RuntimeError('Patched version failed to build.')
375
376 bisect_utils.OutputAnnotationStepClosed()
377
378
379def _RunCommandStepForPerformanceTest(bisect_instance,
380 opts,
381 reset_on_first_run,
382 upload_on_last_run,
383 results_label,
384 run_string):
385 bisect_utils.OutputAnnotationStepStart(run_string)
386
387 results = bisect_instance.RunPerformanceTestAndParseResults(
388 opts.command,
389 opts.metric,
390 reset_on_first_run=reset_on_first_run,
391 upload_on_last_run=upload_on_last_run,
simonhatchbc1d3842015-03-13 19:18:24392 results_label=results_label,
393 allow_flakes=False)
simonhatch88ed7eb52014-12-02 19:01:25394
395 if results[1]:
396 raise RuntimeError('Patched version failed to run performance test.')
397
398 bisect_utils.OutputAnnotationStepClosed()
399
400 return results
401
402
403def _RunPerformanceTest(config):
404 """Runs a performance test with and without the current patch.
405
406 Args:
407 config: Contents of the config file, a dictionary.
408
409 Attempts to build and run the current revision with and without the
410 current patch, with the parameters passed in.
411 """
412 # Bisect script expects to be run from the src directory
413 os.chdir(SRC_DIR)
414
415 opts = _CreateBisectOptionsFromConfig(config)
416 revisions = _ResolveRevisionsFromConfig(config)
417 annotations_dict = _GetStepAnnotationStringsDict(config)
418 b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
419
simonhatchd674bae2014-12-08 21:00:35420 _RunBuildStepForPerformanceTest(b,
421 annotations_dict.get('build1'),
422 annotations_dict.get('sync1'),
423 revisions[0])
simonhatch88ed7eb52014-12-02 19:01:25424
425 results_with_patch = _RunCommandStepForPerformanceTest(
426 b, opts, True, True, annotations_dict['results_label1'],
427 annotations_dict['run1'])
428
429 bisect_utils.OutputAnnotationStepStart('Reverting Patch')
430 # TODO: When this is re-written to recipes, this should use bot_update's
431 # revert mechanism to fully revert the client. But for now, since we know that
432 # the perf try bot currently only supports src/ and src/third_party/WebKit, we
433 # simply reset those two directories.
434 bisect_utils.CheckRunGit(['reset', '--hard'])
435 bisect_utils.CheckRunGit(['reset', '--hard'],
436 os.path.join('third_party', 'WebKit'))
437 bisect_utils.OutputAnnotationStepClosed()
438
simonhatchd674bae2014-12-08 21:00:35439 _RunBuildStepForPerformanceTest(b,
440 annotations_dict.get('build2'),
441 annotations_dict.get('sync2'),
442 revisions[1])
simonhatch88ed7eb52014-12-02 19:01:25443
444 results_without_patch = _RunCommandStepForPerformanceTest(
445 b, opts, False, True, annotations_dict['results_label2'],
sullivandbb5e442014-12-05 15:07:48446 annotations_dict['run2'])
simonhatch88ed7eb52014-12-02 19:01:25447
448 # Find the link to the cloud stored results file.
simonhatch37540432014-12-11 15:25:47449 _ParseAndOutputCloudLinks(
450 results_without_patch, results_with_patch, annotations_dict)
simonhatch88ed7eb52014-12-02 19:01:25451
452
prasadv155d7e52015-04-07 17:06:30453def _SetupAndRunPerformanceTest(config, path_to_goma, is_cq_tryjob=False):
[email protected]774a58c2013-10-22 00:10:24454 """Attempts to build and run the current revision with and without the
455 current patch, with the parameters passed in.
456
457 Args:
458 config: The config read from run-perf-test.cfg.
[email protected]774a58c2013-10-22 00:10:24459 path_to_goma: Path to goma directory.
prasadv155d7e52015-04-07 17:06:30460 is_cq_tryjob: Whether or not the try job was initiated by commit queue.
[email protected]774a58c2013-10-22 00:10:24461
462 Returns:
qyearsley62edbcf2014-09-26 01:19:34463 An exit code: 0 on success, otherwise 1.
[email protected]774a58c2013-10-22 00:10:24464 """
sullivan318e47b7e2014-09-19 17:16:22465 if platform.release() == 'XP':
466 print 'Windows XP is not supported for perf try jobs because it lacks '
467 print 'goma support. Please refer to crbug.com/330900.'
468 return 1
[email protected]774a58c2013-10-22 00:10:24469 try:
[email protected]401c9dc2014-07-25 21:20:42470 with Goma(path_to_goma) as _:
[email protected]774a58c2013-10-22 00:10:24471 config['use_goma'] = bool(path_to_goma)
prasadv43d1aaa2014-08-28 00:39:14472 if config['use_goma']:
473 config['goma_dir'] = os.path.abspath(path_to_goma)
prasadv155d7e52015-04-07 17:06:30474 if not is_cq_tryjob:
475 _RunPerformanceTest(config)
476 else:
477 return _RunBenchmarksForCommitQueue(config)
[email protected]774a58c2013-10-22 00:10:24478 return 0
479 except RuntimeError, e:
simonhatchbc1d3842015-03-13 19:18:24480 bisect_utils.OutputAnnotationStepFailure()
[email protected]774a58c2013-10-22 00:10:24481 bisect_utils.OutputAnnotationStepClosed()
482 _OutputFailedResults('Error: %s' % e.message)
483 return 1
484
485
[email protected]401c9dc2014-07-25 21:20:42486def _RunBisectionScript(
qyearsley62edbcf2014-09-26 01:19:34487 config, working_directory, path_to_goma, path_to_extra_src, dry_run):
488 """Attempts to execute the bisect script with the given parameters.
[email protected]8997afd2013-02-21 17:20:04489
490 Args:
491 config: A dict containing the parameters to pass to the script.
qyearsley62edbcf2014-09-26 01:19:34492 working_directory: A working directory to provide to the bisect script,
493 where it will store it's own copy of the depot.
[email protected]e69075f2013-03-01 02:21:45494 path_to_goma: Path to goma directory.
[email protected]c4d2f1d92013-12-04 06:35:15495 path_to_extra_src: Path to extra source file.
[email protected]63c7f73e2013-10-28 20:02:08496 dry_run: Do a dry run, skipping sync, build, and performance testing steps.
[email protected]8997afd2013-02-21 17:20:04497
498 Returns:
[email protected]401c9dc2014-07-25 21:20:42499 An exit status code: 0 on success, otherwise 1.
[email protected]8997afd2013-02-21 17:20:04500 """
[email protected]401c9dc2014-07-25 21:20:42501 _PrintConfigStep(config)
[email protected]8997afd2013-02-21 17:20:04502
qyearsley35712272014-11-22 01:11:37503 # Construct the basic command with all necessary arguments.
504 cmd = [
505 'python',
506 os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
507 '--command', config['command'],
508 '--good_revision', config['good_revision'],
509 '--bad_revision', config['bad_revision'],
qyearsley35712272014-11-22 01:11:37510 '--working_directory', working_directory,
511 '--output_buildbot_annotations'
512 ]
[email protected]8997afd2013-02-21 17:20:04513
qyearsley35712272014-11-22 01:11:37514 # Add flags for any optional config parameters if given in the config.
515 options = [
qyearsleybd0f02082015-01-28 00:27:02516 ('metric', '--metric'),
qyearsley35712272014-11-22 01:11:37517 ('repeat_count', '--repeat_test_count'),
518 ('truncate_percent', '--truncate_percent'),
519 ('max_time_minutes', '--max_time_minutes'),
520 ('bisect_mode', '--bisect_mode'),
521 ('improvement_direction', '--improvement_direction'),
522 ('bug_id', '--bug_id'),
qyearsley38a6e5a2015-01-12 17:59:24523 ('builder_type', '--builder_type'),
prasadv5557c672015-02-10 20:07:12524 ('target_arch', '--target_arch'),
qyearsley35712272014-11-22 01:11:37525 ]
526 for config_key, flag in options:
527 if config.has_key(config_key):
528 cmd.extend([flag, config[config_key]])
robertocnb20bc1f2014-11-03 19:50:29529
[email protected]bda7af42013-06-10 23:49:42530 cmd.extend(['--build_preference', 'ninja'])
[email protected]2deb10d2013-03-15 19:29:50531
qyearsley35712272014-11-22 01:11:37532 # Possibly set the target platform name based on the browser name in a
533 # Telemetry command.
prasadv1d2dd722015-07-17 00:45:22534 # TODO (prasadv): Remove android-chrome-shell check once we confirm there are
535 # no pending bisect jobs with this in command.
536 if any(item in config['command']
537 for item in ['android-chrome-shell', 'android-chromium']):
qyearsley35712272014-11-22 01:11:37538 cmd.extend(['--target_platform', 'android'])
539 elif 'android-chrome' in config['command']:
540 cmd.extend(['--target_platform', 'android-chrome'])
541 elif 'android' in config['command']:
542 cmd.extend(['--target_platform', 'android'])
[email protected]bda7af42013-06-10 23:49:42543
[email protected]e69075f2013-03-01 02:21:45544 if path_to_goma:
[email protected]401c9dc2014-07-25 21:20:42545 # For Windows XP platforms, goma service is not supported.
[email protected]92a0dddb2014-04-28 06:41:22546 # Moreover we don't compile chrome when gs_bucket flag is set instead
[email protected]401c9dc2014-07-25 21:20:42547 # use builds archives, therefore ignore goma service for Windows XP.
548 # See http://crbug.com/330900.
qyearsley35712272014-11-22 01:11:37549 if platform.release() == 'XP':
[email protected]92a0dddb2014-04-28 06:41:22550 print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
551 'on Windows XP platform. Please refer to crbug.com/330900.')
552 path_to_goma = None
[email protected]e69075f2013-03-01 02:21:45553 cmd.append('--use_goma')
simonhatch88572ab2014-12-23 18:35:06554 cmd.append('--goma_dir')
555 cmd.append(os.path.abspath(path_to_goma))
[email protected]e69075f2013-03-01 02:21:45556
[email protected]c4d2f1d92013-12-04 06:35:15557 if path_to_extra_src:
558 cmd.extend(['--extra_src', path_to_extra_src])
559
[email protected]63c7f73e2013-10-28 20:02:08560 if dry_run:
qyearsley35712272014-11-22 01:11:37561 cmd.extend([
562 '--debug_ignore_build',
563 '--debug_ignore_sync',
564 '--debug_ignore_perf_test'
565 ])
566
[email protected]2deb10d2013-03-15 19:29:50567 cmd = [str(c) for c in cmd]
568
[email protected]401c9dc2014-07-25 21:20:42569 with Goma(path_to_goma) as _:
[email protected]774a58c2013-10-22 00:10:24570 return_code = subprocess.call(cmd)
[email protected]e69075f2013-03-01 02:21:45571
[email protected]8997afd2013-02-21 17:20:04572 if return_code:
qyearsley62edbcf2014-09-26 01:19:34573 print ('Error: bisect_perf_regression.py returned with error %d\n'
[email protected]401c9dc2014-07-25 21:20:42574 % return_code)
[email protected]8997afd2013-02-21 17:20:04575
576 return return_code
577
578
[email protected]401c9dc2014-07-25 21:20:42579def _PrintConfigStep(config):
580 """Prints out the given config, along with Buildbot annotations."""
581 bisect_utils.OutputAnnotationStepStart('Config')
582 print
583 for k, v in config.iteritems():
584 print ' %s : %s' % (k, v)
585 print
586 bisect_utils.OutputAnnotationStepClosed()
[email protected]8997afd2013-02-21 17:20:04587
[email protected]401c9dc2014-07-25 21:20:42588
prasadv155d7e52015-04-07 17:06:30589def _GetBrowserType(bot_platform):
590 """Gets the browser type to be used in the run benchmark command."""
591 if bot_platform == 'android':
prasadv1d2dd722015-07-17 00:45:22592 return 'android-chromium'
prasadv155d7e52015-04-07 17:06:30593 elif 'x64' in bot_platform:
594 return 'release_x64'
595
596 return 'release'
597
598
599
600def _GuessTelemetryTestCommand(bot_platform, test_name=None):
601 """Creates a Telemetry benchmark command based on bot and test name."""
602 command = []
603 # On Windows, Python scripts should be prefixed with the python command.
604 if bot_platform == 'win':
605 command.append('python')
606 command.append('tools/perf/run_benchmark')
607 command.append('-v')
608 command.append('--browser=%s' % _GetBrowserType(bot_platform))
609 if test_name:
610 command.append(test_name)
611
612 return ' '.join(command)
613
614
615def _GetConfigBasedOnPlatform(config, bot_name, test_name):
616 """Generates required options to create BisectPerformanceMetrics instance."""
617 opts_dict = {
618 'command': _GuessTelemetryTestCommand(bot_name, test_name),
619 'target_arch': 'x64' if 'x64' in bot_name else 'ia32',
620 'build_preference': 'ninja',
621 'output_buildbot_annotations': True,
622 'repeat_test_count': 1,
623 'bisect_mode': bisect_utils.BISECT_MODE_RETURN_CODE,
624 }
625
626 if 'use_goma' in config:
627 opts_dict['use_goma'] = config['use_goma']
628 if 'goma_dir' in config:
629 opts_dict['goma_dir'] = config['goma_dir']
prasadv1d2dd722015-07-17 00:45:22630 # TODO (prasadv): Remove android-chrome-shell check once we confirm there are
631 # no pending bisect jobs with this in command.
632 if any(item in opts_dict['command']
633 for item in ['android-chrome-shell', 'android-chromium']):
prasadv155d7e52015-04-07 17:06:30634 opts_dict['target_platform'] = 'android'
635
636 return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
637
638
639def _GetModifiedFilesFromPatch(cwd=None):
640 """Gets list of files modified in the current patch."""
641 log_output = bisect_utils.CheckRunGit(
642 ['diff', '--no-ext-diff', '--name-only', 'HEAD~1'], cwd=cwd)
643 modified_files = log_output.split()
644 return modified_files
645
646
647def _GetAffectedBenchmarkModuleNames():
648 """Gets list of modified benchmark files under tools/perf/benchmarks."""
649 all_affected_files = _GetModifiedFilesFromPatch()
650 modified_benchmarks = []
651 for affected_file in all_affected_files:
Bartosz Fabianowski85a823812015-04-16 10:27:51652 if (affected_file.startswith(PERF_BENCHMARKS_PATH) or
653 affected_file.startswith(PERF_MEASUREMENTS_PATH)):
prasadv155d7e52015-04-07 17:06:30654 benchmark = os.path.basename(os.path.splitext(affected_file)[0])
655 modified_benchmarks.append(benchmark)
656 return modified_benchmarks
657
658
659def _ListAvailableBenchmarks(bot_platform):
660 """Gets all available benchmarks names as a list."""
661 browser_type = _GetBrowserType(bot_platform)
662 if os.path.exists(BENCHMARKS_JSON_FILE):
663 os.remove(BENCHMARKS_JSON_FILE)
664 command = []
665 if 'win' in bot_platform:
666 command.append('python')
667 command.append('tools/perf/run_benchmark')
668 command.extend([
669 'list',
670 '--browser',
671 browser_type,
672 '--json-output',
673 BENCHMARKS_JSON_FILE])
674 try:
675 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(
676 command=command, cwd=SRC_DIR)
677 if return_code:
678 raise RuntimeError('Something went wrong while listing benchmarks. '
679 'Please review the command line: %s.\nERROR: [%s]' %
680 (' '.join(command), output))
681 with open(BENCHMARKS_JSON_FILE) as tests_json:
682 tests_data = json.load(tests_json)
683 if tests_data.get('steps'):
684 return tests_data.get('steps').keys()
685 finally:
686 try:
687 if os.path.exists(BENCHMARKS_JSON_FILE):
688 os.remove(BENCHMARKS_JSON_FILE)
689 except OSError as e:
690 if e.errno != errno.ENOENT:
691 raise
692 return None
693
694
695def _OutputOverallResults(results):
696 """Creates results step and prints results on buildbot job."""
697 test_status = all(current_value == True for current_value in results.values())
698 bisect_utils.OutputAnnotationStepStart(
699 'Results - %s' % ('Passed' if test_status else 'Failed'))
700 print
701 print 'Results of benchmarks:'
702 print
703 for benchmark, result in results.iteritems():
704 print '%s: %s' % (benchmark, 'Passed' if result else 'Failed')
705 if not test_status:
706 bisect_utils.OutputAnnotationStepFailure()
707 bisect_utils.OutputAnnotationStepClosed()
708 # Returns 0 for success and 1 for failure.
709 return 0 if test_status else 1
710
711
712def _RunBenchmark(bisect_instance, opts, bot_name, benchmark_name):
713 """Runs a Telemetry benchmark."""
714 bisect_utils.OutputAnnotationStepStart(benchmark_name)
715 command_to_run = _GuessTelemetryTestCommand(bot_name, benchmark_name)
716 args = shlex.split(command_to_run, posix=not bisect_utils.IsWindowsHost())
717 output, return_code = bisect_utils.RunProcessAndRetrieveOutput(args, SRC_DIR)
718 # A value other than 0 indicates that the test couldn't be run, and results
719 # should also include an error message.
720 if return_code:
721 print ('Error: Something went wrong running the benchmark: %s.'
722 'Please review the command line:%s\n\n%s' %
723 (benchmark_name, command_to_run, output))
724 bisect_utils.OutputAnnotationStepFailure()
725 print output
726 bisect_utils.OutputAnnotationStepClosed()
727 # results[1] contains the return code from subprocess that executes test
728 # command, On successful test run it contains 0 otherwise any non-zero value.
729 return return_code == 0
730
731
732def _RunBenchmarksForCommitQueue(config):
733 """Runs Telemetry benchmark for the commit queue."""
734 os.chdir(SRC_DIR)
735 # To determine the bot platform by reading buildbot name from environment
736 # variable.
737 bot_name = os.environ.get(BUILDBOT_BUILDERNAME)
738 if not bot_name:
739 bot_name = sys.platform
740 bot_name = bot_name.split('_')[0]
741
742 affected_benchmarks = _GetAffectedBenchmarkModuleNames()
743 # Abort if there are no changes to benchmark any existing benchmark files.
744 if not affected_benchmarks:
745 bisect_utils.OutputAnnotationStepStart('Results')
746 print
747 print ('There are no modification to Telemetry benchmarks,'
748 ' aborting the try job.')
749 bisect_utils.OutputAnnotationStepClosed()
750 return 0
751
752 # Bisect script expects to be run from the src directory
753 # Gets required options inorder to create BisectPerformanceMetrics instance.
754 # Since command is a required arg in BisectPerformanceMetrics, we just create
755 # a dummy command for now.
756 opts = _GetConfigBasedOnPlatform(config, bot_name, test_name='')
757 annotations_dict = _GetStepAnnotationStringsDict(config)
758 b = bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
759 _RunBuildStepForPerformanceTest(b,
760 annotations_dict.get('build1'),
761 annotations_dict.get('sync1'),
762 None)
763 available_benchmarks = _ListAvailableBenchmarks(bot_name)
764 overall_results = {}
765 for affected_benchmark in affected_benchmarks:
766 for benchmark in available_benchmarks:
767 if (benchmark.startswith(affected_benchmark) and
768 not benchmark.endswith('reference')):
769 overall_results[benchmark] = _RunBenchmark(b, opts, bot_name, benchmark)
770
771 return _OutputOverallResults(overall_results)
772
773
[email protected]401c9dc2014-07-25 21:20:42774def _OptionParser():
775 """Returns the options parser for run-bisect-perf-regression.py."""
prasadv155d7e52015-04-07 17:06:30776
777 def ConvertJson(option, _, value, parser):
778 """Provides an OptionParser callback to unmarshal a JSON string."""
779 setattr(parser.values, option.dest, json.loads(value))
780
[email protected]8997afd2013-02-21 17:20:04781 usage = ('%prog [options] [-- chromium-options]\n'
qyearsley8b8d5672014-08-23 21:19:14782 'Used by a try bot to run the bisection script using the parameters'
qyearsley5867c6e2014-08-28 19:04:16783 ' provided in the auto_bisect/bisect.cfg file.')
[email protected]8997afd2013-02-21 17:20:04784 parser = optparse.OptionParser(usage=usage)
785 parser.add_option('-w', '--working_directory',
786 type='str',
787 help='A working directory to supply to the bisection '
788 'script, which will use it as the location to checkout '
789 'a copy of the chromium depot.')
[email protected]e69075f2013-03-01 02:21:45790 parser.add_option('-p', '--path_to_goma',
791 type='str',
792 help='Path to goma directory. If this is supplied, goma '
793 'builds will be enabled.')
[email protected]b58276022014-04-10 23:59:19794 parser.add_option('--path_to_config',
795 type='str',
796 help='Path to the config file to use. If this is supplied, '
797 'the bisect script will use this to override the default '
798 'config file path. The script will attempt to load it '
799 'as a bisect config first, then a perf config.')
[email protected]c4d2f1d92013-12-04 06:35:15800 parser.add_option('--extra_src',
801 type='str',
802 help='Path to extra source file. If this is supplied, '
803 'bisect script will use this to override default behavior.')
[email protected]63c7f73e2013-10-28 20:02:08804 parser.add_option('--dry_run',
805 action="store_true",
806 help='The script will perform the full bisect, but '
807 'without syncing, building, or running the performance '
808 'tests.')
prasadv155d7e52015-04-07 17:06:30809 # This argument is passed by buildbot to supply build properties to the bisect
810 # script. Note: Don't change "--build-properties" property name.
811 parser.add_option('--build-properties', action='callback',
812 dest='build_properties',
813 callback=ConvertJson, type='string',
814 nargs=1, default={},
815 help='build properties in JSON format')
816
[email protected]401c9dc2014-07-25 21:20:42817 return parser
[email protected]8997afd2013-02-21 17:20:04818
[email protected]b58276022014-04-10 23:59:19819
[email protected]401c9dc2014-07-25 21:20:42820def main():
821 """Entry point for run-bisect-perf-regression.py.
822
823 Reads the config file, and then tries to either bisect a regression or
824 just run a performance test, depending on the particular config parameters
825 specified in the config file.
826 """
[email protected]401c9dc2014-07-25 21:20:42827 parser = _OptionParser()
828 opts, _ = parser.parse_args()
829
[email protected]401c9dc2014-07-25 21:20:42830 # Use the default config file path unless one was specified.
qyearsley62edbcf2014-09-26 01:19:34831 config_path = BISECT_CONFIG_PATH
[email protected]b58276022014-04-10 23:59:19832 if opts.path_to_config:
[email protected]401c9dc2014-07-25 21:20:42833 config_path = opts.path_to_config
834 config = _LoadConfigFile(config_path)
[email protected]8997afd2013-02-21 17:20:04835
[email protected]401c9dc2014-07-25 21:20:42836 # Check if the config is valid for running bisect job.
[email protected]b58276022014-04-10 23:59:19837 config_is_valid = _ValidateBisectConfigFile(config)
[email protected]8997afd2013-02-21 17:20:04838
[email protected]b58276022014-04-10 23:59:19839 if config and config_is_valid:
[email protected]774a58c2013-10-22 00:10:24840 if not opts.working_directory:
[email protected]401c9dc2014-07-25 21:20:42841 print 'Error: missing required parameter: --working_directory\n'
[email protected]774a58c2013-10-22 00:10:24842 parser.print_help()
843 return 1
844
[email protected]401c9dc2014-07-25 21:20:42845 return _RunBisectionScript(
qyearsley62edbcf2014-09-26 01:19:34846 config, opts.working_directory, opts.path_to_goma, opts.extra_src,
847 opts.dry_run)
[email protected]774a58c2013-10-22 00:10:24848
[email protected]401c9dc2014-07-25 21:20:42849 # If it wasn't valid for running a bisect, then maybe the user wanted
850 # to run a perf test instead of a bisect job. Try reading any possible
851 # perf test config files.
qyearsley62edbcf2014-09-26 01:19:34852 perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
[email protected]401c9dc2014-07-25 21:20:42853 for current_perf_cfg_file in perf_cfg_files:
854 if opts.path_to_config:
855 path_to_perf_cfg = opts.path_to_config
856 else:
857 path_to_perf_cfg = os.path.join(
858 os.path.abspath(os.path.dirname(sys.argv[0])),
859 current_perf_cfg_file)
[email protected]774a58c2013-10-22 00:10:24860
[email protected]401c9dc2014-07-25 21:20:42861 config = _LoadConfigFile(path_to_perf_cfg)
862 config_is_valid = _ValidatePerfConfigFile(config)
[email protected]b3c0ee72013-10-22 21:58:38863
[email protected]401c9dc2014-07-25 21:20:42864 if config and config_is_valid:
qyearsley62edbcf2014-09-26 01:19:34865 return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
[email protected]b3c0ee72013-10-22 21:58:38866
prasadv155d7e52015-04-07 17:06:30867 # If there are no changes to config file, then check if the request is
akuegel8ce00b72015-05-27 08:03:53868 # from the commit queue, if so then run the modified Telemetry benchmarks for
869 # the patch.
prasadv21b44a72015-05-29 16:21:41870 if opts.build_properties.get('requester') in _COMMIT_QUEUE_USERS:
prasadv155d7e52015-04-07 17:06:30871 return _SetupAndRunPerformanceTest(
872 config={}, path_to_goma=opts.path_to_goma, is_cq_tryjob=True)
873
[email protected]401c9dc2014-07-25 21:20:42874 print ('Error: Could not load config file. Double check your changes to '
qyearsley62edbcf2014-09-26 01:19:34875 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
[email protected]401c9dc2014-07-25 21:20:42876 return 1
[email protected]8997afd2013-02-21 17:20:04877
878
879if __name__ == '__main__':
880 sys.exit(main())