blob: ac6d662a88934c8279b6c22387feaaf0ddf2b379 [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
9import collections
[email protected]f7148dd42013-08-20 14:24:5710import logging
[email protected]fbe29322013-07-09 09:03:2611import optparse
12import os
[email protected]6bc1bda22013-07-19 22:08:3713import shutil
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
[email protected]fbe29322013-07-09 09:03:2617
[email protected]f7148dd42013-08-20 14:24:5718from pylib import android_commands
[email protected]fbe29322013-07-09 09:03:2619from pylib import constants
[email protected]c0662e092013-11-12 11:51:2520from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2621from pylib import ports
22from pylib.base import base_test_result
[email protected]6bc1bda22013-07-19 22:08:3723from pylib.base import test_dispatcher
[email protected]6bc1bda22013-07-19 22:08:3724from pylib.gtest import gtest_config
[email protected]2a684222013-08-01 16:59:2225from pylib.gtest import setup as gtest_setup
26from pylib.gtest import test_options as gtest_test_options
[email protected]6b6abac6d2013-10-03 11:56:3827from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1328from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3729from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2230from pylib.instrumentation import test_options as instrumentation_test_options
[email protected]3dbdfa42013-08-08 01:08:1431from pylib.monkey import setup as monkey_setup
32from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4733from pylib.perf import setup as perf_setup
34from pylib.perf import test_options as perf_test_options
35from pylib.perf import test_runner as perf_test_runner
[email protected]6bc1bda22013-07-19 22:08:3736from pylib.uiautomator import setup as uiautomator_setup
[email protected]2a684222013-08-01 16:59:2237from pylib.uiautomator import test_options as uiautomator_test_options
[email protected]803f65a72013-08-20 19:11:3038from pylib.utils import command_option_parser
[email protected]6bc1bda22013-07-19 22:08:3739from pylib.utils import report_results
[email protected]71aec4b2013-11-20 00:35:2440from pylib.utils import reraiser_thread
[email protected]6bc1bda22013-07-19 22:08:3741from pylib.utils import run_tests_helper
[email protected]fbe29322013-07-09 09:03:2642
43
[email protected]fbe29322013-07-09 09:03:2644def AddCommonOptions(option_parser):
45 """Adds all common options to |option_parser|."""
46
[email protected]dfffbcbc2013-09-17 22:06:0147 group = optparse.OptionGroup(option_parser, 'Common Options')
48 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
49 group.add_option('--debug', action='store_const', const='Debug',
50 dest='build_type', default=default_build_type,
51 help=('If set, run test suites under out/Debug. '
52 'Default is env var BUILDTYPE or Debug.'))
53 group.add_option('--release', action='store_const',
54 const='Release', dest='build_type',
55 help=('If set, run test suites under out/Release.'
56 ' Default is env var BUILDTYPE or Debug.'))
57 group.add_option('-c', dest='cleanup_test_files',
58 help='Cleanup test files on the device after run',
59 action='store_true')
60 group.add_option('--num_retries', dest='num_retries', type='int',
61 default=2,
62 help=('Number of retries for a test before '
63 'giving up.'))
64 group.add_option('-v',
65 '--verbose',
66 dest='verbose_count',
67 default=0,
68 action='count',
69 help='Verbose level (multiple times for more)')
70 group.add_option('--tool',
71 dest='tool',
72 help=('Run the test under a tool '
73 '(use --tool help to list them)'))
74 group.add_option('--flakiness-dashboard-server',
75 dest='flakiness_dashboard_server',
76 help=('Address of the server that is hosting the '
77 'Chrome for Android flakiness dashboard.'))
78 group.add_option('--skip-deps-push', dest='push_deps',
79 action='store_false', default=True,
80 help=('Do not push dependencies to the device. '
81 'Use this at own risk for speeding up test '
82 'execution on local machine.'))
83 group.add_option('-d', '--device', dest='test_device',
84 help=('Target device for the test suite '
85 'to run on.'))
86 option_parser.add_option_group(group)
[email protected]fbe29322013-07-09 09:03:2687
88
89def ProcessCommonOptions(options):
90 """Processes and handles all common options."""
[email protected]fbe29322013-07-09 09:03:2691 run_tests_helper.SetLogLevel(options.verbose_count)
[email protected]14b3b1202013-08-15 22:25:2892 constants.SetBuildType(options.build_type)
[email protected]fbe29322013-07-09 09:03:2693
94
[email protected]fbe29322013-07-09 09:03:2695def AddGTestOptions(option_parser):
96 """Adds gtest options to |option_parser|."""
97
98 option_parser.usage = '%prog gtest [options]'
[email protected]dfffbcbc2013-09-17 22:06:0199 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26100 option_parser.example = '%prog gtest -s base_unittests'
101
[email protected]6bc1bda22013-07-19 22:08:37102 # TODO(gkanwar): Make this option required
103 option_parser.add_option('-s', '--suite', dest='suite_name',
[email protected]fbe29322013-07-09 09:03:26104 help=('Executable name of the test suite to run '
105 '(use -s help to list them).'))
[email protected]c53dc4332013-11-20 04:38:03106 option_parser.add_option('-f', '--gtest_filter', '--gtest-filter',
107 dest='test_filter',
[email protected]9e689252013-07-30 20:14:36108 help='googletest-style filter string.')
[email protected]c53dc4332013-11-20 04:38:03109 option_parser.add_option('--gtest_also_run_disabled_tests',
110 '--gtest-also-run-disabled-tests',
[email protected]dfffbcbc2013-09-17 22:06:01111 dest='run_disabled', action='store_true',
112 help='Also run disabled tests if applicable.')
113 option_parser.add_option('-a', '--test-arguments', dest='test_arguments',
114 default='',
[email protected]9e689252013-07-30 20:14:36115 help='Additional arguments to pass to the test.')
116 option_parser.add_option('-t', dest='timeout',
117 help='Timeout to wait for each test',
118 type='int',
119 default=60)
[email protected]5b8b8742014-05-22 08:18:50120 option_parser.add_option('--isolate_file_path',
121 '--isolate-file-path',
122 dest='isolate_file_path',
123 help='.isolate file path to override the default '
124 'path')
[email protected]fbe29322013-07-09 09:03:26125 # TODO(gkanwar): Move these to Common Options once we have the plumbing
126 # in our other test types to handle these commands
[email protected]fbe29322013-07-09 09:03:26127 AddCommonOptions(option_parser)
128
129
[email protected]6b6abac6d2013-10-03 11:56:38130def AddLinkerTestOptions(option_parser):
131 option_parser.usage = '%prog linker'
132 option_parser.commands_dict = {}
133 option_parser.example = '%prog linker'
134
[email protected]98c4feef2013-10-08 01:19:05135 option_parser.add_option('-f', '--gtest-filter', dest='test_filter',
136 help='googletest-style filter string.')
[email protected]6b6abac6d2013-10-03 11:56:38137 AddCommonOptions(option_parser)
138
139
[email protected]6bc1bda22013-07-19 22:08:37140def ProcessGTestOptions(options):
141 """Intercept test suite help to list test suites.
142
143 Args:
144 options: Command line options.
[email protected]6bc1bda22013-07-19 22:08:37145 """
146 if options.suite_name == 'help':
147 print 'Available test suites are:'
[email protected]9e689252013-07-30 20:14:36148 for test_suite in (gtest_config.STABLE_TEST_SUITES +
149 gtest_config.EXPERIMENTAL_TEST_SUITES):
150 print test_suite
[email protected]2a684222013-08-01 16:59:22151 sys.exit(0)
[email protected]6bc1bda22013-07-19 22:08:37152
153 # Convert to a list, assuming all test suites if nothing was specified.
154 # TODO(gkanwar): Require having a test suite
155 if options.suite_name:
156 options.suite_name = [options.suite_name]
157 else:
[email protected]9e689252013-07-30 20:14:36158 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES]
[email protected]6bc1bda22013-07-19 22:08:37159
160
[email protected]fbe29322013-07-09 09:03:26161def AddJavaTestOptions(option_parser):
162 """Adds the Java test options to |option_parser|."""
163
[email protected]dfffbcbc2013-09-17 22:06:01164 option_parser.add_option('-f', '--test-filter', dest='test_filter',
[email protected]fbe29322013-07-09 09:03:26165 help=('Test filter (if not fully qualified, '
166 'will run all matches).'))
167 option_parser.add_option(
168 '-A', '--annotation', dest='annotation_str',
169 help=('Comma-separated list of annotations. Run only tests with any of '
170 'the given annotations. An annotation can be either a key or a '
171 'key-values pair. A test that has no annotation is considered '
172 '"SmallTest".'))
173 option_parser.add_option(
174 '-E', '--exclude-annotation', dest='exclude_annotation_str',
175 help=('Comma-separated list of annotations. Exclude tests with these '
176 'annotations.'))
[email protected]fbe29322013-07-09 09:03:26177 option_parser.add_option('--screenshot', dest='screenshot_failures',
178 action='store_true',
179 help='Capture screenshots of test failures')
180 option_parser.add_option('--save-perf-json', action='store_true',
181 help='Saves the JSON file for each UI Perf test.')
[email protected]37ee0c792013-08-06 19:10:13182 option_parser.add_option('--official-build', action='store_true',
183 help='Run official build tests.')
[email protected]fbe29322013-07-09 09:03:26184 option_parser.add_option('--test_data', action='append', default=[],
185 help=('Each instance defines a directory of test '
186 'data that should be copied to the target(s) '
187 'before running the tests. The argument '
188 'should be of the form <target>:<source>, '
189 '<target> is relative to the device data'
190 'directory, and <source> is relative to the '
191 'chromium build directory.'))
192
193
[email protected]7c53a602014-03-24 16:21:44194def ProcessJavaTestOptions(options):
[email protected]fbe29322013-07-09 09:03:26195 """Processes options/arguments and populates |options| with defaults."""
196
[email protected]fbe29322013-07-09 09:03:26197 if options.annotation_str:
198 options.annotations = options.annotation_str.split(',')
199 elif options.test_filter:
200 options.annotations = []
201 else:
[email protected]6bc1bda22013-07-19 22:08:37202 options.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
203 'EnormousTest']
[email protected]fbe29322013-07-09 09:03:26204
205 if options.exclude_annotation_str:
206 options.exclude_annotations = options.exclude_annotation_str.split(',')
207 else:
208 options.exclude_annotations = []
209
[email protected]fbe29322013-07-09 09:03:26210
211def AddInstrumentationTestOptions(option_parser):
212 """Adds Instrumentation test options to |option_parser|."""
213
214 option_parser.usage = '%prog instrumentation [options]'
[email protected]dfffbcbc2013-09-17 22:06:01215 option_parser.commands_dict = {}
[email protected]fb7ab5e82013-07-26 18:31:20216 option_parser.example = ('%prog instrumentation '
[email protected]efeb59e2014-03-12 01:31:26217 '--test-apk=ChromeShellTest')
[email protected]fbe29322013-07-09 09:03:26218
219 AddJavaTestOptions(option_parser)
220 AddCommonOptions(option_parser)
221
[email protected]dfffbcbc2013-09-17 22:06:01222 option_parser.add_option('-j', '--java-only', action='store_true',
[email protected]37ee0c792013-08-06 19:10:13223 default=False, help='Run only the Java tests.')
[email protected]dfffbcbc2013-09-17 22:06:01224 option_parser.add_option('-p', '--python-only', action='store_true',
[email protected]37ee0c792013-08-06 19:10:13225 default=False,
226 help='Run only the host-driven tests.')
[email protected]a69e85bc2013-08-16 18:07:26227 option_parser.add_option('--host-driven-root',
[email protected]37ee0c792013-08-06 19:10:13228 help='Root of the host-driven tests.')
[email protected]fbe29322013-07-09 09:03:26229 option_parser.add_option('-w', '--wait_debugger', dest='wait_for_debugger',
230 action='store_true',
231 help='Wait for debugger.')
[email protected]fbe29322013-07-09 09:03:26232 option_parser.add_option(
233 '--test-apk', dest='test_apk',
234 help=('The name of the apk containing the tests '
[email protected]ae68d4a2013-09-24 21:57:15235 '(without the .apk extension; e.g. "ContentShellTest").'))
[email protected]803f65a72013-08-20 19:11:30236 option_parser.add_option('--coverage-dir',
237 help=('Directory in which to place all generated '
238 'EMMA coverage files.'))
[email protected]fbe29322013-07-09 09:03:26239
240
241def ProcessInstrumentationOptions(options, error_func):
[email protected]2a684222013-08-01 16:59:22242 """Processes options/arguments and populate |options| with defaults.
243
244 Args:
245 options: optparse.Options object.
246 error_func: Function to call with the error message in case of an error.
247
248 Returns:
249 An InstrumentationOptions named tuple which contains all options relevant to
250 instrumentation tests.
251 """
[email protected]fbe29322013-07-09 09:03:26252
[email protected]7c53a602014-03-24 16:21:44253 ProcessJavaTestOptions(options)
[email protected]fbe29322013-07-09 09:03:26254
[email protected]37ee0c792013-08-06 19:10:13255 if options.java_only and options.python_only:
256 error_func('Options java_only (-j) and python_only (-p) '
257 'are mutually exclusive.')
258 options.run_java_tests = True
259 options.run_python_tests = True
260 if options.java_only:
261 options.run_python_tests = False
262 elif options.python_only:
263 options.run_java_tests = False
264
[email protected]67954f822013-08-14 18:09:08265 if not options.host_driven_root:
[email protected]37ee0c792013-08-06 19:10:13266 options.run_python_tests = False
267
[email protected]fbe29322013-07-09 09:03:26268 if not options.test_apk:
269 error_func('--test-apk must be specified.')
270
[email protected]ae68d4a2013-09-24 21:57:15271
272 options.test_apk_path = os.path.join(constants.GetOutDirectory(),
273 constants.SDK_BUILD_APKS_DIR,
274 '%s.apk' % options.test_apk)
275 options.test_apk_jar_path = os.path.join(
276 constants.GetOutDirectory(),
277 constants.SDK_BUILD_TEST_JAVALIB_DIR,
278 '%s.jar' % options.test_apk)
[email protected]fbe29322013-07-09 09:03:26279
[email protected]5e2f3f62014-06-23 12:31:46280 options.test_support_apk_path = '%sSupport%s' % (
281 os.path.splitext(options.test_apk_path))
282
283
[email protected]2a684222013-08-01 16:59:22284 return instrumentation_test_options.InstrumentationOptions(
[email protected]2a684222013-08-01 16:59:22285 options.tool,
286 options.cleanup_test_files,
287 options.push_deps,
288 options.annotations,
289 options.exclude_annotations,
290 options.test_filter,
291 options.test_data,
292 options.save_perf_json,
293 options.screenshot_failures,
[email protected]2a684222013-08-01 16:59:22294 options.wait_for_debugger,
[email protected]803f65a72013-08-20 19:11:30295 options.coverage_dir,
[email protected]2a684222013-08-01 16:59:22296 options.test_apk,
297 options.test_apk_path,
[email protected]5e2f3f62014-06-23 12:31:46298 options.test_apk_jar_path,
299 options.test_support_apk_path
300 )
[email protected]2a684222013-08-01 16:59:22301
[email protected]fbe29322013-07-09 09:03:26302
303def AddUIAutomatorTestOptions(option_parser):
304 """Adds UI Automator test options to |option_parser|."""
305
306 option_parser.usage = '%prog uiautomator [options]'
[email protected]dfffbcbc2013-09-17 22:06:01307 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26308 option_parser.example = (
[email protected]efeb59e2014-03-12 01:31:26309 '%prog uiautomator --test-jar=chrome_shell_uiautomator_tests'
310 ' --package=chrome_shell')
[email protected]fbe29322013-07-09 09:03:26311 option_parser.add_option(
[email protected]a8886c8a92013-10-08 17:29:30312 '--package',
313 help=('Package under test. Possible values: %s' %
314 constants.PACKAGE_INFO.keys()))
[email protected]fbe29322013-07-09 09:03:26315 option_parser.add_option(
316 '--test-jar', dest='test_jar',
317 help=('The name of the dexed jar containing the tests (without the '
318 '.dex.jar extension). Alternatively, this can be a full path '
319 'to the jar.'))
320
321 AddJavaTestOptions(option_parser)
322 AddCommonOptions(option_parser)
323
324
325def ProcessUIAutomatorOptions(options, error_func):
[email protected]2a684222013-08-01 16:59:22326 """Processes UIAutomator options/arguments.
327
328 Args:
329 options: optparse.Options object.
330 error_func: Function to call with the error message in case of an error.
331
332 Returns:
333 A UIAutomatorOptions named tuple which contains all options relevant to
[email protected]3dbdfa42013-08-08 01:08:14334 uiautomator tests.
[email protected]2a684222013-08-01 16:59:22335 """
[email protected]fbe29322013-07-09 09:03:26336
[email protected]7c53a602014-03-24 16:21:44337 ProcessJavaTestOptions(options)
[email protected]fbe29322013-07-09 09:03:26338
[email protected]a8886c8a92013-10-08 17:29:30339 if not options.package:
340 error_func('--package is required.')
341
342 if options.package not in constants.PACKAGE_INFO:
343 error_func('Invalid package.')
[email protected]fbe29322013-07-09 09:03:26344
345 if not options.test_jar:
346 error_func('--test-jar must be specified.')
347
348 if os.path.exists(options.test_jar):
349 # The dexed JAR is fully qualified, assume the info JAR lives along side.
350 options.uiautomator_jar = options.test_jar
351 else:
352 options.uiautomator_jar = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15353 constants.GetOutDirectory(),
354 constants.SDK_BUILD_JAVALIB_DIR,
[email protected]fbe29322013-07-09 09:03:26355 '%s.dex.jar' % options.test_jar)
356 options.uiautomator_info_jar = (
357 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] +
358 '_java.jar')
359
[email protected]2a684222013-08-01 16:59:22360 return uiautomator_test_options.UIAutomatorOptions(
[email protected]2a684222013-08-01 16:59:22361 options.tool,
362 options.cleanup_test_files,
363 options.push_deps,
364 options.annotations,
365 options.exclude_annotations,
366 options.test_filter,
367 options.test_data,
368 options.save_perf_json,
369 options.screenshot_failures,
[email protected]2a684222013-08-01 16:59:22370 options.uiautomator_jar,
371 options.uiautomator_info_jar,
[email protected]a8886c8a92013-10-08 17:29:30372 options.package)
[email protected]2a684222013-08-01 16:59:22373
[email protected]fbe29322013-07-09 09:03:26374
[email protected]3dbdfa42013-08-08 01:08:14375def AddMonkeyTestOptions(option_parser):
376 """Adds monkey test options to |option_parser|."""
[email protected]fb81b982013-08-09 00:07:12377
378 option_parser.usage = '%prog monkey [options]'
[email protected]dfffbcbc2013-09-17 22:06:01379 option_parser.commands_dict = {}
[email protected]fb81b982013-08-09 00:07:12380 option_parser.example = (
[email protected]efeb59e2014-03-12 01:31:26381 '%prog monkey --package=chrome_shell')
[email protected]fb81b982013-08-09 00:07:12382
[email protected]3dbdfa42013-08-08 01:08:14383 option_parser.add_option(
[email protected]a8886c8a92013-10-08 17:29:30384 '--package',
385 help=('Package under test. Possible values: %s' %
386 constants.PACKAGE_INFO.keys()))
[email protected]3dbdfa42013-08-08 01:08:14387 option_parser.add_option(
388 '--event-count', default=10000, type='int',
389 help='Number of events to generate [default: %default].')
390 option_parser.add_option(
391 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12392 help='A list of allowed categories.')
[email protected]3dbdfa42013-08-08 01:08:14393 option_parser.add_option(
394 '--throttle', default=100, type='int',
395 help='Delay between events (ms) [default: %default]. ')
396 option_parser.add_option(
397 '--seed', type='int',
398 help=('Seed value for pseudo-random generator. Same seed value generates '
399 'the same sequence of events. Seed is randomized by default.'))
400 option_parser.add_option(
401 '--extra-args', default='',
402 help=('String of other args to pass to the command verbatim '
403 '[default: "%default"].'))
404
405 AddCommonOptions(option_parser)
406
407
408def ProcessMonkeyTestOptions(options, error_func):
409 """Processes all monkey test options.
410
411 Args:
412 options: optparse.Options object.
413 error_func: Function to call with the error message in case of an error.
414
415 Returns:
416 A MonkeyOptions named tuple which contains all options relevant to
417 monkey tests.
418 """
[email protected]a8886c8a92013-10-08 17:29:30419 if not options.package:
420 error_func('--package is required.')
421
422 if options.package not in constants.PACKAGE_INFO:
423 error_func('Invalid package.')
[email protected]3dbdfa42013-08-08 01:08:14424
425 category = options.category
426 if category:
427 category = options.category.split(',')
428
429 return monkey_test_options.MonkeyOptions(
[email protected]3dbdfa42013-08-08 01:08:14430 options.verbose_count,
[email protected]a8886c8a92013-10-08 17:29:30431 options.package,
[email protected]3dbdfa42013-08-08 01:08:14432 options.event_count,
433 category,
434 options.throttle,
435 options.seed,
436 options.extra_args)
437
438
[email protected]ec3170b2013-08-14 14:39:47439def AddPerfTestOptions(option_parser):
440 """Adds perf test options to |option_parser|."""
441
442 option_parser.usage = '%prog perf [options]'
[email protected]dfffbcbc2013-09-17 22:06:01443 option_parser.commands_dict = {}
[email protected]def4bce2013-11-12 12:59:52444 option_parser.example = ('%prog perf '
[email protected]ad32f312013-11-13 04:03:29445 '[--single-step -- command args] or '
[email protected]def4bce2013-11-12 12:59:52446 '[--steps perf_steps.json] or '
[email protected]ad32f312013-11-13 04:03:29447 '[--print-step step]')
[email protected]ec3170b2013-08-14 14:39:47448
[email protected]181a5c92013-09-06 17:11:46449 option_parser.add_option(
[email protected]def4bce2013-11-12 12:59:52450 '--single-step',
[email protected]ad32f312013-11-13 04:03:29451 action='store_true',
[email protected]def4bce2013-11-12 12:59:52452 help='Execute the given command with retries, but only print the result '
453 'for the "most successful" round.')
454 option_parser.add_option(
[email protected]181a5c92013-09-06 17:11:46455 '--steps',
[email protected]def4bce2013-11-12 12:59:52456 help='JSON file containing the list of commands to run.')
[email protected]181a5c92013-09-06 17:11:46457 option_parser.add_option(
458 '--flaky-steps',
459 help=('A JSON file containing steps that are flaky '
460 'and will have its exit code ignored.'))
461 option_parser.add_option(
[email protected]61487ed2014-06-09 12:33:56462 '--output-json-list',
463 help='Write a simple list of names from --steps into the given file.')
464 option_parser.add_option(
[email protected]181a5c92013-09-06 17:11:46465 '--print-step',
466 help='The name of a previously executed perf step to print.')
467 option_parser.add_option(
468 '--no-timeout', action='store_true',
469 help=('Do not impose a timeout. Each perf step is responsible for '
470 'implementing the timeout logic.'))
[email protected]650487c2013-09-30 11:40:49471 option_parser.add_option(
472 '-f', '--test-filter',
473 help=('Test filter (will match against the names listed in --steps).'))
474 option_parser.add_option(
475 '--dry-run',
476 action='store_true',
477 help='Just print the steps without executing.')
[email protected]ec3170b2013-08-14 14:39:47478 AddCommonOptions(option_parser)
479
480
[email protected]ad32f312013-11-13 04:03:29481def ProcessPerfTestOptions(options, args, error_func):
[email protected]ec3170b2013-08-14 14:39:47482 """Processes all perf test options.
483
484 Args:
485 options: optparse.Options object.
486 error_func: Function to call with the error message in case of an error.
487
488 Returns:
489 A PerfOptions named tuple which contains all options relevant to
490 perf tests.
491 """
[email protected]def4bce2013-11-12 12:59:52492 # Only one of steps, print_step or single_step must be provided.
493 count = len(filter(None,
494 [options.steps, options.print_step, options.single_step]))
495 if count != 1:
496 error_func('Please specify one of: --steps, --print-step, --single-step.')
[email protected]ad32f312013-11-13 04:03:29497 single_step = None
498 if options.single_step:
499 single_step = ' '.join(args[2:])
[email protected]ec3170b2013-08-14 14:39:47500 return perf_test_options.PerfOptions(
[email protected]61487ed2014-06-09 12:33:56501 options.steps, options.flaky_steps, options.output_json_list,
502 options.print_step, options.no_timeout, options.test_filter,
503 options.dry_run, single_step)
[email protected]ec3170b2013-08-14 14:39:47504
505
[email protected]7c53a602014-03-24 16:21:44506def _RunGTests(options, devices):
[email protected]6bc1bda22013-07-19 22:08:37507 """Subcommand of RunTestsCommands which runs gtests."""
[email protected]2a684222013-08-01 16:59:22508 ProcessGTestOptions(options)
[email protected]6bc1bda22013-07-19 22:08:37509
510 exit_code = 0
511 for suite_name in options.suite_name:
[email protected]2a684222013-08-01 16:59:22512 # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for
513 # the gtest command.
514 gtest_options = gtest_test_options.GTestOptions(
[email protected]2a684222013-08-01 16:59:22515 options.tool,
516 options.cleanup_test_files,
517 options.push_deps,
518 options.test_filter,
[email protected]dfffbcbc2013-09-17 22:06:01519 options.run_disabled,
[email protected]2a684222013-08-01 16:59:22520 options.test_arguments,
521 options.timeout,
[email protected]5b8b8742014-05-22 08:18:50522 options.isolate_file_path,
[email protected]2a684222013-08-01 16:59:22523 suite_name)
[email protected]f7148dd42013-08-20 14:24:57524 runner_factory, tests = gtest_setup.Setup(gtest_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37525
526 results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57527 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]6bc1bda22013-07-19 22:08:37528 num_retries=options.num_retries)
529
530 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
531 exit_code = test_exit_code
532
533 report_results.LogFull(
534 results=results,
535 test_type='Unit test',
536 test_package=suite_name,
[email protected]6bc1bda22013-07-19 22:08:37537 flakiness_server=options.flakiness_dashboard_server)
538
539 if os.path.isdir(constants.ISOLATE_DEPS_DIR):
540 shutil.rmtree(constants.ISOLATE_DEPS_DIR)
541
542 return exit_code
543
544
[email protected]7c53a602014-03-24 16:21:44545def _RunLinkerTests(options, devices):
[email protected]6b6abac6d2013-10-03 11:56:38546 """Subcommand of RunTestsCommands which runs linker tests."""
547 runner_factory, tests = linker_setup.Setup(options, devices)
548
549 results, exit_code = test_dispatcher.RunTests(
550 tests, runner_factory, devices, shard=True, test_timeout=60,
551 num_retries=options.num_retries)
552
553 report_results.LogFull(
554 results=results,
555 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22556 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38557
558 return exit_code
559
560
[email protected]f7148dd42013-08-20 14:24:57561def _RunInstrumentationTests(options, error_func, devices):
[email protected]6bc1bda22013-07-19 22:08:37562 """Subcommand of RunTestsCommands which runs instrumentation tests."""
[email protected]2a684222013-08-01 16:59:22563 instrumentation_options = ProcessInstrumentationOptions(options, error_func)
[email protected]6bc1bda22013-07-19 22:08:37564
[email protected]f7148dd42013-08-20 14:24:57565 if len(devices) > 1 and options.wait_for_debugger:
566 logging.warning('Debugger can not be sharded, using first available device')
567 devices = devices[:1]
568
[email protected]6bc1bda22013-07-19 22:08:37569 results = base_test_result.TestRunResults()
570 exit_code = 0
571
572 if options.run_java_tests:
[email protected]2a684222013-08-01 16:59:22573 runner_factory, tests = instrumentation_setup.Setup(instrumentation_options)
[email protected]6bc1bda22013-07-19 22:08:37574
575 test_results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57576 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]6bc1bda22013-07-19 22:08:37577 num_retries=options.num_retries)
578
579 results.AddTestRunResults(test_results)
580
581 if options.run_python_tests:
[email protected]37ee0c792013-08-06 19:10:13582 runner_factory, tests = host_driven_setup.InstrumentationSetup(
[email protected]67954f822013-08-14 18:09:08583 options.host_driven_root, options.official_build,
[email protected]37ee0c792013-08-06 19:10:13584 instrumentation_options)
585
[email protected]34020022013-08-06 23:35:34586 if tests:
587 test_results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57588 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]34020022013-08-06 23:35:34589 num_retries=options.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37590
[email protected]34020022013-08-06 23:35:34591 results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37592
[email protected]34020022013-08-06 23:35:34593 # Only allow exit code escalation
594 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
595 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37596
597 report_results.LogFull(
598 results=results,
599 test_type='Instrumentation',
600 test_package=os.path.basename(options.test_apk),
601 annotation=options.annotations,
[email protected]6bc1bda22013-07-19 22:08:37602 flakiness_server=options.flakiness_dashboard_server)
603
604 return exit_code
605
606
[email protected]f7148dd42013-08-20 14:24:57607def _RunUIAutomatorTests(options, error_func, devices):
[email protected]6bc1bda22013-07-19 22:08:37608 """Subcommand of RunTestsCommands which runs uiautomator tests."""
[email protected]2a684222013-08-01 16:59:22609 uiautomator_options = ProcessUIAutomatorOptions(options, error_func)
[email protected]6bc1bda22013-07-19 22:08:37610
[email protected]37ee0c792013-08-06 19:10:13611 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options)
[email protected]6bc1bda22013-07-19 22:08:37612
[email protected]37ee0c792013-08-06 19:10:13613 results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57614 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]37ee0c792013-08-06 19:10:13615 num_retries=options.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37616
617 report_results.LogFull(
618 results=results,
619 test_type='UIAutomator',
620 test_package=os.path.basename(options.test_jar),
621 annotation=options.annotations,
[email protected]6bc1bda22013-07-19 22:08:37622 flakiness_server=options.flakiness_dashboard_server)
623
624 return exit_code
625
626
[email protected]f7148dd42013-08-20 14:24:57627def _RunMonkeyTests(options, error_func, devices):
[email protected]3dbdfa42013-08-08 01:08:14628 """Subcommand of RunTestsCommands which runs monkey tests."""
629 monkey_options = ProcessMonkeyTestOptions(options, error_func)
630
631 runner_factory, tests = monkey_setup.Setup(monkey_options)
632
633 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46634 tests, runner_factory, devices, shard=False, test_timeout=None,
635 num_retries=options.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14636
637 report_results.LogFull(
638 results=results,
639 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28640 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14641
642 return exit_code
643
644
[email protected]a72f0752014-06-03 23:52:34645def _RunPerfTests(options, args, error_func):
[email protected]ec3170b2013-08-14 14:39:47646 """Subcommand of RunTestsCommands which runs perf tests."""
[email protected]ad32f312013-11-13 04:03:29647 perf_options = ProcessPerfTestOptions(options, args, error_func)
[email protected]61487ed2014-06-09 12:33:56648
649 # Just save a simple json with a list of test names.
650 if perf_options.output_json_list:
651 return perf_test_runner.OutputJsonList(
652 perf_options.steps, perf_options.output_json_list)
653
[email protected]ad32f312013-11-13 04:03:29654 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47655 if perf_options.print_step:
656 return perf_test_runner.PrintTestOutput(perf_options.print_step)
657
[email protected]a72f0752014-06-03 23:52:34658 runner_factory, tests, devices = perf_setup.Setup(perf_options)
[email protected]ec3170b2013-08-14 14:39:47659
[email protected]a72f0752014-06-03 23:52:34660 # shard=False means that each device will get the full list of tests
661 # and then each one will decide their own affinity.
662 # shard=True means each device will pop the next test available from a queue,
663 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57664 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34665 tests, runner_factory, devices, shard=False, test_timeout=None,
[email protected]181a5c92013-09-06 17:11:46666 num_retries=options.num_retries)
[email protected]ec3170b2013-08-14 14:39:47667
668 report_results.LogFull(
669 results=results,
670 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12671 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52672
673 if perf_options.single_step:
674 return perf_test_runner.PrintTestOutput('single_step')
675
[email protected]11ce8452014-02-17 10:55:03676 perf_test_runner.PrintSummary(tests)
677
[email protected]86184c7b2013-08-15 15:06:57678 # Always return 0 on the sharding stage. Individual tests exit_code
679 # will be returned on the print_step stage.
680 return 0
[email protected]ec3170b2013-08-14 14:39:47681
[email protected]3dbdfa42013-08-08 01:08:14682
[email protected]f7148dd42013-08-20 14:24:57683def _GetAttachedDevices(test_device=None):
684 """Get all attached devices.
685
686 Args:
687 test_device: Name of a specific device to use.
688
689 Returns:
690 A list of attached devices.
691 """
692 attached_devices = []
693
694 attached_devices = android_commands.GetAttachedDevices()
695 if test_device:
696 assert test_device in attached_devices, (
697 'Did not find device %s among attached device. Attached devices: %s'
698 % (test_device, ', '.join(attached_devices)))
699 attached_devices = [test_device]
700
701 assert attached_devices, 'No devices attached.'
702
703 return sorted(attached_devices)
704
705
[email protected]fbe29322013-07-09 09:03:26706def RunTestsCommand(command, options, args, option_parser):
707 """Checks test type and dispatches to the appropriate function.
708
709 Args:
710 command: String indicating the command that was received to trigger
711 this function.
712 options: optparse options dictionary.
713 args: List of extra args from optparse.
714 option_parser: optparse.OptionParser object.
715
716 Returns:
717 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10718
719 Raises:
720 Exception: Unknown command name passed in, or an exception from an
721 individual test runner.
[email protected]fbe29322013-07-09 09:03:26722 """
723
[email protected]d82f0252013-07-12 23:22:57724 # Check for extra arguments
[email protected]ad32f312013-11-13 04:03:29725 if len(args) > 2 and command != 'perf':
[email protected]d82f0252013-07-12 23:22:57726 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:])))
727 return constants.ERROR_EXIT_CODE
[email protected]ad32f312013-11-13 04:03:29728 if command == 'perf':
729 if ((options.single_step and len(args) <= 2) or
730 (not options.single_step and len(args) > 2)):
731 option_parser.error('Unrecognized arguments: %s' % (' '.join(args)))
732 return constants.ERROR_EXIT_CODE
[email protected]d82f0252013-07-12 23:22:57733
[email protected]fbe29322013-07-09 09:03:26734 ProcessCommonOptions(options)
735
[email protected]f7148dd42013-08-20 14:24:57736 devices = _GetAttachedDevices(options.test_device)
737
[email protected]c0662e092013-11-12 11:51:25738 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40739 if not ports.ResetTestServerPortAllocation():
740 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25741
[email protected]fbe29322013-07-09 09:03:26742 if command == 'gtest':
[email protected]7c53a602014-03-24 16:21:44743 return _RunGTests(options, devices)
[email protected]6b6abac6d2013-10-03 11:56:38744 elif command == 'linker':
[email protected]7c53a602014-03-24 16:21:44745 return _RunLinkerTests(options, devices)
[email protected]fbe29322013-07-09 09:03:26746 elif command == 'instrumentation':
[email protected]f7148dd42013-08-20 14:24:57747 return _RunInstrumentationTests(options, option_parser.error, devices)
[email protected]fbe29322013-07-09 09:03:26748 elif command == 'uiautomator':
[email protected]f7148dd42013-08-20 14:24:57749 return _RunUIAutomatorTests(options, option_parser.error, devices)
[email protected]3dbdfa42013-08-08 01:08:14750 elif command == 'monkey':
[email protected]f7148dd42013-08-20 14:24:57751 return _RunMonkeyTests(options, option_parser.error, devices)
[email protected]ec3170b2013-08-14 14:39:47752 elif command == 'perf':
[email protected]a72f0752014-06-03 23:52:34753 return _RunPerfTests(options, args, option_parser.error)
[email protected]fbe29322013-07-09 09:03:26754 else:
[email protected]6bc1bda22013-07-19 22:08:37755 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26756
[email protected]fbe29322013-07-09 09:03:26757
[email protected]7c53a602014-03-24 16:21:44758def HelpCommand(command, _options, args, option_parser):
[email protected]fbe29322013-07-09 09:03:26759 """Display help for a certain command, or overall help.
760
761 Args:
762 command: String indicating the command that was received to trigger
763 this function.
[email protected]7c53a602014-03-24 16:21:44764 options: optparse options dictionary. unused.
[email protected]fbe29322013-07-09 09:03:26765 args: List of extra args from optparse.
766 option_parser: optparse.OptionParser object.
767
768 Returns:
769 Integer indicated exit code.
770 """
771 # If we don't have any args, display overall help
772 if len(args) < 3:
773 option_parser.print_help()
774 return 0
[email protected]d82f0252013-07-12 23:22:57775 # If we have too many args, print an error
776 if len(args) > 3:
777 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[3:])))
778 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:26779
780 command = args[2]
781
782 if command not in VALID_COMMANDS:
783 option_parser.error('Unrecognized command.')
784
785 # Treat the help command as a special case. We don't care about showing a
786 # specific help page for itself.
787 if command == 'help':
788 option_parser.print_help()
789 return 0
790
791 VALID_COMMANDS[command].add_options_func(option_parser)
792 option_parser.usage = '%prog ' + command + ' [options]'
[email protected]dfffbcbc2013-09-17 22:06:01793 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26794 option_parser.print_help()
795
796 return 0
797
798
799# Define a named tuple for the values in the VALID_COMMANDS dictionary so the
800# syntax is a bit prettier. The tuple is two functions: (add options, run
801# command).
802CommandFunctionTuple = collections.namedtuple(
803 'CommandFunctionTuple', ['add_options_func', 'run_command_func'])
804VALID_COMMANDS = {
805 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand),
[email protected]fbe29322013-07-09 09:03:26806 'instrumentation': CommandFunctionTuple(
807 AddInstrumentationTestOptions, RunTestsCommand),
808 'uiautomator': CommandFunctionTuple(
809 AddUIAutomatorTestOptions, RunTestsCommand),
[email protected]3dbdfa42013-08-08 01:08:14810 'monkey': CommandFunctionTuple(
811 AddMonkeyTestOptions, RunTestsCommand),
[email protected]ec3170b2013-08-14 14:39:47812 'perf': CommandFunctionTuple(
813 AddPerfTestOptions, RunTestsCommand),
[email protected]6b6abac6d2013-10-03 11:56:38814 'linker': CommandFunctionTuple(
815 AddLinkerTestOptions, RunTestsCommand),
[email protected]fbe29322013-07-09 09:03:26816 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand)
817 }
818
819
[email protected]7c53a602014-03-24 16:21:44820def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:24821 for thread in threading.enumerate():
822 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:21823
824
[email protected]7c53a602014-03-24 16:21:44825def main():
[email protected]83bb8152013-11-19 15:02:21826 signal.signal(signal.SIGUSR1, DumpThreadStacks)
[email protected]803f65a72013-08-20 19:11:30827 option_parser = command_option_parser.CommandOptionParser(
828 commands_dict=VALID_COMMANDS)
829 return command_option_parser.ParseAndExecute(option_parser)
[email protected]fbe29322013-07-09 09:03:26830
[email protected]fbe29322013-07-09 09:03:26831
832if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:44833 sys.exit(main())