blob: 46fefe943e62cd4e38ea0d2e3ea2629d78db458b [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
9import collections
[email protected]f7148dd42013-08-20 14:24:5710import logging
[email protected]fbe29322013-07-09 09:03:2611import optparse
12import os
[email protected]6bc1bda22013-07-19 22:08:3713import shutil
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
17import traceback
[email protected]fbe29322013-07-09 09:03:2618
[email protected]f7148dd42013-08-20 14:24:5719from pylib import android_commands
[email protected]fbe29322013-07-09 09:03:2620from pylib import constants
[email protected]c0662e092013-11-12 11:51:2521from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2622from pylib import ports
23from pylib.base import base_test_result
[email protected]6bc1bda22013-07-19 22:08:3724from pylib.base import test_dispatcher
[email protected]6bc1bda22013-07-19 22:08:3725from pylib.gtest import gtest_config
[email protected]2a684222013-08-01 16:59:2226from pylib.gtest import setup as gtest_setup
27from pylib.gtest import test_options as gtest_test_options
[email protected]6b6abac6d2013-10-03 11:56:3828from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1329from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3730from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2231from pylib.instrumentation import test_options as instrumentation_test_options
[email protected]3dbdfa42013-08-08 01:08:1432from pylib.monkey import setup as monkey_setup
33from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4734from pylib.perf import setup as perf_setup
35from pylib.perf import test_options as perf_test_options
36from pylib.perf import test_runner as perf_test_runner
[email protected]6bc1bda22013-07-19 22:08:3737from pylib.uiautomator import setup as uiautomator_setup
[email protected]2a684222013-08-01 16:59:2238from pylib.uiautomator import test_options as uiautomator_test_options
[email protected]803f65a72013-08-20 19:11:3039from pylib.utils import command_option_parser
[email protected]6bc1bda22013-07-19 22:08:3740from pylib.utils import report_results
[email protected]71aec4b2013-11-20 00:35:2441from pylib.utils import reraiser_thread
[email protected]6bc1bda22013-07-19 22:08:3742from pylib.utils import run_tests_helper
[email protected]fbe29322013-07-09 09:03:2643
44
[email protected]fbe29322013-07-09 09:03:2645def AddCommonOptions(option_parser):
46 """Adds all common options to |option_parser|."""
47
[email protected]dfffbcbc2013-09-17 22:06:0148 group = optparse.OptionGroup(option_parser, 'Common Options')
49 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
50 group.add_option('--debug', action='store_const', const='Debug',
51 dest='build_type', default=default_build_type,
52 help=('If set, run test suites under out/Debug. '
53 'Default is env var BUILDTYPE or Debug.'))
54 group.add_option('--release', action='store_const',
55 const='Release', dest='build_type',
56 help=('If set, run test suites under out/Release.'
57 ' Default is env var BUILDTYPE or Debug.'))
58 group.add_option('-c', dest='cleanup_test_files',
59 help='Cleanup test files on the device after run',
60 action='store_true')
61 group.add_option('--num_retries', dest='num_retries', type='int',
62 default=2,
63 help=('Number of retries for a test before '
64 'giving up.'))
65 group.add_option('-v',
66 '--verbose',
67 dest='verbose_count',
68 default=0,
69 action='count',
70 help='Verbose level (multiple times for more)')
71 group.add_option('--tool',
72 dest='tool',
73 help=('Run the test under a tool '
74 '(use --tool help to list them)'))
75 group.add_option('--flakiness-dashboard-server',
76 dest='flakiness_dashboard_server',
77 help=('Address of the server that is hosting the '
78 'Chrome for Android flakiness dashboard.'))
79 group.add_option('--skip-deps-push', dest='push_deps',
80 action='store_false', default=True,
81 help=('Do not push dependencies to the device. '
82 'Use this at own risk for speeding up test '
83 'execution on local machine.'))
84 group.add_option('-d', '--device', dest='test_device',
85 help=('Target device for the test suite '
86 'to run on.'))
87 option_parser.add_option_group(group)
[email protected]fbe29322013-07-09 09:03:2688
89
90def ProcessCommonOptions(options):
91 """Processes and handles all common options."""
[email protected]fbe29322013-07-09 09:03:2692 run_tests_helper.SetLogLevel(options.verbose_count)
[email protected]14b3b1202013-08-15 22:25:2893 constants.SetBuildType(options.build_type)
[email protected]fbe29322013-07-09 09:03:2694
95
[email protected]fbe29322013-07-09 09:03:2696def AddGTestOptions(option_parser):
97 """Adds gtest options to |option_parser|."""
98
99 option_parser.usage = '%prog gtest [options]'
[email protected]dfffbcbc2013-09-17 22:06:01100 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26101 option_parser.example = '%prog gtest -s base_unittests'
102
[email protected]6bc1bda22013-07-19 22:08:37103 # TODO(gkanwar): Make this option required
104 option_parser.add_option('-s', '--suite', dest='suite_name',
[email protected]fbe29322013-07-09 09:03:26105 help=('Executable name of the test suite to run '
106 '(use -s help to list them).'))
[email protected]dfffbcbc2013-09-17 22:06:01107 option_parser.add_option('-f', '--gtest-filter', dest='test_filter',
[email protected]9e689252013-07-30 20:14:36108 help='googletest-style filter string.')
[email protected]dfffbcbc2013-09-17 22:06:01109 option_parser.add_option('--gtest-also-run-disabled-tests',
110 dest='run_disabled', action='store_true',
111 help='Also run disabled tests if applicable.')
112 option_parser.add_option('-a', '--test-arguments', dest='test_arguments',
113 default='',
[email protected]9e689252013-07-30 20:14:36114 help='Additional arguments to pass to the test.')
115 option_parser.add_option('-t', dest='timeout',
116 help='Timeout to wait for each test',
117 type='int',
118 default=60)
[email protected]fbe29322013-07-09 09:03:26119 # TODO(gkanwar): Move these to Common Options once we have the plumbing
120 # in our other test types to handle these commands
[email protected]fbe29322013-07-09 09:03:26121 AddCommonOptions(option_parser)
122
123
[email protected]6b6abac6d2013-10-03 11:56:38124def AddLinkerTestOptions(option_parser):
125 option_parser.usage = '%prog linker'
126 option_parser.commands_dict = {}
127 option_parser.example = '%prog linker'
128
[email protected]98c4feef2013-10-08 01:19:05129 option_parser.add_option('-f', '--gtest-filter', dest='test_filter',
130 help='googletest-style filter string.')
[email protected]6b6abac6d2013-10-03 11:56:38131 AddCommonOptions(option_parser)
132
133
[email protected]6bc1bda22013-07-19 22:08:37134def ProcessGTestOptions(options):
135 """Intercept test suite help to list test suites.
136
137 Args:
138 options: Command line options.
[email protected]6bc1bda22013-07-19 22:08:37139 """
140 if options.suite_name == 'help':
141 print 'Available test suites are:'
[email protected]9e689252013-07-30 20:14:36142 for test_suite in (gtest_config.STABLE_TEST_SUITES +
143 gtest_config.EXPERIMENTAL_TEST_SUITES):
144 print test_suite
[email protected]2a684222013-08-01 16:59:22145 sys.exit(0)
[email protected]6bc1bda22013-07-19 22:08:37146
147 # Convert to a list, assuming all test suites if nothing was specified.
148 # TODO(gkanwar): Require having a test suite
149 if options.suite_name:
150 options.suite_name = [options.suite_name]
151 else:
[email protected]9e689252013-07-30 20:14:36152 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES]
[email protected]6bc1bda22013-07-19 22:08:37153
154
[email protected]fbe29322013-07-09 09:03:26155def AddJavaTestOptions(option_parser):
156 """Adds the Java test options to |option_parser|."""
157
[email protected]dfffbcbc2013-09-17 22:06:01158 option_parser.add_option('-f', '--test-filter', dest='test_filter',
[email protected]fbe29322013-07-09 09:03:26159 help=('Test filter (if not fully qualified, '
160 'will run all matches).'))
161 option_parser.add_option(
162 '-A', '--annotation', dest='annotation_str',
163 help=('Comma-separated list of annotations. Run only tests with any of '
164 'the given annotations. An annotation can be either a key or a '
165 'key-values pair. A test that has no annotation is considered '
166 '"SmallTest".'))
167 option_parser.add_option(
168 '-E', '--exclude-annotation', dest='exclude_annotation_str',
169 help=('Comma-separated list of annotations. Exclude tests with these '
170 'annotations.'))
[email protected]fbe29322013-07-09 09:03:26171 option_parser.add_option('--screenshot', dest='screenshot_failures',
172 action='store_true',
173 help='Capture screenshots of test failures')
174 option_parser.add_option('--save-perf-json', action='store_true',
175 help='Saves the JSON file for each UI Perf test.')
[email protected]37ee0c792013-08-06 19:10:13176 option_parser.add_option('--official-build', action='store_true',
177 help='Run official build tests.')
[email protected]fbe29322013-07-09 09:03:26178 option_parser.add_option('--keep_test_server_ports',
179 action='store_true',
180 help=('Indicates the test server ports must be '
181 'kept. When this is run via a sharder '
182 'the test server ports should be kept and '
183 'should not be reset.'))
[email protected]fbe29322013-07-09 09:03:26184 option_parser.add_option('--test_data', action='append', default=[],
185 help=('Each instance defines a directory of test '
186 'data that should be copied to the target(s) '
187 'before running the tests. The argument '
188 'should be of the form <target>:<source>, '
189 '<target> is relative to the device data'
190 'directory, and <source> is relative to the '
191 'chromium build directory.'))
192
193
194def ProcessJavaTestOptions(options, error_func):
195 """Processes options/arguments and populates |options| with defaults."""
196
[email protected]fbe29322013-07-09 09:03:26197 if options.annotation_str:
198 options.annotations = options.annotation_str.split(',')
199 elif options.test_filter:
200 options.annotations = []
201 else:
[email protected]6bc1bda22013-07-19 22:08:37202 options.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
203 'EnormousTest']
[email protected]fbe29322013-07-09 09:03:26204
205 if options.exclude_annotation_str:
206 options.exclude_annotations = options.exclude_annotation_str.split(',')
207 else:
208 options.exclude_annotations = []
209
210 if not options.keep_test_server_ports:
211 if not ports.ResetTestServerPortAllocation():
212 raise Exception('Failed to reset test server port.')
213
214
215def AddInstrumentationTestOptions(option_parser):
216 """Adds Instrumentation test options to |option_parser|."""
217
218 option_parser.usage = '%prog instrumentation [options]'
[email protected]dfffbcbc2013-09-17 22:06:01219 option_parser.commands_dict = {}
[email protected]fb7ab5e82013-07-26 18:31:20220 option_parser.example = ('%prog instrumentation '
[email protected]fbe29322013-07-09 09:03:26221 '--test-apk=ChromiumTestShellTest')
222
223 AddJavaTestOptions(option_parser)
224 AddCommonOptions(option_parser)
225
[email protected]dfffbcbc2013-09-17 22:06:01226 option_parser.add_option('-j', '--java-only', action='store_true',
[email protected]37ee0c792013-08-06 19:10:13227 default=False, help='Run only the Java tests.')
[email protected]dfffbcbc2013-09-17 22:06:01228 option_parser.add_option('-p', '--python-only', action='store_true',
[email protected]37ee0c792013-08-06 19:10:13229 default=False,
230 help='Run only the host-driven tests.')
[email protected]a69e85bc2013-08-16 18:07:26231 option_parser.add_option('--host-driven-root',
[email protected]37ee0c792013-08-06 19:10:13232 help='Root of the host-driven tests.')
[email protected]fbe29322013-07-09 09:03:26233 option_parser.add_option('-w', '--wait_debugger', dest='wait_for_debugger',
234 action='store_true',
235 help='Wait for debugger.')
[email protected]fbe29322013-07-09 09:03:26236 option_parser.add_option(
237 '--test-apk', dest='test_apk',
238 help=('The name of the apk containing the tests '
[email protected]ae68d4a2013-09-24 21:57:15239 '(without the .apk extension; e.g. "ContentShellTest").'))
[email protected]803f65a72013-08-20 19:11:30240 option_parser.add_option('--coverage-dir',
241 help=('Directory in which to place all generated '
242 'EMMA coverage files.'))
[email protected]fbe29322013-07-09 09:03:26243
244
245def ProcessInstrumentationOptions(options, error_func):
[email protected]2a684222013-08-01 16:59:22246 """Processes options/arguments and populate |options| with defaults.
247
248 Args:
249 options: optparse.Options object.
250 error_func: Function to call with the error message in case of an error.
251
252 Returns:
253 An InstrumentationOptions named tuple which contains all options relevant to
254 instrumentation tests.
255 """
[email protected]fbe29322013-07-09 09:03:26256
257 ProcessJavaTestOptions(options, error_func)
258
[email protected]37ee0c792013-08-06 19:10:13259 if options.java_only and options.python_only:
260 error_func('Options java_only (-j) and python_only (-p) '
261 'are mutually exclusive.')
262 options.run_java_tests = True
263 options.run_python_tests = True
264 if options.java_only:
265 options.run_python_tests = False
266 elif options.python_only:
267 options.run_java_tests = False
268
[email protected]67954f822013-08-14 18:09:08269 if not options.host_driven_root:
[email protected]37ee0c792013-08-06 19:10:13270 options.run_python_tests = False
271
[email protected]fbe29322013-07-09 09:03:26272 if not options.test_apk:
273 error_func('--test-apk must be specified.')
274
[email protected]ae68d4a2013-09-24 21:57:15275
276 options.test_apk_path = os.path.join(constants.GetOutDirectory(),
277 constants.SDK_BUILD_APKS_DIR,
278 '%s.apk' % options.test_apk)
279 options.test_apk_jar_path = os.path.join(
280 constants.GetOutDirectory(),
281 constants.SDK_BUILD_TEST_JAVALIB_DIR,
282 '%s.jar' % options.test_apk)
[email protected]fbe29322013-07-09 09:03:26283
[email protected]2a684222013-08-01 16:59:22284 return instrumentation_test_options.InstrumentationOptions(
[email protected]2a684222013-08-01 16:59:22285 options.tool,
286 options.cleanup_test_files,
287 options.push_deps,
288 options.annotations,
289 options.exclude_annotations,
290 options.test_filter,
291 options.test_data,
292 options.save_perf_json,
293 options.screenshot_failures,
[email protected]2a684222013-08-01 16:59:22294 options.wait_for_debugger,
[email protected]803f65a72013-08-20 19:11:30295 options.coverage_dir,
[email protected]2a684222013-08-01 16:59:22296 options.test_apk,
297 options.test_apk_path,
298 options.test_apk_jar_path)
299
[email protected]fbe29322013-07-09 09:03:26300
301def AddUIAutomatorTestOptions(option_parser):
302 """Adds UI Automator test options to |option_parser|."""
303
304 option_parser.usage = '%prog uiautomator [options]'
[email protected]dfffbcbc2013-09-17 22:06:01305 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26306 option_parser.example = (
307 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests'
[email protected]a8886c8a92013-10-08 17:29:30308 ' --package=chromium_test_shell')
[email protected]fbe29322013-07-09 09:03:26309 option_parser.add_option(
[email protected]a8886c8a92013-10-08 17:29:30310 '--package',
311 help=('Package under test. Possible values: %s' %
312 constants.PACKAGE_INFO.keys()))
[email protected]fbe29322013-07-09 09:03:26313 option_parser.add_option(
314 '--test-jar', dest='test_jar',
315 help=('The name of the dexed jar containing the tests (without the '
316 '.dex.jar extension). Alternatively, this can be a full path '
317 'to the jar.'))
318
319 AddJavaTestOptions(option_parser)
320 AddCommonOptions(option_parser)
321
322
323def ProcessUIAutomatorOptions(options, error_func):
[email protected]2a684222013-08-01 16:59:22324 """Processes UIAutomator options/arguments.
325
326 Args:
327 options: optparse.Options object.
328 error_func: Function to call with the error message in case of an error.
329
330 Returns:
331 A UIAutomatorOptions named tuple which contains all options relevant to
[email protected]3dbdfa42013-08-08 01:08:14332 uiautomator tests.
[email protected]2a684222013-08-01 16:59:22333 """
[email protected]fbe29322013-07-09 09:03:26334
335 ProcessJavaTestOptions(options, error_func)
336
[email protected]a8886c8a92013-10-08 17:29:30337 if not options.package:
338 error_func('--package is required.')
339
340 if options.package not in constants.PACKAGE_INFO:
341 error_func('Invalid package.')
[email protected]fbe29322013-07-09 09:03:26342
343 if not options.test_jar:
344 error_func('--test-jar must be specified.')
345
346 if os.path.exists(options.test_jar):
347 # The dexed JAR is fully qualified, assume the info JAR lives along side.
348 options.uiautomator_jar = options.test_jar
349 else:
350 options.uiautomator_jar = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15351 constants.GetOutDirectory(),
352 constants.SDK_BUILD_JAVALIB_DIR,
[email protected]fbe29322013-07-09 09:03:26353 '%s.dex.jar' % options.test_jar)
354 options.uiautomator_info_jar = (
355 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] +
356 '_java.jar')
357
[email protected]2a684222013-08-01 16:59:22358 return uiautomator_test_options.UIAutomatorOptions(
[email protected]2a684222013-08-01 16:59:22359 options.tool,
360 options.cleanup_test_files,
361 options.push_deps,
362 options.annotations,
363 options.exclude_annotations,
364 options.test_filter,
365 options.test_data,
366 options.save_perf_json,
367 options.screenshot_failures,
[email protected]2a684222013-08-01 16:59:22368 options.uiautomator_jar,
369 options.uiautomator_info_jar,
[email protected]a8886c8a92013-10-08 17:29:30370 options.package)
[email protected]2a684222013-08-01 16:59:22371
[email protected]fbe29322013-07-09 09:03:26372
[email protected]3dbdfa42013-08-08 01:08:14373def AddMonkeyTestOptions(option_parser):
374 """Adds monkey test options to |option_parser|."""
[email protected]fb81b982013-08-09 00:07:12375
376 option_parser.usage = '%prog monkey [options]'
[email protected]dfffbcbc2013-09-17 22:06:01377 option_parser.commands_dict = {}
[email protected]fb81b982013-08-09 00:07:12378 option_parser.example = (
[email protected]a8886c8a92013-10-08 17:29:30379 '%prog monkey --package=chromium_test_shell')
[email protected]fb81b982013-08-09 00:07:12380
[email protected]3dbdfa42013-08-08 01:08:14381 option_parser.add_option(
[email protected]a8886c8a92013-10-08 17:29:30382 '--package',
383 help=('Package under test. Possible values: %s' %
384 constants.PACKAGE_INFO.keys()))
[email protected]3dbdfa42013-08-08 01:08:14385 option_parser.add_option(
386 '--event-count', default=10000, type='int',
387 help='Number of events to generate [default: %default].')
388 option_parser.add_option(
389 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12390 help='A list of allowed categories.')
[email protected]3dbdfa42013-08-08 01:08:14391 option_parser.add_option(
392 '--throttle', default=100, type='int',
393 help='Delay between events (ms) [default: %default]. ')
394 option_parser.add_option(
395 '--seed', type='int',
396 help=('Seed value for pseudo-random generator. Same seed value generates '
397 'the same sequence of events. Seed is randomized by default.'))
398 option_parser.add_option(
399 '--extra-args', default='',
400 help=('String of other args to pass to the command verbatim '
401 '[default: "%default"].'))
402
403 AddCommonOptions(option_parser)
404
405
406def ProcessMonkeyTestOptions(options, error_func):
407 """Processes all monkey test options.
408
409 Args:
410 options: optparse.Options object.
411 error_func: Function to call with the error message in case of an error.
412
413 Returns:
414 A MonkeyOptions named tuple which contains all options relevant to
415 monkey tests.
416 """
[email protected]a8886c8a92013-10-08 17:29:30417 if not options.package:
418 error_func('--package is required.')
419
420 if options.package not in constants.PACKAGE_INFO:
421 error_func('Invalid package.')
[email protected]3dbdfa42013-08-08 01:08:14422
423 category = options.category
424 if category:
425 category = options.category.split(',')
426
427 return monkey_test_options.MonkeyOptions(
[email protected]3dbdfa42013-08-08 01:08:14428 options.verbose_count,
[email protected]a8886c8a92013-10-08 17:29:30429 options.package,
[email protected]3dbdfa42013-08-08 01:08:14430 options.event_count,
431 category,
432 options.throttle,
433 options.seed,
434 options.extra_args)
435
436
[email protected]ec3170b2013-08-14 14:39:47437def AddPerfTestOptions(option_parser):
438 """Adds perf test options to |option_parser|."""
439
440 option_parser.usage = '%prog perf [options]'
[email protected]dfffbcbc2013-09-17 22:06:01441 option_parser.commands_dict = {}
[email protected]def4bce2013-11-12 12:59:52442 option_parser.example = ('%prog perf '
[email protected]ad32f312013-11-13 04:03:29443 '[--single-step -- command args] or '
[email protected]def4bce2013-11-12 12:59:52444 '[--steps perf_steps.json] or '
[email protected]ad32f312013-11-13 04:03:29445 '[--print-step step]')
[email protected]ec3170b2013-08-14 14:39:47446
[email protected]181a5c92013-09-06 17:11:46447 option_parser.add_option(
[email protected]def4bce2013-11-12 12:59:52448 '--single-step',
[email protected]ad32f312013-11-13 04:03:29449 action='store_true',
[email protected]def4bce2013-11-12 12:59:52450 help='Execute the given command with retries, but only print the result '
451 'for the "most successful" round.')
452 option_parser.add_option(
[email protected]181a5c92013-09-06 17:11:46453 '--steps',
[email protected]def4bce2013-11-12 12:59:52454 help='JSON file containing the list of commands to run.')
[email protected]181a5c92013-09-06 17:11:46455 option_parser.add_option(
456 '--flaky-steps',
457 help=('A JSON file containing steps that are flaky '
458 'and will have its exit code ignored.'))
459 option_parser.add_option(
460 '--print-step',
461 help='The name of a previously executed perf step to print.')
462 option_parser.add_option(
463 '--no-timeout', action='store_true',
464 help=('Do not impose a timeout. Each perf step is responsible for '
465 'implementing the timeout logic.'))
[email protected]650487c2013-09-30 11:40:49466 option_parser.add_option(
467 '-f', '--test-filter',
468 help=('Test filter (will match against the names listed in --steps).'))
469 option_parser.add_option(
470 '--dry-run',
471 action='store_true',
472 help='Just print the steps without executing.')
[email protected]ec3170b2013-08-14 14:39:47473 AddCommonOptions(option_parser)
474
475
[email protected]ad32f312013-11-13 04:03:29476def ProcessPerfTestOptions(options, args, error_func):
[email protected]ec3170b2013-08-14 14:39:47477 """Processes all perf test options.
478
479 Args:
480 options: optparse.Options object.
481 error_func: Function to call with the error message in case of an error.
482
483 Returns:
484 A PerfOptions named tuple which contains all options relevant to
485 perf tests.
486 """
[email protected]def4bce2013-11-12 12:59:52487 # Only one of steps, print_step or single_step must be provided.
488 count = len(filter(None,
489 [options.steps, options.print_step, options.single_step]))
490 if count != 1:
491 error_func('Please specify one of: --steps, --print-step, --single-step.')
[email protected]ad32f312013-11-13 04:03:29492 single_step = None
493 if options.single_step:
494 single_step = ' '.join(args[2:])
[email protected]ec3170b2013-08-14 14:39:47495 return perf_test_options.PerfOptions(
[email protected]181a5c92013-09-06 17:11:46496 options.steps, options.flaky_steps, options.print_step,
[email protected]def4bce2013-11-12 12:59:52497 options.no_timeout, options.test_filter, options.dry_run,
[email protected]ad32f312013-11-13 04:03:29498 single_step)
[email protected]ec3170b2013-08-14 14:39:47499
500
[email protected]f7148dd42013-08-20 14:24:57501def _RunGTests(options, error_func, devices):
[email protected]6bc1bda22013-07-19 22:08:37502 """Subcommand of RunTestsCommands which runs gtests."""
[email protected]2a684222013-08-01 16:59:22503 ProcessGTestOptions(options)
[email protected]6bc1bda22013-07-19 22:08:37504
505 exit_code = 0
506 for suite_name in options.suite_name:
[email protected]2a684222013-08-01 16:59:22507 # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for
508 # the gtest command.
509 gtest_options = gtest_test_options.GTestOptions(
[email protected]2a684222013-08-01 16:59:22510 options.tool,
511 options.cleanup_test_files,
512 options.push_deps,
513 options.test_filter,
[email protected]dfffbcbc2013-09-17 22:06:01514 options.run_disabled,
[email protected]2a684222013-08-01 16:59:22515 options.test_arguments,
516 options.timeout,
517 suite_name)
[email protected]f7148dd42013-08-20 14:24:57518 runner_factory, tests = gtest_setup.Setup(gtest_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37519
520 results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57521 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]6bc1bda22013-07-19 22:08:37522 num_retries=options.num_retries)
523
524 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
525 exit_code = test_exit_code
526
527 report_results.LogFull(
528 results=results,
529 test_type='Unit test',
530 test_package=suite_name,
[email protected]6bc1bda22013-07-19 22:08:37531 flakiness_server=options.flakiness_dashboard_server)
532
533 if os.path.isdir(constants.ISOLATE_DEPS_DIR):
534 shutil.rmtree(constants.ISOLATE_DEPS_DIR)
535
536 return exit_code
537
538
[email protected]6b6abac6d2013-10-03 11:56:38539def _RunLinkerTests(options, error_func, devices):
540 """Subcommand of RunTestsCommands which runs linker tests."""
541 runner_factory, tests = linker_setup.Setup(options, devices)
542
543 results, exit_code = test_dispatcher.RunTests(
544 tests, runner_factory, devices, shard=True, test_timeout=60,
545 num_retries=options.num_retries)
546
547 report_results.LogFull(
548 results=results,
549 test_type='Linker test',
550 test_package='ContentLinkerTest')
551
552 return exit_code
553
554
[email protected]f7148dd42013-08-20 14:24:57555def _RunInstrumentationTests(options, error_func, devices):
[email protected]6bc1bda22013-07-19 22:08:37556 """Subcommand of RunTestsCommands which runs instrumentation tests."""
[email protected]2a684222013-08-01 16:59:22557 instrumentation_options = ProcessInstrumentationOptions(options, error_func)
[email protected]6bc1bda22013-07-19 22:08:37558
[email protected]f7148dd42013-08-20 14:24:57559 if len(devices) > 1 and options.wait_for_debugger:
560 logging.warning('Debugger can not be sharded, using first available device')
561 devices = devices[:1]
562
[email protected]6bc1bda22013-07-19 22:08:37563 results = base_test_result.TestRunResults()
564 exit_code = 0
565
566 if options.run_java_tests:
[email protected]2a684222013-08-01 16:59:22567 runner_factory, tests = instrumentation_setup.Setup(instrumentation_options)
[email protected]6bc1bda22013-07-19 22:08:37568
569 test_results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57570 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]6bc1bda22013-07-19 22:08:37571 num_retries=options.num_retries)
572
573 results.AddTestRunResults(test_results)
574
575 if options.run_python_tests:
[email protected]37ee0c792013-08-06 19:10:13576 runner_factory, tests = host_driven_setup.InstrumentationSetup(
[email protected]67954f822013-08-14 18:09:08577 options.host_driven_root, options.official_build,
[email protected]37ee0c792013-08-06 19:10:13578 instrumentation_options)
579
[email protected]34020022013-08-06 23:35:34580 if tests:
581 test_results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57582 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]34020022013-08-06 23:35:34583 num_retries=options.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37584
[email protected]34020022013-08-06 23:35:34585 results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37586
[email protected]34020022013-08-06 23:35:34587 # Only allow exit code escalation
588 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
589 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37590
591 report_results.LogFull(
592 results=results,
593 test_type='Instrumentation',
594 test_package=os.path.basename(options.test_apk),
595 annotation=options.annotations,
[email protected]6bc1bda22013-07-19 22:08:37596 flakiness_server=options.flakiness_dashboard_server)
597
598 return exit_code
599
600
[email protected]f7148dd42013-08-20 14:24:57601def _RunUIAutomatorTests(options, error_func, devices):
[email protected]6bc1bda22013-07-19 22:08:37602 """Subcommand of RunTestsCommands which runs uiautomator tests."""
[email protected]2a684222013-08-01 16:59:22603 uiautomator_options = ProcessUIAutomatorOptions(options, error_func)
[email protected]6bc1bda22013-07-19 22:08:37604
[email protected]37ee0c792013-08-06 19:10:13605 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options)
[email protected]6bc1bda22013-07-19 22:08:37606
[email protected]37ee0c792013-08-06 19:10:13607 results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57608 tests, runner_factory, devices, shard=True, test_timeout=None,
[email protected]37ee0c792013-08-06 19:10:13609 num_retries=options.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37610
611 report_results.LogFull(
612 results=results,
613 test_type='UIAutomator',
614 test_package=os.path.basename(options.test_jar),
615 annotation=options.annotations,
[email protected]6bc1bda22013-07-19 22:08:37616 flakiness_server=options.flakiness_dashboard_server)
617
618 return exit_code
619
620
[email protected]f7148dd42013-08-20 14:24:57621def _RunMonkeyTests(options, error_func, devices):
[email protected]3dbdfa42013-08-08 01:08:14622 """Subcommand of RunTestsCommands which runs monkey tests."""
623 monkey_options = ProcessMonkeyTestOptions(options, error_func)
624
625 runner_factory, tests = monkey_setup.Setup(monkey_options)
626
627 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46628 tests, runner_factory, devices, shard=False, test_timeout=None,
629 num_retries=options.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14630
631 report_results.LogFull(
632 results=results,
633 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28634 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14635
636 return exit_code
637
638
[email protected]ad32f312013-11-13 04:03:29639def _RunPerfTests(options, args, error_func, devices):
[email protected]ec3170b2013-08-14 14:39:47640 """Subcommand of RunTestsCommands which runs perf tests."""
[email protected]ad32f312013-11-13 04:03:29641 perf_options = ProcessPerfTestOptions(options, args, error_func)
642 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47643 if perf_options.print_step:
644 return perf_test_runner.PrintTestOutput(perf_options.print_step)
645
646 runner_factory, tests = perf_setup.Setup(perf_options)
647
[email protected]86184c7b2013-08-15 15:06:57648 results, _ = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46649 tests, runner_factory, devices, shard=True, test_timeout=None,
650 num_retries=options.num_retries)
[email protected]ec3170b2013-08-14 14:39:47651
652 report_results.LogFull(
653 results=results,
654 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12655 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52656
657 if perf_options.single_step:
658 return perf_test_runner.PrintTestOutput('single_step')
659
[email protected]86184c7b2013-08-15 15:06:57660 # Always return 0 on the sharding stage. Individual tests exit_code
661 # will be returned on the print_step stage.
662 return 0
[email protected]ec3170b2013-08-14 14:39:47663
[email protected]3dbdfa42013-08-08 01:08:14664
[email protected]f7148dd42013-08-20 14:24:57665def _GetAttachedDevices(test_device=None):
666 """Get all attached devices.
667
668 Args:
669 test_device: Name of a specific device to use.
670
671 Returns:
672 A list of attached devices.
673 """
674 attached_devices = []
675
676 attached_devices = android_commands.GetAttachedDevices()
677 if test_device:
678 assert test_device in attached_devices, (
679 'Did not find device %s among attached device. Attached devices: %s'
680 % (test_device, ', '.join(attached_devices)))
681 attached_devices = [test_device]
682
683 assert attached_devices, 'No devices attached.'
684
685 return sorted(attached_devices)
686
687
[email protected]fbe29322013-07-09 09:03:26688def RunTestsCommand(command, options, args, option_parser):
689 """Checks test type and dispatches to the appropriate function.
690
691 Args:
692 command: String indicating the command that was received to trigger
693 this function.
694 options: optparse options dictionary.
695 args: List of extra args from optparse.
696 option_parser: optparse.OptionParser object.
697
698 Returns:
699 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10700
701 Raises:
702 Exception: Unknown command name passed in, or an exception from an
703 individual test runner.
[email protected]fbe29322013-07-09 09:03:26704 """
705
[email protected]d82f0252013-07-12 23:22:57706 # Check for extra arguments
[email protected]ad32f312013-11-13 04:03:29707 if len(args) > 2 and command != 'perf':
[email protected]d82f0252013-07-12 23:22:57708 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:])))
709 return constants.ERROR_EXIT_CODE
[email protected]ad32f312013-11-13 04:03:29710 if command == 'perf':
711 if ((options.single_step and len(args) <= 2) or
712 (not options.single_step and len(args) > 2)):
713 option_parser.error('Unrecognized arguments: %s' % (' '.join(args)))
714 return constants.ERROR_EXIT_CODE
[email protected]d82f0252013-07-12 23:22:57715
[email protected]fbe29322013-07-09 09:03:26716 ProcessCommonOptions(options)
717
[email protected]f7148dd42013-08-20 14:24:57718 devices = _GetAttachedDevices(options.test_device)
719
[email protected]c0662e092013-11-12 11:51:25720 forwarder.Forwarder.RemoveHostLog()
721
[email protected]fbe29322013-07-09 09:03:26722 if command == 'gtest':
[email protected]f7148dd42013-08-20 14:24:57723 return _RunGTests(options, option_parser.error, devices)
[email protected]6b6abac6d2013-10-03 11:56:38724 elif command == 'linker':
725 return _RunLinkerTests(options, option_parser.error, devices)
[email protected]fbe29322013-07-09 09:03:26726 elif command == 'instrumentation':
[email protected]f7148dd42013-08-20 14:24:57727 return _RunInstrumentationTests(options, option_parser.error, devices)
[email protected]fbe29322013-07-09 09:03:26728 elif command == 'uiautomator':
[email protected]f7148dd42013-08-20 14:24:57729 return _RunUIAutomatorTests(options, option_parser.error, devices)
[email protected]3dbdfa42013-08-08 01:08:14730 elif command == 'monkey':
[email protected]f7148dd42013-08-20 14:24:57731 return _RunMonkeyTests(options, option_parser.error, devices)
[email protected]ec3170b2013-08-14 14:39:47732 elif command == 'perf':
[email protected]ad32f312013-11-13 04:03:29733 return _RunPerfTests(options, args, option_parser.error, devices)
[email protected]fbe29322013-07-09 09:03:26734 else:
[email protected]6bc1bda22013-07-19 22:08:37735 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26736
[email protected]fbe29322013-07-09 09:03:26737
738def HelpCommand(command, options, args, option_parser):
739 """Display help for a certain command, or overall help.
740
741 Args:
742 command: String indicating the command that was received to trigger
743 this function.
744 options: optparse options dictionary.
745 args: List of extra args from optparse.
746 option_parser: optparse.OptionParser object.
747
748 Returns:
749 Integer indicated exit code.
750 """
751 # If we don't have any args, display overall help
752 if len(args) < 3:
753 option_parser.print_help()
754 return 0
[email protected]d82f0252013-07-12 23:22:57755 # If we have too many args, print an error
756 if len(args) > 3:
757 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[3:])))
758 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:26759
760 command = args[2]
761
762 if command not in VALID_COMMANDS:
763 option_parser.error('Unrecognized command.')
764
765 # Treat the help command as a special case. We don't care about showing a
766 # specific help page for itself.
767 if command == 'help':
768 option_parser.print_help()
769 return 0
770
771 VALID_COMMANDS[command].add_options_func(option_parser)
772 option_parser.usage = '%prog ' + command + ' [options]'
[email protected]dfffbcbc2013-09-17 22:06:01773 option_parser.commands_dict = {}
[email protected]fbe29322013-07-09 09:03:26774 option_parser.print_help()
775
776 return 0
777
778
779# Define a named tuple for the values in the VALID_COMMANDS dictionary so the
780# syntax is a bit prettier. The tuple is two functions: (add options, run
781# command).
782CommandFunctionTuple = collections.namedtuple(
783 'CommandFunctionTuple', ['add_options_func', 'run_command_func'])
784VALID_COMMANDS = {
785 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand),
[email protected]fbe29322013-07-09 09:03:26786 'instrumentation': CommandFunctionTuple(
787 AddInstrumentationTestOptions, RunTestsCommand),
788 'uiautomator': CommandFunctionTuple(
789 AddUIAutomatorTestOptions, RunTestsCommand),
[email protected]3dbdfa42013-08-08 01:08:14790 'monkey': CommandFunctionTuple(
791 AddMonkeyTestOptions, RunTestsCommand),
[email protected]ec3170b2013-08-14 14:39:47792 'perf': CommandFunctionTuple(
793 AddPerfTestOptions, RunTestsCommand),
[email protected]6b6abac6d2013-10-03 11:56:38794 'linker': CommandFunctionTuple(
795 AddLinkerTestOptions, RunTestsCommand),
[email protected]fbe29322013-07-09 09:03:26796 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand)
797 }
798
799
[email protected]83bb8152013-11-19 15:02:21800def DumpThreadStacks(signal, frame):
[email protected]71aec4b2013-11-20 00:35:24801 for thread in threading.enumerate():
802 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:21803
804
[email protected]fbe29322013-07-09 09:03:26805def main(argv):
[email protected]83bb8152013-11-19 15:02:21806 signal.signal(signal.SIGUSR1, DumpThreadStacks)
[email protected]803f65a72013-08-20 19:11:30807 option_parser = command_option_parser.CommandOptionParser(
808 commands_dict=VALID_COMMANDS)
809 return command_option_parser.ParseAndExecute(option_parser)
[email protected]fbe29322013-07-09 09:03:26810
[email protected]fbe29322013-07-09 09:03:26811
812if __name__ == '__main__':
813 sys.exit(main(sys.argv))