blob: 0b1dc746357d230762be93da2f42a4d13886c7a1 [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
jama47ca85c2014-12-03 18:38:079import argparse
[email protected]fbe29322013-07-09 09:03:2610import collections
jbudorickeb7ea71c2015-09-28 16:40:2011import itertools
[email protected]f7148dd42013-08-20 14:24:5712import logging
[email protected]fbe29322013-07-09 09:03:2613import os
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
jbudorick256fd532014-10-24 01:50:1317import unittest
[email protected]fbe29322013-07-09 09:03:2618
jbudorick061629442015-09-03 18:00:5719from devil import base_error
20from devil.android import apk_helper
21from devil.android import device_blacklist
22from devil.android import device_errors
23from devil.android import device_utils
24from devil.android import ports
25from devil.utils import reraiser_thread
26from devil.utils import run_tests_helper
27
[email protected]fbe29322013-07-09 09:03:2628from pylib import constants
[email protected]c0662e092013-11-12 11:51:2529from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2630from pylib.base import base_test_result
jbudorick66dc3722014-11-06 21:33:5131from pylib.base import environment_factory
[email protected]6bc1bda22013-07-19 22:08:3732from pylib.base import test_dispatcher
jbudorick66dc3722014-11-06 21:33:5133from pylib.base import test_instance_factory
34from pylib.base import test_run_factory
[email protected]6b6abac6d2013-10-03 11:56:3835from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1336from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3737from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2238from pylib.instrumentation import test_options as instrumentation_test_options
jbudorick9a6b7b332014-09-20 00:01:0739from pylib.junit import setup as junit_setup
40from pylib.junit import test_dispatcher as junit_dispatcher
[email protected]3dbdfa42013-08-08 01:08:1441from pylib.monkey import setup as monkey_setup
42from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4743from pylib.perf import setup as perf_setup
44from pylib.perf import test_options as perf_test_options
45from pylib.perf import test_runner as perf_test_runner
jbudorickb8c42072014-12-01 18:07:5446from pylib.results import json_results
47from pylib.results import report_results
[email protected]fbe29322013-07-09 09:03:2648
49
jama47ca85c2014-12-03 18:38:0750def AddCommonOptions(parser):
51 """Adds all common options to |parser|."""
[email protected]fbe29322013-07-09 09:03:2652
jama47ca85c2014-12-03 18:38:0753 group = parser.add_argument_group('Common Options')
54
[email protected]dfffbcbc2013-09-17 22:06:0155 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
jama47ca85c2014-12-03 18:38:0756
57 debug_or_release_group = group.add_mutually_exclusive_group()
58 debug_or_release_group.add_argument(
59 '--debug', action='store_const', const='Debug', dest='build_type',
60 default=default_build_type,
61 help=('If set, run test suites under out/Debug. '
62 'Default is env var BUILDTYPE or Debug.'))
63 debug_or_release_group.add_argument(
64 '--release', action='store_const', const='Release', dest='build_type',
65 help=('If set, run test suites under out/Release. '
66 'Default is env var BUILDTYPE or Debug.'))
67
68 group.add_argument('--build-directory', dest='build_directory',
69 help=('Path to the directory in which build files are'
70 ' located (should not include build type)'))
71 group.add_argument('--output-directory', dest='output_directory',
72 help=('Path to the directory in which build files are'
73 ' located (must include build type). This will take'
74 ' precedence over --debug, --release and'
75 ' --build-directory'))
agrieveddb11f12015-10-23 17:03:4376 group.add_argument('--num_retries', '--num-retries', dest='num_retries',
77 type=int, default=2,
jama47ca85c2014-12-03 18:38:0778 help=('Number of retries for a test before '
79 'giving up (default: %(default)s).'))
80 group.add_argument('-v',
81 '--verbose',
82 dest='verbose_count',
83 default=0,
84 action='count',
85 help='Verbose level (multiple times for more)')
86 group.add_argument('--flakiness-dashboard-server',
87 dest='flakiness_dashboard_server',
88 help=('Address of the server that is hosting the '
89 'Chrome for Android flakiness dashboard.'))
90 group.add_argument('--enable-platform-mode', action='store_true',
91 help=('Run the test scripts in platform mode, which '
92 'conceptually separates the test runner from the '
93 '"device" (local or remote, real or emulated) on '
94 'which the tests are running. [experimental]'))
95 group.add_argument('-e', '--environment', default='local',
96 choices=constants.VALID_ENVIRONMENTS,
97 help='Test environment to run in (default: %(default)s).')
98 group.add_argument('--adb-path',
99 help=('Specify the absolute path of the adb binary that '
100 'should be used.'))
101 group.add_argument('--json-results-file', dest='json_results_file',
102 help='If set, will dump results in JSON form '
103 'to specified file.')
[email protected]fbe29322013-07-09 09:03:26104
jama47ca85c2014-12-03 18:38:07105def ProcessCommonOptions(args):
[email protected]fbe29322013-07-09 09:03:26106 """Processes and handles all common options."""
jama47ca85c2014-12-03 18:38:07107 run_tests_helper.SetLogLevel(args.verbose_count)
108 constants.SetBuildType(args.build_type)
109 if args.build_directory:
110 constants.SetBuildDirectory(args.build_directory)
111 if args.output_directory:
mikecase0aea9c52015-04-30 00:12:33112 constants.SetOutputDirectory(args.output_directory)
jama47ca85c2014-12-03 18:38:07113 if args.adb_path:
114 constants.SetAdbPath(args.adb_path)
mikecase48e16bf2014-11-19 22:46:45115 # Some things such as Forwarder require ADB to be in the environment path.
116 adb_dir = os.path.dirname(constants.GetAdbPath())
117 if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep):
118 os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
[email protected]fbe29322013-07-09 09:03:26119
120
rnephew5c499782014-12-12 19:08:55121def AddRemoteDeviceOptions(parser):
122 group = parser.add_argument_group('Remote Device Options')
123
rnephewefe44b42015-02-04 04:45:15124 group.add_argument('--trigger',
jbudoricke6c560152015-01-13 23:49:28125 help=('Only triggers the test if set. Stores test_run_id '
126 'in given file path. '))
rnephewefe44b42015-02-04 04:45:15127 group.add_argument('--collect',
jbudoricke6c560152015-01-13 23:49:28128 help=('Only collects the test results if set. '
129 'Gets test_run_id from given file path.'))
rnephewefe44b42015-02-04 04:45:15130 group.add_argument('--remote-device', action='append',
jbudoricke6c560152015-01-13 23:49:28131 help='Device type to run test on.')
rnephewefe44b42015-02-04 04:45:15132 group.add_argument('--results-path',
jbudoricke6c560152015-01-13 23:49:28133 help='File path to download results to.')
rnephew7f1e2052014-12-12 23:00:11134 group.add_argument('--api-protocol',
jbudoricke6c560152015-01-13 23:49:28135 help='HTTP protocol to use. (http or https)')
rnephewefe44b42015-02-04 04:45:15136 group.add_argument('--api-address',
137 help='Address to send HTTP requests.')
138 group.add_argument('--api-port',
139 help='Port to send HTTP requests to.')
140 group.add_argument('--runner-type',
jbudoricke6c560152015-01-13 23:49:28141 help='Type of test to run as.')
rnephewefe44b42015-02-04 04:45:15142 group.add_argument('--runner-package',
143 help='Package name of test.')
144 group.add_argument('--device-type',
rnephewa46fc562015-01-23 16:00:14145 choices=constants.VALID_DEVICE_TYPES,
146 help=('Type of device to run on. iOS or android'))
rnephewefe44b42015-02-04 04:45:15147 group.add_argument('--device-oem', action='append',
148 help='Device OEM to run on.')
149 group.add_argument('--remote-device-file',
150 help=('File with JSON to select remote device. '
151 'Overrides all other flags.'))
rnephewc9ae8f52015-02-13 03:02:55152 group.add_argument('--remote-device-timeout', type=int,
153 help='Times to retry finding remote device')
mikecase520cbbb52015-04-21 18:51:18154 group.add_argument('--network-config', type=int,
155 help='Integer that specifies the network environment '
156 'that the tests will be run in.')
mikecaseddfa35d2015-10-28 01:14:27157 group.add_argument('--test-timeout', type=int,
158 help='Test run timeout in seconds.')
rnephewefe44b42015-02-04 04:45:15159
160 device_os_group = group.add_mutually_exclusive_group()
161 device_os_group.add_argument('--remote-device-minimum-os',
162 help='Minimum OS on device.')
163 device_os_group.add_argument('--remote-device-os', action='append',
164 help='OS to have on the device.')
rnephew5c499782014-12-12 19:08:55165
166 api_secret_group = group.add_mutually_exclusive_group()
167 api_secret_group.add_argument('--api-secret', default='',
jbudoricke6c560152015-01-13 23:49:28168 help='API secret for remote devices.')
rnephew5c499782014-12-12 19:08:55169 api_secret_group.add_argument('--api-secret-file', default='',
jbudoricke6c560152015-01-13 23:49:28170 help='Path to file that contains API secret.')
rnephew5c499782014-12-12 19:08:55171
172 api_key_group = group.add_mutually_exclusive_group()
173 api_key_group.add_argument('--api-key', default='',
jbudoricke6c560152015-01-13 23:49:28174 help='API key for remote devices.')
rnephew5c499782014-12-12 19:08:55175 api_key_group.add_argument('--api-key-file', default='',
jbudoricke6c560152015-01-13 23:49:28176 help='Path to file that contains API key.')
rnephew5c499782014-12-12 19:08:55177
178
jama47ca85c2014-12-03 18:38:07179def AddDeviceOptions(parser):
180 """Adds device options to |parser|."""
181 group = parser.add_argument_group(title='Device Options')
jama47ca85c2014-12-03 18:38:07182 group.add_argument('--tool',
183 dest='tool',
184 help=('Run the test under a tool '
185 '(use --tool help to list them)'))
186 group.add_argument('-d', '--device', dest='test_device',
187 help=('Target device for the test suite '
188 'to run on.'))
jbudorickdde688fb2015-08-27 03:00:17189 group.add_argument('--blacklist-file', help='Device blacklist file.')
agrievea538a142015-10-09 15:45:56190 group.add_argument('--enable-device-cache', action='store_true',
191 help='Cache device state to disk between runs')
agrieve1a02e582015-10-15 21:35:39192 group.add_argument('--incremental-install', action='store_true',
193 help='Use an _incremental apk.')
agrieve8bcb52e2015-10-20 19:38:33194 group.add_argument('--enable-concurrent-adb', action='store_true',
195 help='Run multiple adb commands at the same time, even '
196 'for the same device.')
jbudorick256fd532014-10-24 01:50:13197
198
jama47ca85c2014-12-03 18:38:07199def AddGTestOptions(parser):
200 """Adds gtest options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26201
jama47ca85c2014-12-03 18:38:07202 group = parser.add_argument_group('GTest Options')
jbudorick15cdcd52014-12-03 19:58:49203 group.add_argument('-s', '--suite', dest='suite_name',
jama47ca85c2014-12-03 18:38:07204 nargs='+', metavar='SUITE_NAME', required=True,
jbudorick277f2312015-09-24 16:37:43205 help='Executable name of the test suite to run.')
jama47ca85c2014-12-03 18:38:07206 group.add_argument('--gtest_also_run_disabled_tests',
207 '--gtest-also-run-disabled-tests',
208 dest='run_disabled', action='store_true',
209 help='Also run disabled tests if applicable.')
210 group.add_argument('-a', '--test-arguments', dest='test_arguments',
211 default='',
212 help='Additional arguments to pass to the test.')
jbudorick24616eb2015-10-06 02:40:57213 group.add_argument('-t', '--shard-timeout',
214 dest='shard_timeout', type=int, default=60,
jama47ca85c2014-12-03 18:38:07215 help='Timeout to wait for each test '
216 '(default: %(default)s).')
217 group.add_argument('--isolate_file_path',
218 '--isolate-file-path',
219 dest='isolate_file_path',
220 help='.isolate file path to override the default '
221 'path')
jbudorick5ee45892015-06-10 18:46:22222 group.add_argument('--app-data-file', action='append', dest='app_data_files',
223 help='A file path relative to the app data directory '
224 'that should be saved to the host.')
225 group.add_argument('--app-data-file-dir',
226 help='Host directory to which app data files will be'
227 ' saved. Used with --app-data-file.')
mlliud7f9fe92015-06-15 19:36:56228 group.add_argument('--delete-stale-data', dest='delete_stale_data',
229 action='store_true',
230 help='Delete stale test data on the device.')
jbudorickeb7ea71c2015-09-28 16:40:20231 group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat',
232 dest='repeat', type=int, default=0,
233 help='Number of times to repeat the specified set of '
234 'tests.')
jbudorick442a6932015-02-03 03:01:15235
236 filter_group = group.add_mutually_exclusive_group()
237 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
238 dest='test_filter',
239 help='googletest-style filter string.')
240 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
241 help='Path to file that contains googletest-style '
242 'filter strings. (Lines will be joined with '
243 '":" to create a single filter string.)')
244
jama47ca85c2014-12-03 18:38:07245 AddDeviceOptions(parser)
246 AddCommonOptions(parser)
rnephew5c499782014-12-12 19:08:55247 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26248
249
jama47ca85c2014-12-03 18:38:07250def AddLinkerTestOptions(parser):
251 group = parser.add_argument_group('Linker Test Options')
252 group.add_argument('-f', '--gtest-filter', dest='test_filter',
253 help='googletest-style filter string.')
254 AddCommonOptions(parser)
255 AddDeviceOptions(parser)
[email protected]6b6abac6d2013-10-03 11:56:38256
257
jama47ca85c2014-12-03 18:38:07258def AddJavaTestOptions(argument_group):
[email protected]fbe29322013-07-09 09:03:26259 """Adds the Java test options to |option_parser|."""
260
jama47ca85c2014-12-03 18:38:07261 argument_group.add_argument(
262 '-f', '--test-filter', dest='test_filter',
263 help=('Test filter (if not fully qualified, will run all matches).'))
264 argument_group.add_argument(
jbudorickeb7ea71c2015-09-28 16:40:20265 '--repeat', dest='repeat', type=int, default=0,
266 help='Number of times to repeat the specified set of tests.')
267 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26268 '-A', '--annotation', dest='annotation_str',
269 help=('Comma-separated list of annotations. Run only tests with any of '
270 'the given annotations. An annotation can be either a key or a '
271 'key-values pair. A test that has no annotation is considered '
272 '"SmallTest".'))
jama47ca85c2014-12-03 18:38:07273 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26274 '-E', '--exclude-annotation', dest='exclude_annotation_str',
275 help=('Comma-separated list of annotations. Exclude tests with these '
276 'annotations.'))
jama47ca85c2014-12-03 18:38:07277 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59278 '--screenshot', dest='screenshot_failures', action='store_true',
279 help='Capture screenshots of test failures')
jama47ca85c2014-12-03 18:38:07280 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59281 '--save-perf-json', action='store_true',
282 help='Saves the JSON file for each UI Perf test.')
jama47ca85c2014-12-03 18:38:07283 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59284 '--official-build', action='store_true', help='Run official build tests.')
jama47ca85c2014-12-03 18:38:07285 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59286 '--test_data', '--test-data', action='append', default=[],
287 help=('Each instance defines a directory of test data that should be '
288 'copied to the target(s) before running the tests. The argument '
289 'should be of the form <target>:<source>, <target> is relative to '
290 'the device data directory, and <source> is relative to the '
291 'chromium build directory.'))
davileen98efad12015-01-05 19:48:21292 argument_group.add_argument(
293 '--disable-dalvik-asserts', dest='set_asserts', action='store_false',
294 default=True, help='Removes the dalvik.vm.enableassertions property')
295
[email protected]fbe29322013-07-09 09:03:26296
297
jama47ca85c2014-12-03 18:38:07298def ProcessJavaTestOptions(args):
[email protected]fbe29322013-07-09 09:03:26299 """Processes options/arguments and populates |options| with defaults."""
300
jama47ca85c2014-12-03 18:38:07301 # TODO(jbudorick): Handle most of this function in argparse.
302 if args.annotation_str:
303 args.annotations = args.annotation_str.split(',')
304 elif args.test_filter:
305 args.annotations = []
[email protected]fbe29322013-07-09 09:03:26306 else:
jama47ca85c2014-12-03 18:38:07307 args.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
308 'EnormousTest', 'IntegrationTest']
[email protected]fbe29322013-07-09 09:03:26309
jama47ca85c2014-12-03 18:38:07310 if args.exclude_annotation_str:
311 args.exclude_annotations = args.exclude_annotation_str.split(',')
[email protected]fbe29322013-07-09 09:03:26312 else:
jama47ca85c2014-12-03 18:38:07313 args.exclude_annotations = []
[email protected]fbe29322013-07-09 09:03:26314
[email protected]fbe29322013-07-09 09:03:26315
jama47ca85c2014-12-03 18:38:07316def AddInstrumentationTestOptions(parser):
317 """Adds Instrumentation test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26318
jama47ca85c2014-12-03 18:38:07319 parser.usage = '%(prog)s [options]'
[email protected]fbe29322013-07-09 09:03:26320
jama47ca85c2014-12-03 18:38:07321 group = parser.add_argument_group('Instrumentation Test Options')
322 AddJavaTestOptions(group)
[email protected]fbe29322013-07-09 09:03:26323
jama47ca85c2014-12-03 18:38:07324 java_or_python_group = group.add_mutually_exclusive_group()
325 java_or_python_group.add_argument(
326 '-j', '--java-only', action='store_false',
327 dest='run_python_tests', default=True, help='Run only the Java tests.')
328 java_or_python_group.add_argument(
329 '-p', '--python-only', action='store_false',
330 dest='run_java_tests', default=True,
331 help='Run only the host-driven tests.')
332
333 group.add_argument('--host-driven-root',
334 help='Root of the host-driven tests.')
335 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger',
336 action='store_true',
337 help='Wait for debugger.')
jbudorick911be58d2015-01-13 02:51:06338 group.add_argument('--apk-under-test', dest='apk_under_test',
339 help=('the name of the apk under test.'))
jama47ca85c2014-12-03 18:38:07340 group.add_argument('--test-apk', dest='test_apk', required=True,
341 help=('The name of the apk containing the tests '
342 '(without the .apk extension; '
343 'e.g. "ContentShellTest").'))
mikecasee7258622015-09-29 13:47:35344 group.add_argument('--additional-apk', action='append',
mikecase8c4ab302015-09-29 17:03:29345 dest='additional_apks', default=[],
mikecasee7258622015-09-29 13:47:35346 help='Additional apk that must be installed on '
347 'the device when the tests are run')
jama47ca85c2014-12-03 18:38:07348 group.add_argument('--coverage-dir',
349 help=('Directory in which to place all generated '
350 'EMMA coverage files.'))
351 group.add_argument('--device-flags', dest='device_flags', default='',
352 help='The relative filepath to a file containing '
353 'command-line flags to set on the device')
jbudorick911be58d2015-01-13 02:51:06354 group.add_argument('--device-flags-file', default='',
355 help='The relative filepath to a file containing '
356 'command-line flags to set on the device')
jama47ca85c2014-12-03 18:38:07357 group.add_argument('--isolate_file_path',
358 '--isolate-file-path',
359 dest='isolate_file_path',
360 help='.isolate file path to override the default '
361 'path')
mlliud7f9fe92015-06-15 19:36:56362 group.add_argument('--delete-stale-data', dest='delete_stale_data',
363 action='store_true',
364 help='Delete stale test data on the device.')
jama47ca85c2014-12-03 18:38:07365
366 AddCommonOptions(parser)
367 AddDeviceOptions(parser)
rnephewe416dff2015-01-21 21:26:37368 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26369
370
jama47ca85c2014-12-03 18:38:07371def ProcessInstrumentationOptions(args):
[email protected]2a684222013-08-01 16:59:22372 """Processes options/arguments and populate |options| with defaults.
373
374 Args:
jama47ca85c2014-12-03 18:38:07375 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22376
377 Returns:
378 An InstrumentationOptions named tuple which contains all options relevant to
379 instrumentation tests.
380 """
[email protected]fbe29322013-07-09 09:03:26381
jama47ca85c2014-12-03 18:38:07382 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26383
jama47ca85c2014-12-03 18:38:07384 if not args.host_driven_root:
385 args.run_python_tests = False
[email protected]37ee0c792013-08-06 19:10:13386
jbudorick9ef3f9552015-10-20 22:58:33387 if os.path.exists(args.test_apk):
388 args.test_apk_path = args.test_apk
389 args.test_apk, _ = os.path.splitext(os.path.basename(args.test_apk))
390 else:
391 args.test_apk_path = os.path.join(
392 constants.GetOutDirectory(),
393 constants.SDK_BUILD_APKS_DIR,
394 '%s.apk' % args.test_apk)
395
jama47ca85c2014-12-03 18:38:07396 args.test_apk_jar_path = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15397 constants.GetOutDirectory(),
398 constants.SDK_BUILD_TEST_JAVALIB_DIR,
jbudorick9ef3f9552015-10-20 22:58:33399 '%s.jar' % args.test_apk)
yusufo72c598c02015-07-16 23:40:20400 args.test_support_apk_path = '%sSupport%s' % (
401 os.path.splitext(args.test_apk_path))
[email protected]5e2f3f62014-06-23 12:31:46402
jama47ca85c2014-12-03 18:38:07403 args.test_runner = apk_helper.GetInstrumentationName(args.test_apk_path)
[email protected]5e2f3f62014-06-23 12:31:46404
jama47ca85c2014-12-03 18:38:07405 # TODO(jbudorick): Get rid of InstrumentationOptions.
[email protected]2a684222013-08-01 16:59:22406 return instrumentation_test_options.InstrumentationOptions(
jama47ca85c2014-12-03 18:38:07407 args.tool,
jama47ca85c2014-12-03 18:38:07408 args.annotations,
409 args.exclude_annotations,
410 args.test_filter,
411 args.test_data,
412 args.save_perf_json,
413 args.screenshot_failures,
414 args.wait_for_debugger,
415 args.coverage_dir,
416 args.test_apk,
417 args.test_apk_path,
418 args.test_apk_jar_path,
419 args.test_runner,
420 args.test_support_apk_path,
421 args.device_flags,
davileen98efad12015-01-05 19:48:21422 args.isolate_file_path,
mlliud7f9fe92015-06-15 19:36:56423 args.set_asserts,
424 args.delete_stale_data
[email protected]5e2f3f62014-06-23 12:31:46425 )
[email protected]2a684222013-08-01 16:59:22426
[email protected]fbe29322013-07-09 09:03:26427
jama47ca85c2014-12-03 18:38:07428def AddUIAutomatorTestOptions(parser):
429 """Adds UI Automator test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26430
jama47ca85c2014-12-03 18:38:07431 group = parser.add_argument_group('UIAutomator Test Options')
432 AddJavaTestOptions(group)
433 group.add_argument(
434 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
435 metavar='PACKAGE', help='Package under test.')
436 group.add_argument(
437 '--test-jar', dest='test_jar', required=True,
[email protected]fbe29322013-07-09 09:03:26438 help=('The name of the dexed jar containing the tests (without the '
439 '.dex.jar extension). Alternatively, this can be a full path '
440 'to the jar.'))
441
jama47ca85c2014-12-03 18:38:07442 AddCommonOptions(parser)
443 AddDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26444
445
jama47ca85c2014-12-03 18:38:07446def AddJUnitTestOptions(parser):
447 """Adds junit test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07448
jama47ca85c2014-12-03 18:38:07449 group = parser.add_argument_group('JUnit Test Options')
450 group.add_argument(
451 '-s', '--test-suite', dest='test_suite', required=True,
jbudorick9a6b7b332014-09-20 00:01:07452 help=('JUnit test suite to run.'))
jama47ca85c2014-12-03 18:38:07453 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07454 '-f', '--test-filter', dest='test_filter',
455 help='Filters tests googletest-style.')
jama47ca85c2014-12-03 18:38:07456 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07457 '--package-filter', dest='package_filter',
458 help='Filters tests by package.')
jama47ca85c2014-12-03 18:38:07459 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07460 '--runner-filter', dest='runner_filter',
461 help='Filters tests by runner class. Must be fully qualified.')
jama47ca85c2014-12-03 18:38:07462 group.add_argument(
463 '--sdk-version', dest='sdk_version', type=int,
jbudorick9a6b7b332014-09-20 00:01:07464 help='The Android SDK version.')
jama47ca85c2014-12-03 18:38:07465 AddCommonOptions(parser)
jbudorick9a6b7b332014-09-20 00:01:07466
467
jama47ca85c2014-12-03 18:38:07468def AddMonkeyTestOptions(parser):
469 """Adds monkey test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07470
jama47ca85c2014-12-03 18:38:07471 group = parser.add_argument_group('Monkey Test Options')
472 group.add_argument(
473 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
474 metavar='PACKAGE', help='Package under test.')
475 group.add_argument(
476 '--event-count', default=10000, type=int,
477 help='Number of events to generate (default: %(default)s).')
478 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14479 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12480 help='A list of allowed categories.')
jama47ca85c2014-12-03 18:38:07481 group.add_argument(
482 '--throttle', default=100, type=int,
483 help='Delay between events (ms) (default: %(default)s). ')
484 group.add_argument(
485 '--seed', type=int,
[email protected]3dbdfa42013-08-08 01:08:14486 help=('Seed value for pseudo-random generator. Same seed value generates '
487 'the same sequence of events. Seed is randomized by default.'))
jama47ca85c2014-12-03 18:38:07488 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14489 '--extra-args', default='',
jama47ca85c2014-12-03 18:38:07490 help=('String of other args to pass to the command verbatim.'))
[email protected]3dbdfa42013-08-08 01:08:14491
jama47ca85c2014-12-03 18:38:07492 AddCommonOptions(parser)
493 AddDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14494
jama47ca85c2014-12-03 18:38:07495def ProcessMonkeyTestOptions(args):
[email protected]3dbdfa42013-08-08 01:08:14496 """Processes all monkey test options.
497
498 Args:
jama47ca85c2014-12-03 18:38:07499 args: argparse.Namespace object.
[email protected]3dbdfa42013-08-08 01:08:14500
501 Returns:
502 A MonkeyOptions named tuple which contains all options relevant to
503 monkey tests.
504 """
jama47ca85c2014-12-03 18:38:07505 # TODO(jbudorick): Handle this directly in argparse with nargs='+'
506 category = args.category
[email protected]3dbdfa42013-08-08 01:08:14507 if category:
jama47ca85c2014-12-03 18:38:07508 category = args.category.split(',')
[email protected]3dbdfa42013-08-08 01:08:14509
jama47ca85c2014-12-03 18:38:07510 # TODO(jbudorick): Get rid of MonkeyOptions.
[email protected]3dbdfa42013-08-08 01:08:14511 return monkey_test_options.MonkeyOptions(
jama47ca85c2014-12-03 18:38:07512 args.verbose_count,
513 args.package,
514 args.event_count,
[email protected]3dbdfa42013-08-08 01:08:14515 category,
jama47ca85c2014-12-03 18:38:07516 args.throttle,
517 args.seed,
518 args.extra_args)
[email protected]3dbdfa42013-08-08 01:08:14519
rnephew5c499782014-12-12 19:08:55520def AddUirobotTestOptions(parser):
521 """Adds uirobot test options to |option_parser|."""
522 group = parser.add_argument_group('Uirobot Test Options')
523
rnephewefe44b42015-02-04 04:45:15524 group.add_argument('--app-under-test', required=True,
525 help='APK to run tests on.')
rnephew5c499782014-12-12 19:08:55526 group.add_argument(
mikecaseafa43842015-10-19 23:04:12527 '--repeat', dest='repeat', type=int, default=0,
528 help='Number of times to repeat the uirobot test.')
529 group.add_argument(
rnephew5c499782014-12-12 19:08:55530 '--minutes', default=5, type=int,
jbudorick676b1202015-02-06 22:02:27531 help='Number of minutes to run uirobot test [default: %(default)s].')
rnephew5c499782014-12-12 19:08:55532
533 AddCommonOptions(parser)
534 AddDeviceOptions(parser)
535 AddRemoteDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14536
jama47ca85c2014-12-03 18:38:07537def AddPerfTestOptions(parser):
538 """Adds perf test options to |parser|."""
[email protected]ec3170b2013-08-14 14:39:47539
jama47ca85c2014-12-03 18:38:07540 group = parser.add_argument_group('Perf Test Options')
[email protected]ec3170b2013-08-14 14:39:47541
jama47ca85c2014-12-03 18:38:07542 class SingleStepAction(argparse.Action):
543 def __call__(self, parser, namespace, values, option_string=None):
544 if values and not namespace.single_step:
545 parser.error('single step command provided, '
546 'but --single-step not specified.')
547 elif namespace.single_step and not values:
548 parser.error('--single-step specified, '
549 'but no single step command provided.')
550 setattr(namespace, self.dest, values)
551
552 step_group = group.add_mutually_exclusive_group(required=True)
553 # TODO(jbudorick): Revise --single-step to use argparse.REMAINDER.
554 # This requires removing "--" from client calls.
555 step_group.add_argument(
556 '--single-step', action='store_true',
[email protected]def4bce2013-11-12 12:59:52557 help='Execute the given command with retries, but only print the result '
558 'for the "most successful" round.')
jama47ca85c2014-12-03 18:38:07559 step_group.add_argument(
[email protected]181a5c92013-09-06 17:11:46560 '--steps',
[email protected]def4bce2013-11-12 12:59:52561 help='JSON file containing the list of commands to run.')
jama47ca85c2014-12-03 18:38:07562 step_group.add_argument(
563 '--print-step',
564 help='The name of a previously executed perf step to print.')
565
566 group.add_argument(
peterbd4e73d2014-12-03 15:47:36567 '--output-json-list',
568 help='Write a simple list of names from --steps into the given file.')
jama47ca85c2014-12-03 18:38:07569 group.add_argument(
peterbd4e73d2014-12-03 15:47:36570 '--collect-chartjson-data',
571 action='store_true',
572 help='Cache the chartjson output from each step for later use.')
jama47ca85c2014-12-03 18:38:07573 group.add_argument(
peterbd4e73d2014-12-03 15:47:36574 '--output-chartjson-data',
575 default='',
576 help='Write out chartjson into the given file.')
jama47ca85c2014-12-03 18:38:07577 group.add_argument(
perezju67cf7f12015-09-29 11:39:05578 '--get-output-dir-archive', metavar='FILENAME',
579 help='Write the chached output directory archived by a step into the'
580 ' given ZIP file.')
581 group.add_argument(
jama47ca85c2014-12-03 18:38:07582 '--flaky-steps',
583 help=('A JSON file containing steps that are flaky '
584 'and will have its exit code ignored.'))
585 group.add_argument(
[email protected]181a5c92013-09-06 17:11:46586 '--no-timeout', action='store_true',
587 help=('Do not impose a timeout. Each perf step is responsible for '
588 'implementing the timeout logic.'))
jama47ca85c2014-12-03 18:38:07589 group.add_argument(
[email protected]650487c2013-09-30 11:40:49590 '-f', '--test-filter',
591 help=('Test filter (will match against the names listed in --steps).'))
jama47ca85c2014-12-03 18:38:07592 group.add_argument(
593 '--dry-run', action='store_true',
[email protected]650487c2013-09-30 11:40:49594 help='Just print the steps without executing.')
jbudorick5cfff872015-07-01 18:46:13595 # Uses 0.1 degrees C because that's what Android does.
596 group.add_argument(
597 '--max-battery-temp', type=int,
598 help='Only start tests when the battery is at or below the given '
599 'temperature (0.1 C)')
jama47ca85c2014-12-03 18:38:07600 group.add_argument('single_step_command', nargs='*', action=SingleStepAction,
601 help='If --single-step is specified, the command to run.')
rnephewdde05da82015-07-09 20:31:01602 group.add_argument('--min-battery-level', type=int,
603 help='Only starts tests when the battery is charged above '
604 'given level.')
jama47ca85c2014-12-03 18:38:07605 AddCommonOptions(parser)
606 AddDeviceOptions(parser)
[email protected]ec3170b2013-08-14 14:39:47607
608
jama47ca85c2014-12-03 18:38:07609def ProcessPerfTestOptions(args):
[email protected]ec3170b2013-08-14 14:39:47610 """Processes all perf test options.
611
612 Args:
jama47ca85c2014-12-03 18:38:07613 args: argparse.Namespace object.
[email protected]ec3170b2013-08-14 14:39:47614
615 Returns:
616 A PerfOptions named tuple which contains all options relevant to
617 perf tests.
618 """
jama47ca85c2014-12-03 18:38:07619 # TODO(jbudorick): Move single_step handling down into the perf tests.
620 if args.single_step:
621 args.single_step = ' '.join(args.single_step_command)
622 # TODO(jbudorick): Get rid of PerfOptions.
[email protected]ec3170b2013-08-14 14:39:47623 return perf_test_options.PerfOptions(
jama47ca85c2014-12-03 18:38:07624 args.steps, args.flaky_steps, args.output_json_list,
625 args.print_step, args.no_timeout, args.test_filter,
626 args.dry_run, args.single_step, args.collect_chartjson_data,
perezju67cf7f12015-09-29 11:39:05627 args.output_chartjson_data, args.get_output_dir_archive,
628 args.max_battery_temp, args.min_battery_level)
[email protected]ec3170b2013-08-14 14:39:47629
630
jama47ca85c2014-12-03 18:38:07631def AddPythonTestOptions(parser):
632 group = parser.add_argument_group('Python Test Options')
633 group.add_argument(
634 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME',
635 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(),
636 help='Name of the test suite to run.')
637 AddCommonOptions(parser)
jbudorick256fd532014-10-24 01:50:13638
639
jama47ca85c2014-12-03 18:38:07640def _RunLinkerTests(args, devices):
[email protected]6b6abac6d2013-10-03 11:56:38641 """Subcommand of RunTestsCommands which runs linker tests."""
jama47ca85c2014-12-03 18:38:07642 runner_factory, tests = linker_setup.Setup(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38643
644 results, exit_code = test_dispatcher.RunTests(
645 tests, runner_factory, devices, shard=True, test_timeout=60,
jama47ca85c2014-12-03 18:38:07646 num_retries=args.num_retries)
[email protected]6b6abac6d2013-10-03 11:56:38647
648 report_results.LogFull(
649 results=results,
650 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22651 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38652
jama47ca85c2014-12-03 18:38:07653 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20654 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54655
[email protected]6b6abac6d2013-10-03 11:56:38656 return exit_code
657
658
jama47ca85c2014-12-03 18:38:07659def _RunInstrumentationTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37660 """Subcommand of RunTestsCommands which runs instrumentation tests."""
jbudorick58b4d362015-09-08 16:44:59661 logging.info('_RunInstrumentationTests(%s, %s)', str(args), str(devices))
[email protected]6bc1bda22013-07-19 22:08:37662
jama47ca85c2014-12-03 18:38:07663 instrumentation_options = ProcessInstrumentationOptions(args)
664
665 if len(devices) > 1 and args.wait_for_debugger:
[email protected]f7148dd42013-08-20 14:24:57666 logging.warning('Debugger can not be sharded, using first available device')
667 devices = devices[:1]
668
[email protected]6bc1bda22013-07-19 22:08:37669 results = base_test_result.TestRunResults()
670 exit_code = 0
671
jama47ca85c2014-12-03 18:38:07672 if args.run_java_tests:
jbudorickeb7ea71c2015-09-28 16:40:20673 java_runner_factory, java_tests = instrumentation_setup.Setup(
mikecase526d68e2014-11-19 20:02:05674 instrumentation_options, devices)
jbudorickeb7ea71c2015-09-28 16:40:20675 else:
676 java_runner_factory = None
677 java_tests = None
[email protected]6bc1bda22013-07-19 22:08:37678
jama47ca85c2014-12-03 18:38:07679 if args.run_python_tests:
jbudorickeb7ea71c2015-09-28 16:40:20680 py_runner_factory, py_tests = host_driven_setup.InstrumentationSetup(
jama47ca85c2014-12-03 18:38:07681 args.host_driven_root, args.official_build,
[email protected]37ee0c792013-08-06 19:10:13682 instrumentation_options)
jbudorickeb7ea71c2015-09-28 16:40:20683 else:
684 py_runner_factory = None
685 py_tests = None
[email protected]37ee0c792013-08-06 19:10:13686
jbudorickeb7ea71c2015-09-28 16:40:20687 results = []
688 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
689 else itertools.count())
690 for _ in repetitions:
691 iteration_results = base_test_result.TestRunResults()
692 if java_tests:
[email protected]34020022013-08-06 23:35:34693 test_results, test_exit_code = test_dispatcher.RunTests(
jbudorickeb7ea71c2015-09-28 16:40:20694 java_tests, java_runner_factory, devices, shard=True,
695 test_timeout=None, num_retries=args.num_retries)
696 iteration_results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37697
[email protected]34020022013-08-06 23:35:34698 # Only allow exit code escalation
699 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
700 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37701
jbudorickeb7ea71c2015-09-28 16:40:20702 if py_tests:
703 test_results, test_exit_code = test_dispatcher.RunTests(
704 py_tests, py_runner_factory, devices, shard=True, test_timeout=None,
705 num_retries=args.num_retries)
706 iteration_results.AddTestRunResults(test_results)
[email protected]4f777ca2014-08-08 01:45:59707
jbudorickeb7ea71c2015-09-28 16:40:20708 # Only allow exit code escalation
709 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
710 exit_code = test_exit_code
711
712 results.append(iteration_results)
713 report_results.LogFull(
714 results=iteration_results,
715 test_type='Instrumentation',
716 test_package=os.path.basename(args.test_apk),
717 annotation=args.annotations,
718 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37719
jama47ca85c2014-12-03 18:38:07720 if args.json_results_file:
721 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54722
[email protected]6bc1bda22013-07-19 22:08:37723 return exit_code
724
725
jama47ca85c2014-12-03 18:38:07726def _RunJUnitTests(args):
jbudorick9a6b7b332014-09-20 00:01:07727 """Subcommand of RunTestsCommand which runs junit tests."""
jama47ca85c2014-12-03 18:38:07728 runner_factory, tests = junit_setup.Setup(args)
mikecasec638a072015-04-01 16:35:35729 results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
730
731 report_results.LogFull(
732 results=results,
733 test_type='JUnit',
734 test_package=args.test_suite)
735
mikecase572401b2015-04-09 02:28:57736 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20737 json_results.GenerateJsonResultsFile([results], args.json_results_file)
mikecase572401b2015-04-09 02:28:57738
jbudorick9a6b7b332014-09-20 00:01:07739 return exit_code
740
741
jama47ca85c2014-12-03 18:38:07742def _RunMonkeyTests(args, devices):
[email protected]3dbdfa42013-08-08 01:08:14743 """Subcommand of RunTestsCommands which runs monkey tests."""
jama47ca85c2014-12-03 18:38:07744 monkey_options = ProcessMonkeyTestOptions(args)
[email protected]3dbdfa42013-08-08 01:08:14745
746 runner_factory, tests = monkey_setup.Setup(monkey_options)
747
748 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46749 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07750 num_retries=args.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14751
752 report_results.LogFull(
753 results=results,
754 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28755 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14756
jama47ca85c2014-12-03 18:38:07757 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20758 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54759
[email protected]3dbdfa42013-08-08 01:08:14760 return exit_code
761
762
jbudorickdde688fb2015-08-27 03:00:17763def _RunPerfTests(args, active_devices):
[email protected]ec3170b2013-08-14 14:39:47764 """Subcommand of RunTestsCommands which runs perf tests."""
jama47ca85c2014-12-03 18:38:07765 perf_options = ProcessPerfTestOptions(args)
[email protected]61487ed2014-06-09 12:33:56766
767 # Just save a simple json with a list of test names.
768 if perf_options.output_json_list:
769 return perf_test_runner.OutputJsonList(
770 perf_options.steps, perf_options.output_json_list)
771
[email protected]ad32f312013-11-13 04:03:29772 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47773 if perf_options.print_step:
simonhatch9b9256d2015-01-07 18:03:42774 return perf_test_runner.PrintTestOutput(
perezju67cf7f12015-09-29 11:39:05775 perf_options.print_step, perf_options.output_chartjson_data,
776 perf_options.get_output_dir_archive)
[email protected]ec3170b2013-08-14 14:39:47777
jbudorickdde688fb2015-08-27 03:00:17778 runner_factory, tests, devices = perf_setup.Setup(
779 perf_options, active_devices)
[email protected]ec3170b2013-08-14 14:39:47780
[email protected]a72f0752014-06-03 23:52:34781 # shard=False means that each device will get the full list of tests
782 # and then each one will decide their own affinity.
783 # shard=True means each device will pop the next test available from a queue,
784 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57785 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34786 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07787 num_retries=args.num_retries)
[email protected]ec3170b2013-08-14 14:39:47788
789 report_results.LogFull(
790 results=results,
791 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12792 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52793
jama47ca85c2014-12-03 18:38:07794 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20795 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54796
[email protected]def4bce2013-11-12 12:59:52797 if perf_options.single_step:
798 return perf_test_runner.PrintTestOutput('single_step')
799
[email protected]11ce8452014-02-17 10:55:03800 perf_test_runner.PrintSummary(tests)
801
[email protected]86184c7b2013-08-15 15:06:57802 # Always return 0 on the sharding stage. Individual tests exit_code
803 # will be returned on the print_step stage.
804 return 0
[email protected]ec3170b2013-08-14 14:39:47805
[email protected]3dbdfa42013-08-08 01:08:14806
jama47ca85c2014-12-03 18:38:07807def _RunPythonTests(args):
jbudorick256fd532014-10-24 01:50:13808 """Subcommand of RunTestsCommand which runs python unit tests."""
jama47ca85c2014-12-03 18:38:07809 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
jbudorick256fd532014-10-24 01:50:13810 suite_path = suite_vars['path']
811 suite_test_modules = suite_vars['test_modules']
812
813 sys.path = [suite_path] + sys.path
814 try:
815 suite = unittest.TestSuite()
816 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
817 for m in suite_test_modules)
jama47ca85c2014-12-03 18:38:07818 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
jbudorick256fd532014-10-24 01:50:13819 return 0 if runner.run(suite).wasSuccessful() else 1
820 finally:
821 sys.path = sys.path[1:]
822
823
agrievea538a142015-10-09 15:45:56824def _GetAttachedDevices(blacklist_file, test_device, enable_cache):
[email protected]f7148dd42013-08-20 14:24:57825 """Get all attached devices.
826
827 Args:
agrievea538a142015-10-09 15:45:56828 blacklist_file: Path to device blacklist.
[email protected]f7148dd42013-08-20 14:24:57829 test_device: Name of a specific device to use.
agrievea538a142015-10-09 15:45:56830 enable_cache: Whether to enable checksum caching.
[email protected]f7148dd42013-08-20 14:24:57831
832 Returns:
833 A list of attached devices.
834 """
jbudoricka583ba32015-09-11 17:23:19835 blacklist = (device_blacklist.Blacklist(blacklist_file)
836 if blacklist_file
837 else None)
jbudorickdde688fb2015-08-27 03:00:17838
agrievea538a142015-10-09 15:45:56839 attached_devices = device_utils.DeviceUtils.HealthyDevices(
840 blacklist, enable_device_files_cache=enable_cache)
aberent6a02a6182015-04-29 11:07:55841 if test_device:
jbudorick4551d0dc2015-04-29 16:07:06842 test_device = [d for d in attached_devices if d == test_device]
843 if not test_device:
844 raise device_errors.DeviceUnreachableError(
845 'Did not find device %s among attached device. Attached devices: %s'
846 % (test_device, ', '.join(attached_devices)))
847 return test_device
aberent6a02a6182015-04-29 11:07:55848
jbudorick4551d0dc2015-04-29 16:07:06849 else:
850 if not attached_devices:
851 raise device_errors.NoDevicesError()
852 return sorted(attached_devices)
[email protected]f7148dd42013-08-20 14:24:57853
854
jbudorick58b4d362015-09-08 16:44:59855def RunTestsCommand(args, parser): # pylint: disable=too-many-return-statements
[email protected]fbe29322013-07-09 09:03:26856 """Checks test type and dispatches to the appropriate function.
857
858 Args:
jama47ca85c2014-12-03 18:38:07859 args: argparse.Namespace object.
860 parser: argparse.ArgumentParser object.
[email protected]fbe29322013-07-09 09:03:26861
862 Returns:
863 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10864
865 Raises:
866 Exception: Unknown command name passed in, or an exception from an
867 individual test runner.
[email protected]fbe29322013-07-09 09:03:26868 """
jama47ca85c2014-12-03 18:38:07869 command = args.command
[email protected]fbe29322013-07-09 09:03:26870
jama47ca85c2014-12-03 18:38:07871 ProcessCommonOptions(args)
[email protected]d82f0252013-07-12 23:22:57872
jama47ca85c2014-12-03 18:38:07873 if args.enable_platform_mode:
rnephew5c499782014-12-12 19:08:55874 return RunTestsInPlatformMode(args, parser)
jbudorick66dc3722014-11-06 21:33:51875
[email protected]c0662e092013-11-12 11:51:25876 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40877 if not ports.ResetTestServerPortAllocation():
878 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25879
agrieve18930bd2015-10-09 17:41:42880 def get_devices():
881 return _GetAttachedDevices(args.blacklist_file, args.test_device,
882 args.enable_device_cache)
883
[email protected]fbe29322013-07-09 09:03:26884 if command == 'gtest':
jbudorick566592ab2015-09-21 15:32:47885 return RunTestsInPlatformMode(args, parser)
[email protected]6b6abac6d2013-10-03 11:56:38886 elif command == 'linker':
agrieve18930bd2015-10-09 17:41:42887 return _RunLinkerTests(args, get_devices())
[email protected]fbe29322013-07-09 09:03:26888 elif command == 'instrumentation':
agrieve18930bd2015-10-09 17:41:42889 return _RunInstrumentationTests(args, get_devices())
jbudorick9a6b7b332014-09-20 00:01:07890 elif command == 'junit':
jama47ca85c2014-12-03 18:38:07891 return _RunJUnitTests(args)
[email protected]3dbdfa42013-08-08 01:08:14892 elif command == 'monkey':
agrieve18930bd2015-10-09 17:41:42893 return _RunMonkeyTests(args, get_devices())
[email protected]ec3170b2013-08-14 14:39:47894 elif command == 'perf':
agrieve18930bd2015-10-09 17:41:42895 return _RunPerfTests(args, get_devices())
jbudorick256fd532014-10-24 01:50:13896 elif command == 'python':
jama47ca85c2014-12-03 18:38:07897 return _RunPythonTests(args)
[email protected]fbe29322013-07-09 09:03:26898 else:
[email protected]6bc1bda22013-07-19 22:08:37899 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26900
[email protected]fbe29322013-07-09 09:03:26901
jbudorick66dc3722014-11-06 21:33:51902_SUPPORTED_IN_PLATFORM_MODE = [
903 # TODO(jbudorick): Add support for more test types.
jbudorick911be58d2015-01-13 02:51:06904 'gtest',
905 'instrumentation',
906 'uirobot',
jbudorick66dc3722014-11-06 21:33:51907]
908
909
jama47ca85c2014-12-03 18:38:07910def RunTestsInPlatformMode(args, parser):
jbudorick66dc3722014-11-06 21:33:51911
jbudorick566592ab2015-09-21 15:32:47912 def infra_error(message):
913 parser.exit(status=constants.INFRA_EXIT_CODE, message=message)
jbudorickb9b0ada2015-09-17 22:52:58914
jbudorick566592ab2015-09-21 15:32:47915 if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
916 infra_error('%s is not yet supported in platform mode' % args.command)
917
918 with environment_factory.CreateEnvironment(args, infra_error) as env:
919 with test_instance_factory.CreateTestInstance(args, infra_error) as test:
jbudorick66dc3722014-11-06 21:33:51920 with test_run_factory.CreateTestRun(
jbudorick566592ab2015-09-21 15:32:47921 args, env, test, infra_error) as test_run:
jbudorickeb7ea71c2015-09-28 16:40:20922 results = []
923 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
924 else itertools.count())
925 for _ in repetitions:
926 iteration_results = test_run.RunTests()
jbudorick66dc3722014-11-06 21:33:51927
jbudorickeb7ea71c2015-09-28 16:40:20928 if iteration_results is not None:
jbudorickd4f77982015-09-28 21:09:18929 results.append(iteration_results)
jbudorickeb7ea71c2015-09-28 16:40:20930 report_results.LogFull(
931 results=iteration_results,
932 test_type=test.TestType(),
933 test_package=test_run.TestPackage(),
934 annotation=getattr(args, 'annotations', None),
935 flakiness_server=getattr(args, 'flakiness_dashboard_server',
936 None))
jbudorick66dc3722014-11-06 21:33:51937
jama47ca85c2014-12-03 18:38:07938 if args.json_results_file:
jbudorickb8c42072014-12-01 18:07:54939 json_results.GenerateJsonResultsFile(
jama47ca85c2014-12-03 18:38:07940 results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54941
jbudorickeb7ea71c2015-09-28 16:40:20942 return (0 if all(r.DidRunPass() for r in results)
943 else constants.ERROR_EXIT_CODE)
jbudorick66dc3722014-11-06 21:33:51944
945
jama47ca85c2014-12-03 18:38:07946CommandConfigTuple = collections.namedtuple(
947 'CommandConfigTuple',
948 ['add_options_func', 'help_txt'])
[email protected]fbe29322013-07-09 09:03:26949VALID_COMMANDS = {
jama47ca85c2014-12-03 18:38:07950 'gtest': CommandConfigTuple(
951 AddGTestOptions,
952 'googletest-based C++ tests'),
953 'instrumentation': CommandConfigTuple(
954 AddInstrumentationTestOptions,
955 'InstrumentationTestCase-based Java tests'),
jama47ca85c2014-12-03 18:38:07956 'junit': CommandConfigTuple(
957 AddJUnitTestOptions,
958 'JUnit4-based Java tests'),
959 'monkey': CommandConfigTuple(
960 AddMonkeyTestOptions,
961 "Tests based on Android's monkey"),
962 'perf': CommandConfigTuple(
963 AddPerfTestOptions,
964 'Performance tests'),
965 'python': CommandConfigTuple(
966 AddPythonTestOptions,
967 'Python tests based on unittest.TestCase'),
968 'linker': CommandConfigTuple(
969 AddLinkerTestOptions,
970 'Linker tests'),
rnephew5c499782014-12-12 19:08:55971 'uirobot': CommandConfigTuple(
972 AddUirobotTestOptions,
973 'Uirobot test'),
jama47ca85c2014-12-03 18:38:07974}
[email protected]fbe29322013-07-09 09:03:26975
976
[email protected]7c53a602014-03-24 16:21:44977def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:24978 for thread in threading.enumerate():
979 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:21980
981
[email protected]7c53a602014-03-24 16:21:44982def main():
[email protected]83bb8152013-11-19 15:02:21983 signal.signal(signal.SIGUSR1, DumpThreadStacks)
jama47ca85c2014-12-03 18:38:07984
985 parser = argparse.ArgumentParser()
986 command_parsers = parser.add_subparsers(title='test types',
987 dest='command')
988
989 for test_type, config in sorted(VALID_COMMANDS.iteritems(),
990 key=lambda x: x[0]):
991 subparser = command_parsers.add_parser(
992 test_type, usage='%(prog)s [options]', help=config.help_txt)
993 config.add_options_func(subparser)
994
995 args = parser.parse_args()
mikecasee74051022015-02-26 23:08:22996
997 try:
998 return RunTestsCommand(args, parser)
999 except base_error.BaseError as e:
1000 logging.exception('Error occurred.')
1001 if e.is_infra_error:
1002 return constants.INFRA_EXIT_CODE
mswecce6732015-06-06 00:31:331003 return constants.ERROR_EXIT_CODE
mikecasee74051022015-02-26 23:08:221004 except: # pylint: disable=W0702
1005 logging.exception('Unrecognized error occurred.')
1006 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:261007
[email protected]fbe29322013-07-09 09:03:261008
1009if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:441010 sys.exit(main())