blob: 467cfba261072f2c6a46ea277aafa36d9f6aeeeb [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
jama47ca85c2014-12-03 18:38:079import argparse
[email protected]fbe29322013-07-09 09:03:2610import collections
jbudorickeb7ea71c2015-09-28 16:40:2011import itertools
[email protected]f7148dd42013-08-20 14:24:5712import logging
[email protected]fbe29322013-07-09 09:03:2613import os
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
jbudorick256fd532014-10-24 01:50:1317import unittest
[email protected]fbe29322013-07-09 09:03:2618
jbudorick0c2a94a2015-12-04 14:27:4319import devil_chromium
jbudorick061629442015-09-03 18:00:5720from devil import base_error
jbudoricke29693be2015-12-07 15:53:2321from devil import devil_env
jbudorick061629442015-09-03 18:00:5722from devil.android import apk_helper
23from devil.android import device_blacklist
24from devil.android import device_errors
25from devil.android import device_utils
jbudorickd645efe82015-12-11 19:09:5926from devil.android import forwarder
jbudorick061629442015-09-03 18:00:5727from devil.android import ports
28from devil.utils import reraiser_thread
29from devil.utils import run_tests_helper
30
[email protected]fbe29322013-07-09 09:03:2631from pylib import constants
jbudorickd28554a2016-01-11 16:22:5932from pylib.constants import host_paths
[email protected]fbe29322013-07-09 09:03:2633from pylib.base import base_test_result
jbudorick66dc3722014-11-06 21:33:5134from pylib.base import environment_factory
[email protected]6bc1bda22013-07-19 22:08:3735from pylib.base import test_dispatcher
jbudorick66dc3722014-11-06 21:33:5136from pylib.base import test_instance_factory
37from pylib.base import test_run_factory
[email protected]6b6abac6d2013-10-03 11:56:3838from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1339from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3740from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2241from pylib.instrumentation import test_options as instrumentation_test_options
jbudorick9a6b7b332014-09-20 00:01:0742from pylib.junit import setup as junit_setup
43from pylib.junit import test_dispatcher as junit_dispatcher
[email protected]3dbdfa42013-08-08 01:08:1444from pylib.monkey import setup as monkey_setup
45from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4746from pylib.perf import setup as perf_setup
47from pylib.perf import test_options as perf_test_options
48from pylib.perf import test_runner as perf_test_runner
jbudorickb8c42072014-12-01 18:07:5449from pylib.results import json_results
50from pylib.results import report_results
[email protected]fbe29322013-07-09 09:03:2651
52
jbudorick0c2a94a2015-12-04 14:27:4353_DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
jbudorickd28554a2016-01-11 16:22:5954 host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))
jbudorick0c2a94a2015-12-04 14:27:4355
56
jama47ca85c2014-12-03 18:38:0757def AddCommonOptions(parser):
58 """Adds all common options to |parser|."""
[email protected]fbe29322013-07-09 09:03:2659
jama47ca85c2014-12-03 18:38:0760 group = parser.add_argument_group('Common Options')
61
[email protected]dfffbcbc2013-09-17 22:06:0162 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
jama47ca85c2014-12-03 18:38:0763
64 debug_or_release_group = group.add_mutually_exclusive_group()
65 debug_or_release_group.add_argument(
66 '--debug', action='store_const', const='Debug', dest='build_type',
67 default=default_build_type,
68 help=('If set, run test suites under out/Debug. '
69 'Default is env var BUILDTYPE or Debug.'))
70 debug_or_release_group.add_argument(
71 '--release', action='store_const', const='Release', dest='build_type',
72 help=('If set, run test suites under out/Release. '
73 'Default is env var BUILDTYPE or Debug.'))
74
75 group.add_argument('--build-directory', dest='build_directory',
76 help=('Path to the directory in which build files are'
77 ' located (should not include build type)'))
78 group.add_argument('--output-directory', dest='output_directory',
79 help=('Path to the directory in which build files are'
80 ' located (must include build type). This will take'
81 ' precedence over --debug, --release and'
82 ' --build-directory'))
agrieveddb11f12015-10-23 17:03:4383 group.add_argument('--num_retries', '--num-retries', dest='num_retries',
84 type=int, default=2,
jama47ca85c2014-12-03 18:38:0785 help=('Number of retries for a test before '
86 'giving up (default: %(default)s).'))
87 group.add_argument('-v',
88 '--verbose',
89 dest='verbose_count',
90 default=0,
91 action='count',
92 help='Verbose level (multiple times for more)')
93 group.add_argument('--flakiness-dashboard-server',
94 dest='flakiness_dashboard_server',
95 help=('Address of the server that is hosting the '
96 'Chrome for Android flakiness dashboard.'))
97 group.add_argument('--enable-platform-mode', action='store_true',
98 help=('Run the test scripts in platform mode, which '
99 'conceptually separates the test runner from the '
100 '"device" (local or remote, real or emulated) on '
101 'which the tests are running. [experimental]'))
102 group.add_argument('-e', '--environment', default='local',
103 choices=constants.VALID_ENVIRONMENTS,
104 help='Test environment to run in (default: %(default)s).')
105 group.add_argument('--adb-path',
106 help=('Specify the absolute path of the adb binary that '
107 'should be used.'))
stipa5733b52015-12-02 08:17:19108 group.add_argument('--json-results-file', '--test-launcher-summary-output',
109 dest='json_results_file',
jama47ca85c2014-12-03 18:38:07110 help='If set, will dump results in JSON form '
111 'to specified file.')
mikecasef205c8362016-01-11 20:47:46112 group.add_argument('--logcat-output-dir',
113 help='If set, will dump logcats recorded during test run '
114 'to directory. File names will be the device ids.')
[email protected]fbe29322013-07-09 09:03:26115
jama47ca85c2014-12-03 18:38:07116def ProcessCommonOptions(args):
[email protected]fbe29322013-07-09 09:03:26117 """Processes and handles all common options."""
jama47ca85c2014-12-03 18:38:07118 run_tests_helper.SetLogLevel(args.verbose_count)
119 constants.SetBuildType(args.build_type)
120 if args.build_directory:
121 constants.SetBuildDirectory(args.build_directory)
122 if args.output_directory:
mikecase0aea9c52015-04-30 00:12:33123 constants.SetOutputDirectory(args.output_directory)
jbudorick0c2a94a2015-12-04 14:27:43124
125 devil_custom_deps = None
jama47ca85c2014-12-03 18:38:07126 if args.adb_path:
jbudorick0c2a94a2015-12-04 14:27:43127 devil_custom_deps = {
128 'adb': {
jbudoricke29693be2015-12-07 15:53:23129 devil_env.GetPlatform(): [args.adb_path]
jbudorick0c2a94a2015-12-04 14:27:43130 }
131 }
132
133 devil_chromium.Initialize(
134 output_directory=constants.GetOutDirectory(),
135 custom_deps=devil_custom_deps)
136
mikecase48e16bf2014-11-19 22:46:45137 # Some things such as Forwarder require ADB to be in the environment path.
138 adb_dir = os.path.dirname(constants.GetAdbPath())
139 if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep):
140 os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
[email protected]fbe29322013-07-09 09:03:26141
142
rnephew5c499782014-12-12 19:08:55143def AddRemoteDeviceOptions(parser):
144 group = parser.add_argument_group('Remote Device Options')
145
rnephewefe44b42015-02-04 04:45:15146 group.add_argument('--trigger',
jbudoricke6c560152015-01-13 23:49:28147 help=('Only triggers the test if set. Stores test_run_id '
148 'in given file path. '))
rnephewefe44b42015-02-04 04:45:15149 group.add_argument('--collect',
jbudoricke6c560152015-01-13 23:49:28150 help=('Only collects the test results if set. '
151 'Gets test_run_id from given file path.'))
rnephewefe44b42015-02-04 04:45:15152 group.add_argument('--remote-device', action='append',
jbudoricke6c560152015-01-13 23:49:28153 help='Device type to run test on.')
rnephewefe44b42015-02-04 04:45:15154 group.add_argument('--results-path',
jbudoricke6c560152015-01-13 23:49:28155 help='File path to download results to.')
rnephew7f1e2052014-12-12 23:00:11156 group.add_argument('--api-protocol',
jbudoricke6c560152015-01-13 23:49:28157 help='HTTP protocol to use. (http or https)')
rnephewefe44b42015-02-04 04:45:15158 group.add_argument('--api-address',
159 help='Address to send HTTP requests.')
160 group.add_argument('--api-port',
161 help='Port to send HTTP requests to.')
162 group.add_argument('--runner-type',
jbudoricke6c560152015-01-13 23:49:28163 help='Type of test to run as.')
rnephewefe44b42015-02-04 04:45:15164 group.add_argument('--runner-package',
165 help='Package name of test.')
166 group.add_argument('--device-type',
rnephewa46fc562015-01-23 16:00:14167 choices=constants.VALID_DEVICE_TYPES,
168 help=('Type of device to run on. iOS or android'))
rnephewefe44b42015-02-04 04:45:15169 group.add_argument('--device-oem', action='append',
170 help='Device OEM to run on.')
171 group.add_argument('--remote-device-file',
172 help=('File with JSON to select remote device. '
173 'Overrides all other flags.'))
rnephewc9ae8f52015-02-13 03:02:55174 group.add_argument('--remote-device-timeout', type=int,
175 help='Times to retry finding remote device')
mikecase520cbbb52015-04-21 18:51:18176 group.add_argument('--network-config', type=int,
177 help='Integer that specifies the network environment '
178 'that the tests will be run in.')
mikecaseddfa35d2015-10-28 01:14:27179 group.add_argument('--test-timeout', type=int,
180 help='Test run timeout in seconds.')
rnephewefe44b42015-02-04 04:45:15181
182 device_os_group = group.add_mutually_exclusive_group()
183 device_os_group.add_argument('--remote-device-minimum-os',
184 help='Minimum OS on device.')
185 device_os_group.add_argument('--remote-device-os', action='append',
186 help='OS to have on the device.')
rnephew5c499782014-12-12 19:08:55187
188 api_secret_group = group.add_mutually_exclusive_group()
189 api_secret_group.add_argument('--api-secret', default='',
jbudoricke6c560152015-01-13 23:49:28190 help='API secret for remote devices.')
rnephew5c499782014-12-12 19:08:55191 api_secret_group.add_argument('--api-secret-file', default='',
jbudoricke6c560152015-01-13 23:49:28192 help='Path to file that contains API secret.')
rnephew5c499782014-12-12 19:08:55193
194 api_key_group = group.add_mutually_exclusive_group()
195 api_key_group.add_argument('--api-key', default='',
jbudoricke6c560152015-01-13 23:49:28196 help='API key for remote devices.')
rnephew5c499782014-12-12 19:08:55197 api_key_group.add_argument('--api-key-file', default='',
jbudoricke6c560152015-01-13 23:49:28198 help='Path to file that contains API key.')
rnephew5c499782014-12-12 19:08:55199
200
jama47ca85c2014-12-03 18:38:07201def AddDeviceOptions(parser):
202 """Adds device options to |parser|."""
203 group = parser.add_argument_group(title='Device Options')
jama47ca85c2014-12-03 18:38:07204 group.add_argument('--tool',
205 dest='tool',
206 help=('Run the test under a tool '
207 '(use --tool help to list them)'))
208 group.add_argument('-d', '--device', dest='test_device',
209 help=('Target device for the test suite '
210 'to run on.'))
jbudorickdde688fb2015-08-27 03:00:17211 group.add_argument('--blacklist-file', help='Device blacklist file.')
agrievea538a142015-10-09 15:45:56212 group.add_argument('--enable-device-cache', action='store_true',
213 help='Cache device state to disk between runs')
agrieve1a02e582015-10-15 21:35:39214 group.add_argument('--incremental-install', action='store_true',
215 help='Use an _incremental apk.')
agrieve8bcb52e2015-10-20 19:38:33216 group.add_argument('--enable-concurrent-adb', action='store_true',
217 help='Run multiple adb commands at the same time, even '
218 'for the same device.')
jbudorick256fd532014-10-24 01:50:13219
220
jama47ca85c2014-12-03 18:38:07221def AddGTestOptions(parser):
222 """Adds gtest options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26223
jama47ca85c2014-12-03 18:38:07224 group = parser.add_argument_group('GTest Options')
jbudorick15cdcd52014-12-03 19:58:49225 group.add_argument('-s', '--suite', dest='suite_name',
jama47ca85c2014-12-03 18:38:07226 nargs='+', metavar='SUITE_NAME', required=True,
jbudorick277f2312015-09-24 16:37:43227 help='Executable name of the test suite to run.')
jama47ca85c2014-12-03 18:38:07228 group.add_argument('--gtest_also_run_disabled_tests',
229 '--gtest-also-run-disabled-tests',
230 dest='run_disabled', action='store_true',
231 help='Also run disabled tests if applicable.')
232 group.add_argument('-a', '--test-arguments', dest='test_arguments',
233 default='',
234 help='Additional arguments to pass to the test.')
jbudorick24616eb2015-10-06 02:40:57235 group.add_argument('-t', '--shard-timeout',
mikecase68bcfc82016-01-22 20:14:38236 dest='shard_timeout', type=int, default=120,
jama47ca85c2014-12-03 18:38:07237 help='Timeout to wait for each test '
238 '(default: %(default)s).')
239 group.add_argument('--isolate_file_path',
240 '--isolate-file-path',
241 dest='isolate_file_path',
242 help='.isolate file path to override the default '
243 'path')
jbudorick5ee45892015-06-10 18:46:22244 group.add_argument('--app-data-file', action='append', dest='app_data_files',
245 help='A file path relative to the app data directory '
246 'that should be saved to the host.')
247 group.add_argument('--app-data-file-dir',
248 help='Host directory to which app data files will be'
249 ' saved. Used with --app-data-file.')
mlliud7f9fe92015-06-15 19:36:56250 group.add_argument('--delete-stale-data', dest='delete_stale_data',
251 action='store_true',
252 help='Delete stale test data on the device.')
jbudorickeb7ea71c2015-09-28 16:40:20253 group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat',
254 dest='repeat', type=int, default=0,
255 help='Number of times to repeat the specified set of '
256 'tests.')
alexandermonta3f03bf2015-12-02 18:56:45257 group.add_argument('--break-on-failure', '--break_on_failure',
258 dest='break_on_failure', action='store_true',
259 help='Whether to break on failure.')
jbudorick442a6932015-02-03 03:01:15260
261 filter_group = group.add_mutually_exclusive_group()
262 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
263 dest='test_filter',
264 help='googletest-style filter string.')
265 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
266 help='Path to file that contains googletest-style '
267 'filter strings. (Lines will be joined with '
268 '":" to create a single filter string.)')
269
jama47ca85c2014-12-03 18:38:07270 AddDeviceOptions(parser)
271 AddCommonOptions(parser)
rnephew5c499782014-12-12 19:08:55272 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26273
274
jama47ca85c2014-12-03 18:38:07275def AddLinkerTestOptions(parser):
276 group = parser.add_argument_group('Linker Test Options')
277 group.add_argument('-f', '--gtest-filter', dest='test_filter',
278 help='googletest-style filter string.')
279 AddCommonOptions(parser)
280 AddDeviceOptions(parser)
[email protected]6b6abac6d2013-10-03 11:56:38281
282
jama47ca85c2014-12-03 18:38:07283def AddJavaTestOptions(argument_group):
[email protected]fbe29322013-07-09 09:03:26284 """Adds the Java test options to |option_parser|."""
285
jama47ca85c2014-12-03 18:38:07286 argument_group.add_argument(
287 '-f', '--test-filter', dest='test_filter',
288 help=('Test filter (if not fully qualified, will run all matches).'))
289 argument_group.add_argument(
jbudorickeb7ea71c2015-09-28 16:40:20290 '--repeat', dest='repeat', type=int, default=0,
291 help='Number of times to repeat the specified set of tests.')
292 argument_group.add_argument(
alexandermonta3f03bf2015-12-02 18:56:45293 '--break-on-failure', '--break_on_failure',
294 dest='break_on_failure', action='store_true',
295 help='Whether to break on failure.')
296 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26297 '-A', '--annotation', dest='annotation_str',
298 help=('Comma-separated list of annotations. Run only tests with any of '
299 'the given annotations. An annotation can be either a key or a '
300 'key-values pair. A test that has no annotation is considered '
301 '"SmallTest".'))
jama47ca85c2014-12-03 18:38:07302 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26303 '-E', '--exclude-annotation', dest='exclude_annotation_str',
304 help=('Comma-separated list of annotations. Exclude tests with these '
305 'annotations.'))
jama47ca85c2014-12-03 18:38:07306 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59307 '--screenshot', dest='screenshot_failures', action='store_true',
308 help='Capture screenshots of test failures')
jama47ca85c2014-12-03 18:38:07309 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59310 '--save-perf-json', action='store_true',
311 help='Saves the JSON file for each UI Perf test.')
jama47ca85c2014-12-03 18:38:07312 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59313 '--official-build', action='store_true', help='Run official build tests.')
jama47ca85c2014-12-03 18:38:07314 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59315 '--test_data', '--test-data', action='append', default=[],
316 help=('Each instance defines a directory of test data that should be '
317 'copied to the target(s) before running the tests. The argument '
318 'should be of the form <target>:<source>, <target> is relative to '
319 'the device data directory, and <source> is relative to the '
320 'chromium build directory.'))
davileen98efad12015-01-05 19:48:21321 argument_group.add_argument(
322 '--disable-dalvik-asserts', dest='set_asserts', action='store_false',
323 default=True, help='Removes the dalvik.vm.enableassertions property')
324
[email protected]fbe29322013-07-09 09:03:26325
326
jama47ca85c2014-12-03 18:38:07327def ProcessJavaTestOptions(args):
[email protected]fbe29322013-07-09 09:03:26328 """Processes options/arguments and populates |options| with defaults."""
329
jama47ca85c2014-12-03 18:38:07330 # TODO(jbudorick): Handle most of this function in argparse.
331 if args.annotation_str:
332 args.annotations = args.annotation_str.split(',')
333 elif args.test_filter:
334 args.annotations = []
[email protected]fbe29322013-07-09 09:03:26335 else:
jama47ca85c2014-12-03 18:38:07336 args.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
337 'EnormousTest', 'IntegrationTest']
[email protected]fbe29322013-07-09 09:03:26338
jama47ca85c2014-12-03 18:38:07339 if args.exclude_annotation_str:
340 args.exclude_annotations = args.exclude_annotation_str.split(',')
[email protected]fbe29322013-07-09 09:03:26341 else:
jama47ca85c2014-12-03 18:38:07342 args.exclude_annotations = []
[email protected]fbe29322013-07-09 09:03:26343
[email protected]fbe29322013-07-09 09:03:26344
jama47ca85c2014-12-03 18:38:07345def AddInstrumentationTestOptions(parser):
346 """Adds Instrumentation test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26347
jama47ca85c2014-12-03 18:38:07348 parser.usage = '%(prog)s [options]'
[email protected]fbe29322013-07-09 09:03:26349
jama47ca85c2014-12-03 18:38:07350 group = parser.add_argument_group('Instrumentation Test Options')
351 AddJavaTestOptions(group)
[email protected]fbe29322013-07-09 09:03:26352
jama47ca85c2014-12-03 18:38:07353 java_or_python_group = group.add_mutually_exclusive_group()
354 java_or_python_group.add_argument(
355 '-j', '--java-only', action='store_false',
356 dest='run_python_tests', default=True, help='Run only the Java tests.')
357 java_or_python_group.add_argument(
358 '-p', '--python-only', action='store_false',
359 dest='run_java_tests', default=True,
360 help='Run only the host-driven tests.')
361
362 group.add_argument('--host-driven-root',
363 help='Root of the host-driven tests.')
364 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger',
365 action='store_true',
366 help='Wait for debugger.')
jbudorick911be58d2015-01-13 02:51:06367 group.add_argument('--apk-under-test', dest='apk_under_test',
368 help=('the name of the apk under test.'))
jama47ca85c2014-12-03 18:38:07369 group.add_argument('--test-apk', dest='test_apk', required=True,
370 help=('The name of the apk containing the tests '
371 '(without the .apk extension; '
372 'e.g. "ContentShellTest").'))
mikecasee7258622015-09-29 13:47:35373 group.add_argument('--additional-apk', action='append',
mikecase8c4ab302015-09-29 17:03:29374 dest='additional_apks', default=[],
mikecasee7258622015-09-29 13:47:35375 help='Additional apk that must be installed on '
376 'the device when the tests are run')
jama47ca85c2014-12-03 18:38:07377 group.add_argument('--coverage-dir',
378 help=('Directory in which to place all generated '
379 'EMMA coverage files.'))
380 group.add_argument('--device-flags', dest='device_flags', default='',
381 help='The relative filepath to a file containing '
382 'command-line flags to set on the device')
jbudorick911be58d2015-01-13 02:51:06383 group.add_argument('--device-flags-file', default='',
384 help='The relative filepath to a file containing '
385 'command-line flags to set on the device')
jama47ca85c2014-12-03 18:38:07386 group.add_argument('--isolate_file_path',
387 '--isolate-file-path',
388 dest='isolate_file_path',
389 help='.isolate file path to override the default '
390 'path')
mlliud7f9fe92015-06-15 19:36:56391 group.add_argument('--delete-stale-data', dest='delete_stale_data',
392 action='store_true',
393 help='Delete stale test data on the device.')
jbudorickede49722015-11-25 05:16:34394 group.add_argument('--timeout-scale', type=float,
395 help='Factor by which timeouts should be scaled.')
wnwen0ec4ebb2016-01-13 19:12:59396 group.add_argument('--strict-mode', dest='strict_mode', default='off',
wnwen2b56c152016-01-12 16:40:10397 help='StrictMode command-line flag set on the device, '
398 'death/testing to kill the process, off to stop '
399 'checking, flash to flash only. Default testing.')
jama47ca85c2014-12-03 18:38:07400
401 AddCommonOptions(parser)
402 AddDeviceOptions(parser)
rnephewe416dff2015-01-21 21:26:37403 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26404
405
jama47ca85c2014-12-03 18:38:07406def ProcessInstrumentationOptions(args):
[email protected]2a684222013-08-01 16:59:22407 """Processes options/arguments and populate |options| with defaults.
408
409 Args:
jama47ca85c2014-12-03 18:38:07410 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22411
412 Returns:
413 An InstrumentationOptions named tuple which contains all options relevant to
414 instrumentation tests.
415 """
[email protected]fbe29322013-07-09 09:03:26416
jama47ca85c2014-12-03 18:38:07417 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26418
jama47ca85c2014-12-03 18:38:07419 if not args.host_driven_root:
420 args.run_python_tests = False
[email protected]37ee0c792013-08-06 19:10:13421
jbudorick9ef3f9552015-10-20 22:58:33422 if os.path.exists(args.test_apk):
423 args.test_apk_path = args.test_apk
424 args.test_apk, _ = os.path.splitext(os.path.basename(args.test_apk))
425 else:
426 args.test_apk_path = os.path.join(
427 constants.GetOutDirectory(),
428 constants.SDK_BUILD_APKS_DIR,
429 '%s.apk' % args.test_apk)
430
jama47ca85c2014-12-03 18:38:07431 args.test_apk_jar_path = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15432 constants.GetOutDirectory(),
433 constants.SDK_BUILD_TEST_JAVALIB_DIR,
jbudorick9ef3f9552015-10-20 22:58:33434 '%s.jar' % args.test_apk)
yusufo72c598c02015-07-16 23:40:20435 args.test_support_apk_path = '%sSupport%s' % (
436 os.path.splitext(args.test_apk_path))
[email protected]5e2f3f62014-06-23 12:31:46437
jama47ca85c2014-12-03 18:38:07438 args.test_runner = apk_helper.GetInstrumentationName(args.test_apk_path)
[email protected]5e2f3f62014-06-23 12:31:46439
jama47ca85c2014-12-03 18:38:07440 # TODO(jbudorick): Get rid of InstrumentationOptions.
[email protected]2a684222013-08-01 16:59:22441 return instrumentation_test_options.InstrumentationOptions(
jama47ca85c2014-12-03 18:38:07442 args.tool,
jama47ca85c2014-12-03 18:38:07443 args.annotations,
444 args.exclude_annotations,
445 args.test_filter,
446 args.test_data,
447 args.save_perf_json,
448 args.screenshot_failures,
449 args.wait_for_debugger,
450 args.coverage_dir,
451 args.test_apk,
452 args.test_apk_path,
453 args.test_apk_jar_path,
454 args.test_runner,
455 args.test_support_apk_path,
456 args.device_flags,
davileen98efad12015-01-05 19:48:21457 args.isolate_file_path,
mlliud7f9fe92015-06-15 19:36:56458 args.set_asserts,
jbudorickede49722015-11-25 05:16:34459 args.delete_stale_data,
jbudorick248e31a2016-01-06 16:28:11460 args.timeout_scale,
461 args.apk_under_test,
wnwen2b56c152016-01-12 16:40:10462 args.additional_apks,
463 args.strict_mode)
[email protected]2a684222013-08-01 16:59:22464
[email protected]fbe29322013-07-09 09:03:26465
jama47ca85c2014-12-03 18:38:07466def AddUIAutomatorTestOptions(parser):
467 """Adds UI Automator test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26468
jama47ca85c2014-12-03 18:38:07469 group = parser.add_argument_group('UIAutomator Test Options')
470 AddJavaTestOptions(group)
471 group.add_argument(
472 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
473 metavar='PACKAGE', help='Package under test.')
474 group.add_argument(
475 '--test-jar', dest='test_jar', required=True,
[email protected]fbe29322013-07-09 09:03:26476 help=('The name of the dexed jar containing the tests (without the '
477 '.dex.jar extension). Alternatively, this can be a full path '
478 'to the jar.'))
479
jama47ca85c2014-12-03 18:38:07480 AddCommonOptions(parser)
481 AddDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26482
483
jama47ca85c2014-12-03 18:38:07484def AddJUnitTestOptions(parser):
485 """Adds junit test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07486
jama47ca85c2014-12-03 18:38:07487 group = parser.add_argument_group('JUnit Test Options')
488 group.add_argument(
489 '-s', '--test-suite', dest='test_suite', required=True,
jbudorick9a6b7b332014-09-20 00:01:07490 help=('JUnit test suite to run.'))
jama47ca85c2014-12-03 18:38:07491 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07492 '-f', '--test-filter', dest='test_filter',
493 help='Filters tests googletest-style.')
jama47ca85c2014-12-03 18:38:07494 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07495 '--package-filter', dest='package_filter',
496 help='Filters tests by package.')
jama47ca85c2014-12-03 18:38:07497 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07498 '--runner-filter', dest='runner_filter',
499 help='Filters tests by runner class. Must be fully qualified.')
jama47ca85c2014-12-03 18:38:07500 group.add_argument(
501 '--sdk-version', dest='sdk_version', type=int,
jbudorick9a6b7b332014-09-20 00:01:07502 help='The Android SDK version.')
jama47ca85c2014-12-03 18:38:07503 AddCommonOptions(parser)
jbudorick9a6b7b332014-09-20 00:01:07504
505
jama47ca85c2014-12-03 18:38:07506def AddMonkeyTestOptions(parser):
507 """Adds monkey test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07508
jama47ca85c2014-12-03 18:38:07509 group = parser.add_argument_group('Monkey Test Options')
510 group.add_argument(
511 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
512 metavar='PACKAGE', help='Package under test.')
513 group.add_argument(
514 '--event-count', default=10000, type=int,
515 help='Number of events to generate (default: %(default)s).')
516 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14517 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12518 help='A list of allowed categories.')
jama47ca85c2014-12-03 18:38:07519 group.add_argument(
520 '--throttle', default=100, type=int,
521 help='Delay between events (ms) (default: %(default)s). ')
522 group.add_argument(
523 '--seed', type=int,
[email protected]3dbdfa42013-08-08 01:08:14524 help=('Seed value for pseudo-random generator. Same seed value generates '
525 'the same sequence of events. Seed is randomized by default.'))
jama47ca85c2014-12-03 18:38:07526 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14527 '--extra-args', default='',
jama47ca85c2014-12-03 18:38:07528 help=('String of other args to pass to the command verbatim.'))
[email protected]3dbdfa42013-08-08 01:08:14529
jama47ca85c2014-12-03 18:38:07530 AddCommonOptions(parser)
531 AddDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14532
jama47ca85c2014-12-03 18:38:07533def ProcessMonkeyTestOptions(args):
[email protected]3dbdfa42013-08-08 01:08:14534 """Processes all monkey test options.
535
536 Args:
jama47ca85c2014-12-03 18:38:07537 args: argparse.Namespace object.
[email protected]3dbdfa42013-08-08 01:08:14538
539 Returns:
540 A MonkeyOptions named tuple which contains all options relevant to
541 monkey tests.
542 """
jama47ca85c2014-12-03 18:38:07543 # TODO(jbudorick): Handle this directly in argparse with nargs='+'
544 category = args.category
[email protected]3dbdfa42013-08-08 01:08:14545 if category:
jama47ca85c2014-12-03 18:38:07546 category = args.category.split(',')
[email protected]3dbdfa42013-08-08 01:08:14547
jama47ca85c2014-12-03 18:38:07548 # TODO(jbudorick): Get rid of MonkeyOptions.
[email protected]3dbdfa42013-08-08 01:08:14549 return monkey_test_options.MonkeyOptions(
jama47ca85c2014-12-03 18:38:07550 args.verbose_count,
551 args.package,
552 args.event_count,
[email protected]3dbdfa42013-08-08 01:08:14553 category,
jama47ca85c2014-12-03 18:38:07554 args.throttle,
555 args.seed,
556 args.extra_args)
[email protected]3dbdfa42013-08-08 01:08:14557
rnephew5c499782014-12-12 19:08:55558def AddUirobotTestOptions(parser):
559 """Adds uirobot test options to |option_parser|."""
560 group = parser.add_argument_group('Uirobot Test Options')
561
rnephewefe44b42015-02-04 04:45:15562 group.add_argument('--app-under-test', required=True,
563 help='APK to run tests on.')
rnephew5c499782014-12-12 19:08:55564 group.add_argument(
mikecaseafa43842015-10-19 23:04:12565 '--repeat', dest='repeat', type=int, default=0,
566 help='Number of times to repeat the uirobot test.')
567 group.add_argument(
rnephew5c499782014-12-12 19:08:55568 '--minutes', default=5, type=int,
jbudorick676b1202015-02-06 22:02:27569 help='Number of minutes to run uirobot test [default: %(default)s].')
rnephew5c499782014-12-12 19:08:55570
571 AddCommonOptions(parser)
572 AddDeviceOptions(parser)
573 AddRemoteDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14574
jama47ca85c2014-12-03 18:38:07575def AddPerfTestOptions(parser):
576 """Adds perf test options to |parser|."""
[email protected]ec3170b2013-08-14 14:39:47577
jama47ca85c2014-12-03 18:38:07578 group = parser.add_argument_group('Perf Test Options')
[email protected]ec3170b2013-08-14 14:39:47579
jama47ca85c2014-12-03 18:38:07580 class SingleStepAction(argparse.Action):
581 def __call__(self, parser, namespace, values, option_string=None):
582 if values and not namespace.single_step:
583 parser.error('single step command provided, '
584 'but --single-step not specified.')
585 elif namespace.single_step and not values:
586 parser.error('--single-step specified, '
587 'but no single step command provided.')
588 setattr(namespace, self.dest, values)
589
590 step_group = group.add_mutually_exclusive_group(required=True)
591 # TODO(jbudorick): Revise --single-step to use argparse.REMAINDER.
592 # This requires removing "--" from client calls.
593 step_group.add_argument(
594 '--single-step', action='store_true',
[email protected]def4bce2013-11-12 12:59:52595 help='Execute the given command with retries, but only print the result '
596 'for the "most successful" round.')
jama47ca85c2014-12-03 18:38:07597 step_group.add_argument(
[email protected]181a5c92013-09-06 17:11:46598 '--steps',
[email protected]def4bce2013-11-12 12:59:52599 help='JSON file containing the list of commands to run.')
jama47ca85c2014-12-03 18:38:07600 step_group.add_argument(
601 '--print-step',
602 help='The name of a previously executed perf step to print.')
603
604 group.add_argument(
peterbd4e73d2014-12-03 15:47:36605 '--output-json-list',
606 help='Write a simple list of names from --steps into the given file.')
jama47ca85c2014-12-03 18:38:07607 group.add_argument(
peterbd4e73d2014-12-03 15:47:36608 '--collect-chartjson-data',
609 action='store_true',
610 help='Cache the chartjson output from each step for later use.')
jama47ca85c2014-12-03 18:38:07611 group.add_argument(
peterbd4e73d2014-12-03 15:47:36612 '--output-chartjson-data',
613 default='',
614 help='Write out chartjson into the given file.')
jama47ca85c2014-12-03 18:38:07615 group.add_argument(
perezju67cf7f12015-09-29 11:39:05616 '--get-output-dir-archive', metavar='FILENAME',
617 help='Write the chached output directory archived by a step into the'
618 ' given ZIP file.')
619 group.add_argument(
jama47ca85c2014-12-03 18:38:07620 '--flaky-steps',
621 help=('A JSON file containing steps that are flaky '
622 'and will have its exit code ignored.'))
623 group.add_argument(
[email protected]181a5c92013-09-06 17:11:46624 '--no-timeout', action='store_true',
625 help=('Do not impose a timeout. Each perf step is responsible for '
626 'implementing the timeout logic.'))
jama47ca85c2014-12-03 18:38:07627 group.add_argument(
[email protected]650487c2013-09-30 11:40:49628 '-f', '--test-filter',
629 help=('Test filter (will match against the names listed in --steps).'))
jama47ca85c2014-12-03 18:38:07630 group.add_argument(
631 '--dry-run', action='store_true',
[email protected]650487c2013-09-30 11:40:49632 help='Just print the steps without executing.')
jbudorick5cfff872015-07-01 18:46:13633 # Uses 0.1 degrees C because that's what Android does.
634 group.add_argument(
635 '--max-battery-temp', type=int,
636 help='Only start tests when the battery is at or below the given '
637 'temperature (0.1 C)')
jama47ca85c2014-12-03 18:38:07638 group.add_argument('single_step_command', nargs='*', action=SingleStepAction,
639 help='If --single-step is specified, the command to run.')
rnephewdde05da82015-07-09 20:31:01640 group.add_argument('--min-battery-level', type=int,
641 help='Only starts tests when the battery is charged above '
642 'given level.')
jama47ca85c2014-12-03 18:38:07643 AddCommonOptions(parser)
644 AddDeviceOptions(parser)
[email protected]ec3170b2013-08-14 14:39:47645
646
jama47ca85c2014-12-03 18:38:07647def ProcessPerfTestOptions(args):
[email protected]ec3170b2013-08-14 14:39:47648 """Processes all perf test options.
649
650 Args:
jama47ca85c2014-12-03 18:38:07651 args: argparse.Namespace object.
[email protected]ec3170b2013-08-14 14:39:47652
653 Returns:
654 A PerfOptions named tuple which contains all options relevant to
655 perf tests.
656 """
jama47ca85c2014-12-03 18:38:07657 # TODO(jbudorick): Move single_step handling down into the perf tests.
658 if args.single_step:
659 args.single_step = ' '.join(args.single_step_command)
660 # TODO(jbudorick): Get rid of PerfOptions.
[email protected]ec3170b2013-08-14 14:39:47661 return perf_test_options.PerfOptions(
jama47ca85c2014-12-03 18:38:07662 args.steps, args.flaky_steps, args.output_json_list,
663 args.print_step, args.no_timeout, args.test_filter,
664 args.dry_run, args.single_step, args.collect_chartjson_data,
perezju67cf7f12015-09-29 11:39:05665 args.output_chartjson_data, args.get_output_dir_archive,
666 args.max_battery_temp, args.min_battery_level)
[email protected]ec3170b2013-08-14 14:39:47667
668
jama47ca85c2014-12-03 18:38:07669def AddPythonTestOptions(parser):
670 group = parser.add_argument_group('Python Test Options')
671 group.add_argument(
672 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME',
673 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(),
674 help='Name of the test suite to run.')
675 AddCommonOptions(parser)
jbudorick256fd532014-10-24 01:50:13676
677
jama47ca85c2014-12-03 18:38:07678def _RunLinkerTests(args, devices):
[email protected]6b6abac6d2013-10-03 11:56:38679 """Subcommand of RunTestsCommands which runs linker tests."""
jama47ca85c2014-12-03 18:38:07680 runner_factory, tests = linker_setup.Setup(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38681
682 results, exit_code = test_dispatcher.RunTests(
683 tests, runner_factory, devices, shard=True, test_timeout=60,
jama47ca85c2014-12-03 18:38:07684 num_retries=args.num_retries)
[email protected]6b6abac6d2013-10-03 11:56:38685
686 report_results.LogFull(
687 results=results,
688 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22689 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38690
jama47ca85c2014-12-03 18:38:07691 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20692 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54693
[email protected]6b6abac6d2013-10-03 11:56:38694 return exit_code
695
696
jama47ca85c2014-12-03 18:38:07697def _RunInstrumentationTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37698 """Subcommand of RunTestsCommands which runs instrumentation tests."""
jbudorick58b4d362015-09-08 16:44:59699 logging.info('_RunInstrumentationTests(%s, %s)', str(args), str(devices))
[email protected]6bc1bda22013-07-19 22:08:37700
jama47ca85c2014-12-03 18:38:07701 instrumentation_options = ProcessInstrumentationOptions(args)
702
703 if len(devices) > 1 and args.wait_for_debugger:
[email protected]f7148dd42013-08-20 14:24:57704 logging.warning('Debugger can not be sharded, using first available device')
705 devices = devices[:1]
706
[email protected]6bc1bda22013-07-19 22:08:37707 results = base_test_result.TestRunResults()
708 exit_code = 0
709
jama47ca85c2014-12-03 18:38:07710 if args.run_java_tests:
jbudorickeb7ea71c2015-09-28 16:40:20711 java_runner_factory, java_tests = instrumentation_setup.Setup(
mikecase526d68e2014-11-19 20:02:05712 instrumentation_options, devices)
jbudorickeb7ea71c2015-09-28 16:40:20713 else:
714 java_runner_factory = None
715 java_tests = None
[email protected]6bc1bda22013-07-19 22:08:37716
jama47ca85c2014-12-03 18:38:07717 if args.run_python_tests:
jbudorickeb7ea71c2015-09-28 16:40:20718 py_runner_factory, py_tests = host_driven_setup.InstrumentationSetup(
jama47ca85c2014-12-03 18:38:07719 args.host_driven_root, args.official_build,
[email protected]37ee0c792013-08-06 19:10:13720 instrumentation_options)
jbudorickeb7ea71c2015-09-28 16:40:20721 else:
722 py_runner_factory = None
723 py_tests = None
[email protected]37ee0c792013-08-06 19:10:13724
jbudorickeb7ea71c2015-09-28 16:40:20725 results = []
726 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
727 else itertools.count())
alexandermonta3f03bf2015-12-02 18:56:45728
alexandermonte2cbe022015-12-16 04:58:58729 code_counts = {constants.INFRA_EXIT_CODE: 0,
730 constants.ERROR_EXIT_CODE: 0,
731 constants.WARNING_EXIT_CODE: 0,
732 0: 0}
733
alexandermonta3f03bf2015-12-02 18:56:45734 def _escalate_code(old, new):
735 for x in (constants.INFRA_EXIT_CODE,
736 constants.ERROR_EXIT_CODE,
737 constants.WARNING_EXIT_CODE):
738 if x in (old, new):
739 return x
740 return 0
741
jbudorickeb7ea71c2015-09-28 16:40:20742 for _ in repetitions:
743 iteration_results = base_test_result.TestRunResults()
744 if java_tests:
[email protected]34020022013-08-06 23:35:34745 test_results, test_exit_code = test_dispatcher.RunTests(
jbudorickeb7ea71c2015-09-28 16:40:20746 java_tests, java_runner_factory, devices, shard=True,
747 test_timeout=None, num_retries=args.num_retries)
748 iteration_results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37749
alexandermonte2cbe022015-12-16 04:58:58750 code_counts[test_exit_code] += 1
alexandermonta3f03bf2015-12-02 18:56:45751 exit_code = _escalate_code(exit_code, test_exit_code)
[email protected]6bc1bda22013-07-19 22:08:37752
jbudorickeb7ea71c2015-09-28 16:40:20753 if py_tests:
754 test_results, test_exit_code = test_dispatcher.RunTests(
755 py_tests, py_runner_factory, devices, shard=True, test_timeout=None,
756 num_retries=args.num_retries)
757 iteration_results.AddTestRunResults(test_results)
[email protected]4f777ca2014-08-08 01:45:59758
alexandermonte2cbe022015-12-16 04:58:58759 code_counts[test_exit_code] += 1
alexandermonta3f03bf2015-12-02 18:56:45760 exit_code = _escalate_code(exit_code, test_exit_code)
jbudorickeb7ea71c2015-09-28 16:40:20761
762 results.append(iteration_results)
763 report_results.LogFull(
764 results=iteration_results,
765 test_type='Instrumentation',
766 test_package=os.path.basename(args.test_apk),
767 annotation=args.annotations,
768 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37769
alexandermonte2cbe022015-12-16 04:58:58770
alexandermonta3f03bf2015-12-02 18:56:45771 if args.break_on_failure and exit_code in (constants.ERROR_EXIT_CODE,
772 constants.INFRA_EXIT_CODE):
773 break
774
alexandermonte2cbe022015-12-16 04:58:58775 logging.critical('Instr tests: %s success, %s infra, %s errors, %s warnings',
776 str(code_counts[0]),
777 str(code_counts[constants.INFRA_EXIT_CODE]),
778 str(code_counts[constants.ERROR_EXIT_CODE]),
779 str(code_counts[constants.WARNING_EXIT_CODE]))
780
jama47ca85c2014-12-03 18:38:07781 if args.json_results_file:
782 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54783
[email protected]6bc1bda22013-07-19 22:08:37784 return exit_code
785
786
jama47ca85c2014-12-03 18:38:07787def _RunJUnitTests(args):
jbudorick9a6b7b332014-09-20 00:01:07788 """Subcommand of RunTestsCommand which runs junit tests."""
jama47ca85c2014-12-03 18:38:07789 runner_factory, tests = junit_setup.Setup(args)
mikecasec638a072015-04-01 16:35:35790 results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
791
792 report_results.LogFull(
793 results=results,
794 test_type='JUnit',
795 test_package=args.test_suite)
796
mikecase572401b2015-04-09 02:28:57797 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20798 json_results.GenerateJsonResultsFile([results], args.json_results_file)
mikecase572401b2015-04-09 02:28:57799
jbudorick9a6b7b332014-09-20 00:01:07800 return exit_code
801
802
jama47ca85c2014-12-03 18:38:07803def _RunMonkeyTests(args, devices):
[email protected]3dbdfa42013-08-08 01:08:14804 """Subcommand of RunTestsCommands which runs monkey tests."""
jama47ca85c2014-12-03 18:38:07805 monkey_options = ProcessMonkeyTestOptions(args)
[email protected]3dbdfa42013-08-08 01:08:14806
807 runner_factory, tests = monkey_setup.Setup(monkey_options)
808
809 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46810 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07811 num_retries=args.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14812
813 report_results.LogFull(
814 results=results,
815 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28816 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14817
jama47ca85c2014-12-03 18:38:07818 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20819 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54820
[email protected]3dbdfa42013-08-08 01:08:14821 return exit_code
822
823
jbudorickdde688fb2015-08-27 03:00:17824def _RunPerfTests(args, active_devices):
[email protected]ec3170b2013-08-14 14:39:47825 """Subcommand of RunTestsCommands which runs perf tests."""
jama47ca85c2014-12-03 18:38:07826 perf_options = ProcessPerfTestOptions(args)
[email protected]61487ed2014-06-09 12:33:56827
828 # Just save a simple json with a list of test names.
829 if perf_options.output_json_list:
830 return perf_test_runner.OutputJsonList(
831 perf_options.steps, perf_options.output_json_list)
832
[email protected]ad32f312013-11-13 04:03:29833 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47834 if perf_options.print_step:
simonhatch9b9256d2015-01-07 18:03:42835 return perf_test_runner.PrintTestOutput(
perezju67cf7f12015-09-29 11:39:05836 perf_options.print_step, perf_options.output_chartjson_data,
837 perf_options.get_output_dir_archive)
[email protected]ec3170b2013-08-14 14:39:47838
jbudorickdde688fb2015-08-27 03:00:17839 runner_factory, tests, devices = perf_setup.Setup(
840 perf_options, active_devices)
[email protected]ec3170b2013-08-14 14:39:47841
[email protected]a72f0752014-06-03 23:52:34842 # shard=False means that each device will get the full list of tests
843 # and then each one will decide their own affinity.
844 # shard=True means each device will pop the next test available from a queue,
845 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57846 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34847 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07848 num_retries=args.num_retries)
[email protected]ec3170b2013-08-14 14:39:47849
850 report_results.LogFull(
851 results=results,
852 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12853 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52854
jama47ca85c2014-12-03 18:38:07855 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20856 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54857
[email protected]def4bce2013-11-12 12:59:52858 if perf_options.single_step:
859 return perf_test_runner.PrintTestOutput('single_step')
860
[email protected]11ce8452014-02-17 10:55:03861 perf_test_runner.PrintSummary(tests)
862
[email protected]86184c7b2013-08-15 15:06:57863 # Always return 0 on the sharding stage. Individual tests exit_code
864 # will be returned on the print_step stage.
865 return 0
[email protected]ec3170b2013-08-14 14:39:47866
[email protected]3dbdfa42013-08-08 01:08:14867
jama47ca85c2014-12-03 18:38:07868def _RunPythonTests(args):
jbudorick256fd532014-10-24 01:50:13869 """Subcommand of RunTestsCommand which runs python unit tests."""
jama47ca85c2014-12-03 18:38:07870 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
jbudorick256fd532014-10-24 01:50:13871 suite_path = suite_vars['path']
872 suite_test_modules = suite_vars['test_modules']
873
874 sys.path = [suite_path] + sys.path
875 try:
876 suite = unittest.TestSuite()
877 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
878 for m in suite_test_modules)
jama47ca85c2014-12-03 18:38:07879 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
jbudorick256fd532014-10-24 01:50:13880 return 0 if runner.run(suite).wasSuccessful() else 1
881 finally:
882 sys.path = sys.path[1:]
883
884
agrievea538a142015-10-09 15:45:56885def _GetAttachedDevices(blacklist_file, test_device, enable_cache):
[email protected]f7148dd42013-08-20 14:24:57886 """Get all attached devices.
887
888 Args:
agrievea538a142015-10-09 15:45:56889 blacklist_file: Path to device blacklist.
[email protected]f7148dd42013-08-20 14:24:57890 test_device: Name of a specific device to use.
agrievea538a142015-10-09 15:45:56891 enable_cache: Whether to enable checksum caching.
[email protected]f7148dd42013-08-20 14:24:57892
893 Returns:
894 A list of attached devices.
895 """
jbudoricka583ba32015-09-11 17:23:19896 blacklist = (device_blacklist.Blacklist(blacklist_file)
897 if blacklist_file
898 else None)
jbudorickdde688fb2015-08-27 03:00:17899
agrievea538a142015-10-09 15:45:56900 attached_devices = device_utils.DeviceUtils.HealthyDevices(
901 blacklist, enable_device_files_cache=enable_cache)
aberent6a02a6182015-04-29 11:07:55902 if test_device:
jbudorick4551d0dc2015-04-29 16:07:06903 test_device = [d for d in attached_devices if d == test_device]
904 if not test_device:
905 raise device_errors.DeviceUnreachableError(
906 'Did not find device %s among attached device. Attached devices: %s'
907 % (test_device, ', '.join(attached_devices)))
908 return test_device
aberent6a02a6182015-04-29 11:07:55909
jbudorick4551d0dc2015-04-29 16:07:06910 else:
911 if not attached_devices:
912 raise device_errors.NoDevicesError()
913 return sorted(attached_devices)
[email protected]f7148dd42013-08-20 14:24:57914
915
jbudorick58b4d362015-09-08 16:44:59916def RunTestsCommand(args, parser): # pylint: disable=too-many-return-statements
[email protected]fbe29322013-07-09 09:03:26917 """Checks test type and dispatches to the appropriate function.
918
919 Args:
jama47ca85c2014-12-03 18:38:07920 args: argparse.Namespace object.
921 parser: argparse.ArgumentParser object.
[email protected]fbe29322013-07-09 09:03:26922
923 Returns:
924 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10925
926 Raises:
927 Exception: Unknown command name passed in, or an exception from an
928 individual test runner.
[email protected]fbe29322013-07-09 09:03:26929 """
jama47ca85c2014-12-03 18:38:07930 command = args.command
[email protected]fbe29322013-07-09 09:03:26931
jama47ca85c2014-12-03 18:38:07932 ProcessCommonOptions(args)
[email protected]d82f0252013-07-12 23:22:57933
jama47ca85c2014-12-03 18:38:07934 if args.enable_platform_mode:
rnephew5c499782014-12-12 19:08:55935 return RunTestsInPlatformMode(args, parser)
jbudorick66dc3722014-11-06 21:33:51936
[email protected]c0662e092013-11-12 11:51:25937 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40938 if not ports.ResetTestServerPortAllocation():
939 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25940
agrieve18930bd2015-10-09 17:41:42941 def get_devices():
942 return _GetAttachedDevices(args.blacklist_file, args.test_device,
943 args.enable_device_cache)
944
[email protected]fbe29322013-07-09 09:03:26945 if command == 'gtest':
jbudorick566592ab2015-09-21 15:32:47946 return RunTestsInPlatformMode(args, parser)
[email protected]6b6abac6d2013-10-03 11:56:38947 elif command == 'linker':
agrieve18930bd2015-10-09 17:41:42948 return _RunLinkerTests(args, get_devices())
[email protected]fbe29322013-07-09 09:03:26949 elif command == 'instrumentation':
agrieve18930bd2015-10-09 17:41:42950 return _RunInstrumentationTests(args, get_devices())
jbudorick9a6b7b332014-09-20 00:01:07951 elif command == 'junit':
jama47ca85c2014-12-03 18:38:07952 return _RunJUnitTests(args)
[email protected]3dbdfa42013-08-08 01:08:14953 elif command == 'monkey':
agrieve18930bd2015-10-09 17:41:42954 return _RunMonkeyTests(args, get_devices())
[email protected]ec3170b2013-08-14 14:39:47955 elif command == 'perf':
agrieve18930bd2015-10-09 17:41:42956 return _RunPerfTests(args, get_devices())
jbudorick256fd532014-10-24 01:50:13957 elif command == 'python':
jama47ca85c2014-12-03 18:38:07958 return _RunPythonTests(args)
[email protected]fbe29322013-07-09 09:03:26959 else:
[email protected]6bc1bda22013-07-19 22:08:37960 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26961
[email protected]fbe29322013-07-09 09:03:26962
jbudorick66dc3722014-11-06 21:33:51963_SUPPORTED_IN_PLATFORM_MODE = [
964 # TODO(jbudorick): Add support for more test types.
jbudorick911be58d2015-01-13 02:51:06965 'gtest',
966 'instrumentation',
967 'uirobot',
jbudorick66dc3722014-11-06 21:33:51968]
969
970
jama47ca85c2014-12-03 18:38:07971def RunTestsInPlatformMode(args, parser):
jbudorick66dc3722014-11-06 21:33:51972
jbudorick566592ab2015-09-21 15:32:47973 def infra_error(message):
974 parser.exit(status=constants.INFRA_EXIT_CODE, message=message)
jbudorickb9b0ada2015-09-17 22:52:58975
jbudorick566592ab2015-09-21 15:32:47976 if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
977 infra_error('%s is not yet supported in platform mode' % args.command)
978
979 with environment_factory.CreateEnvironment(args, infra_error) as env:
980 with test_instance_factory.CreateTestInstance(args, infra_error) as test:
jbudorick66dc3722014-11-06 21:33:51981 with test_run_factory.CreateTestRun(
jbudorick566592ab2015-09-21 15:32:47982 args, env, test, infra_error) as test_run:
jbudorickeb7ea71c2015-09-28 16:40:20983 results = []
984 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
985 else itertools.count())
alexandermonte2cbe022015-12-16 04:58:58986 result_counts = collections.defaultdict(
987 lambda: collections.defaultdict(int))
988 iteration_count = 0
jbudorickeb7ea71c2015-09-28 16:40:20989 for _ in repetitions:
990 iteration_results = test_run.RunTests()
jbudorickeb7ea71c2015-09-28 16:40:20991 if iteration_results is not None:
alexandermonte2cbe022015-12-16 04:58:58992 iteration_count += 1
jbudorickd4f77982015-09-28 21:09:18993 results.append(iteration_results)
alexandermonte2cbe022015-12-16 04:58:58994 for r in iteration_results.GetAll():
995 result_counts[r.GetName()][r.GetType()] += 1
jbudorickeb7ea71c2015-09-28 16:40:20996 report_results.LogFull(
997 results=iteration_results,
998 test_type=test.TestType(),
999 test_package=test_run.TestPackage(),
1000 annotation=getattr(args, 'annotations', None),
1001 flakiness_server=getattr(args, 'flakiness_dashboard_server',
1002 None))
alexandermonta3f03bf2015-12-02 18:56:451003 if args.break_on_failure and not iteration_results.DidRunPass():
1004 break
jbudorick66dc3722014-11-06 21:33:511005
alexandermonte2cbe022015-12-16 04:58:581006 if iteration_count > 1:
1007 # display summary results
1008 # only display results for a test if at least one test did not pass
1009 all_pass = 0
1010 tot_tests = 0
1011 for test_name in result_counts:
1012 tot_tests += 1
1013 if any(result_counts[test_name][x] for x in (
1014 base_test_result.ResultType.FAIL,
1015 base_test_result.ResultType.CRASH,
1016 base_test_result.ResultType.TIMEOUT,
1017 base_test_result.ResultType.UNKNOWN)):
1018 logging.critical(
1019 '%s: %s',
1020 test_name,
1021 ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
1022 for i in base_test_result.ResultType.GetTypes()))
1023 else:
1024 all_pass += 1
1025
1026 logging.critical('%s of %s tests passed in all %s runs',
1027 str(all_pass),
1028 str(tot_tests),
1029 str(iteration_count))
1030
jama47ca85c2014-12-03 18:38:071031 if args.json_results_file:
jbudorickb8c42072014-12-01 18:07:541032 json_results.GenerateJsonResultsFile(
jama47ca85c2014-12-03 18:38:071033 results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:541034
jbudorickeb7ea71c2015-09-28 16:40:201035 return (0 if all(r.DidRunPass() for r in results)
1036 else constants.ERROR_EXIT_CODE)
jbudorick66dc3722014-11-06 21:33:511037
1038
jama47ca85c2014-12-03 18:38:071039CommandConfigTuple = collections.namedtuple(
1040 'CommandConfigTuple',
1041 ['add_options_func', 'help_txt'])
[email protected]fbe29322013-07-09 09:03:261042VALID_COMMANDS = {
jama47ca85c2014-12-03 18:38:071043 'gtest': CommandConfigTuple(
1044 AddGTestOptions,
1045 'googletest-based C++ tests'),
1046 'instrumentation': CommandConfigTuple(
1047 AddInstrumentationTestOptions,
1048 'InstrumentationTestCase-based Java tests'),
jama47ca85c2014-12-03 18:38:071049 'junit': CommandConfigTuple(
1050 AddJUnitTestOptions,
1051 'JUnit4-based Java tests'),
1052 'monkey': CommandConfigTuple(
1053 AddMonkeyTestOptions,
1054 "Tests based on Android's monkey"),
1055 'perf': CommandConfigTuple(
1056 AddPerfTestOptions,
1057 'Performance tests'),
1058 'python': CommandConfigTuple(
1059 AddPythonTestOptions,
1060 'Python tests based on unittest.TestCase'),
1061 'linker': CommandConfigTuple(
1062 AddLinkerTestOptions,
1063 'Linker tests'),
rnephew5c499782014-12-12 19:08:551064 'uirobot': CommandConfigTuple(
1065 AddUirobotTestOptions,
1066 'Uirobot test'),
jama47ca85c2014-12-03 18:38:071067}
[email protected]fbe29322013-07-09 09:03:261068
1069
[email protected]7c53a602014-03-24 16:21:441070def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:241071 for thread in threading.enumerate():
1072 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:211073
1074
[email protected]7c53a602014-03-24 16:21:441075def main():
[email protected]83bb8152013-11-19 15:02:211076 signal.signal(signal.SIGUSR1, DumpThreadStacks)
jama47ca85c2014-12-03 18:38:071077
1078 parser = argparse.ArgumentParser()
1079 command_parsers = parser.add_subparsers(title='test types',
1080 dest='command')
1081
1082 for test_type, config in sorted(VALID_COMMANDS.iteritems(),
1083 key=lambda x: x[0]):
1084 subparser = command_parsers.add_parser(
1085 test_type, usage='%(prog)s [options]', help=config.help_txt)
1086 config.add_options_func(subparser)
1087
1088 args = parser.parse_args()
mikecasee74051022015-02-26 23:08:221089
1090 try:
1091 return RunTestsCommand(args, parser)
1092 except base_error.BaseError as e:
1093 logging.exception('Error occurred.')
1094 if e.is_infra_error:
1095 return constants.INFRA_EXIT_CODE
mswecce6732015-06-06 00:31:331096 return constants.ERROR_EXIT_CODE
mikecasee74051022015-02-26 23:08:221097 except: # pylint: disable=W0702
1098 logging.exception('Unrecognized error occurred.')
1099 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:261100
[email protected]fbe29322013-07-09 09:03:261101
1102if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:441103 sys.exit(main())