blob: 4a61aaca4850edf8a5a87bc1f96716c42179cb3e [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
jama47ca85c2014-12-03 18:38:079import argparse
[email protected]fbe29322013-07-09 09:03:2610import collections
jbudorickeb7ea71c2015-09-28 16:40:2011import itertools
[email protected]f7148dd42013-08-20 14:24:5712import logging
[email protected]fbe29322013-07-09 09:03:2613import os
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
jbudorick256fd532014-10-24 01:50:1317import unittest
[email protected]fbe29322013-07-09 09:03:2618
jbudorick061629442015-09-03 18:00:5719from devil import base_error
20from devil.android import apk_helper
21from devil.android import device_blacklist
22from devil.android import device_errors
23from devil.android import device_utils
24from devil.android import ports
25from devil.utils import reraiser_thread
26from devil.utils import run_tests_helper
27
[email protected]fbe29322013-07-09 09:03:2628from pylib import constants
[email protected]c0662e092013-11-12 11:51:2529from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2630from pylib.base import base_test_result
jbudorick66dc3722014-11-06 21:33:5131from pylib.base import environment_factory
[email protected]6bc1bda22013-07-19 22:08:3732from pylib.base import test_dispatcher
jbudorick66dc3722014-11-06 21:33:5133from pylib.base import test_instance_factory
34from pylib.base import test_run_factory
[email protected]6b6abac6d2013-10-03 11:56:3835from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1336from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3737from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2238from pylib.instrumentation import test_options as instrumentation_test_options
jbudorick9a6b7b332014-09-20 00:01:0739from pylib.junit import setup as junit_setup
40from pylib.junit import test_dispatcher as junit_dispatcher
[email protected]3dbdfa42013-08-08 01:08:1441from pylib.monkey import setup as monkey_setup
42from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4743from pylib.perf import setup as perf_setup
44from pylib.perf import test_options as perf_test_options
45from pylib.perf import test_runner as perf_test_runner
jbudorickb8c42072014-12-01 18:07:5446from pylib.results import json_results
47from pylib.results import report_results
[email protected]6bc1bda22013-07-19 22:08:3748from pylib.uiautomator import setup as uiautomator_setup
[email protected]2a684222013-08-01 16:59:2249from pylib.uiautomator import test_options as uiautomator_test_options
[email protected]fbe29322013-07-09 09:03:2650
51
jama47ca85c2014-12-03 18:38:0752def AddCommonOptions(parser):
53 """Adds all common options to |parser|."""
[email protected]fbe29322013-07-09 09:03:2654
jama47ca85c2014-12-03 18:38:0755 group = parser.add_argument_group('Common Options')
56
[email protected]dfffbcbc2013-09-17 22:06:0157 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
jama47ca85c2014-12-03 18:38:0758
59 debug_or_release_group = group.add_mutually_exclusive_group()
60 debug_or_release_group.add_argument(
61 '--debug', action='store_const', const='Debug', dest='build_type',
62 default=default_build_type,
63 help=('If set, run test suites under out/Debug. '
64 'Default is env var BUILDTYPE or Debug.'))
65 debug_or_release_group.add_argument(
66 '--release', action='store_const', const='Release', dest='build_type',
67 help=('If set, run test suites under out/Release. '
68 'Default is env var BUILDTYPE or Debug.'))
69
70 group.add_argument('--build-directory', dest='build_directory',
71 help=('Path to the directory in which build files are'
72 ' located (should not include build type)'))
73 group.add_argument('--output-directory', dest='output_directory',
74 help=('Path to the directory in which build files are'
75 ' located (must include build type). This will take'
76 ' precedence over --debug, --release and'
77 ' --build-directory'))
78 group.add_argument('--num_retries', dest='num_retries', type=int, default=2,
79 help=('Number of retries for a test before '
80 'giving up (default: %(default)s).'))
81 group.add_argument('-v',
82 '--verbose',
83 dest='verbose_count',
84 default=0,
85 action='count',
86 help='Verbose level (multiple times for more)')
87 group.add_argument('--flakiness-dashboard-server',
88 dest='flakiness_dashboard_server',
89 help=('Address of the server that is hosting the '
90 'Chrome for Android flakiness dashboard.'))
91 group.add_argument('--enable-platform-mode', action='store_true',
92 help=('Run the test scripts in platform mode, which '
93 'conceptually separates the test runner from the '
94 '"device" (local or remote, real or emulated) on '
95 'which the tests are running. [experimental]'))
96 group.add_argument('-e', '--environment', default='local',
97 choices=constants.VALID_ENVIRONMENTS,
98 help='Test environment to run in (default: %(default)s).')
99 group.add_argument('--adb-path',
100 help=('Specify the absolute path of the adb binary that '
101 'should be used.'))
102 group.add_argument('--json-results-file', dest='json_results_file',
103 help='If set, will dump results in JSON form '
104 'to specified file.')
[email protected]fbe29322013-07-09 09:03:26105
jama47ca85c2014-12-03 18:38:07106def ProcessCommonOptions(args):
[email protected]fbe29322013-07-09 09:03:26107 """Processes and handles all common options."""
jama47ca85c2014-12-03 18:38:07108 run_tests_helper.SetLogLevel(args.verbose_count)
109 constants.SetBuildType(args.build_type)
110 if args.build_directory:
111 constants.SetBuildDirectory(args.build_directory)
112 if args.output_directory:
mikecase0aea9c52015-04-30 00:12:33113 constants.SetOutputDirectory(args.output_directory)
jama47ca85c2014-12-03 18:38:07114 if args.adb_path:
115 constants.SetAdbPath(args.adb_path)
mikecase48e16bf2014-11-19 22:46:45116 # Some things such as Forwarder require ADB to be in the environment path.
117 adb_dir = os.path.dirname(constants.GetAdbPath())
118 if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep):
119 os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
[email protected]fbe29322013-07-09 09:03:26120
121
rnephew5c499782014-12-12 19:08:55122def AddRemoteDeviceOptions(parser):
123 group = parser.add_argument_group('Remote Device Options')
124
rnephewefe44b42015-02-04 04:45:15125 group.add_argument('--trigger',
jbudoricke6c560152015-01-13 23:49:28126 help=('Only triggers the test if set. Stores test_run_id '
127 'in given file path. '))
rnephewefe44b42015-02-04 04:45:15128 group.add_argument('--collect',
jbudoricke6c560152015-01-13 23:49:28129 help=('Only collects the test results if set. '
130 'Gets test_run_id from given file path.'))
rnephewefe44b42015-02-04 04:45:15131 group.add_argument('--remote-device', action='append',
jbudoricke6c560152015-01-13 23:49:28132 help='Device type to run test on.')
rnephewefe44b42015-02-04 04:45:15133 group.add_argument('--results-path',
jbudoricke6c560152015-01-13 23:49:28134 help='File path to download results to.')
rnephew7f1e2052014-12-12 23:00:11135 group.add_argument('--api-protocol',
jbudoricke6c560152015-01-13 23:49:28136 help='HTTP protocol to use. (http or https)')
rnephewefe44b42015-02-04 04:45:15137 group.add_argument('--api-address',
138 help='Address to send HTTP requests.')
139 group.add_argument('--api-port',
140 help='Port to send HTTP requests to.')
141 group.add_argument('--runner-type',
jbudoricke6c560152015-01-13 23:49:28142 help='Type of test to run as.')
rnephewefe44b42015-02-04 04:45:15143 group.add_argument('--runner-package',
144 help='Package name of test.')
145 group.add_argument('--device-type',
rnephewa46fc562015-01-23 16:00:14146 choices=constants.VALID_DEVICE_TYPES,
147 help=('Type of device to run on. iOS or android'))
rnephewefe44b42015-02-04 04:45:15148 group.add_argument('--device-oem', action='append',
149 help='Device OEM to run on.')
150 group.add_argument('--remote-device-file',
151 help=('File with JSON to select remote device. '
152 'Overrides all other flags.'))
rnephewc9ae8f52015-02-13 03:02:55153 group.add_argument('--remote-device-timeout', type=int,
154 help='Times to retry finding remote device')
mikecase520cbbb52015-04-21 18:51:18155 group.add_argument('--network-config', type=int,
156 help='Integer that specifies the network environment '
157 'that the tests will be run in.')
rnephewefe44b42015-02-04 04:45:15158
159 device_os_group = group.add_mutually_exclusive_group()
160 device_os_group.add_argument('--remote-device-minimum-os',
161 help='Minimum OS on device.')
162 device_os_group.add_argument('--remote-device-os', action='append',
163 help='OS to have on the device.')
rnephew5c499782014-12-12 19:08:55164
165 api_secret_group = group.add_mutually_exclusive_group()
166 api_secret_group.add_argument('--api-secret', default='',
jbudoricke6c560152015-01-13 23:49:28167 help='API secret for remote devices.')
rnephew5c499782014-12-12 19:08:55168 api_secret_group.add_argument('--api-secret-file', default='',
jbudoricke6c560152015-01-13 23:49:28169 help='Path to file that contains API secret.')
rnephew5c499782014-12-12 19:08:55170
171 api_key_group = group.add_mutually_exclusive_group()
172 api_key_group.add_argument('--api-key', default='',
jbudoricke6c560152015-01-13 23:49:28173 help='API key for remote devices.')
rnephew5c499782014-12-12 19:08:55174 api_key_group.add_argument('--api-key-file', default='',
jbudoricke6c560152015-01-13 23:49:28175 help='Path to file that contains API key.')
rnephew5c499782014-12-12 19:08:55176
177
jama47ca85c2014-12-03 18:38:07178def AddDeviceOptions(parser):
179 """Adds device options to |parser|."""
180 group = parser.add_argument_group(title='Device Options')
jama47ca85c2014-12-03 18:38:07181 group.add_argument('--tool',
182 dest='tool',
183 help=('Run the test under a tool '
184 '(use --tool help to list them)'))
185 group.add_argument('-d', '--device', dest='test_device',
186 help=('Target device for the test suite '
187 'to run on.'))
jbudorickdde688fb2015-08-27 03:00:17188 group.add_argument('--blacklist-file', help='Device blacklist file.')
jbudorick256fd532014-10-24 01:50:13189
190
jama47ca85c2014-12-03 18:38:07191def AddGTestOptions(parser):
192 """Adds gtest options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26193
jama47ca85c2014-12-03 18:38:07194 group = parser.add_argument_group('GTest Options')
jbudorick15cdcd52014-12-03 19:58:49195 group.add_argument('-s', '--suite', dest='suite_name',
jama47ca85c2014-12-03 18:38:07196 nargs='+', metavar='SUITE_NAME', required=True,
jbudorick277f2312015-09-24 16:37:43197 help='Executable name of the test suite to run.')
jama47ca85c2014-12-03 18:38:07198 group.add_argument('--gtest_also_run_disabled_tests',
199 '--gtest-also-run-disabled-tests',
200 dest='run_disabled', action='store_true',
201 help='Also run disabled tests if applicable.')
202 group.add_argument('-a', '--test-arguments', dest='test_arguments',
203 default='',
204 help='Additional arguments to pass to the test.')
205 group.add_argument('-t', dest='timeout', type=int, default=60,
206 help='Timeout to wait for each test '
207 '(default: %(default)s).')
208 group.add_argument('--isolate_file_path',
209 '--isolate-file-path',
210 dest='isolate_file_path',
211 help='.isolate file path to override the default '
212 'path')
jbudorick5ee45892015-06-10 18:46:22213 group.add_argument('--app-data-file', action='append', dest='app_data_files',
214 help='A file path relative to the app data directory '
215 'that should be saved to the host.')
216 group.add_argument('--app-data-file-dir',
217 help='Host directory to which app data files will be'
218 ' saved. Used with --app-data-file.')
mlliud7f9fe92015-06-15 19:36:56219 group.add_argument('--delete-stale-data', dest='delete_stale_data',
220 action='store_true',
221 help='Delete stale test data on the device.')
jbudorickeb7ea71c2015-09-28 16:40:20222 group.add_argument('--repeat', '--gtest_repeat', '--gtest-repeat',
223 dest='repeat', type=int, default=0,
224 help='Number of times to repeat the specified set of '
225 'tests.')
jbudorick442a6932015-02-03 03:01:15226
227 filter_group = group.add_mutually_exclusive_group()
228 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
229 dest='test_filter',
230 help='googletest-style filter string.')
231 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
232 help='Path to file that contains googletest-style '
233 'filter strings. (Lines will be joined with '
234 '":" to create a single filter string.)')
235
jama47ca85c2014-12-03 18:38:07236 AddDeviceOptions(parser)
237 AddCommonOptions(parser)
rnephew5c499782014-12-12 19:08:55238 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26239
240
jama47ca85c2014-12-03 18:38:07241def AddLinkerTestOptions(parser):
242 group = parser.add_argument_group('Linker Test Options')
243 group.add_argument('-f', '--gtest-filter', dest='test_filter',
244 help='googletest-style filter string.')
245 AddCommonOptions(parser)
246 AddDeviceOptions(parser)
[email protected]6b6abac6d2013-10-03 11:56:38247
248
jama47ca85c2014-12-03 18:38:07249def AddJavaTestOptions(argument_group):
[email protected]fbe29322013-07-09 09:03:26250 """Adds the Java test options to |option_parser|."""
251
jama47ca85c2014-12-03 18:38:07252 argument_group.add_argument(
253 '-f', '--test-filter', dest='test_filter',
254 help=('Test filter (if not fully qualified, will run all matches).'))
255 argument_group.add_argument(
jbudorickeb7ea71c2015-09-28 16:40:20256 '--repeat', dest='repeat', type=int, default=0,
257 help='Number of times to repeat the specified set of tests.')
258 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26259 '-A', '--annotation', dest='annotation_str',
260 help=('Comma-separated list of annotations. Run only tests with any of '
261 'the given annotations. An annotation can be either a key or a '
262 'key-values pair. A test that has no annotation is considered '
263 '"SmallTest".'))
jama47ca85c2014-12-03 18:38:07264 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26265 '-E', '--exclude-annotation', dest='exclude_annotation_str',
266 help=('Comma-separated list of annotations. Exclude tests with these '
267 'annotations.'))
jama47ca85c2014-12-03 18:38:07268 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59269 '--screenshot', dest='screenshot_failures', action='store_true',
270 help='Capture screenshots of test failures')
jama47ca85c2014-12-03 18:38:07271 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59272 '--save-perf-json', action='store_true',
273 help='Saves the JSON file for each UI Perf test.')
jama47ca85c2014-12-03 18:38:07274 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59275 '--official-build', action='store_true', help='Run official build tests.')
jama47ca85c2014-12-03 18:38:07276 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59277 '--test_data', '--test-data', action='append', default=[],
278 help=('Each instance defines a directory of test data that should be '
279 'copied to the target(s) before running the tests. The argument '
280 'should be of the form <target>:<source>, <target> is relative to '
281 'the device data directory, and <source> is relative to the '
282 'chromium build directory.'))
davileen98efad12015-01-05 19:48:21283 argument_group.add_argument(
284 '--disable-dalvik-asserts', dest='set_asserts', action='store_false',
285 default=True, help='Removes the dalvik.vm.enableassertions property')
286
[email protected]fbe29322013-07-09 09:03:26287
288
jama47ca85c2014-12-03 18:38:07289def ProcessJavaTestOptions(args):
[email protected]fbe29322013-07-09 09:03:26290 """Processes options/arguments and populates |options| with defaults."""
291
jama47ca85c2014-12-03 18:38:07292 # TODO(jbudorick): Handle most of this function in argparse.
293 if args.annotation_str:
294 args.annotations = args.annotation_str.split(',')
295 elif args.test_filter:
296 args.annotations = []
[email protected]fbe29322013-07-09 09:03:26297 else:
jama47ca85c2014-12-03 18:38:07298 args.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
299 'EnormousTest', 'IntegrationTest']
[email protected]fbe29322013-07-09 09:03:26300
jama47ca85c2014-12-03 18:38:07301 if args.exclude_annotation_str:
302 args.exclude_annotations = args.exclude_annotation_str.split(',')
[email protected]fbe29322013-07-09 09:03:26303 else:
jama47ca85c2014-12-03 18:38:07304 args.exclude_annotations = []
[email protected]fbe29322013-07-09 09:03:26305
[email protected]fbe29322013-07-09 09:03:26306
jama47ca85c2014-12-03 18:38:07307def AddInstrumentationTestOptions(parser):
308 """Adds Instrumentation test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26309
jama47ca85c2014-12-03 18:38:07310 parser.usage = '%(prog)s [options]'
[email protected]fbe29322013-07-09 09:03:26311
jama47ca85c2014-12-03 18:38:07312 group = parser.add_argument_group('Instrumentation Test Options')
313 AddJavaTestOptions(group)
[email protected]fbe29322013-07-09 09:03:26314
jama47ca85c2014-12-03 18:38:07315 java_or_python_group = group.add_mutually_exclusive_group()
316 java_or_python_group.add_argument(
317 '-j', '--java-only', action='store_false',
318 dest='run_python_tests', default=True, help='Run only the Java tests.')
319 java_or_python_group.add_argument(
320 '-p', '--python-only', action='store_false',
321 dest='run_java_tests', default=True,
322 help='Run only the host-driven tests.')
323
324 group.add_argument('--host-driven-root',
325 help='Root of the host-driven tests.')
326 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger',
327 action='store_true',
328 help='Wait for debugger.')
jbudorick911be58d2015-01-13 02:51:06329 group.add_argument('--apk-under-test', dest='apk_under_test',
330 help=('the name of the apk under test.'))
jama47ca85c2014-12-03 18:38:07331 group.add_argument('--test-apk', dest='test_apk', required=True,
332 help=('The name of the apk containing the tests '
333 '(without the .apk extension; '
334 'e.g. "ContentShellTest").'))
mikecasee7258622015-09-29 13:47:35335 group.add_argument('--additional-apk', action='append',
336 dest='additional_apks',
337 help='Additional apk that must be installed on '
338 'the device when the tests are run')
jama47ca85c2014-12-03 18:38:07339 group.add_argument('--coverage-dir',
340 help=('Directory in which to place all generated '
341 'EMMA coverage files.'))
342 group.add_argument('--device-flags', dest='device_flags', default='',
343 help='The relative filepath to a file containing '
344 'command-line flags to set on the device')
jbudorick911be58d2015-01-13 02:51:06345 group.add_argument('--device-flags-file', default='',
346 help='The relative filepath to a file containing '
347 'command-line flags to set on the device')
jama47ca85c2014-12-03 18:38:07348 group.add_argument('--isolate_file_path',
349 '--isolate-file-path',
350 dest='isolate_file_path',
351 help='.isolate file path to override the default '
352 'path')
mlliud7f9fe92015-06-15 19:36:56353 group.add_argument('--delete-stale-data', dest='delete_stale_data',
354 action='store_true',
355 help='Delete stale test data on the device.')
jama47ca85c2014-12-03 18:38:07356
357 AddCommonOptions(parser)
358 AddDeviceOptions(parser)
rnephewe416dff2015-01-21 21:26:37359 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26360
361
jama47ca85c2014-12-03 18:38:07362def ProcessInstrumentationOptions(args):
[email protected]2a684222013-08-01 16:59:22363 """Processes options/arguments and populate |options| with defaults.
364
365 Args:
jama47ca85c2014-12-03 18:38:07366 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22367
368 Returns:
369 An InstrumentationOptions named tuple which contains all options relevant to
370 instrumentation tests.
371 """
[email protected]fbe29322013-07-09 09:03:26372
jama47ca85c2014-12-03 18:38:07373 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26374
jama47ca85c2014-12-03 18:38:07375 if not args.host_driven_root:
376 args.run_python_tests = False
[email protected]37ee0c792013-08-06 19:10:13377
jama47ca85c2014-12-03 18:38:07378 args.test_apk_path = os.path.join(
[email protected]2eea4872014-07-28 23:06:17379 constants.GetOutDirectory(),
380 constants.SDK_BUILD_APKS_DIR,
jama47ca85c2014-12-03 18:38:07381 '%s.apk' % args.test_apk)
382 args.test_apk_jar_path = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15383 constants.GetOutDirectory(),
384 constants.SDK_BUILD_TEST_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07385 '%s.jar' % args.test_apk)
yusufo72c598c02015-07-16 23:40:20386 args.test_support_apk_path = '%sSupport%s' % (
387 os.path.splitext(args.test_apk_path))
[email protected]5e2f3f62014-06-23 12:31:46388
jama47ca85c2014-12-03 18:38:07389 args.test_runner = apk_helper.GetInstrumentationName(args.test_apk_path)
[email protected]5e2f3f62014-06-23 12:31:46390
jama47ca85c2014-12-03 18:38:07391 # TODO(jbudorick): Get rid of InstrumentationOptions.
[email protected]2a684222013-08-01 16:59:22392 return instrumentation_test_options.InstrumentationOptions(
jama47ca85c2014-12-03 18:38:07393 args.tool,
jama47ca85c2014-12-03 18:38:07394 args.annotations,
395 args.exclude_annotations,
396 args.test_filter,
397 args.test_data,
398 args.save_perf_json,
399 args.screenshot_failures,
400 args.wait_for_debugger,
401 args.coverage_dir,
402 args.test_apk,
403 args.test_apk_path,
404 args.test_apk_jar_path,
405 args.test_runner,
406 args.test_support_apk_path,
407 args.device_flags,
davileen98efad12015-01-05 19:48:21408 args.isolate_file_path,
mlliud7f9fe92015-06-15 19:36:56409 args.set_asserts,
410 args.delete_stale_data
[email protected]5e2f3f62014-06-23 12:31:46411 )
[email protected]2a684222013-08-01 16:59:22412
[email protected]fbe29322013-07-09 09:03:26413
jama47ca85c2014-12-03 18:38:07414def AddUIAutomatorTestOptions(parser):
415 """Adds UI Automator test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26416
jama47ca85c2014-12-03 18:38:07417 group = parser.add_argument_group('UIAutomator Test Options')
418 AddJavaTestOptions(group)
419 group.add_argument(
420 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
421 metavar='PACKAGE', help='Package under test.')
422 group.add_argument(
423 '--test-jar', dest='test_jar', required=True,
[email protected]fbe29322013-07-09 09:03:26424 help=('The name of the dexed jar containing the tests (without the '
425 '.dex.jar extension). Alternatively, this can be a full path '
426 'to the jar.'))
427
jama47ca85c2014-12-03 18:38:07428 AddCommonOptions(parser)
429 AddDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26430
431
jama47ca85c2014-12-03 18:38:07432def ProcessUIAutomatorOptions(args):
[email protected]2a684222013-08-01 16:59:22433 """Processes UIAutomator options/arguments.
434
435 Args:
jama47ca85c2014-12-03 18:38:07436 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22437
438 Returns:
439 A UIAutomatorOptions named tuple which contains all options relevant to
[email protected]3dbdfa42013-08-08 01:08:14440 uiautomator tests.
[email protected]2a684222013-08-01 16:59:22441 """
[email protected]fbe29322013-07-09 09:03:26442
jama47ca85c2014-12-03 18:38:07443 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26444
jama47ca85c2014-12-03 18:38:07445 if os.path.exists(args.test_jar):
[email protected]fbe29322013-07-09 09:03:26446 # The dexed JAR is fully qualified, assume the info JAR lives along side.
jama47ca85c2014-12-03 18:38:07447 args.uiautomator_jar = args.test_jar
[email protected]fbe29322013-07-09 09:03:26448 else:
jama47ca85c2014-12-03 18:38:07449 args.uiautomator_jar = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15450 constants.GetOutDirectory(),
451 constants.SDK_BUILD_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07452 '%s.dex.jar' % args.test_jar)
453 args.uiautomator_info_jar = (
454 args.uiautomator_jar[:args.uiautomator_jar.find('.dex.jar')] +
[email protected]fbe29322013-07-09 09:03:26455 '_java.jar')
456
[email protected]2a684222013-08-01 16:59:22457 return uiautomator_test_options.UIAutomatorOptions(
jama47ca85c2014-12-03 18:38:07458 args.tool,
jama47ca85c2014-12-03 18:38:07459 args.annotations,
460 args.exclude_annotations,
461 args.test_filter,
462 args.test_data,
463 args.save_perf_json,
464 args.screenshot_failures,
465 args.uiautomator_jar,
466 args.uiautomator_info_jar,
davileen98efad12015-01-05 19:48:21467 args.package,
468 args.set_asserts)
[email protected]2a684222013-08-01 16:59:22469
[email protected]fbe29322013-07-09 09:03:26470
jama47ca85c2014-12-03 18:38:07471def AddJUnitTestOptions(parser):
472 """Adds junit test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07473
jama47ca85c2014-12-03 18:38:07474 group = parser.add_argument_group('JUnit Test Options')
475 group.add_argument(
476 '-s', '--test-suite', dest='test_suite', required=True,
jbudorick9a6b7b332014-09-20 00:01:07477 help=('JUnit test suite to run.'))
jama47ca85c2014-12-03 18:38:07478 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07479 '-f', '--test-filter', dest='test_filter',
480 help='Filters tests googletest-style.')
jama47ca85c2014-12-03 18:38:07481 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07482 '--package-filter', dest='package_filter',
483 help='Filters tests by package.')
jama47ca85c2014-12-03 18:38:07484 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07485 '--runner-filter', dest='runner_filter',
486 help='Filters tests by runner class. Must be fully qualified.')
jama47ca85c2014-12-03 18:38:07487 group.add_argument(
488 '--sdk-version', dest='sdk_version', type=int,
jbudorick9a6b7b332014-09-20 00:01:07489 help='The Android SDK version.')
jama47ca85c2014-12-03 18:38:07490 AddCommonOptions(parser)
jbudorick9a6b7b332014-09-20 00:01:07491
492
jama47ca85c2014-12-03 18:38:07493def AddMonkeyTestOptions(parser):
494 """Adds monkey test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07495
jama47ca85c2014-12-03 18:38:07496 group = parser.add_argument_group('Monkey Test Options')
497 group.add_argument(
498 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
499 metavar='PACKAGE', help='Package under test.')
500 group.add_argument(
501 '--event-count', default=10000, type=int,
502 help='Number of events to generate (default: %(default)s).')
503 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14504 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12505 help='A list of allowed categories.')
jama47ca85c2014-12-03 18:38:07506 group.add_argument(
507 '--throttle', default=100, type=int,
508 help='Delay between events (ms) (default: %(default)s). ')
509 group.add_argument(
510 '--seed', type=int,
[email protected]3dbdfa42013-08-08 01:08:14511 help=('Seed value for pseudo-random generator. Same seed value generates '
512 'the same sequence of events. Seed is randomized by default.'))
jama47ca85c2014-12-03 18:38:07513 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14514 '--extra-args', default='',
jama47ca85c2014-12-03 18:38:07515 help=('String of other args to pass to the command verbatim.'))
[email protected]3dbdfa42013-08-08 01:08:14516
jama47ca85c2014-12-03 18:38:07517 AddCommonOptions(parser)
518 AddDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14519
jama47ca85c2014-12-03 18:38:07520def ProcessMonkeyTestOptions(args):
[email protected]3dbdfa42013-08-08 01:08:14521 """Processes all monkey test options.
522
523 Args:
jama47ca85c2014-12-03 18:38:07524 args: argparse.Namespace object.
[email protected]3dbdfa42013-08-08 01:08:14525
526 Returns:
527 A MonkeyOptions named tuple which contains all options relevant to
528 monkey tests.
529 """
jama47ca85c2014-12-03 18:38:07530 # TODO(jbudorick): Handle this directly in argparse with nargs='+'
531 category = args.category
[email protected]3dbdfa42013-08-08 01:08:14532 if category:
jama47ca85c2014-12-03 18:38:07533 category = args.category.split(',')
[email protected]3dbdfa42013-08-08 01:08:14534
jama47ca85c2014-12-03 18:38:07535 # TODO(jbudorick): Get rid of MonkeyOptions.
[email protected]3dbdfa42013-08-08 01:08:14536 return monkey_test_options.MonkeyOptions(
jama47ca85c2014-12-03 18:38:07537 args.verbose_count,
538 args.package,
539 args.event_count,
[email protected]3dbdfa42013-08-08 01:08:14540 category,
jama47ca85c2014-12-03 18:38:07541 args.throttle,
542 args.seed,
543 args.extra_args)
[email protected]3dbdfa42013-08-08 01:08:14544
rnephew5c499782014-12-12 19:08:55545def AddUirobotTestOptions(parser):
546 """Adds uirobot test options to |option_parser|."""
547 group = parser.add_argument_group('Uirobot Test Options')
548
rnephewefe44b42015-02-04 04:45:15549 group.add_argument('--app-under-test', required=True,
550 help='APK to run tests on.')
rnephew5c499782014-12-12 19:08:55551 group.add_argument(
552 '--minutes', default=5, type=int,
jbudorick676b1202015-02-06 22:02:27553 help='Number of minutes to run uirobot test [default: %(default)s].')
rnephew5c499782014-12-12 19:08:55554
555 AddCommonOptions(parser)
556 AddDeviceOptions(parser)
557 AddRemoteDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14558
jama47ca85c2014-12-03 18:38:07559def AddPerfTestOptions(parser):
560 """Adds perf test options to |parser|."""
[email protected]ec3170b2013-08-14 14:39:47561
jama47ca85c2014-12-03 18:38:07562 group = parser.add_argument_group('Perf Test Options')
[email protected]ec3170b2013-08-14 14:39:47563
jama47ca85c2014-12-03 18:38:07564 class SingleStepAction(argparse.Action):
565 def __call__(self, parser, namespace, values, option_string=None):
566 if values and not namespace.single_step:
567 parser.error('single step command provided, '
568 'but --single-step not specified.')
569 elif namespace.single_step and not values:
570 parser.error('--single-step specified, '
571 'but no single step command provided.')
572 setattr(namespace, self.dest, values)
573
574 step_group = group.add_mutually_exclusive_group(required=True)
575 # TODO(jbudorick): Revise --single-step to use argparse.REMAINDER.
576 # This requires removing "--" from client calls.
577 step_group.add_argument(
578 '--single-step', action='store_true',
[email protected]def4bce2013-11-12 12:59:52579 help='Execute the given command with retries, but only print the result '
580 'for the "most successful" round.')
jama47ca85c2014-12-03 18:38:07581 step_group.add_argument(
[email protected]181a5c92013-09-06 17:11:46582 '--steps',
[email protected]def4bce2013-11-12 12:59:52583 help='JSON file containing the list of commands to run.')
jama47ca85c2014-12-03 18:38:07584 step_group.add_argument(
585 '--print-step',
586 help='The name of a previously executed perf step to print.')
587
588 group.add_argument(
peterbd4e73d2014-12-03 15:47:36589 '--output-json-list',
590 help='Write a simple list of names from --steps into the given file.')
jama47ca85c2014-12-03 18:38:07591 group.add_argument(
peterbd4e73d2014-12-03 15:47:36592 '--collect-chartjson-data',
593 action='store_true',
594 help='Cache the chartjson output from each step for later use.')
jama47ca85c2014-12-03 18:38:07595 group.add_argument(
peterbd4e73d2014-12-03 15:47:36596 '--output-chartjson-data',
597 default='',
598 help='Write out chartjson into the given file.')
jama47ca85c2014-12-03 18:38:07599 group.add_argument(
perezju67cf7f12015-09-29 11:39:05600 '--get-output-dir-archive', metavar='FILENAME',
601 help='Write the chached output directory archived by a step into the'
602 ' given ZIP file.')
603 group.add_argument(
jama47ca85c2014-12-03 18:38:07604 '--flaky-steps',
605 help=('A JSON file containing steps that are flaky '
606 'and will have its exit code ignored.'))
607 group.add_argument(
[email protected]181a5c92013-09-06 17:11:46608 '--no-timeout', action='store_true',
609 help=('Do not impose a timeout. Each perf step is responsible for '
610 'implementing the timeout logic.'))
jama47ca85c2014-12-03 18:38:07611 group.add_argument(
[email protected]650487c2013-09-30 11:40:49612 '-f', '--test-filter',
613 help=('Test filter (will match against the names listed in --steps).'))
jama47ca85c2014-12-03 18:38:07614 group.add_argument(
615 '--dry-run', action='store_true',
[email protected]650487c2013-09-30 11:40:49616 help='Just print the steps without executing.')
jbudorick5cfff872015-07-01 18:46:13617 # Uses 0.1 degrees C because that's what Android does.
618 group.add_argument(
619 '--max-battery-temp', type=int,
620 help='Only start tests when the battery is at or below the given '
621 'temperature (0.1 C)')
jama47ca85c2014-12-03 18:38:07622 group.add_argument('single_step_command', nargs='*', action=SingleStepAction,
623 help='If --single-step is specified, the command to run.')
rnephewdde05da82015-07-09 20:31:01624 group.add_argument('--min-battery-level', type=int,
625 help='Only starts tests when the battery is charged above '
626 'given level.')
jama47ca85c2014-12-03 18:38:07627 AddCommonOptions(parser)
628 AddDeviceOptions(parser)
[email protected]ec3170b2013-08-14 14:39:47629
630
jama47ca85c2014-12-03 18:38:07631def ProcessPerfTestOptions(args):
[email protected]ec3170b2013-08-14 14:39:47632 """Processes all perf test options.
633
634 Args:
jama47ca85c2014-12-03 18:38:07635 args: argparse.Namespace object.
[email protected]ec3170b2013-08-14 14:39:47636
637 Returns:
638 A PerfOptions named tuple which contains all options relevant to
639 perf tests.
640 """
jama47ca85c2014-12-03 18:38:07641 # TODO(jbudorick): Move single_step handling down into the perf tests.
642 if args.single_step:
643 args.single_step = ' '.join(args.single_step_command)
644 # TODO(jbudorick): Get rid of PerfOptions.
[email protected]ec3170b2013-08-14 14:39:47645 return perf_test_options.PerfOptions(
jama47ca85c2014-12-03 18:38:07646 args.steps, args.flaky_steps, args.output_json_list,
647 args.print_step, args.no_timeout, args.test_filter,
648 args.dry_run, args.single_step, args.collect_chartjson_data,
perezju67cf7f12015-09-29 11:39:05649 args.output_chartjson_data, args.get_output_dir_archive,
650 args.max_battery_temp, args.min_battery_level)
[email protected]ec3170b2013-08-14 14:39:47651
652
jama47ca85c2014-12-03 18:38:07653def AddPythonTestOptions(parser):
654 group = parser.add_argument_group('Python Test Options')
655 group.add_argument(
656 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME',
657 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(),
658 help='Name of the test suite to run.')
659 AddCommonOptions(parser)
jbudorick256fd532014-10-24 01:50:13660
661
jama47ca85c2014-12-03 18:38:07662def _RunLinkerTests(args, devices):
[email protected]6b6abac6d2013-10-03 11:56:38663 """Subcommand of RunTestsCommands which runs linker tests."""
jama47ca85c2014-12-03 18:38:07664 runner_factory, tests = linker_setup.Setup(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38665
666 results, exit_code = test_dispatcher.RunTests(
667 tests, runner_factory, devices, shard=True, test_timeout=60,
jama47ca85c2014-12-03 18:38:07668 num_retries=args.num_retries)
[email protected]6b6abac6d2013-10-03 11:56:38669
670 report_results.LogFull(
671 results=results,
672 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22673 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38674
jama47ca85c2014-12-03 18:38:07675 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20676 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54677
[email protected]6b6abac6d2013-10-03 11:56:38678 return exit_code
679
680
jama47ca85c2014-12-03 18:38:07681def _RunInstrumentationTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37682 """Subcommand of RunTestsCommands which runs instrumentation tests."""
jbudorick58b4d362015-09-08 16:44:59683 logging.info('_RunInstrumentationTests(%s, %s)', str(args), str(devices))
[email protected]6bc1bda22013-07-19 22:08:37684
jama47ca85c2014-12-03 18:38:07685 instrumentation_options = ProcessInstrumentationOptions(args)
686
687 if len(devices) > 1 and args.wait_for_debugger:
[email protected]f7148dd42013-08-20 14:24:57688 logging.warning('Debugger can not be sharded, using first available device')
689 devices = devices[:1]
690
[email protected]6bc1bda22013-07-19 22:08:37691 results = base_test_result.TestRunResults()
692 exit_code = 0
693
jama47ca85c2014-12-03 18:38:07694 if args.run_java_tests:
jbudorickeb7ea71c2015-09-28 16:40:20695 java_runner_factory, java_tests = instrumentation_setup.Setup(
mikecase526d68e2014-11-19 20:02:05696 instrumentation_options, devices)
jbudorickeb7ea71c2015-09-28 16:40:20697 else:
698 java_runner_factory = None
699 java_tests = None
[email protected]6bc1bda22013-07-19 22:08:37700
jama47ca85c2014-12-03 18:38:07701 if args.run_python_tests:
jbudorickeb7ea71c2015-09-28 16:40:20702 py_runner_factory, py_tests = host_driven_setup.InstrumentationSetup(
jama47ca85c2014-12-03 18:38:07703 args.host_driven_root, args.official_build,
[email protected]37ee0c792013-08-06 19:10:13704 instrumentation_options)
jbudorickeb7ea71c2015-09-28 16:40:20705 else:
706 py_runner_factory = None
707 py_tests = None
[email protected]37ee0c792013-08-06 19:10:13708
jbudorickeb7ea71c2015-09-28 16:40:20709 results = []
710 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
711 else itertools.count())
712 for _ in repetitions:
713 iteration_results = base_test_result.TestRunResults()
714 if java_tests:
[email protected]34020022013-08-06 23:35:34715 test_results, test_exit_code = test_dispatcher.RunTests(
jbudorickeb7ea71c2015-09-28 16:40:20716 java_tests, java_runner_factory, devices, shard=True,
717 test_timeout=None, num_retries=args.num_retries)
718 iteration_results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37719
[email protected]34020022013-08-06 23:35:34720 # Only allow exit code escalation
721 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
722 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37723
jbudorickeb7ea71c2015-09-28 16:40:20724 if py_tests:
725 test_results, test_exit_code = test_dispatcher.RunTests(
726 py_tests, py_runner_factory, devices, shard=True, test_timeout=None,
727 num_retries=args.num_retries)
728 iteration_results.AddTestRunResults(test_results)
[email protected]4f777ca2014-08-08 01:45:59729
jbudorickeb7ea71c2015-09-28 16:40:20730 # Only allow exit code escalation
731 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
732 exit_code = test_exit_code
733
734 results.append(iteration_results)
735 report_results.LogFull(
736 results=iteration_results,
737 test_type='Instrumentation',
738 test_package=os.path.basename(args.test_apk),
739 annotation=args.annotations,
740 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37741
jama47ca85c2014-12-03 18:38:07742 if args.json_results_file:
743 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54744
[email protected]6bc1bda22013-07-19 22:08:37745 return exit_code
746
747
jama47ca85c2014-12-03 18:38:07748def _RunUIAutomatorTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37749 """Subcommand of RunTestsCommands which runs uiautomator tests."""
jama47ca85c2014-12-03 18:38:07750 uiautomator_options = ProcessUIAutomatorOptions(args)
[email protected]6bc1bda22013-07-19 22:08:37751
jbudorickdde688fb2015-08-27 03:00:17752 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37753
[email protected]37ee0c792013-08-06 19:10:13754 results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57755 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07756 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37757
758 report_results.LogFull(
759 results=results,
760 test_type='UIAutomator',
jama47ca85c2014-12-03 18:38:07761 test_package=os.path.basename(args.test_jar),
762 annotation=args.annotations,
763 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37764
jama47ca85c2014-12-03 18:38:07765 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20766 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54767
[email protected]6bc1bda22013-07-19 22:08:37768 return exit_code
769
770
jama47ca85c2014-12-03 18:38:07771def _RunJUnitTests(args):
jbudorick9a6b7b332014-09-20 00:01:07772 """Subcommand of RunTestsCommand which runs junit tests."""
jama47ca85c2014-12-03 18:38:07773 runner_factory, tests = junit_setup.Setup(args)
mikecasec638a072015-04-01 16:35:35774 results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
775
776 report_results.LogFull(
777 results=results,
778 test_type='JUnit',
779 test_package=args.test_suite)
780
mikecase572401b2015-04-09 02:28:57781 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20782 json_results.GenerateJsonResultsFile([results], args.json_results_file)
mikecase572401b2015-04-09 02:28:57783
jbudorick9a6b7b332014-09-20 00:01:07784 return exit_code
785
786
jama47ca85c2014-12-03 18:38:07787def _RunMonkeyTests(args, devices):
[email protected]3dbdfa42013-08-08 01:08:14788 """Subcommand of RunTestsCommands which runs monkey tests."""
jama47ca85c2014-12-03 18:38:07789 monkey_options = ProcessMonkeyTestOptions(args)
[email protected]3dbdfa42013-08-08 01:08:14790
791 runner_factory, tests = monkey_setup.Setup(monkey_options)
792
793 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46794 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07795 num_retries=args.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14796
797 report_results.LogFull(
798 results=results,
799 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28800 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14801
jama47ca85c2014-12-03 18:38:07802 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20803 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54804
[email protected]3dbdfa42013-08-08 01:08:14805 return exit_code
806
807
jbudorickdde688fb2015-08-27 03:00:17808def _RunPerfTests(args, active_devices):
[email protected]ec3170b2013-08-14 14:39:47809 """Subcommand of RunTestsCommands which runs perf tests."""
jama47ca85c2014-12-03 18:38:07810 perf_options = ProcessPerfTestOptions(args)
[email protected]61487ed2014-06-09 12:33:56811
812 # Just save a simple json with a list of test names.
813 if perf_options.output_json_list:
814 return perf_test_runner.OutputJsonList(
815 perf_options.steps, perf_options.output_json_list)
816
[email protected]ad32f312013-11-13 04:03:29817 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47818 if perf_options.print_step:
simonhatch9b9256d2015-01-07 18:03:42819 return perf_test_runner.PrintTestOutput(
perezju67cf7f12015-09-29 11:39:05820 perf_options.print_step, perf_options.output_chartjson_data,
821 perf_options.get_output_dir_archive)
[email protected]ec3170b2013-08-14 14:39:47822
jbudorickdde688fb2015-08-27 03:00:17823 runner_factory, tests, devices = perf_setup.Setup(
824 perf_options, active_devices)
[email protected]ec3170b2013-08-14 14:39:47825
[email protected]a72f0752014-06-03 23:52:34826 # shard=False means that each device will get the full list of tests
827 # and then each one will decide their own affinity.
828 # shard=True means each device will pop the next test available from a queue,
829 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57830 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34831 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07832 num_retries=args.num_retries)
[email protected]ec3170b2013-08-14 14:39:47833
834 report_results.LogFull(
835 results=results,
836 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12837 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52838
jama47ca85c2014-12-03 18:38:07839 if args.json_results_file:
jbudorickeb7ea71c2015-09-28 16:40:20840 json_results.GenerateJsonResultsFile([results], args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54841
[email protected]def4bce2013-11-12 12:59:52842 if perf_options.single_step:
843 return perf_test_runner.PrintTestOutput('single_step')
844
[email protected]11ce8452014-02-17 10:55:03845 perf_test_runner.PrintSummary(tests)
846
[email protected]86184c7b2013-08-15 15:06:57847 # Always return 0 on the sharding stage. Individual tests exit_code
848 # will be returned on the print_step stage.
849 return 0
[email protected]ec3170b2013-08-14 14:39:47850
[email protected]3dbdfa42013-08-08 01:08:14851
jama47ca85c2014-12-03 18:38:07852def _RunPythonTests(args):
jbudorick256fd532014-10-24 01:50:13853 """Subcommand of RunTestsCommand which runs python unit tests."""
jama47ca85c2014-12-03 18:38:07854 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
jbudorick256fd532014-10-24 01:50:13855 suite_path = suite_vars['path']
856 suite_test_modules = suite_vars['test_modules']
857
858 sys.path = [suite_path] + sys.path
859 try:
860 suite = unittest.TestSuite()
861 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
862 for m in suite_test_modules)
jama47ca85c2014-12-03 18:38:07863 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
jbudorick256fd532014-10-24 01:50:13864 return 0 if runner.run(suite).wasSuccessful() else 1
865 finally:
866 sys.path = sys.path[1:]
867
868
jbudorickdde688fb2015-08-27 03:00:17869def _GetAttachedDevices(blacklist_file, test_device):
[email protected]f7148dd42013-08-20 14:24:57870 """Get all attached devices.
871
872 Args:
873 test_device: Name of a specific device to use.
874
875 Returns:
876 A list of attached devices.
877 """
jbudoricka583ba32015-09-11 17:23:19878 blacklist = (device_blacklist.Blacklist(blacklist_file)
879 if blacklist_file
880 else None)
jbudorickdde688fb2015-08-27 03:00:17881
jbudorickdde688fb2015-08-27 03:00:17882 attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist)
aberent6a02a6182015-04-29 11:07:55883 if test_device:
jbudorick4551d0dc2015-04-29 16:07:06884 test_device = [d for d in attached_devices if d == test_device]
885 if not test_device:
886 raise device_errors.DeviceUnreachableError(
887 'Did not find device %s among attached device. Attached devices: %s'
888 % (test_device, ', '.join(attached_devices)))
889 return test_device
aberent6a02a6182015-04-29 11:07:55890
jbudorick4551d0dc2015-04-29 16:07:06891 else:
892 if not attached_devices:
893 raise device_errors.NoDevicesError()
894 return sorted(attached_devices)
[email protected]f7148dd42013-08-20 14:24:57895
896
jbudorick58b4d362015-09-08 16:44:59897def RunTestsCommand(args, parser): # pylint: disable=too-many-return-statements
[email protected]fbe29322013-07-09 09:03:26898 """Checks test type and dispatches to the appropriate function.
899
900 Args:
jama47ca85c2014-12-03 18:38:07901 args: argparse.Namespace object.
902 parser: argparse.ArgumentParser object.
[email protected]fbe29322013-07-09 09:03:26903
904 Returns:
905 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10906
907 Raises:
908 Exception: Unknown command name passed in, or an exception from an
909 individual test runner.
[email protected]fbe29322013-07-09 09:03:26910 """
jama47ca85c2014-12-03 18:38:07911 command = args.command
[email protected]fbe29322013-07-09 09:03:26912
jama47ca85c2014-12-03 18:38:07913 ProcessCommonOptions(args)
[email protected]d82f0252013-07-12 23:22:57914
jama47ca85c2014-12-03 18:38:07915 if args.enable_platform_mode:
rnephew5c499782014-12-12 19:08:55916 return RunTestsInPlatformMode(args, parser)
jbudorick66dc3722014-11-06 21:33:51917
918 if command in constants.LOCAL_MACHINE_TESTS:
jbudorick256fd532014-10-24 01:50:13919 devices = []
920 else:
jbudorickdde688fb2015-08-27 03:00:17921 devices = _GetAttachedDevices(args.blacklist_file, args.test_device)
[email protected]f7148dd42013-08-20 14:24:57922
[email protected]c0662e092013-11-12 11:51:25923 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40924 if not ports.ResetTestServerPortAllocation():
925 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25926
[email protected]fbe29322013-07-09 09:03:26927 if command == 'gtest':
jbudorick566592ab2015-09-21 15:32:47928 return RunTestsInPlatformMode(args, parser)
[email protected]6b6abac6d2013-10-03 11:56:38929 elif command == 'linker':
jama47ca85c2014-12-03 18:38:07930 return _RunLinkerTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26931 elif command == 'instrumentation':
jama47ca85c2014-12-03 18:38:07932 return _RunInstrumentationTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26933 elif command == 'uiautomator':
jama47ca85c2014-12-03 18:38:07934 return _RunUIAutomatorTests(args, devices)
jbudorick9a6b7b332014-09-20 00:01:07935 elif command == 'junit':
jama47ca85c2014-12-03 18:38:07936 return _RunJUnitTests(args)
[email protected]3dbdfa42013-08-08 01:08:14937 elif command == 'monkey':
jama47ca85c2014-12-03 18:38:07938 return _RunMonkeyTests(args, devices)
[email protected]ec3170b2013-08-14 14:39:47939 elif command == 'perf':
jbudorickdde688fb2015-08-27 03:00:17940 return _RunPerfTests(args, devices)
jbudorick256fd532014-10-24 01:50:13941 elif command == 'python':
jama47ca85c2014-12-03 18:38:07942 return _RunPythonTests(args)
[email protected]fbe29322013-07-09 09:03:26943 else:
[email protected]6bc1bda22013-07-19 22:08:37944 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26945
[email protected]fbe29322013-07-09 09:03:26946
jbudorick66dc3722014-11-06 21:33:51947_SUPPORTED_IN_PLATFORM_MODE = [
948 # TODO(jbudorick): Add support for more test types.
jbudorick911be58d2015-01-13 02:51:06949 'gtest',
950 'instrumentation',
951 'uirobot',
jbudorick66dc3722014-11-06 21:33:51952]
953
954
jama47ca85c2014-12-03 18:38:07955def RunTestsInPlatformMode(args, parser):
jbudorick66dc3722014-11-06 21:33:51956
jbudorick566592ab2015-09-21 15:32:47957 def infra_error(message):
958 parser.exit(status=constants.INFRA_EXIT_CODE, message=message)
jbudorickb9b0ada2015-09-17 22:52:58959
jbudorick566592ab2015-09-21 15:32:47960 if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
961 infra_error('%s is not yet supported in platform mode' % args.command)
962
963 with environment_factory.CreateEnvironment(args, infra_error) as env:
964 with test_instance_factory.CreateTestInstance(args, infra_error) as test:
jbudorick66dc3722014-11-06 21:33:51965 with test_run_factory.CreateTestRun(
jbudorick566592ab2015-09-21 15:32:47966 args, env, test, infra_error) as test_run:
jbudorickeb7ea71c2015-09-28 16:40:20967 results = []
968 repetitions = (xrange(args.repeat + 1) if args.repeat >= 0
969 else itertools.count())
970 for _ in repetitions:
971 iteration_results = test_run.RunTests()
jbudorick66dc3722014-11-06 21:33:51972
jbudorickeb7ea71c2015-09-28 16:40:20973 if iteration_results is not None:
jbudorickd4f77982015-09-28 21:09:18974 results.append(iteration_results)
jbudorickeb7ea71c2015-09-28 16:40:20975 report_results.LogFull(
976 results=iteration_results,
977 test_type=test.TestType(),
978 test_package=test_run.TestPackage(),
979 annotation=getattr(args, 'annotations', None),
980 flakiness_server=getattr(args, 'flakiness_dashboard_server',
981 None))
jbudorick66dc3722014-11-06 21:33:51982
jama47ca85c2014-12-03 18:38:07983 if args.json_results_file:
jbudorickb8c42072014-12-01 18:07:54984 json_results.GenerateJsonResultsFile(
jama47ca85c2014-12-03 18:38:07985 results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54986
jbudorickeb7ea71c2015-09-28 16:40:20987 return (0 if all(r.DidRunPass() for r in results)
988 else constants.ERROR_EXIT_CODE)
jbudorick66dc3722014-11-06 21:33:51989
990
jama47ca85c2014-12-03 18:38:07991CommandConfigTuple = collections.namedtuple(
992 'CommandConfigTuple',
993 ['add_options_func', 'help_txt'])
[email protected]fbe29322013-07-09 09:03:26994VALID_COMMANDS = {
jama47ca85c2014-12-03 18:38:07995 'gtest': CommandConfigTuple(
996 AddGTestOptions,
997 'googletest-based C++ tests'),
998 'instrumentation': CommandConfigTuple(
999 AddInstrumentationTestOptions,
1000 'InstrumentationTestCase-based Java tests'),
1001 'uiautomator': CommandConfigTuple(
1002 AddUIAutomatorTestOptions,
1003 "Tests that run via Android's uiautomator command"),
1004 'junit': CommandConfigTuple(
1005 AddJUnitTestOptions,
1006 'JUnit4-based Java tests'),
1007 'monkey': CommandConfigTuple(
1008 AddMonkeyTestOptions,
1009 "Tests based on Android's monkey"),
1010 'perf': CommandConfigTuple(
1011 AddPerfTestOptions,
1012 'Performance tests'),
1013 'python': CommandConfigTuple(
1014 AddPythonTestOptions,
1015 'Python tests based on unittest.TestCase'),
1016 'linker': CommandConfigTuple(
1017 AddLinkerTestOptions,
1018 'Linker tests'),
rnephew5c499782014-12-12 19:08:551019 'uirobot': CommandConfigTuple(
1020 AddUirobotTestOptions,
1021 'Uirobot test'),
jama47ca85c2014-12-03 18:38:071022}
[email protected]fbe29322013-07-09 09:03:261023
1024
[email protected]7c53a602014-03-24 16:21:441025def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:241026 for thread in threading.enumerate():
1027 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:211028
1029
[email protected]7c53a602014-03-24 16:21:441030def main():
[email protected]83bb8152013-11-19 15:02:211031 signal.signal(signal.SIGUSR1, DumpThreadStacks)
jama47ca85c2014-12-03 18:38:071032
1033 parser = argparse.ArgumentParser()
1034 command_parsers = parser.add_subparsers(title='test types',
1035 dest='command')
1036
1037 for test_type, config in sorted(VALID_COMMANDS.iteritems(),
1038 key=lambda x: x[0]):
1039 subparser = command_parsers.add_parser(
1040 test_type, usage='%(prog)s [options]', help=config.help_txt)
1041 config.add_options_func(subparser)
1042
1043 args = parser.parse_args()
mikecasee74051022015-02-26 23:08:221044
1045 try:
1046 return RunTestsCommand(args, parser)
1047 except base_error.BaseError as e:
1048 logging.exception('Error occurred.')
1049 if e.is_infra_error:
1050 return constants.INFRA_EXIT_CODE
mswecce6732015-06-06 00:31:331051 return constants.ERROR_EXIT_CODE
mikecasee74051022015-02-26 23:08:221052 except: # pylint: disable=W0702
1053 logging.exception('Unrecognized error occurred.')
1054 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:261055
[email protected]fbe29322013-07-09 09:03:261056
1057if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:441058 sys.exit(main())