blob: c69b531b8ed3389d6ae37931cdc08d355ab4736b [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
jama47ca85c2014-12-03 18:38:079import argparse
[email protected]fbe29322013-07-09 09:03:2610import collections
[email protected]f7148dd42013-08-20 14:24:5711import logging
[email protected]fbe29322013-07-09 09:03:2612import os
[email protected]6bc1bda22013-07-19 22:08:3713import shutil
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
jbudorick256fd532014-10-24 01:50:1317import unittest
[email protected]fbe29322013-07-09 09:03:2618
[email protected]fbe29322013-07-09 09:03:2619from pylib import constants
[email protected]c0662e092013-11-12 11:51:2520from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2621from pylib import ports
22from pylib.base import base_test_result
jbudorick66dc3722014-11-06 21:33:5123from pylib.base import environment_factory
[email protected]6bc1bda22013-07-19 22:08:3724from pylib.base import test_dispatcher
jbudorick66dc3722014-11-06 21:33:5125from pylib.base import test_instance_factory
26from pylib.base import test_run_factory
jbudorick4551d0dc2015-04-29 16:07:0627from pylib.device import device_errors
28from pylib.device import device_utils
[email protected]6bc1bda22013-07-19 22:08:3729from pylib.gtest import gtest_config
jbudorick590a3d72015-06-12 23:53:3130# TODO(jbudorick): Remove this once we stop selectively enabling platform mode.
31from pylib.gtest import gtest_test_instance
[email protected]2a684222013-08-01 16:59:2232from pylib.gtest import setup as gtest_setup
33from pylib.gtest import test_options as gtest_test_options
[email protected]6b6abac6d2013-10-03 11:56:3834from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1335from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3736from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2237from pylib.instrumentation import test_options as instrumentation_test_options
jbudorick9a6b7b332014-09-20 00:01:0738from pylib.junit import setup as junit_setup
39from pylib.junit import test_dispatcher as junit_dispatcher
[email protected]3dbdfa42013-08-08 01:08:1440from pylib.monkey import setup as monkey_setup
41from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4742from pylib.perf import setup as perf_setup
43from pylib.perf import test_options as perf_test_options
44from pylib.perf import test_runner as perf_test_runner
jbudorickb8c42072014-12-01 18:07:5445from pylib.results import json_results
46from pylib.results import report_results
[email protected]6bc1bda22013-07-19 22:08:3747from pylib.uiautomator import setup as uiautomator_setup
[email protected]2a684222013-08-01 16:59:2248from pylib.uiautomator import test_options as uiautomator_test_options
[email protected]2eea4872014-07-28 23:06:1749from pylib.utils import apk_helper
mikecasee74051022015-02-26 23:08:2250from pylib.utils import base_error
[email protected]71aec4b2013-11-20 00:35:2451from pylib.utils import reraiser_thread
[email protected]6bc1bda22013-07-19 22:08:3752from pylib.utils import run_tests_helper
[email protected]fbe29322013-07-09 09:03:2653
54
jama47ca85c2014-12-03 18:38:0755def AddCommonOptions(parser):
56 """Adds all common options to |parser|."""
[email protected]fbe29322013-07-09 09:03:2657
jama47ca85c2014-12-03 18:38:0758 group = parser.add_argument_group('Common Options')
59
[email protected]dfffbcbc2013-09-17 22:06:0160 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
jama47ca85c2014-12-03 18:38:0761
62 debug_or_release_group = group.add_mutually_exclusive_group()
63 debug_or_release_group.add_argument(
64 '--debug', action='store_const', const='Debug', dest='build_type',
65 default=default_build_type,
66 help=('If set, run test suites under out/Debug. '
67 'Default is env var BUILDTYPE or Debug.'))
68 debug_or_release_group.add_argument(
69 '--release', action='store_const', const='Release', dest='build_type',
70 help=('If set, run test suites under out/Release. '
71 'Default is env var BUILDTYPE or Debug.'))
72
73 group.add_argument('--build-directory', dest='build_directory',
74 help=('Path to the directory in which build files are'
75 ' located (should not include build type)'))
76 group.add_argument('--output-directory', dest='output_directory',
77 help=('Path to the directory in which build files are'
78 ' located (must include build type). This will take'
79 ' precedence over --debug, --release and'
80 ' --build-directory'))
81 group.add_argument('--num_retries', dest='num_retries', type=int, default=2,
82 help=('Number of retries for a test before '
83 'giving up (default: %(default)s).'))
84 group.add_argument('-v',
85 '--verbose',
86 dest='verbose_count',
87 default=0,
88 action='count',
89 help='Verbose level (multiple times for more)')
90 group.add_argument('--flakiness-dashboard-server',
91 dest='flakiness_dashboard_server',
92 help=('Address of the server that is hosting the '
93 'Chrome for Android flakiness dashboard.'))
94 group.add_argument('--enable-platform-mode', action='store_true',
95 help=('Run the test scripts in platform mode, which '
96 'conceptually separates the test runner from the '
97 '"device" (local or remote, real or emulated) on '
98 'which the tests are running. [experimental]'))
99 group.add_argument('-e', '--environment', default='local',
100 choices=constants.VALID_ENVIRONMENTS,
101 help='Test environment to run in (default: %(default)s).')
102 group.add_argument('--adb-path',
103 help=('Specify the absolute path of the adb binary that '
104 'should be used.'))
105 group.add_argument('--json-results-file', dest='json_results_file',
106 help='If set, will dump results in JSON form '
107 'to specified file.')
[email protected]fbe29322013-07-09 09:03:26108
jama47ca85c2014-12-03 18:38:07109def ProcessCommonOptions(args):
[email protected]fbe29322013-07-09 09:03:26110 """Processes and handles all common options."""
jama47ca85c2014-12-03 18:38:07111 run_tests_helper.SetLogLevel(args.verbose_count)
112 constants.SetBuildType(args.build_type)
113 if args.build_directory:
114 constants.SetBuildDirectory(args.build_directory)
115 if args.output_directory:
mikecase0aea9c52015-04-30 00:12:33116 constants.SetOutputDirectory(args.output_directory)
jama47ca85c2014-12-03 18:38:07117 if args.adb_path:
118 constants.SetAdbPath(args.adb_path)
mikecase48e16bf2014-11-19 22:46:45119 # Some things such as Forwarder require ADB to be in the environment path.
120 adb_dir = os.path.dirname(constants.GetAdbPath())
121 if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep):
122 os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
[email protected]fbe29322013-07-09 09:03:26123
124
rnephew5c499782014-12-12 19:08:55125def AddRemoteDeviceOptions(parser):
126 group = parser.add_argument_group('Remote Device Options')
127
rnephewefe44b42015-02-04 04:45:15128 group.add_argument('--trigger',
jbudoricke6c560152015-01-13 23:49:28129 help=('Only triggers the test if set. Stores test_run_id '
130 'in given file path. '))
rnephewefe44b42015-02-04 04:45:15131 group.add_argument('--collect',
jbudoricke6c560152015-01-13 23:49:28132 help=('Only collects the test results if set. '
133 'Gets test_run_id from given file path.'))
rnephewefe44b42015-02-04 04:45:15134 group.add_argument('--remote-device', action='append',
jbudoricke6c560152015-01-13 23:49:28135 help='Device type to run test on.')
rnephewefe44b42015-02-04 04:45:15136 group.add_argument('--results-path',
jbudoricke6c560152015-01-13 23:49:28137 help='File path to download results to.')
rnephew7f1e2052014-12-12 23:00:11138 group.add_argument('--api-protocol',
jbudoricke6c560152015-01-13 23:49:28139 help='HTTP protocol to use. (http or https)')
rnephewefe44b42015-02-04 04:45:15140 group.add_argument('--api-address',
141 help='Address to send HTTP requests.')
142 group.add_argument('--api-port',
143 help='Port to send HTTP requests to.')
144 group.add_argument('--runner-type',
jbudoricke6c560152015-01-13 23:49:28145 help='Type of test to run as.')
rnephewefe44b42015-02-04 04:45:15146 group.add_argument('--runner-package',
147 help='Package name of test.')
148 group.add_argument('--device-type',
rnephewa46fc562015-01-23 16:00:14149 choices=constants.VALID_DEVICE_TYPES,
150 help=('Type of device to run on. iOS or android'))
rnephewefe44b42015-02-04 04:45:15151 group.add_argument('--device-oem', action='append',
152 help='Device OEM to run on.')
153 group.add_argument('--remote-device-file',
154 help=('File with JSON to select remote device. '
155 'Overrides all other flags.'))
rnephewc9ae8f52015-02-13 03:02:55156 group.add_argument('--remote-device-timeout', type=int,
157 help='Times to retry finding remote device')
mikecase520cbbb52015-04-21 18:51:18158 group.add_argument('--network-config', type=int,
159 help='Integer that specifies the network environment '
160 'that the tests will be run in.')
rnephewefe44b42015-02-04 04:45:15161
162 device_os_group = group.add_mutually_exclusive_group()
163 device_os_group.add_argument('--remote-device-minimum-os',
164 help='Minimum OS on device.')
165 device_os_group.add_argument('--remote-device-os', action='append',
166 help='OS to have on the device.')
rnephew5c499782014-12-12 19:08:55167
168 api_secret_group = group.add_mutually_exclusive_group()
169 api_secret_group.add_argument('--api-secret', default='',
jbudoricke6c560152015-01-13 23:49:28170 help='API secret for remote devices.')
rnephew5c499782014-12-12 19:08:55171 api_secret_group.add_argument('--api-secret-file', default='',
jbudoricke6c560152015-01-13 23:49:28172 help='Path to file that contains API secret.')
rnephew5c499782014-12-12 19:08:55173
174 api_key_group = group.add_mutually_exclusive_group()
175 api_key_group.add_argument('--api-key', default='',
jbudoricke6c560152015-01-13 23:49:28176 help='API key for remote devices.')
rnephew5c499782014-12-12 19:08:55177 api_key_group.add_argument('--api-key-file', default='',
jbudoricke6c560152015-01-13 23:49:28178 help='Path to file that contains API key.')
rnephew5c499782014-12-12 19:08:55179
180
jama47ca85c2014-12-03 18:38:07181def AddDeviceOptions(parser):
182 """Adds device options to |parser|."""
183 group = parser.add_argument_group(title='Device Options')
jama47ca85c2014-12-03 18:38:07184 group.add_argument('--tool',
185 dest='tool',
186 help=('Run the test under a tool '
187 '(use --tool help to list them)'))
188 group.add_argument('-d', '--device', dest='test_device',
189 help=('Target device for the test suite '
190 'to run on.'))
jbudorick256fd532014-10-24 01:50:13191
192
jama47ca85c2014-12-03 18:38:07193def AddGTestOptions(parser):
194 """Adds gtest options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26195
jama47ca85c2014-12-03 18:38:07196 gtest_suites = list(gtest_config.STABLE_TEST_SUITES
197 + gtest_config.EXPERIMENTAL_TEST_SUITES)
[email protected]fbe29322013-07-09 09:03:26198
jama47ca85c2014-12-03 18:38:07199 group = parser.add_argument_group('GTest Options')
jbudorick15cdcd52014-12-03 19:58:49200 group.add_argument('-s', '--suite', dest='suite_name',
jama47ca85c2014-12-03 18:38:07201 nargs='+', metavar='SUITE_NAME', required=True,
jbudorick15cdcd52014-12-03 19:58:49202 help=('Executable name of the test suite to run. '
203 'Available suites include (but are not limited to): '
204 '%s' % ', '.join('"%s"' % s for s in gtest_suites)))
jama47ca85c2014-12-03 18:38:07205 group.add_argument('--gtest_also_run_disabled_tests',
206 '--gtest-also-run-disabled-tests',
207 dest='run_disabled', action='store_true',
208 help='Also run disabled tests if applicable.')
209 group.add_argument('-a', '--test-arguments', dest='test_arguments',
210 default='',
211 help='Additional arguments to pass to the test.')
212 group.add_argument('-t', dest='timeout', type=int, default=60,
213 help='Timeout to wait for each test '
214 '(default: %(default)s).')
215 group.add_argument('--isolate_file_path',
216 '--isolate-file-path',
217 dest='isolate_file_path',
218 help='.isolate file path to override the default '
219 'path')
jbudorick5ee45892015-06-10 18:46:22220 group.add_argument('--app-data-file', action='append', dest='app_data_files',
221 help='A file path relative to the app data directory '
222 'that should be saved to the host.')
223 group.add_argument('--app-data-file-dir',
224 help='Host directory to which app data files will be'
225 ' saved. Used with --app-data-file.')
jbudorick442a6932015-02-03 03:01:15226
227 filter_group = group.add_mutually_exclusive_group()
228 filter_group.add_argument('-f', '--gtest_filter', '--gtest-filter',
229 dest='test_filter',
230 help='googletest-style filter string.')
231 filter_group.add_argument('--gtest-filter-file', dest='test_filter_file',
232 help='Path to file that contains googletest-style '
233 'filter strings. (Lines will be joined with '
234 '":" to create a single filter string.)')
235
jama47ca85c2014-12-03 18:38:07236 AddDeviceOptions(parser)
237 AddCommonOptions(parser)
rnephew5c499782014-12-12 19:08:55238 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26239
240
jama47ca85c2014-12-03 18:38:07241def AddLinkerTestOptions(parser):
242 group = parser.add_argument_group('Linker Test Options')
243 group.add_argument('-f', '--gtest-filter', dest='test_filter',
244 help='googletest-style filter string.')
245 AddCommonOptions(parser)
246 AddDeviceOptions(parser)
[email protected]6b6abac6d2013-10-03 11:56:38247
248
jama47ca85c2014-12-03 18:38:07249def AddJavaTestOptions(argument_group):
[email protected]fbe29322013-07-09 09:03:26250 """Adds the Java test options to |option_parser|."""
251
jama47ca85c2014-12-03 18:38:07252 argument_group.add_argument(
253 '-f', '--test-filter', dest='test_filter',
254 help=('Test filter (if not fully qualified, will run all matches).'))
255 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26256 '-A', '--annotation', dest='annotation_str',
257 help=('Comma-separated list of annotations. Run only tests with any of '
258 'the given annotations. An annotation can be either a key or a '
259 'key-values pair. A test that has no annotation is considered '
260 '"SmallTest".'))
jama47ca85c2014-12-03 18:38:07261 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26262 '-E', '--exclude-annotation', dest='exclude_annotation_str',
263 help=('Comma-separated list of annotations. Exclude tests with these '
264 'annotations.'))
jama47ca85c2014-12-03 18:38:07265 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59266 '--screenshot', dest='screenshot_failures', action='store_true',
267 help='Capture screenshots of test failures')
jama47ca85c2014-12-03 18:38:07268 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59269 '--save-perf-json', action='store_true',
270 help='Saves the JSON file for each UI Perf test.')
jama47ca85c2014-12-03 18:38:07271 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59272 '--official-build', action='store_true', help='Run official build tests.')
jama47ca85c2014-12-03 18:38:07273 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59274 '--test_data', '--test-data', action='append', default=[],
275 help=('Each instance defines a directory of test data that should be '
276 'copied to the target(s) before running the tests. The argument '
277 'should be of the form <target>:<source>, <target> is relative to '
278 'the device data directory, and <source> is relative to the '
279 'chromium build directory.'))
davileen98efad12015-01-05 19:48:21280 argument_group.add_argument(
281 '--disable-dalvik-asserts', dest='set_asserts', action='store_false',
282 default=True, help='Removes the dalvik.vm.enableassertions property')
283
[email protected]fbe29322013-07-09 09:03:26284
285
jama47ca85c2014-12-03 18:38:07286def ProcessJavaTestOptions(args):
[email protected]fbe29322013-07-09 09:03:26287 """Processes options/arguments and populates |options| with defaults."""
288
jama47ca85c2014-12-03 18:38:07289 # TODO(jbudorick): Handle most of this function in argparse.
290 if args.annotation_str:
291 args.annotations = args.annotation_str.split(',')
292 elif args.test_filter:
293 args.annotations = []
[email protected]fbe29322013-07-09 09:03:26294 else:
jama47ca85c2014-12-03 18:38:07295 args.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
296 'EnormousTest', 'IntegrationTest']
[email protected]fbe29322013-07-09 09:03:26297
jama47ca85c2014-12-03 18:38:07298 if args.exclude_annotation_str:
299 args.exclude_annotations = args.exclude_annotation_str.split(',')
[email protected]fbe29322013-07-09 09:03:26300 else:
jama47ca85c2014-12-03 18:38:07301 args.exclude_annotations = []
[email protected]fbe29322013-07-09 09:03:26302
[email protected]fbe29322013-07-09 09:03:26303
jama47ca85c2014-12-03 18:38:07304def AddInstrumentationTestOptions(parser):
305 """Adds Instrumentation test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26306
jama47ca85c2014-12-03 18:38:07307 parser.usage = '%(prog)s [options]'
[email protected]fbe29322013-07-09 09:03:26308
jama47ca85c2014-12-03 18:38:07309 group = parser.add_argument_group('Instrumentation Test Options')
310 AddJavaTestOptions(group)
[email protected]fbe29322013-07-09 09:03:26311
jama47ca85c2014-12-03 18:38:07312 java_or_python_group = group.add_mutually_exclusive_group()
313 java_or_python_group.add_argument(
314 '-j', '--java-only', action='store_false',
315 dest='run_python_tests', default=True, help='Run only the Java tests.')
316 java_or_python_group.add_argument(
317 '-p', '--python-only', action='store_false',
318 dest='run_java_tests', default=True,
319 help='Run only the host-driven tests.')
320
321 group.add_argument('--host-driven-root',
322 help='Root of the host-driven tests.')
323 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger',
324 action='store_true',
325 help='Wait for debugger.')
jbudorick911be58d2015-01-13 02:51:06326 group.add_argument('--apk-under-test', dest='apk_under_test',
327 help=('the name of the apk under test.'))
jama47ca85c2014-12-03 18:38:07328 group.add_argument('--test-apk', dest='test_apk', required=True,
329 help=('The name of the apk containing the tests '
330 '(without the .apk extension; '
331 'e.g. "ContentShellTest").'))
332 group.add_argument('--coverage-dir',
333 help=('Directory in which to place all generated '
334 'EMMA coverage files.'))
335 group.add_argument('--device-flags', dest='device_flags', default='',
336 help='The relative filepath to a file containing '
337 'command-line flags to set on the device')
jbudorick911be58d2015-01-13 02:51:06338 group.add_argument('--device-flags-file', default='',
339 help='The relative filepath to a file containing '
340 'command-line flags to set on the device')
jama47ca85c2014-12-03 18:38:07341 group.add_argument('--isolate_file_path',
342 '--isolate-file-path',
343 dest='isolate_file_path',
344 help='.isolate file path to override the default '
345 'path')
346
347 AddCommonOptions(parser)
348 AddDeviceOptions(parser)
rnephewe416dff2015-01-21 21:26:37349 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26350
351
jama47ca85c2014-12-03 18:38:07352def ProcessInstrumentationOptions(args):
[email protected]2a684222013-08-01 16:59:22353 """Processes options/arguments and populate |options| with defaults.
354
355 Args:
jama47ca85c2014-12-03 18:38:07356 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22357
358 Returns:
359 An InstrumentationOptions named tuple which contains all options relevant to
360 instrumentation tests.
361 """
[email protected]fbe29322013-07-09 09:03:26362
jama47ca85c2014-12-03 18:38:07363 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26364
jama47ca85c2014-12-03 18:38:07365 if not args.host_driven_root:
366 args.run_python_tests = False
[email protected]37ee0c792013-08-06 19:10:13367
jama47ca85c2014-12-03 18:38:07368 args.test_apk_path = os.path.join(
[email protected]2eea4872014-07-28 23:06:17369 constants.GetOutDirectory(),
370 constants.SDK_BUILD_APKS_DIR,
jama47ca85c2014-12-03 18:38:07371 '%s.apk' % args.test_apk)
372 args.test_apk_jar_path = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15373 constants.GetOutDirectory(),
374 constants.SDK_BUILD_TEST_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07375 '%s.jar' % args.test_apk)
376 args.test_support_apk_path = '%sSupport%s' % (
377 os.path.splitext(args.test_apk_path))
[email protected]5e2f3f62014-06-23 12:31:46378
jama47ca85c2014-12-03 18:38:07379 args.test_runner = apk_helper.GetInstrumentationName(args.test_apk_path)
[email protected]5e2f3f62014-06-23 12:31:46380
jama47ca85c2014-12-03 18:38:07381 # TODO(jbudorick): Get rid of InstrumentationOptions.
[email protected]2a684222013-08-01 16:59:22382 return instrumentation_test_options.InstrumentationOptions(
jama47ca85c2014-12-03 18:38:07383 args.tool,
jama47ca85c2014-12-03 18:38:07384 args.annotations,
385 args.exclude_annotations,
386 args.test_filter,
387 args.test_data,
388 args.save_perf_json,
389 args.screenshot_failures,
390 args.wait_for_debugger,
391 args.coverage_dir,
392 args.test_apk,
393 args.test_apk_path,
394 args.test_apk_jar_path,
395 args.test_runner,
396 args.test_support_apk_path,
397 args.device_flags,
davileen98efad12015-01-05 19:48:21398 args.isolate_file_path,
399 args.set_asserts
[email protected]5e2f3f62014-06-23 12:31:46400 )
[email protected]2a684222013-08-01 16:59:22401
[email protected]fbe29322013-07-09 09:03:26402
jama47ca85c2014-12-03 18:38:07403def AddUIAutomatorTestOptions(parser):
404 """Adds UI Automator test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26405
jama47ca85c2014-12-03 18:38:07406 group = parser.add_argument_group('UIAutomator Test Options')
407 AddJavaTestOptions(group)
408 group.add_argument(
409 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
410 metavar='PACKAGE', help='Package under test.')
411 group.add_argument(
412 '--test-jar', dest='test_jar', required=True,
[email protected]fbe29322013-07-09 09:03:26413 help=('The name of the dexed jar containing the tests (without the '
414 '.dex.jar extension). Alternatively, this can be a full path '
415 'to the jar.'))
416
jama47ca85c2014-12-03 18:38:07417 AddCommonOptions(parser)
418 AddDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26419
420
jama47ca85c2014-12-03 18:38:07421def ProcessUIAutomatorOptions(args):
[email protected]2a684222013-08-01 16:59:22422 """Processes UIAutomator options/arguments.
423
424 Args:
jama47ca85c2014-12-03 18:38:07425 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22426
427 Returns:
428 A UIAutomatorOptions named tuple which contains all options relevant to
[email protected]3dbdfa42013-08-08 01:08:14429 uiautomator tests.
[email protected]2a684222013-08-01 16:59:22430 """
[email protected]fbe29322013-07-09 09:03:26431
jama47ca85c2014-12-03 18:38:07432 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26433
jama47ca85c2014-12-03 18:38:07434 if os.path.exists(args.test_jar):
[email protected]fbe29322013-07-09 09:03:26435 # The dexed JAR is fully qualified, assume the info JAR lives along side.
jama47ca85c2014-12-03 18:38:07436 args.uiautomator_jar = args.test_jar
[email protected]fbe29322013-07-09 09:03:26437 else:
jama47ca85c2014-12-03 18:38:07438 args.uiautomator_jar = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15439 constants.GetOutDirectory(),
440 constants.SDK_BUILD_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07441 '%s.dex.jar' % args.test_jar)
442 args.uiautomator_info_jar = (
443 args.uiautomator_jar[:args.uiautomator_jar.find('.dex.jar')] +
[email protected]fbe29322013-07-09 09:03:26444 '_java.jar')
445
[email protected]2a684222013-08-01 16:59:22446 return uiautomator_test_options.UIAutomatorOptions(
jama47ca85c2014-12-03 18:38:07447 args.tool,
jama47ca85c2014-12-03 18:38:07448 args.annotations,
449 args.exclude_annotations,
450 args.test_filter,
451 args.test_data,
452 args.save_perf_json,
453 args.screenshot_failures,
454 args.uiautomator_jar,
455 args.uiautomator_info_jar,
davileen98efad12015-01-05 19:48:21456 args.package,
457 args.set_asserts)
[email protected]2a684222013-08-01 16:59:22458
[email protected]fbe29322013-07-09 09:03:26459
jama47ca85c2014-12-03 18:38:07460def AddJUnitTestOptions(parser):
461 """Adds junit test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07462
jama47ca85c2014-12-03 18:38:07463 group = parser.add_argument_group('JUnit Test Options')
464 group.add_argument(
465 '-s', '--test-suite', dest='test_suite', required=True,
jbudorick9a6b7b332014-09-20 00:01:07466 help=('JUnit test suite to run.'))
jama47ca85c2014-12-03 18:38:07467 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07468 '-f', '--test-filter', dest='test_filter',
469 help='Filters tests googletest-style.')
jama47ca85c2014-12-03 18:38:07470 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07471 '--package-filter', dest='package_filter',
472 help='Filters tests by package.')
jama47ca85c2014-12-03 18:38:07473 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07474 '--runner-filter', dest='runner_filter',
475 help='Filters tests by runner class. Must be fully qualified.')
jama47ca85c2014-12-03 18:38:07476 group.add_argument(
477 '--sdk-version', dest='sdk_version', type=int,
jbudorick9a6b7b332014-09-20 00:01:07478 help='The Android SDK version.')
jama47ca85c2014-12-03 18:38:07479 AddCommonOptions(parser)
jbudorick9a6b7b332014-09-20 00:01:07480
481
jama47ca85c2014-12-03 18:38:07482def AddMonkeyTestOptions(parser):
483 """Adds monkey test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07484
jama47ca85c2014-12-03 18:38:07485 group = parser.add_argument_group('Monkey Test Options')
486 group.add_argument(
487 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
488 metavar='PACKAGE', help='Package under test.')
489 group.add_argument(
490 '--event-count', default=10000, type=int,
491 help='Number of events to generate (default: %(default)s).')
492 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14493 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12494 help='A list of allowed categories.')
jama47ca85c2014-12-03 18:38:07495 group.add_argument(
496 '--throttle', default=100, type=int,
497 help='Delay between events (ms) (default: %(default)s). ')
498 group.add_argument(
499 '--seed', type=int,
[email protected]3dbdfa42013-08-08 01:08:14500 help=('Seed value for pseudo-random generator. Same seed value generates '
501 'the same sequence of events. Seed is randomized by default.'))
jama47ca85c2014-12-03 18:38:07502 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14503 '--extra-args', default='',
jama47ca85c2014-12-03 18:38:07504 help=('String of other args to pass to the command verbatim.'))
[email protected]3dbdfa42013-08-08 01:08:14505
jama47ca85c2014-12-03 18:38:07506 AddCommonOptions(parser)
507 AddDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14508
jama47ca85c2014-12-03 18:38:07509def ProcessMonkeyTestOptions(args):
[email protected]3dbdfa42013-08-08 01:08:14510 """Processes all monkey test options.
511
512 Args:
jama47ca85c2014-12-03 18:38:07513 args: argparse.Namespace object.
[email protected]3dbdfa42013-08-08 01:08:14514
515 Returns:
516 A MonkeyOptions named tuple which contains all options relevant to
517 monkey tests.
518 """
jama47ca85c2014-12-03 18:38:07519 # TODO(jbudorick): Handle this directly in argparse with nargs='+'
520 category = args.category
[email protected]3dbdfa42013-08-08 01:08:14521 if category:
jama47ca85c2014-12-03 18:38:07522 category = args.category.split(',')
[email protected]3dbdfa42013-08-08 01:08:14523
jama47ca85c2014-12-03 18:38:07524 # TODO(jbudorick): Get rid of MonkeyOptions.
[email protected]3dbdfa42013-08-08 01:08:14525 return monkey_test_options.MonkeyOptions(
jama47ca85c2014-12-03 18:38:07526 args.verbose_count,
527 args.package,
528 args.event_count,
[email protected]3dbdfa42013-08-08 01:08:14529 category,
jama47ca85c2014-12-03 18:38:07530 args.throttle,
531 args.seed,
532 args.extra_args)
[email protected]3dbdfa42013-08-08 01:08:14533
rnephew5c499782014-12-12 19:08:55534def AddUirobotTestOptions(parser):
535 """Adds uirobot test options to |option_parser|."""
536 group = parser.add_argument_group('Uirobot Test Options')
537
rnephewefe44b42015-02-04 04:45:15538 group.add_argument('--app-under-test', required=True,
539 help='APK to run tests on.')
rnephew5c499782014-12-12 19:08:55540 group.add_argument(
541 '--minutes', default=5, type=int,
jbudorick676b1202015-02-06 22:02:27542 help='Number of minutes to run uirobot test [default: %(default)s].')
rnephew5c499782014-12-12 19:08:55543
544 AddCommonOptions(parser)
545 AddDeviceOptions(parser)
546 AddRemoteDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14547
jama47ca85c2014-12-03 18:38:07548def AddPerfTestOptions(parser):
549 """Adds perf test options to |parser|."""
[email protected]ec3170b2013-08-14 14:39:47550
jama47ca85c2014-12-03 18:38:07551 group = parser.add_argument_group('Perf Test Options')
[email protected]ec3170b2013-08-14 14:39:47552
jama47ca85c2014-12-03 18:38:07553 class SingleStepAction(argparse.Action):
554 def __call__(self, parser, namespace, values, option_string=None):
555 if values and not namespace.single_step:
556 parser.error('single step command provided, '
557 'but --single-step not specified.')
558 elif namespace.single_step and not values:
559 parser.error('--single-step specified, '
560 'but no single step command provided.')
561 setattr(namespace, self.dest, values)
562
563 step_group = group.add_mutually_exclusive_group(required=True)
564 # TODO(jbudorick): Revise --single-step to use argparse.REMAINDER.
565 # This requires removing "--" from client calls.
566 step_group.add_argument(
567 '--single-step', action='store_true',
[email protected]def4bce2013-11-12 12:59:52568 help='Execute the given command with retries, but only print the result '
569 'for the "most successful" round.')
jama47ca85c2014-12-03 18:38:07570 step_group.add_argument(
[email protected]181a5c92013-09-06 17:11:46571 '--steps',
[email protected]def4bce2013-11-12 12:59:52572 help='JSON file containing the list of commands to run.')
jama47ca85c2014-12-03 18:38:07573 step_group.add_argument(
574 '--print-step',
575 help='The name of a previously executed perf step to print.')
576
577 group.add_argument(
peterbd4e73d2014-12-03 15:47:36578 '--output-json-list',
579 help='Write a simple list of names from --steps into the given file.')
jama47ca85c2014-12-03 18:38:07580 group.add_argument(
peterbd4e73d2014-12-03 15:47:36581 '--collect-chartjson-data',
582 action='store_true',
583 help='Cache the chartjson output from each step for later use.')
jama47ca85c2014-12-03 18:38:07584 group.add_argument(
peterbd4e73d2014-12-03 15:47:36585 '--output-chartjson-data',
586 default='',
587 help='Write out chartjson into the given file.')
jama47ca85c2014-12-03 18:38:07588 group.add_argument(
589 '--flaky-steps',
590 help=('A JSON file containing steps that are flaky '
591 'and will have its exit code ignored.'))
592 group.add_argument(
[email protected]181a5c92013-09-06 17:11:46593 '--no-timeout', action='store_true',
594 help=('Do not impose a timeout. Each perf step is responsible for '
595 'implementing the timeout logic.'))
jama47ca85c2014-12-03 18:38:07596 group.add_argument(
[email protected]650487c2013-09-30 11:40:49597 '-f', '--test-filter',
598 help=('Test filter (will match against the names listed in --steps).'))
jama47ca85c2014-12-03 18:38:07599 group.add_argument(
600 '--dry-run', action='store_true',
[email protected]650487c2013-09-30 11:40:49601 help='Just print the steps without executing.')
jama47ca85c2014-12-03 18:38:07602 group.add_argument('single_step_command', nargs='*', action=SingleStepAction,
603 help='If --single-step is specified, the command to run.')
604 AddCommonOptions(parser)
605 AddDeviceOptions(parser)
[email protected]ec3170b2013-08-14 14:39:47606
607
jama47ca85c2014-12-03 18:38:07608def ProcessPerfTestOptions(args):
[email protected]ec3170b2013-08-14 14:39:47609 """Processes all perf test options.
610
611 Args:
jama47ca85c2014-12-03 18:38:07612 args: argparse.Namespace object.
[email protected]ec3170b2013-08-14 14:39:47613
614 Returns:
615 A PerfOptions named tuple which contains all options relevant to
616 perf tests.
617 """
jama47ca85c2014-12-03 18:38:07618 # TODO(jbudorick): Move single_step handling down into the perf tests.
619 if args.single_step:
620 args.single_step = ' '.join(args.single_step_command)
621 # TODO(jbudorick): Get rid of PerfOptions.
[email protected]ec3170b2013-08-14 14:39:47622 return perf_test_options.PerfOptions(
jama47ca85c2014-12-03 18:38:07623 args.steps, args.flaky_steps, args.output_json_list,
624 args.print_step, args.no_timeout, args.test_filter,
625 args.dry_run, args.single_step, args.collect_chartjson_data,
626 args.output_chartjson_data)
[email protected]ec3170b2013-08-14 14:39:47627
628
jama47ca85c2014-12-03 18:38:07629def AddPythonTestOptions(parser):
630 group = parser.add_argument_group('Python Test Options')
631 group.add_argument(
632 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME',
633 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(),
634 help='Name of the test suite to run.')
635 AddCommonOptions(parser)
jbudorick256fd532014-10-24 01:50:13636
637
jama47ca85c2014-12-03 18:38:07638def _RunGTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37639 """Subcommand of RunTestsCommands which runs gtests."""
[email protected]6bc1bda22013-07-19 22:08:37640 exit_code = 0
jama47ca85c2014-12-03 18:38:07641 for suite_name in args.suite_name:
642 # TODO(jbudorick): Either deprecate multi-suite or move its handling down
643 # into the gtest code.
[email protected]2a684222013-08-01 16:59:22644 gtest_options = gtest_test_options.GTestOptions(
jama47ca85c2014-12-03 18:38:07645 args.tool,
jama47ca85c2014-12-03 18:38:07646 args.test_filter,
647 args.run_disabled,
648 args.test_arguments,
649 args.timeout,
650 args.isolate_file_path,
jbudorick5ee45892015-06-10 18:46:22651 suite_name,
652 args.app_data_files,
653 args.app_data_file_dir)
[email protected]f7148dd42013-08-20 14:24:57654 runner_factory, tests = gtest_setup.Setup(gtest_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37655
656 results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57657 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07658 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37659
660 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
661 exit_code = test_exit_code
662
663 report_results.LogFull(
664 results=results,
665 test_type='Unit test',
666 test_package=suite_name,
jama47ca85c2014-12-03 18:38:07667 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37668
jama47ca85c2014-12-03 18:38:07669 if args.json_results_file:
670 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54671
[email protected]6bc1bda22013-07-19 22:08:37672 return exit_code
673
674
jama47ca85c2014-12-03 18:38:07675def _RunLinkerTests(args, devices):
[email protected]6b6abac6d2013-10-03 11:56:38676 """Subcommand of RunTestsCommands which runs linker tests."""
jama47ca85c2014-12-03 18:38:07677 runner_factory, tests = linker_setup.Setup(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38678
679 results, exit_code = test_dispatcher.RunTests(
680 tests, runner_factory, devices, shard=True, test_timeout=60,
jama47ca85c2014-12-03 18:38:07681 num_retries=args.num_retries)
[email protected]6b6abac6d2013-10-03 11:56:38682
683 report_results.LogFull(
684 results=results,
685 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22686 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38687
jama47ca85c2014-12-03 18:38:07688 if args.json_results_file:
689 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54690
[email protected]6b6abac6d2013-10-03 11:56:38691 return exit_code
692
693
jama47ca85c2014-12-03 18:38:07694def _RunInstrumentationTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37695 """Subcommand of RunTestsCommands which runs instrumentation tests."""
jama47ca85c2014-12-03 18:38:07696 logging.info('_RunInstrumentationTests(%s, %s)' % (str(args), str(devices)))
[email protected]6bc1bda22013-07-19 22:08:37697
jama47ca85c2014-12-03 18:38:07698 instrumentation_options = ProcessInstrumentationOptions(args)
699
700 if len(devices) > 1 and args.wait_for_debugger:
[email protected]f7148dd42013-08-20 14:24:57701 logging.warning('Debugger can not be sharded, using first available device')
702 devices = devices[:1]
703
[email protected]6bc1bda22013-07-19 22:08:37704 results = base_test_result.TestRunResults()
705 exit_code = 0
706
jama47ca85c2014-12-03 18:38:07707 if args.run_java_tests:
mikecase526d68e2014-11-19 20:02:05708 runner_factory, tests = instrumentation_setup.Setup(
709 instrumentation_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37710
711 test_results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57712 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07713 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37714
715 results.AddTestRunResults(test_results)
716
jama47ca85c2014-12-03 18:38:07717 if args.run_python_tests:
[email protected]37ee0c792013-08-06 19:10:13718 runner_factory, tests = host_driven_setup.InstrumentationSetup(
jama47ca85c2014-12-03 18:38:07719 args.host_driven_root, args.official_build,
[email protected]37ee0c792013-08-06 19:10:13720 instrumentation_options)
721
[email protected]34020022013-08-06 23:35:34722 if tests:
723 test_results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57724 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07725 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37726
[email protected]34020022013-08-06 23:35:34727 results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37728
[email protected]34020022013-08-06 23:35:34729 # Only allow exit code escalation
730 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
731 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37732
jama47ca85c2014-12-03 18:38:07733 if args.device_flags:
734 args.device_flags = os.path.join(constants.DIR_SOURCE_ROOT,
735 args.device_flags)
[email protected]4f777ca2014-08-08 01:45:59736
[email protected]6bc1bda22013-07-19 22:08:37737 report_results.LogFull(
738 results=results,
739 test_type='Instrumentation',
jama47ca85c2014-12-03 18:38:07740 test_package=os.path.basename(args.test_apk),
741 annotation=args.annotations,
742 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37743
jama47ca85c2014-12-03 18:38:07744 if args.json_results_file:
745 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54746
[email protected]6bc1bda22013-07-19 22:08:37747 return exit_code
748
749
jama47ca85c2014-12-03 18:38:07750def _RunUIAutomatorTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37751 """Subcommand of RunTestsCommands which runs uiautomator tests."""
jama47ca85c2014-12-03 18:38:07752 uiautomator_options = ProcessUIAutomatorOptions(args)
[email protected]6bc1bda22013-07-19 22:08:37753
[email protected]37ee0c792013-08-06 19:10:13754 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options)
[email protected]6bc1bda22013-07-19 22:08:37755
[email protected]37ee0c792013-08-06 19:10:13756 results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57757 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07758 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37759
760 report_results.LogFull(
761 results=results,
762 test_type='UIAutomator',
jama47ca85c2014-12-03 18:38:07763 test_package=os.path.basename(args.test_jar),
764 annotation=args.annotations,
765 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37766
jama47ca85c2014-12-03 18:38:07767 if args.json_results_file:
768 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54769
[email protected]6bc1bda22013-07-19 22:08:37770 return exit_code
771
772
jama47ca85c2014-12-03 18:38:07773def _RunJUnitTests(args):
jbudorick9a6b7b332014-09-20 00:01:07774 """Subcommand of RunTestsCommand which runs junit tests."""
jama47ca85c2014-12-03 18:38:07775 runner_factory, tests = junit_setup.Setup(args)
mikecasec638a072015-04-01 16:35:35776 results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
777
778 report_results.LogFull(
779 results=results,
780 test_type='JUnit',
781 test_package=args.test_suite)
782
mikecase572401b2015-04-09 02:28:57783 if args.json_results_file:
784 json_results.GenerateJsonResultsFile(results, args.json_results_file)
785
jbudorick9a6b7b332014-09-20 00:01:07786 return exit_code
787
788
jama47ca85c2014-12-03 18:38:07789def _RunMonkeyTests(args, devices):
[email protected]3dbdfa42013-08-08 01:08:14790 """Subcommand of RunTestsCommands which runs monkey tests."""
jama47ca85c2014-12-03 18:38:07791 monkey_options = ProcessMonkeyTestOptions(args)
[email protected]3dbdfa42013-08-08 01:08:14792
793 runner_factory, tests = monkey_setup.Setup(monkey_options)
794
795 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46796 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07797 num_retries=args.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14798
799 report_results.LogFull(
800 results=results,
801 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28802 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14803
jama47ca85c2014-12-03 18:38:07804 if args.json_results_file:
805 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54806
[email protected]3dbdfa42013-08-08 01:08:14807 return exit_code
808
809
jama47ca85c2014-12-03 18:38:07810def _RunPerfTests(args):
[email protected]ec3170b2013-08-14 14:39:47811 """Subcommand of RunTestsCommands which runs perf tests."""
jama47ca85c2014-12-03 18:38:07812 perf_options = ProcessPerfTestOptions(args)
[email protected]61487ed2014-06-09 12:33:56813
814 # Just save a simple json with a list of test names.
815 if perf_options.output_json_list:
816 return perf_test_runner.OutputJsonList(
817 perf_options.steps, perf_options.output_json_list)
818
[email protected]ad32f312013-11-13 04:03:29819 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47820 if perf_options.print_step:
simonhatch9b9256d2015-01-07 18:03:42821 return perf_test_runner.PrintTestOutput(
822 perf_options.print_step, perf_options.output_chartjson_data)
[email protected]ec3170b2013-08-14 14:39:47823
[email protected]a72f0752014-06-03 23:52:34824 runner_factory, tests, devices = perf_setup.Setup(perf_options)
[email protected]ec3170b2013-08-14 14:39:47825
[email protected]a72f0752014-06-03 23:52:34826 # shard=False means that each device will get the full list of tests
827 # and then each one will decide their own affinity.
828 # shard=True means each device will pop the next test available from a queue,
829 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57830 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34831 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07832 num_retries=args.num_retries)
[email protected]ec3170b2013-08-14 14:39:47833
834 report_results.LogFull(
835 results=results,
836 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12837 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52838
jama47ca85c2014-12-03 18:38:07839 if args.json_results_file:
840 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54841
[email protected]def4bce2013-11-12 12:59:52842 if perf_options.single_step:
843 return perf_test_runner.PrintTestOutput('single_step')
844
[email protected]11ce8452014-02-17 10:55:03845 perf_test_runner.PrintSummary(tests)
846
[email protected]86184c7b2013-08-15 15:06:57847 # Always return 0 on the sharding stage. Individual tests exit_code
848 # will be returned on the print_step stage.
849 return 0
[email protected]ec3170b2013-08-14 14:39:47850
[email protected]3dbdfa42013-08-08 01:08:14851
jama47ca85c2014-12-03 18:38:07852def _RunPythonTests(args):
jbudorick256fd532014-10-24 01:50:13853 """Subcommand of RunTestsCommand which runs python unit tests."""
jama47ca85c2014-12-03 18:38:07854 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
jbudorick256fd532014-10-24 01:50:13855 suite_path = suite_vars['path']
856 suite_test_modules = suite_vars['test_modules']
857
858 sys.path = [suite_path] + sys.path
859 try:
860 suite = unittest.TestSuite()
861 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
862 for m in suite_test_modules)
jama47ca85c2014-12-03 18:38:07863 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
jbudorick256fd532014-10-24 01:50:13864 return 0 if runner.run(suite).wasSuccessful() else 1
865 finally:
866 sys.path = sys.path[1:]
867
868
[email protected]f7148dd42013-08-20 14:24:57869def _GetAttachedDevices(test_device=None):
870 """Get all attached devices.
871
872 Args:
873 test_device: Name of a specific device to use.
874
875 Returns:
876 A list of attached devices.
877 """
jbudorick4551d0dc2015-04-29 16:07:06878 attached_devices = device_utils.DeviceUtils.HealthyDevices()
aberent6a02a6182015-04-29 11:07:55879 if test_device:
jbudorick4551d0dc2015-04-29 16:07:06880 test_device = [d for d in attached_devices if d == test_device]
881 if not test_device:
882 raise device_errors.DeviceUnreachableError(
883 'Did not find device %s among attached device. Attached devices: %s'
884 % (test_device, ', '.join(attached_devices)))
885 return test_device
aberent6a02a6182015-04-29 11:07:55886
jbudorick4551d0dc2015-04-29 16:07:06887 else:
888 if not attached_devices:
889 raise device_errors.NoDevicesError()
890 return sorted(attached_devices)
[email protected]f7148dd42013-08-20 14:24:57891
892
jama47ca85c2014-12-03 18:38:07893def RunTestsCommand(args, parser):
[email protected]fbe29322013-07-09 09:03:26894 """Checks test type and dispatches to the appropriate function.
895
896 Args:
jama47ca85c2014-12-03 18:38:07897 args: argparse.Namespace object.
898 parser: argparse.ArgumentParser object.
[email protected]fbe29322013-07-09 09:03:26899
900 Returns:
901 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10902
903 Raises:
904 Exception: Unknown command name passed in, or an exception from an
905 individual test runner.
[email protected]fbe29322013-07-09 09:03:26906 """
jama47ca85c2014-12-03 18:38:07907 command = args.command
[email protected]fbe29322013-07-09 09:03:26908
jama47ca85c2014-12-03 18:38:07909 ProcessCommonOptions(args)
[email protected]d82f0252013-07-12 23:22:57910
jama47ca85c2014-12-03 18:38:07911 if args.enable_platform_mode:
rnephew5c499782014-12-12 19:08:55912 return RunTestsInPlatformMode(args, parser)
jbudorick66dc3722014-11-06 21:33:51913
914 if command in constants.LOCAL_MACHINE_TESTS:
jbudorick256fd532014-10-24 01:50:13915 devices = []
916 else:
jama47ca85c2014-12-03 18:38:07917 devices = _GetAttachedDevices(args.test_device)
[email protected]f7148dd42013-08-20 14:24:57918
[email protected]c0662e092013-11-12 11:51:25919 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40920 if not ports.ResetTestServerPortAllocation():
921 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25922
[email protected]fbe29322013-07-09 09:03:26923 if command == 'gtest':
jbudorick590a3d72015-06-12 23:53:31924 if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES:
925 return RunTestsInPlatformMode(args, parser)
jama47ca85c2014-12-03 18:38:07926 return _RunGTests(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38927 elif command == 'linker':
jama47ca85c2014-12-03 18:38:07928 return _RunLinkerTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26929 elif command == 'instrumentation':
jama47ca85c2014-12-03 18:38:07930 return _RunInstrumentationTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26931 elif command == 'uiautomator':
jama47ca85c2014-12-03 18:38:07932 return _RunUIAutomatorTests(args, devices)
jbudorick9a6b7b332014-09-20 00:01:07933 elif command == 'junit':
jama47ca85c2014-12-03 18:38:07934 return _RunJUnitTests(args)
[email protected]3dbdfa42013-08-08 01:08:14935 elif command == 'monkey':
jama47ca85c2014-12-03 18:38:07936 return _RunMonkeyTests(args, devices)
[email protected]ec3170b2013-08-14 14:39:47937 elif command == 'perf':
jama47ca85c2014-12-03 18:38:07938 return _RunPerfTests(args)
jbudorick256fd532014-10-24 01:50:13939 elif command == 'python':
jama47ca85c2014-12-03 18:38:07940 return _RunPythonTests(args)
[email protected]fbe29322013-07-09 09:03:26941 else:
[email protected]6bc1bda22013-07-19 22:08:37942 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26943
[email protected]fbe29322013-07-09 09:03:26944
jbudorick66dc3722014-11-06 21:33:51945_SUPPORTED_IN_PLATFORM_MODE = [
946 # TODO(jbudorick): Add support for more test types.
jbudorick911be58d2015-01-13 02:51:06947 'gtest',
948 'instrumentation',
949 'uirobot',
jbudorick66dc3722014-11-06 21:33:51950]
951
952
jama47ca85c2014-12-03 18:38:07953def RunTestsInPlatformMode(args, parser):
jbudorick66dc3722014-11-06 21:33:51954
jama47ca85c2014-12-03 18:38:07955 if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
956 parser.error('%s is not yet supported in platform mode' % args.command)
jbudorick66dc3722014-11-06 21:33:51957
jama47ca85c2014-12-03 18:38:07958 with environment_factory.CreateEnvironment(args, parser.error) as env:
959 with test_instance_factory.CreateTestInstance(args, parser.error) as test:
jbudorick66dc3722014-11-06 21:33:51960 with test_run_factory.CreateTestRun(
jama47ca85c2014-12-03 18:38:07961 args, env, test, parser.error) as test_run:
jbudorick66dc3722014-11-06 21:33:51962 results = test_run.RunTests()
963
jbudorick911be58d2015-01-13 02:51:06964 if args.environment == 'remote_device' and args.trigger:
rnephew5c499782014-12-12 19:08:55965 return 0 # Not returning results, only triggering.
966
jbudorick66dc3722014-11-06 21:33:51967 report_results.LogFull(
968 results=results,
969 test_type=test.TestType(),
970 test_package=test_run.TestPackage(),
jbudorickf9543672014-12-08 22:36:33971 annotation=getattr(args, 'annotations', None),
972 flakiness_server=getattr(args, 'flakiness_dashboard_server', None))
jbudorick66dc3722014-11-06 21:33:51973
jama47ca85c2014-12-03 18:38:07974 if args.json_results_file:
jbudorickb8c42072014-12-01 18:07:54975 json_results.GenerateJsonResultsFile(
jama47ca85c2014-12-03 18:38:07976 results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54977
mikecasee74051022015-02-26 23:08:22978 return 0 if results.DidRunPass() else constants.ERROR_EXIT_CODE
jbudorick66dc3722014-11-06 21:33:51979
980
jama47ca85c2014-12-03 18:38:07981CommandConfigTuple = collections.namedtuple(
982 'CommandConfigTuple',
983 ['add_options_func', 'help_txt'])
[email protected]fbe29322013-07-09 09:03:26984VALID_COMMANDS = {
jama47ca85c2014-12-03 18:38:07985 'gtest': CommandConfigTuple(
986 AddGTestOptions,
987 'googletest-based C++ tests'),
988 'instrumentation': CommandConfigTuple(
989 AddInstrumentationTestOptions,
990 'InstrumentationTestCase-based Java tests'),
991 'uiautomator': CommandConfigTuple(
992 AddUIAutomatorTestOptions,
993 "Tests that run via Android's uiautomator command"),
994 'junit': CommandConfigTuple(
995 AddJUnitTestOptions,
996 'JUnit4-based Java tests'),
997 'monkey': CommandConfigTuple(
998 AddMonkeyTestOptions,
999 "Tests based on Android's monkey"),
1000 'perf': CommandConfigTuple(
1001 AddPerfTestOptions,
1002 'Performance tests'),
1003 'python': CommandConfigTuple(
1004 AddPythonTestOptions,
1005 'Python tests based on unittest.TestCase'),
1006 'linker': CommandConfigTuple(
1007 AddLinkerTestOptions,
1008 'Linker tests'),
rnephew5c499782014-12-12 19:08:551009 'uirobot': CommandConfigTuple(
1010 AddUirobotTestOptions,
1011 'Uirobot test'),
jama47ca85c2014-12-03 18:38:071012}
[email protected]fbe29322013-07-09 09:03:261013
1014
[email protected]7c53a602014-03-24 16:21:441015def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:241016 for thread in threading.enumerate():
1017 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:211018
1019
[email protected]7c53a602014-03-24 16:21:441020def main():
[email protected]83bb8152013-11-19 15:02:211021 signal.signal(signal.SIGUSR1, DumpThreadStacks)
jama47ca85c2014-12-03 18:38:071022
1023 parser = argparse.ArgumentParser()
1024 command_parsers = parser.add_subparsers(title='test types',
1025 dest='command')
1026
1027 for test_type, config in sorted(VALID_COMMANDS.iteritems(),
1028 key=lambda x: x[0]):
1029 subparser = command_parsers.add_parser(
1030 test_type, usage='%(prog)s [options]', help=config.help_txt)
1031 config.add_options_func(subparser)
1032
1033 args = parser.parse_args()
mikecasee74051022015-02-26 23:08:221034
1035 try:
1036 return RunTestsCommand(args, parser)
1037 except base_error.BaseError as e:
1038 logging.exception('Error occurred.')
1039 if e.is_infra_error:
1040 return constants.INFRA_EXIT_CODE
mswecce6732015-06-06 00:31:331041 return constants.ERROR_EXIT_CODE
mikecasee74051022015-02-26 23:08:221042 except: # pylint: disable=W0702
1043 logging.exception('Unrecognized error occurred.')
1044 return constants.ERROR_EXIT_CODE
[email protected]fbe29322013-07-09 09:03:261045
[email protected]fbe29322013-07-09 09:03:261046
1047if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:441048 sys.exit(main())