blob: 4164bd8bb0079c3a947cffe2e7081ea927dabe85 [file] [log] [blame]
[email protected]fbe29322013-07-09 09:03:261#!/usr/bin/env python
2#
3# Copyright 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
[email protected]181a5c92013-09-06 17:11:467"""Runs all types of tests from one unified interface."""
[email protected]fbe29322013-07-09 09:03:268
jama47ca85c2014-12-03 18:38:079import argparse
[email protected]fbe29322013-07-09 09:03:2610import collections
[email protected]f7148dd42013-08-20 14:24:5711import logging
[email protected]fbe29322013-07-09 09:03:2612import os
[email protected]6bc1bda22013-07-19 22:08:3713import shutil
[email protected]83bb8152013-11-19 15:02:2114import signal
[email protected]fbe29322013-07-09 09:03:2615import sys
[email protected]83bb8152013-11-19 15:02:2116import threading
jbudorick256fd532014-10-24 01:50:1317import unittest
[email protected]fbe29322013-07-09 09:03:2618
[email protected]f7148dd42013-08-20 14:24:5719from pylib import android_commands
[email protected]fbe29322013-07-09 09:03:2620from pylib import constants
[email protected]c0662e092013-11-12 11:51:2521from pylib import forwarder
[email protected]fbe29322013-07-09 09:03:2622from pylib import ports
23from pylib.base import base_test_result
jbudorick66dc3722014-11-06 21:33:5124from pylib.base import environment_factory
[email protected]6bc1bda22013-07-19 22:08:3725from pylib.base import test_dispatcher
jbudorick66dc3722014-11-06 21:33:5126from pylib.base import test_instance_factory
27from pylib.base import test_run_factory
[email protected]6bc1bda22013-07-19 22:08:3728from pylib.gtest import gtest_config
[email protected]2a684222013-08-01 16:59:2229from pylib.gtest import setup as gtest_setup
30from pylib.gtest import test_options as gtest_test_options
[email protected]6b6abac6d2013-10-03 11:56:3831from pylib.linker import setup as linker_setup
[email protected]37ee0c792013-08-06 19:10:1332from pylib.host_driven import setup as host_driven_setup
[email protected]6bc1bda22013-07-19 22:08:3733from pylib.instrumentation import setup as instrumentation_setup
[email protected]2a684222013-08-01 16:59:2234from pylib.instrumentation import test_options as instrumentation_test_options
jbudorick9a6b7b332014-09-20 00:01:0735from pylib.junit import setup as junit_setup
36from pylib.junit import test_dispatcher as junit_dispatcher
[email protected]3dbdfa42013-08-08 01:08:1437from pylib.monkey import setup as monkey_setup
38from pylib.monkey import test_options as monkey_test_options
[email protected]ec3170b2013-08-14 14:39:4739from pylib.perf import setup as perf_setup
40from pylib.perf import test_options as perf_test_options
41from pylib.perf import test_runner as perf_test_runner
jbudorickb8c42072014-12-01 18:07:5442from pylib.results import json_results
43from pylib.results import report_results
[email protected]6bc1bda22013-07-19 22:08:3744from pylib.uiautomator import setup as uiautomator_setup
[email protected]2a684222013-08-01 16:59:2245from pylib.uiautomator import test_options as uiautomator_test_options
[email protected]2eea4872014-07-28 23:06:1746from pylib.utils import apk_helper
[email protected]71aec4b2013-11-20 00:35:2447from pylib.utils import reraiser_thread
[email protected]6bc1bda22013-07-19 22:08:3748from pylib.utils import run_tests_helper
[email protected]fbe29322013-07-09 09:03:2649
50
jama47ca85c2014-12-03 18:38:0751def AddCommonOptions(parser):
52 """Adds all common options to |parser|."""
[email protected]fbe29322013-07-09 09:03:2653
jama47ca85c2014-12-03 18:38:0754 group = parser.add_argument_group('Common Options')
55
[email protected]dfffbcbc2013-09-17 22:06:0156 default_build_type = os.environ.get('BUILDTYPE', 'Debug')
jama47ca85c2014-12-03 18:38:0757
58 debug_or_release_group = group.add_mutually_exclusive_group()
59 debug_or_release_group.add_argument(
60 '--debug', action='store_const', const='Debug', dest='build_type',
61 default=default_build_type,
62 help=('If set, run test suites under out/Debug. '
63 'Default is env var BUILDTYPE or Debug.'))
64 debug_or_release_group.add_argument(
65 '--release', action='store_const', const='Release', dest='build_type',
66 help=('If set, run test suites under out/Release. '
67 'Default is env var BUILDTYPE or Debug.'))
68
69 group.add_argument('--build-directory', dest='build_directory',
70 help=('Path to the directory in which build files are'
71 ' located (should not include build type)'))
72 group.add_argument('--output-directory', dest='output_directory',
73 help=('Path to the directory in which build files are'
74 ' located (must include build type). This will take'
75 ' precedence over --debug, --release and'
76 ' --build-directory'))
77 group.add_argument('--num_retries', dest='num_retries', type=int, default=2,
78 help=('Number of retries for a test before '
79 'giving up (default: %(default)s).'))
80 group.add_argument('-v',
81 '--verbose',
82 dest='verbose_count',
83 default=0,
84 action='count',
85 help='Verbose level (multiple times for more)')
86 group.add_argument('--flakiness-dashboard-server',
87 dest='flakiness_dashboard_server',
88 help=('Address of the server that is hosting the '
89 'Chrome for Android flakiness dashboard.'))
90 group.add_argument('--enable-platform-mode', action='store_true',
91 help=('Run the test scripts in platform mode, which '
92 'conceptually separates the test runner from the '
93 '"device" (local or remote, real or emulated) on '
94 'which the tests are running. [experimental]'))
95 group.add_argument('-e', '--environment', default='local',
96 choices=constants.VALID_ENVIRONMENTS,
97 help='Test environment to run in (default: %(default)s).')
98 group.add_argument('--adb-path',
99 help=('Specify the absolute path of the adb binary that '
100 'should be used.'))
101 group.add_argument('--json-results-file', dest='json_results_file',
102 help='If set, will dump results in JSON form '
103 'to specified file.')
[email protected]fbe29322013-07-09 09:03:26104
105
jama47ca85c2014-12-03 18:38:07106def ProcessCommonOptions(args):
[email protected]fbe29322013-07-09 09:03:26107 """Processes and handles all common options."""
jama47ca85c2014-12-03 18:38:07108 run_tests_helper.SetLogLevel(args.verbose_count)
109 constants.SetBuildType(args.build_type)
110 if args.build_directory:
111 constants.SetBuildDirectory(args.build_directory)
112 if args.output_directory:
113 constants.SetOutputDirectort(args.output_directory)
114 if args.adb_path:
115 constants.SetAdbPath(args.adb_path)
mikecase48e16bf2014-11-19 22:46:45116 # Some things such as Forwarder require ADB to be in the environment path.
117 adb_dir = os.path.dirname(constants.GetAdbPath())
118 if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep):
119 os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
[email protected]fbe29322013-07-09 09:03:26120
121
rnephew5c499782014-12-12 19:08:55122def AddRemoteDeviceOptions(parser):
123 group = parser.add_argument_group('Remote Device Options')
124
125 group.add_argument('--trigger', default='',
jbudoricke6c560152015-01-13 23:49:28126 help=('Only triggers the test if set. Stores test_run_id '
127 'in given file path. '))
rnephew5c499782014-12-12 19:08:55128 group.add_argument('--collect', default='',
jbudoricke6c560152015-01-13 23:49:28129 help=('Only collects the test results if set. '
130 'Gets test_run_id from given file path.'))
rnephewb74dd0632015-01-07 19:34:53131 group.add_argument('--remote-device', default='',
jbudoricke6c560152015-01-13 23:49:28132 help='Device type to run test on.')
rnephewb74dd0632015-01-07 19:34:53133 group.add_argument('--remote-device-os', default='',
jbudoricke6c560152015-01-13 23:49:28134 help='OS to have on the device.')
rnephew5c499782014-12-12 19:08:55135 group.add_argument('--results-path', default='',
jbudoricke6c560152015-01-13 23:49:28136 help='File path to download results to.')
rnephew7f1e2052014-12-12 23:00:11137 group.add_argument('--api-protocol',
jbudoricke6c560152015-01-13 23:49:28138 help='HTTP protocol to use. (http or https)')
139 group.add_argument('--api-address', help='Address to send HTTP requests.')
140 group.add_argument('--api-port', help='Port to send HTTP requests to.')
rnephew5c499782014-12-12 19:08:55141 group.add_argument('--runner-type', default='',
jbudoricke6c560152015-01-13 23:49:28142 help='Type of test to run as.')
rnephew5c499782014-12-12 19:08:55143 group.add_argument('--runner-package', default='',
jbudoricke6c560152015-01-13 23:49:28144 help='Package name of test.')
rnephewe416dff2015-01-21 21:26:37145 group.add_argument('--app-under-test', default='',
jbudoricke6c560152015-01-13 23:49:28146 help='APK to run tests on.')
rnephew5c499782014-12-12 19:08:55147
148 api_secret_group = group.add_mutually_exclusive_group()
149 api_secret_group.add_argument('--api-secret', default='',
jbudoricke6c560152015-01-13 23:49:28150 help='API secret for remote devices.')
rnephew5c499782014-12-12 19:08:55151 api_secret_group.add_argument('--api-secret-file', default='',
jbudoricke6c560152015-01-13 23:49:28152 help='Path to file that contains API secret.')
rnephew5c499782014-12-12 19:08:55153
154 api_key_group = group.add_mutually_exclusive_group()
155 api_key_group.add_argument('--api-key', default='',
jbudoricke6c560152015-01-13 23:49:28156 help='API key for remote devices.')
rnephew5c499782014-12-12 19:08:55157 api_key_group.add_argument('--api-key-file', default='',
jbudoricke6c560152015-01-13 23:49:28158 help='Path to file that contains API key.')
rnephew5c499782014-12-12 19:08:55159
160
jama47ca85c2014-12-03 18:38:07161def AddDeviceOptions(parser):
162 """Adds device options to |parser|."""
163 group = parser.add_argument_group(title='Device Options')
164 group.add_argument('-c', dest='cleanup_test_files',
165 help='Cleanup test files on the device after run',
166 action='store_true')
167 group.add_argument('--tool',
168 dest='tool',
169 help=('Run the test under a tool '
170 '(use --tool help to list them)'))
171 group.add_argument('-d', '--device', dest='test_device',
172 help=('Target device for the test suite '
173 'to run on.'))
jbudorick256fd532014-10-24 01:50:13174
175
jama47ca85c2014-12-03 18:38:07176def AddGTestOptions(parser):
177 """Adds gtest options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26178
jama47ca85c2014-12-03 18:38:07179 gtest_suites = list(gtest_config.STABLE_TEST_SUITES
180 + gtest_config.EXPERIMENTAL_TEST_SUITES)
[email protected]fbe29322013-07-09 09:03:26181
jama47ca85c2014-12-03 18:38:07182 group = parser.add_argument_group('GTest Options')
jbudorick15cdcd52014-12-03 19:58:49183 group.add_argument('-s', '--suite', dest='suite_name',
jama47ca85c2014-12-03 18:38:07184 nargs='+', metavar='SUITE_NAME', required=True,
jbudorick15cdcd52014-12-03 19:58:49185 help=('Executable name of the test suite to run. '
186 'Available suites include (but are not limited to): '
187 '%s' % ', '.join('"%s"' % s for s in gtest_suites)))
jama47ca85c2014-12-03 18:38:07188 group.add_argument('-f', '--gtest_filter', '--gtest-filter',
189 dest='test_filter',
190 help='googletest-style filter string.')
191 group.add_argument('--gtest_also_run_disabled_tests',
192 '--gtest-also-run-disabled-tests',
193 dest='run_disabled', action='store_true',
194 help='Also run disabled tests if applicable.')
195 group.add_argument('-a', '--test-arguments', dest='test_arguments',
196 default='',
197 help='Additional arguments to pass to the test.')
198 group.add_argument('-t', dest='timeout', type=int, default=60,
199 help='Timeout to wait for each test '
200 '(default: %(default)s).')
201 group.add_argument('--isolate_file_path',
202 '--isolate-file-path',
203 dest='isolate_file_path',
204 help='.isolate file path to override the default '
205 'path')
206 AddDeviceOptions(parser)
207 AddCommonOptions(parser)
rnephew5c499782014-12-12 19:08:55208 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26209
210
jama47ca85c2014-12-03 18:38:07211def AddLinkerTestOptions(parser):
212 group = parser.add_argument_group('Linker Test Options')
213 group.add_argument('-f', '--gtest-filter', dest='test_filter',
214 help='googletest-style filter string.')
215 AddCommonOptions(parser)
216 AddDeviceOptions(parser)
[email protected]6b6abac6d2013-10-03 11:56:38217
218
jama47ca85c2014-12-03 18:38:07219def AddJavaTestOptions(argument_group):
[email protected]fbe29322013-07-09 09:03:26220 """Adds the Java test options to |option_parser|."""
221
jama47ca85c2014-12-03 18:38:07222 argument_group.add_argument(
223 '-f', '--test-filter', dest='test_filter',
224 help=('Test filter (if not fully qualified, will run all matches).'))
225 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26226 '-A', '--annotation', dest='annotation_str',
227 help=('Comma-separated list of annotations. Run only tests with any of '
228 'the given annotations. An annotation can be either a key or a '
229 'key-values pair. A test that has no annotation is considered '
230 '"SmallTest".'))
jama47ca85c2014-12-03 18:38:07231 argument_group.add_argument(
[email protected]fbe29322013-07-09 09:03:26232 '-E', '--exclude-annotation', dest='exclude_annotation_str',
233 help=('Comma-separated list of annotations. Exclude tests with these '
234 'annotations.'))
jama47ca85c2014-12-03 18:38:07235 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59236 '--screenshot', dest='screenshot_failures', action='store_true',
237 help='Capture screenshots of test failures')
jama47ca85c2014-12-03 18:38:07238 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59239 '--save-perf-json', action='store_true',
240 help='Saves the JSON file for each UI Perf test.')
jama47ca85c2014-12-03 18:38:07241 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59242 '--official-build', action='store_true', help='Run official build tests.')
jama47ca85c2014-12-03 18:38:07243 argument_group.add_argument(
jbudorickcbcc115d2014-09-18 17:50:59244 '--test_data', '--test-data', action='append', default=[],
245 help=('Each instance defines a directory of test data that should be '
246 'copied to the target(s) before running the tests. The argument '
247 'should be of the form <target>:<source>, <target> is relative to '
248 'the device data directory, and <source> is relative to the '
249 'chromium build directory.'))
davileen98efad12015-01-05 19:48:21250 argument_group.add_argument(
251 '--disable-dalvik-asserts', dest='set_asserts', action='store_false',
252 default=True, help='Removes the dalvik.vm.enableassertions property')
253
[email protected]fbe29322013-07-09 09:03:26254
255
jama47ca85c2014-12-03 18:38:07256def ProcessJavaTestOptions(args):
[email protected]fbe29322013-07-09 09:03:26257 """Processes options/arguments and populates |options| with defaults."""
258
jama47ca85c2014-12-03 18:38:07259 # TODO(jbudorick): Handle most of this function in argparse.
260 if args.annotation_str:
261 args.annotations = args.annotation_str.split(',')
262 elif args.test_filter:
263 args.annotations = []
[email protected]fbe29322013-07-09 09:03:26264 else:
jama47ca85c2014-12-03 18:38:07265 args.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
266 'EnormousTest', 'IntegrationTest']
[email protected]fbe29322013-07-09 09:03:26267
jama47ca85c2014-12-03 18:38:07268 if args.exclude_annotation_str:
269 args.exclude_annotations = args.exclude_annotation_str.split(',')
[email protected]fbe29322013-07-09 09:03:26270 else:
jama47ca85c2014-12-03 18:38:07271 args.exclude_annotations = []
[email protected]fbe29322013-07-09 09:03:26272
[email protected]fbe29322013-07-09 09:03:26273
jama47ca85c2014-12-03 18:38:07274def AddInstrumentationTestOptions(parser):
275 """Adds Instrumentation test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26276
jama47ca85c2014-12-03 18:38:07277 parser.usage = '%(prog)s [options]'
[email protected]fbe29322013-07-09 09:03:26278
jama47ca85c2014-12-03 18:38:07279 group = parser.add_argument_group('Instrumentation Test Options')
280 AddJavaTestOptions(group)
[email protected]fbe29322013-07-09 09:03:26281
jama47ca85c2014-12-03 18:38:07282 java_or_python_group = group.add_mutually_exclusive_group()
283 java_or_python_group.add_argument(
284 '-j', '--java-only', action='store_false',
285 dest='run_python_tests', default=True, help='Run only the Java tests.')
286 java_or_python_group.add_argument(
287 '-p', '--python-only', action='store_false',
288 dest='run_java_tests', default=True,
289 help='Run only the host-driven tests.')
290
291 group.add_argument('--host-driven-root',
292 help='Root of the host-driven tests.')
293 group.add_argument('-w', '--wait_debugger', dest='wait_for_debugger',
294 action='store_true',
295 help='Wait for debugger.')
jbudorick911be58d2015-01-13 02:51:06296 group.add_argument('--apk-under-test', dest='apk_under_test',
297 help=('the name of the apk under test.'))
jama47ca85c2014-12-03 18:38:07298 group.add_argument('--test-apk', dest='test_apk', required=True,
299 help=('The name of the apk containing the tests '
300 '(without the .apk extension; '
301 'e.g. "ContentShellTest").'))
302 group.add_argument('--coverage-dir',
303 help=('Directory in which to place all generated '
304 'EMMA coverage files.'))
305 group.add_argument('--device-flags', dest='device_flags', default='',
306 help='The relative filepath to a file containing '
307 'command-line flags to set on the device')
jbudorick911be58d2015-01-13 02:51:06308 group.add_argument('--device-flags-file', default='',
309 help='The relative filepath to a file containing '
310 'command-line flags to set on the device')
jama47ca85c2014-12-03 18:38:07311 group.add_argument('--isolate_file_path',
312 '--isolate-file-path',
313 dest='isolate_file_path',
314 help='.isolate file path to override the default '
315 'path')
316
317 AddCommonOptions(parser)
318 AddDeviceOptions(parser)
rnephewe416dff2015-01-21 21:26:37319 AddRemoteDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26320
321
jama47ca85c2014-12-03 18:38:07322def ProcessInstrumentationOptions(args):
[email protected]2a684222013-08-01 16:59:22323 """Processes options/arguments and populate |options| with defaults.
324
325 Args:
jama47ca85c2014-12-03 18:38:07326 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22327
328 Returns:
329 An InstrumentationOptions named tuple which contains all options relevant to
330 instrumentation tests.
331 """
[email protected]fbe29322013-07-09 09:03:26332
jama47ca85c2014-12-03 18:38:07333 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26334
jama47ca85c2014-12-03 18:38:07335 if not args.host_driven_root:
336 args.run_python_tests = False
[email protected]37ee0c792013-08-06 19:10:13337
jama47ca85c2014-12-03 18:38:07338 args.test_apk_path = os.path.join(
[email protected]2eea4872014-07-28 23:06:17339 constants.GetOutDirectory(),
340 constants.SDK_BUILD_APKS_DIR,
jama47ca85c2014-12-03 18:38:07341 '%s.apk' % args.test_apk)
342 args.test_apk_jar_path = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15343 constants.GetOutDirectory(),
344 constants.SDK_BUILD_TEST_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07345 '%s.jar' % args.test_apk)
346 args.test_support_apk_path = '%sSupport%s' % (
347 os.path.splitext(args.test_apk_path))
[email protected]5e2f3f62014-06-23 12:31:46348
jama47ca85c2014-12-03 18:38:07349 args.test_runner = apk_helper.GetInstrumentationName(args.test_apk_path)
[email protected]5e2f3f62014-06-23 12:31:46350
jama47ca85c2014-12-03 18:38:07351 # TODO(jbudorick): Get rid of InstrumentationOptions.
[email protected]2a684222013-08-01 16:59:22352 return instrumentation_test_options.InstrumentationOptions(
jama47ca85c2014-12-03 18:38:07353 args.tool,
354 args.cleanup_test_files,
355 args.annotations,
356 args.exclude_annotations,
357 args.test_filter,
358 args.test_data,
359 args.save_perf_json,
360 args.screenshot_failures,
361 args.wait_for_debugger,
362 args.coverage_dir,
363 args.test_apk,
364 args.test_apk_path,
365 args.test_apk_jar_path,
366 args.test_runner,
367 args.test_support_apk_path,
368 args.device_flags,
davileen98efad12015-01-05 19:48:21369 args.isolate_file_path,
370 args.set_asserts
[email protected]5e2f3f62014-06-23 12:31:46371 )
[email protected]2a684222013-08-01 16:59:22372
[email protected]fbe29322013-07-09 09:03:26373
jama47ca85c2014-12-03 18:38:07374def AddUIAutomatorTestOptions(parser):
375 """Adds UI Automator test options to |parser|."""
[email protected]fbe29322013-07-09 09:03:26376
jama47ca85c2014-12-03 18:38:07377 group = parser.add_argument_group('UIAutomator Test Options')
378 AddJavaTestOptions(group)
379 group.add_argument(
380 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
381 metavar='PACKAGE', help='Package under test.')
382 group.add_argument(
383 '--test-jar', dest='test_jar', required=True,
[email protected]fbe29322013-07-09 09:03:26384 help=('The name of the dexed jar containing the tests (without the '
385 '.dex.jar extension). Alternatively, this can be a full path '
386 'to the jar.'))
387
jama47ca85c2014-12-03 18:38:07388 AddCommonOptions(parser)
389 AddDeviceOptions(parser)
[email protected]fbe29322013-07-09 09:03:26390
391
jama47ca85c2014-12-03 18:38:07392def ProcessUIAutomatorOptions(args):
[email protected]2a684222013-08-01 16:59:22393 """Processes UIAutomator options/arguments.
394
395 Args:
jama47ca85c2014-12-03 18:38:07396 args: argparse.Namespace object.
[email protected]2a684222013-08-01 16:59:22397
398 Returns:
399 A UIAutomatorOptions named tuple which contains all options relevant to
[email protected]3dbdfa42013-08-08 01:08:14400 uiautomator tests.
[email protected]2a684222013-08-01 16:59:22401 """
[email protected]fbe29322013-07-09 09:03:26402
jama47ca85c2014-12-03 18:38:07403 ProcessJavaTestOptions(args)
[email protected]fbe29322013-07-09 09:03:26404
jama47ca85c2014-12-03 18:38:07405 if os.path.exists(args.test_jar):
[email protected]fbe29322013-07-09 09:03:26406 # The dexed JAR is fully qualified, assume the info JAR lives along side.
jama47ca85c2014-12-03 18:38:07407 args.uiautomator_jar = args.test_jar
[email protected]fbe29322013-07-09 09:03:26408 else:
jama47ca85c2014-12-03 18:38:07409 args.uiautomator_jar = os.path.join(
[email protected]ae68d4a2013-09-24 21:57:15410 constants.GetOutDirectory(),
411 constants.SDK_BUILD_JAVALIB_DIR,
jama47ca85c2014-12-03 18:38:07412 '%s.dex.jar' % args.test_jar)
413 args.uiautomator_info_jar = (
414 args.uiautomator_jar[:args.uiautomator_jar.find('.dex.jar')] +
[email protected]fbe29322013-07-09 09:03:26415 '_java.jar')
416
[email protected]2a684222013-08-01 16:59:22417 return uiautomator_test_options.UIAutomatorOptions(
jama47ca85c2014-12-03 18:38:07418 args.tool,
419 args.cleanup_test_files,
420 args.annotations,
421 args.exclude_annotations,
422 args.test_filter,
423 args.test_data,
424 args.save_perf_json,
425 args.screenshot_failures,
426 args.uiautomator_jar,
427 args.uiautomator_info_jar,
davileen98efad12015-01-05 19:48:21428 args.package,
429 args.set_asserts)
[email protected]2a684222013-08-01 16:59:22430
[email protected]fbe29322013-07-09 09:03:26431
jama47ca85c2014-12-03 18:38:07432def AddJUnitTestOptions(parser):
433 """Adds junit test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07434
jama47ca85c2014-12-03 18:38:07435 group = parser.add_argument_group('JUnit Test Options')
436 group.add_argument(
437 '-s', '--test-suite', dest='test_suite', required=True,
jbudorick9a6b7b332014-09-20 00:01:07438 help=('JUnit test suite to run.'))
jama47ca85c2014-12-03 18:38:07439 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07440 '-f', '--test-filter', dest='test_filter',
441 help='Filters tests googletest-style.')
jama47ca85c2014-12-03 18:38:07442 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07443 '--package-filter', dest='package_filter',
444 help='Filters tests by package.')
jama47ca85c2014-12-03 18:38:07445 group.add_argument(
jbudorick9a6b7b332014-09-20 00:01:07446 '--runner-filter', dest='runner_filter',
447 help='Filters tests by runner class. Must be fully qualified.')
jama47ca85c2014-12-03 18:38:07448 group.add_argument(
449 '--sdk-version', dest='sdk_version', type=int,
jbudorick9a6b7b332014-09-20 00:01:07450 help='The Android SDK version.')
jama47ca85c2014-12-03 18:38:07451 AddCommonOptions(parser)
jbudorick9a6b7b332014-09-20 00:01:07452
453
jama47ca85c2014-12-03 18:38:07454def AddMonkeyTestOptions(parser):
455 """Adds monkey test options to |parser|."""
jbudorick9a6b7b332014-09-20 00:01:07456
jama47ca85c2014-12-03 18:38:07457 group = parser.add_argument_group('Monkey Test Options')
458 group.add_argument(
459 '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
460 metavar='PACKAGE', help='Package under test.')
461 group.add_argument(
462 '--event-count', default=10000, type=int,
463 help='Number of events to generate (default: %(default)s).')
464 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14465 '--category', default='',
[email protected]fb81b982013-08-09 00:07:12466 help='A list of allowed categories.')
jama47ca85c2014-12-03 18:38:07467 group.add_argument(
468 '--throttle', default=100, type=int,
469 help='Delay between events (ms) (default: %(default)s). ')
470 group.add_argument(
471 '--seed', type=int,
[email protected]3dbdfa42013-08-08 01:08:14472 help=('Seed value for pseudo-random generator. Same seed value generates '
473 'the same sequence of events. Seed is randomized by default.'))
jama47ca85c2014-12-03 18:38:07474 group.add_argument(
[email protected]3dbdfa42013-08-08 01:08:14475 '--extra-args', default='',
jama47ca85c2014-12-03 18:38:07476 help=('String of other args to pass to the command verbatim.'))
[email protected]3dbdfa42013-08-08 01:08:14477
jama47ca85c2014-12-03 18:38:07478 AddCommonOptions(parser)
479 AddDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14480
jama47ca85c2014-12-03 18:38:07481def ProcessMonkeyTestOptions(args):
[email protected]3dbdfa42013-08-08 01:08:14482 """Processes all monkey test options.
483
484 Args:
jama47ca85c2014-12-03 18:38:07485 args: argparse.Namespace object.
[email protected]3dbdfa42013-08-08 01:08:14486
487 Returns:
488 A MonkeyOptions named tuple which contains all options relevant to
489 monkey tests.
490 """
jama47ca85c2014-12-03 18:38:07491 # TODO(jbudorick): Handle this directly in argparse with nargs='+'
492 category = args.category
[email protected]3dbdfa42013-08-08 01:08:14493 if category:
jama47ca85c2014-12-03 18:38:07494 category = args.category.split(',')
[email protected]3dbdfa42013-08-08 01:08:14495
jama47ca85c2014-12-03 18:38:07496 # TODO(jbudorick): Get rid of MonkeyOptions.
[email protected]3dbdfa42013-08-08 01:08:14497 return monkey_test_options.MonkeyOptions(
jama47ca85c2014-12-03 18:38:07498 args.verbose_count,
499 args.package,
500 args.event_count,
[email protected]3dbdfa42013-08-08 01:08:14501 category,
jama47ca85c2014-12-03 18:38:07502 args.throttle,
503 args.seed,
504 args.extra_args)
[email protected]3dbdfa42013-08-08 01:08:14505
rnephew5c499782014-12-12 19:08:55506def AddUirobotTestOptions(parser):
507 """Adds uirobot test options to |option_parser|."""
508 group = parser.add_argument_group('Uirobot Test Options')
509
510 group.add_argument(
511 '--minutes', default=5, type=int,
512 help='Number of minutes to run uirobot test [default: %default].')
513
514 AddCommonOptions(parser)
515 AddDeviceOptions(parser)
516 AddRemoteDeviceOptions(parser)
[email protected]3dbdfa42013-08-08 01:08:14517
jama47ca85c2014-12-03 18:38:07518def AddPerfTestOptions(parser):
519 """Adds perf test options to |parser|."""
[email protected]ec3170b2013-08-14 14:39:47520
jama47ca85c2014-12-03 18:38:07521 group = parser.add_argument_group('Perf Test Options')
[email protected]ec3170b2013-08-14 14:39:47522
jama47ca85c2014-12-03 18:38:07523 class SingleStepAction(argparse.Action):
524 def __call__(self, parser, namespace, values, option_string=None):
525 if values and not namespace.single_step:
526 parser.error('single step command provided, '
527 'but --single-step not specified.')
528 elif namespace.single_step and not values:
529 parser.error('--single-step specified, '
530 'but no single step command provided.')
531 setattr(namespace, self.dest, values)
532
533 step_group = group.add_mutually_exclusive_group(required=True)
534 # TODO(jbudorick): Revise --single-step to use argparse.REMAINDER.
535 # This requires removing "--" from client calls.
536 step_group.add_argument(
537 '--single-step', action='store_true',
[email protected]def4bce2013-11-12 12:59:52538 help='Execute the given command with retries, but only print the result '
539 'for the "most successful" round.')
jama47ca85c2014-12-03 18:38:07540 step_group.add_argument(
[email protected]181a5c92013-09-06 17:11:46541 '--steps',
[email protected]def4bce2013-11-12 12:59:52542 help='JSON file containing the list of commands to run.')
jama47ca85c2014-12-03 18:38:07543 step_group.add_argument(
544 '--print-step',
545 help='The name of a previously executed perf step to print.')
546
547 group.add_argument(
peterbd4e73d2014-12-03 15:47:36548 '--output-json-list',
549 help='Write a simple list of names from --steps into the given file.')
jama47ca85c2014-12-03 18:38:07550 group.add_argument(
peterbd4e73d2014-12-03 15:47:36551 '--collect-chartjson-data',
552 action='store_true',
553 help='Cache the chartjson output from each step for later use.')
jama47ca85c2014-12-03 18:38:07554 group.add_argument(
peterbd4e73d2014-12-03 15:47:36555 '--output-chartjson-data',
556 default='',
557 help='Write out chartjson into the given file.')
jama47ca85c2014-12-03 18:38:07558 group.add_argument(
559 '--flaky-steps',
560 help=('A JSON file containing steps that are flaky '
561 'and will have its exit code ignored.'))
562 group.add_argument(
[email protected]181a5c92013-09-06 17:11:46563 '--no-timeout', action='store_true',
564 help=('Do not impose a timeout. Each perf step is responsible for '
565 'implementing the timeout logic.'))
jama47ca85c2014-12-03 18:38:07566 group.add_argument(
[email protected]650487c2013-09-30 11:40:49567 '-f', '--test-filter',
568 help=('Test filter (will match against the names listed in --steps).'))
jama47ca85c2014-12-03 18:38:07569 group.add_argument(
570 '--dry-run', action='store_true',
[email protected]650487c2013-09-30 11:40:49571 help='Just print the steps without executing.')
jama47ca85c2014-12-03 18:38:07572 group.add_argument('single_step_command', nargs='*', action=SingleStepAction,
573 help='If --single-step is specified, the command to run.')
574 AddCommonOptions(parser)
575 AddDeviceOptions(parser)
[email protected]ec3170b2013-08-14 14:39:47576
577
jama47ca85c2014-12-03 18:38:07578def ProcessPerfTestOptions(args):
[email protected]ec3170b2013-08-14 14:39:47579 """Processes all perf test options.
580
581 Args:
jama47ca85c2014-12-03 18:38:07582 args: argparse.Namespace object.
[email protected]ec3170b2013-08-14 14:39:47583
584 Returns:
585 A PerfOptions named tuple which contains all options relevant to
586 perf tests.
587 """
jama47ca85c2014-12-03 18:38:07588 # TODO(jbudorick): Move single_step handling down into the perf tests.
589 if args.single_step:
590 args.single_step = ' '.join(args.single_step_command)
591 # TODO(jbudorick): Get rid of PerfOptions.
[email protected]ec3170b2013-08-14 14:39:47592 return perf_test_options.PerfOptions(
jama47ca85c2014-12-03 18:38:07593 args.steps, args.flaky_steps, args.output_json_list,
594 args.print_step, args.no_timeout, args.test_filter,
595 args.dry_run, args.single_step, args.collect_chartjson_data,
596 args.output_chartjson_data)
[email protected]ec3170b2013-08-14 14:39:47597
598
jama47ca85c2014-12-03 18:38:07599def AddPythonTestOptions(parser):
600 group = parser.add_argument_group('Python Test Options')
601 group.add_argument(
602 '-s', '--suite', dest='suite_name', metavar='SUITE_NAME',
603 choices=constants.PYTHON_UNIT_TEST_SUITES.keys(),
604 help='Name of the test suite to run.')
605 AddCommonOptions(parser)
jbudorick256fd532014-10-24 01:50:13606
607
jama47ca85c2014-12-03 18:38:07608def _RunGTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37609 """Subcommand of RunTestsCommands which runs gtests."""
[email protected]6bc1bda22013-07-19 22:08:37610 exit_code = 0
jama47ca85c2014-12-03 18:38:07611 for suite_name in args.suite_name:
612 # TODO(jbudorick): Either deprecate multi-suite or move its handling down
613 # into the gtest code.
[email protected]2a684222013-08-01 16:59:22614 gtest_options = gtest_test_options.GTestOptions(
jama47ca85c2014-12-03 18:38:07615 args.tool,
616 args.cleanup_test_files,
617 args.test_filter,
618 args.run_disabled,
619 args.test_arguments,
620 args.timeout,
621 args.isolate_file_path,
[email protected]2a684222013-08-01 16:59:22622 suite_name)
[email protected]f7148dd42013-08-20 14:24:57623 runner_factory, tests = gtest_setup.Setup(gtest_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37624
625 results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57626 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07627 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37628
629 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
630 exit_code = test_exit_code
631
632 report_results.LogFull(
633 results=results,
634 test_type='Unit test',
635 test_package=suite_name,
jama47ca85c2014-12-03 18:38:07636 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37637
jama47ca85c2014-12-03 18:38:07638 if args.json_results_file:
639 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54640
[email protected]6bc1bda22013-07-19 22:08:37641 if os.path.isdir(constants.ISOLATE_DEPS_DIR):
642 shutil.rmtree(constants.ISOLATE_DEPS_DIR)
643
644 return exit_code
645
646
jama47ca85c2014-12-03 18:38:07647def _RunLinkerTests(args, devices):
[email protected]6b6abac6d2013-10-03 11:56:38648 """Subcommand of RunTestsCommands which runs linker tests."""
jama47ca85c2014-12-03 18:38:07649 runner_factory, tests = linker_setup.Setup(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38650
651 results, exit_code = test_dispatcher.RunTests(
652 tests, runner_factory, devices, shard=True, test_timeout=60,
jama47ca85c2014-12-03 18:38:07653 num_retries=args.num_retries)
[email protected]6b6abac6d2013-10-03 11:56:38654
655 report_results.LogFull(
656 results=results,
657 test_type='Linker test',
[email protected]93c9f9b2014-02-10 16:19:22658 test_package='ChromiumLinkerTest')
[email protected]6b6abac6d2013-10-03 11:56:38659
jama47ca85c2014-12-03 18:38:07660 if args.json_results_file:
661 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54662
[email protected]6b6abac6d2013-10-03 11:56:38663 return exit_code
664
665
jama47ca85c2014-12-03 18:38:07666def _RunInstrumentationTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37667 """Subcommand of RunTestsCommands which runs instrumentation tests."""
jama47ca85c2014-12-03 18:38:07668 logging.info('_RunInstrumentationTests(%s, %s)' % (str(args), str(devices)))
[email protected]6bc1bda22013-07-19 22:08:37669
jama47ca85c2014-12-03 18:38:07670 instrumentation_options = ProcessInstrumentationOptions(args)
671
672 if len(devices) > 1 and args.wait_for_debugger:
[email protected]f7148dd42013-08-20 14:24:57673 logging.warning('Debugger can not be sharded, using first available device')
674 devices = devices[:1]
675
[email protected]6bc1bda22013-07-19 22:08:37676 results = base_test_result.TestRunResults()
677 exit_code = 0
678
jama47ca85c2014-12-03 18:38:07679 if args.run_java_tests:
mikecase526d68e2014-11-19 20:02:05680 runner_factory, tests = instrumentation_setup.Setup(
681 instrumentation_options, devices)
[email protected]6bc1bda22013-07-19 22:08:37682
683 test_results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57684 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07685 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37686
687 results.AddTestRunResults(test_results)
688
jama47ca85c2014-12-03 18:38:07689 if args.run_python_tests:
[email protected]37ee0c792013-08-06 19:10:13690 runner_factory, tests = host_driven_setup.InstrumentationSetup(
jama47ca85c2014-12-03 18:38:07691 args.host_driven_root, args.official_build,
[email protected]37ee0c792013-08-06 19:10:13692 instrumentation_options)
693
[email protected]34020022013-08-06 23:35:34694 if tests:
695 test_results, test_exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57696 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07697 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37698
[email protected]34020022013-08-06 23:35:34699 results.AddTestRunResults(test_results)
[email protected]6bc1bda22013-07-19 22:08:37700
[email protected]34020022013-08-06 23:35:34701 # Only allow exit code escalation
702 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
703 exit_code = test_exit_code
[email protected]6bc1bda22013-07-19 22:08:37704
jama47ca85c2014-12-03 18:38:07705 if args.device_flags:
706 args.device_flags = os.path.join(constants.DIR_SOURCE_ROOT,
707 args.device_flags)
[email protected]4f777ca2014-08-08 01:45:59708
[email protected]6bc1bda22013-07-19 22:08:37709 report_results.LogFull(
710 results=results,
711 test_type='Instrumentation',
jama47ca85c2014-12-03 18:38:07712 test_package=os.path.basename(args.test_apk),
713 annotation=args.annotations,
714 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37715
jama47ca85c2014-12-03 18:38:07716 if args.json_results_file:
717 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54718
[email protected]6bc1bda22013-07-19 22:08:37719 return exit_code
720
721
jama47ca85c2014-12-03 18:38:07722def _RunUIAutomatorTests(args, devices):
[email protected]6bc1bda22013-07-19 22:08:37723 """Subcommand of RunTestsCommands which runs uiautomator tests."""
jama47ca85c2014-12-03 18:38:07724 uiautomator_options = ProcessUIAutomatorOptions(args)
[email protected]6bc1bda22013-07-19 22:08:37725
[email protected]37ee0c792013-08-06 19:10:13726 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options)
[email protected]6bc1bda22013-07-19 22:08:37727
[email protected]37ee0c792013-08-06 19:10:13728 results, exit_code = test_dispatcher.RunTests(
[email protected]f7148dd42013-08-20 14:24:57729 tests, runner_factory, devices, shard=True, test_timeout=None,
jama47ca85c2014-12-03 18:38:07730 num_retries=args.num_retries)
[email protected]6bc1bda22013-07-19 22:08:37731
732 report_results.LogFull(
733 results=results,
734 test_type='UIAutomator',
jama47ca85c2014-12-03 18:38:07735 test_package=os.path.basename(args.test_jar),
736 annotation=args.annotations,
737 flakiness_server=args.flakiness_dashboard_server)
[email protected]6bc1bda22013-07-19 22:08:37738
jama47ca85c2014-12-03 18:38:07739 if args.json_results_file:
740 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54741
[email protected]6bc1bda22013-07-19 22:08:37742 return exit_code
743
744
jama47ca85c2014-12-03 18:38:07745def _RunJUnitTests(args):
jbudorick9a6b7b332014-09-20 00:01:07746 """Subcommand of RunTestsCommand which runs junit tests."""
jama47ca85c2014-12-03 18:38:07747 runner_factory, tests = junit_setup.Setup(args)
jbudorick9a6b7b332014-09-20 00:01:07748 _, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
jbudorick9a6b7b332014-09-20 00:01:07749 return exit_code
750
751
jama47ca85c2014-12-03 18:38:07752def _RunMonkeyTests(args, devices):
[email protected]3dbdfa42013-08-08 01:08:14753 """Subcommand of RunTestsCommands which runs monkey tests."""
jama47ca85c2014-12-03 18:38:07754 monkey_options = ProcessMonkeyTestOptions(args)
[email protected]3dbdfa42013-08-08 01:08:14755
756 runner_factory, tests = monkey_setup.Setup(monkey_options)
757
758 results, exit_code = test_dispatcher.RunTests(
[email protected]181a5c92013-09-06 17:11:46759 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07760 num_retries=args.num_retries)
[email protected]3dbdfa42013-08-08 01:08:14761
762 report_results.LogFull(
763 results=results,
764 test_type='Monkey',
[email protected]14b3b1202013-08-15 22:25:28765 test_package='Monkey')
[email protected]3dbdfa42013-08-08 01:08:14766
jama47ca85c2014-12-03 18:38:07767 if args.json_results_file:
768 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54769
[email protected]3dbdfa42013-08-08 01:08:14770 return exit_code
771
772
jama47ca85c2014-12-03 18:38:07773def _RunPerfTests(args):
[email protected]ec3170b2013-08-14 14:39:47774 """Subcommand of RunTestsCommands which runs perf tests."""
jama47ca85c2014-12-03 18:38:07775 perf_options = ProcessPerfTestOptions(args)
[email protected]61487ed2014-06-09 12:33:56776
777 # Just save a simple json with a list of test names.
778 if perf_options.output_json_list:
779 return perf_test_runner.OutputJsonList(
780 perf_options.steps, perf_options.output_json_list)
781
[email protected]ad32f312013-11-13 04:03:29782 # Just print the results from a single previously executed step.
[email protected]ec3170b2013-08-14 14:39:47783 if perf_options.print_step:
simonhatch9b9256d2015-01-07 18:03:42784 return perf_test_runner.PrintTestOutput(
785 perf_options.print_step, perf_options.output_chartjson_data)
[email protected]ec3170b2013-08-14 14:39:47786
[email protected]a72f0752014-06-03 23:52:34787 runner_factory, tests, devices = perf_setup.Setup(perf_options)
[email protected]ec3170b2013-08-14 14:39:47788
[email protected]a72f0752014-06-03 23:52:34789 # shard=False means that each device will get the full list of tests
790 # and then each one will decide their own affinity.
791 # shard=True means each device will pop the next test available from a queue,
792 # which increases throughput but have no affinity.
[email protected]86184c7b2013-08-15 15:06:57793 results, _ = test_dispatcher.RunTests(
[email protected]a72f0752014-06-03 23:52:34794 tests, runner_factory, devices, shard=False, test_timeout=None,
jama47ca85c2014-12-03 18:38:07795 num_retries=args.num_retries)
[email protected]ec3170b2013-08-14 14:39:47796
797 report_results.LogFull(
798 results=results,
799 test_type='Perf',
[email protected]865a47a2013-08-16 14:01:12800 test_package='Perf')
[email protected]def4bce2013-11-12 12:59:52801
jama47ca85c2014-12-03 18:38:07802 if args.json_results_file:
803 json_results.GenerateJsonResultsFile(results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54804
[email protected]def4bce2013-11-12 12:59:52805 if perf_options.single_step:
806 return perf_test_runner.PrintTestOutput('single_step')
807
[email protected]11ce8452014-02-17 10:55:03808 perf_test_runner.PrintSummary(tests)
809
[email protected]86184c7b2013-08-15 15:06:57810 # Always return 0 on the sharding stage. Individual tests exit_code
811 # will be returned on the print_step stage.
812 return 0
[email protected]ec3170b2013-08-14 14:39:47813
[email protected]3dbdfa42013-08-08 01:08:14814
jama47ca85c2014-12-03 18:38:07815def _RunPythonTests(args):
jbudorick256fd532014-10-24 01:50:13816 """Subcommand of RunTestsCommand which runs python unit tests."""
jama47ca85c2014-12-03 18:38:07817 suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
jbudorick256fd532014-10-24 01:50:13818 suite_path = suite_vars['path']
819 suite_test_modules = suite_vars['test_modules']
820
821 sys.path = [suite_path] + sys.path
822 try:
823 suite = unittest.TestSuite()
824 suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
825 for m in suite_test_modules)
jama47ca85c2014-12-03 18:38:07826 runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
jbudorick256fd532014-10-24 01:50:13827 return 0 if runner.run(suite).wasSuccessful() else 1
828 finally:
829 sys.path = sys.path[1:]
830
831
[email protected]f7148dd42013-08-20 14:24:57832def _GetAttachedDevices(test_device=None):
833 """Get all attached devices.
834
835 Args:
836 test_device: Name of a specific device to use.
837
838 Returns:
839 A list of attached devices.
840 """
841 attached_devices = []
842
843 attached_devices = android_commands.GetAttachedDevices()
844 if test_device:
845 assert test_device in attached_devices, (
846 'Did not find device %s among attached device. Attached devices: %s'
847 % (test_device, ', '.join(attached_devices)))
848 attached_devices = [test_device]
849
850 assert attached_devices, 'No devices attached.'
851
852 return sorted(attached_devices)
853
854
jama47ca85c2014-12-03 18:38:07855def RunTestsCommand(args, parser):
[email protected]fbe29322013-07-09 09:03:26856 """Checks test type and dispatches to the appropriate function.
857
858 Args:
jama47ca85c2014-12-03 18:38:07859 args: argparse.Namespace object.
860 parser: argparse.ArgumentParser object.
[email protected]fbe29322013-07-09 09:03:26861
862 Returns:
863 Integer indicated exit code.
[email protected]b3873892013-07-10 04:57:10864
865 Raises:
866 Exception: Unknown command name passed in, or an exception from an
867 individual test runner.
[email protected]fbe29322013-07-09 09:03:26868 """
jama47ca85c2014-12-03 18:38:07869 command = args.command
[email protected]fbe29322013-07-09 09:03:26870
jama47ca85c2014-12-03 18:38:07871 ProcessCommonOptions(args)
[email protected]d82f0252013-07-12 23:22:57872
jama47ca85c2014-12-03 18:38:07873 if args.enable_platform_mode:
rnephew5c499782014-12-12 19:08:55874 return RunTestsInPlatformMode(args, parser)
jbudorick66dc3722014-11-06 21:33:51875
876 if command in constants.LOCAL_MACHINE_TESTS:
jbudorick256fd532014-10-24 01:50:13877 devices = []
878 else:
jama47ca85c2014-12-03 18:38:07879 devices = _GetAttachedDevices(args.test_device)
[email protected]f7148dd42013-08-20 14:24:57880
[email protected]c0662e092013-11-12 11:51:25881 forwarder.Forwarder.RemoveHostLog()
[email protected]6b11583b2013-11-21 16:18:40882 if not ports.ResetTestServerPortAllocation():
883 raise Exception('Failed to reset test server port.')
[email protected]c0662e092013-11-12 11:51:25884
[email protected]fbe29322013-07-09 09:03:26885 if command == 'gtest':
jama47ca85c2014-12-03 18:38:07886 return _RunGTests(args, devices)
[email protected]6b6abac6d2013-10-03 11:56:38887 elif command == 'linker':
jama47ca85c2014-12-03 18:38:07888 return _RunLinkerTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26889 elif command == 'instrumentation':
jama47ca85c2014-12-03 18:38:07890 return _RunInstrumentationTests(args, devices)
[email protected]fbe29322013-07-09 09:03:26891 elif command == 'uiautomator':
jama47ca85c2014-12-03 18:38:07892 return _RunUIAutomatorTests(args, devices)
jbudorick9a6b7b332014-09-20 00:01:07893 elif command == 'junit':
jama47ca85c2014-12-03 18:38:07894 return _RunJUnitTests(args)
[email protected]3dbdfa42013-08-08 01:08:14895 elif command == 'monkey':
jama47ca85c2014-12-03 18:38:07896 return _RunMonkeyTests(args, devices)
[email protected]ec3170b2013-08-14 14:39:47897 elif command == 'perf':
jama47ca85c2014-12-03 18:38:07898 return _RunPerfTests(args)
jbudorick256fd532014-10-24 01:50:13899 elif command == 'python':
jama47ca85c2014-12-03 18:38:07900 return _RunPythonTests(args)
[email protected]fbe29322013-07-09 09:03:26901 else:
[email protected]6bc1bda22013-07-19 22:08:37902 raise Exception('Unknown test type.')
[email protected]fbe29322013-07-09 09:03:26903
[email protected]fbe29322013-07-09 09:03:26904
jbudorick66dc3722014-11-06 21:33:51905_SUPPORTED_IN_PLATFORM_MODE = [
906 # TODO(jbudorick): Add support for more test types.
jbudorick911be58d2015-01-13 02:51:06907 'gtest',
908 'instrumentation',
909 'uirobot',
jbudorick66dc3722014-11-06 21:33:51910]
911
912
jama47ca85c2014-12-03 18:38:07913def RunTestsInPlatformMode(args, parser):
jbudorick66dc3722014-11-06 21:33:51914
jama47ca85c2014-12-03 18:38:07915 if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
916 parser.error('%s is not yet supported in platform mode' % args.command)
jbudorick66dc3722014-11-06 21:33:51917
jama47ca85c2014-12-03 18:38:07918 with environment_factory.CreateEnvironment(args, parser.error) as env:
919 with test_instance_factory.CreateTestInstance(args, parser.error) as test:
jbudorick66dc3722014-11-06 21:33:51920 with test_run_factory.CreateTestRun(
jama47ca85c2014-12-03 18:38:07921 args, env, test, parser.error) as test_run:
jbudorick66dc3722014-11-06 21:33:51922 results = test_run.RunTests()
923
jbudorick911be58d2015-01-13 02:51:06924 if args.environment == 'remote_device' and args.trigger:
rnephew5c499782014-12-12 19:08:55925 return 0 # Not returning results, only triggering.
926
jbudorick66dc3722014-11-06 21:33:51927 report_results.LogFull(
928 results=results,
929 test_type=test.TestType(),
930 test_package=test_run.TestPackage(),
jbudorickf9543672014-12-08 22:36:33931 annotation=getattr(args, 'annotations', None),
932 flakiness_server=getattr(args, 'flakiness_dashboard_server', None))
jbudorick66dc3722014-11-06 21:33:51933
jama47ca85c2014-12-03 18:38:07934 if args.json_results_file:
jbudorickb8c42072014-12-01 18:07:54935 json_results.GenerateJsonResultsFile(
jama47ca85c2014-12-03 18:38:07936 results, args.json_results_file)
jbudorickb8c42072014-12-01 18:07:54937
rnephew5c499782014-12-12 19:08:55938 return 0 if results.DidRunPass() else 1
jbudorick66dc3722014-11-06 21:33:51939
940
jama47ca85c2014-12-03 18:38:07941CommandConfigTuple = collections.namedtuple(
942 'CommandConfigTuple',
943 ['add_options_func', 'help_txt'])
[email protected]fbe29322013-07-09 09:03:26944VALID_COMMANDS = {
jama47ca85c2014-12-03 18:38:07945 'gtest': CommandConfigTuple(
946 AddGTestOptions,
947 'googletest-based C++ tests'),
948 'instrumentation': CommandConfigTuple(
949 AddInstrumentationTestOptions,
950 'InstrumentationTestCase-based Java tests'),
951 'uiautomator': CommandConfigTuple(
952 AddUIAutomatorTestOptions,
953 "Tests that run via Android's uiautomator command"),
954 'junit': CommandConfigTuple(
955 AddJUnitTestOptions,
956 'JUnit4-based Java tests'),
957 'monkey': CommandConfigTuple(
958 AddMonkeyTestOptions,
959 "Tests based on Android's monkey"),
960 'perf': CommandConfigTuple(
961 AddPerfTestOptions,
962 'Performance tests'),
963 'python': CommandConfigTuple(
964 AddPythonTestOptions,
965 'Python tests based on unittest.TestCase'),
966 'linker': CommandConfigTuple(
967 AddLinkerTestOptions,
968 'Linker tests'),
rnephew5c499782014-12-12 19:08:55969 'uirobot': CommandConfigTuple(
970 AddUirobotTestOptions,
971 'Uirobot test'),
jama47ca85c2014-12-03 18:38:07972}
[email protected]fbe29322013-07-09 09:03:26973
974
[email protected]7c53a602014-03-24 16:21:44975def DumpThreadStacks(_signal, _frame):
[email protected]71aec4b2013-11-20 00:35:24976 for thread in threading.enumerate():
977 reraiser_thread.LogThreadStack(thread)
[email protected]83bb8152013-11-19 15:02:21978
979
[email protected]7c53a602014-03-24 16:21:44980def main():
[email protected]83bb8152013-11-19 15:02:21981 signal.signal(signal.SIGUSR1, DumpThreadStacks)
jama47ca85c2014-12-03 18:38:07982
983 parser = argparse.ArgumentParser()
984 command_parsers = parser.add_subparsers(title='test types',
985 dest='command')
986
987 for test_type, config in sorted(VALID_COMMANDS.iteritems(),
988 key=lambda x: x[0]):
989 subparser = command_parsers.add_parser(
990 test_type, usage='%(prog)s [options]', help=config.help_txt)
991 config.add_options_func(subparser)
992
993 args = parser.parse_args()
jbudorick15cdcd52014-12-03 19:58:49994 return RunTestsCommand(args, parser)
[email protected]fbe29322013-07-09 09:03:26995
[email protected]fbe29322013-07-09 09:03:26996
997if __name__ == '__main__':
[email protected]7c53a602014-03-24 16:21:44998 sys.exit(main())