[iOS test runners]Modularizing and updating test_runners.
Changes in this cl:
- added test_apps(Gtest and EGtests) to reduce code duplication (
filling xctest_run files for EG1 and EG2 tests, filtering tests,
creating commands to run tests).
-Removes shard_xctest method and other EG sharding code from test_runner
because shards only use in xcodebuild_runner.
- updated *runner.py to support test_apps
- replaced `iossim` with `xcodebuild` command.
In following cls:
1. iossim parameter will be removed in all test-runners
and unittests and then
2. ios/api.py remove iossim.
Bug: 1011498
Change-Id: I9966ba1152eabeee56457fa8db3b6e9f585df3c9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1961166
Commit-Queue: Maksym Onufriienko <[email protected]>
Reviewed-by: Rohit Rao <[email protected]>
Cr-Commit-Position: refs/heads/master@{#742746}
diff --git a/ios/build/bots/scripts/iossim_util_test.py b/ios/build/bots/scripts/iossim_util_test.py
index c235faf9..5b9253d 100644
--- a/ios/build/bots/scripts/iossim_util_test.py
+++ b/ios/build/bots/scripts/iossim_util_test.py
@@ -8,82 +8,12 @@
import test_runner
import test_runner_test
-SIMULATORS_LIST = {
- 'devices': {
- 'com.apple.CoreSimulator.SimRuntime.iOS-11-4': [{
- 'isAvailable': True,
- 'name': 'iPhone 5s',
- 'state': 'Shutdown',
- 'udid': 'E4E66320-177A-450A-9BA1-488D85B7278E'
- }],
- 'com.apple.CoreSimulator.SimRuntime.iOS-13-2': [
- {
- 'isAvailable': True,
- 'name': 'iPhone X',
- 'state': 'Shutdown',
- 'udid': 'E4E66321-177A-450A-9BA1-488D85B7278E'
- },
- {
- 'isAvailable': True,
- 'name': 'iPhone 11',
- 'state': 'Shutdown',
- 'udid': 'A4E66321-177A-450A-9BA1-488D85B7278E'
- }
- ]
- },
- 'devicetypes': [
- {
- 'name': 'iPhone 5s',
- 'bundlePath': '/path/iPhone 4s/Content',
- 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-5s'
- },
- {
- 'name': 'iPhone X',
- 'bundlePath': '/path/iPhone X/Content',
- 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-X'
- },
- {
- 'name': 'iPhone 11',
- 'bundlePath': '/path/iPhone 11/Content',
- 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-11'
- },
- ],
- 'pairs': [],
- 'runtimes': [
- {
- "buildversion": "15F79",
- "bundlePath": "/path/Runtimes/iOS 11.4.simruntime",
- "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-11-4",
- "isAvailable": True,
- "name": "iOS 11.4",
- "version": "11.4"
- },
- {
- "buildversion": "17A844",
- "bundlePath": "/path/Runtimes/iOS 13.1.simruntime",
- "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-1",
- "isAvailable": True,
- "name": "iOS 13.1",
- "version": "13.1"
- },
- {
- "buildversion": "17B102",
- "bundlePath": "/path/Runtimes/iOS.simruntime",
- "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-2",
- "isAvailable": True,
- "name": "iOS 13.2",
- "version": "13.2.2"
- },
- ]
-}
-
class GetiOSSimUtil(test_runner_test.TestCase):
"""Tests for iossim_util.py."""
def setUp(self):
super(GetiOSSimUtil, self).setUp()
- self.mock(iossim_util, 'get_simulator_list', lambda: SIMULATORS_LIST)
def test_get_simulator_runtime_by_version(self):
"""Ensures correctness of filter."""
diff --git a/ios/build/bots/scripts/test_apps.py b/ios/build/bots/scripts/test_apps.py
new file mode 100644
index 0000000..0d03030
--- /dev/null
+++ b/ios/build/bots/scripts/test_apps.py
@@ -0,0 +1,309 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Test apps for running tests using xcodebuild."""
+
+import os
+import plistlib
+import time
+
+import test_runner
+
+
+#TODO(crbug.com/1046911): Remove usage of KIF filters.
+def get_kif_test_filter(tests, invert=False):
+ """Returns the KIF test filter to filter the given test cases.
+
+ Args:
+ tests: List of test cases to filter.
+ invert: Whether to invert the filter or not. Inverted, the filter will match
+ everything except the given test cases.
+
+ Returns:
+ A string which can be supplied to GKIF_SCENARIO_FILTER.
+ """
+ # A pipe-separated list of test cases with the "KIF." prefix omitted.
+ # e.g. NAME:a|b|c matches KIF.a, KIF.b, KIF.c.
+ # e.g. -NAME:a|b|c matches everything except KIF.a, KIF.b, KIF.c.
+ test_filter = '|'.join(test.split('KIF.', 1)[-1] for test in tests)
+ if invert:
+ return '-NAME:%s' % test_filter
+ return 'NAME:%s' % test_filter
+
+
+def get_gtest_filter(tests, invert=False):
+ """Returns the GTest filter to filter the given test cases.
+
+ Args:
+ tests: List of test cases to filter.
+ invert: Whether to invert the filter or not. Inverted, the filter will match
+ everything except the given test cases.
+
+ Returns:
+ A string which can be supplied to --gtest_filter.
+ """
+ # A colon-separated list of tests cases.
+ # e.g. a:b:c matches a, b, c.
+ # e.g. -a:b:c matches everything except a, b, c.
+ test_filter = ':'.join(test for test in tests)
+ if invert:
+ return '-%s' % test_filter
+ return test_filter
+
+
+class GTestsApp(object):
+ """Gtests app to run.
+
+ Stores data about egtests:
+ test_app: full path to an app.
+ """
+
+ def __init__(self,
+ test_app,
+ included_tests=None,
+ excluded_tests=None,
+ test_args=None,
+ env_vars=None,
+ host_app_path=None):
+ """Initialize Egtests.
+
+ Args:
+ test_app: (str) full path to egtests app.
+ included_tests: (list) Specific tests to run
+ E.g.
+ [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
+ excluded_tests: (list) Specific tests not to run
+ E.g.
+ [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
+ test_args: List of strings to pass as arguments to the test when
+ launching.
+ env_vars: List of environment variables to pass to the test itself.
+
+ Raises:
+ AppNotFoundError: If the given app does not exist
+ """
+ if not os.path.exists(test_app):
+ raise test_runner.AppNotFoundError(test_app)
+ self.test_app_path = test_app
+ self.project_path = os.path.dirname(self.test_app_path)
+ self.test_args = test_args or []
+ self.env_vars = {}
+ for env_var in env_vars or []:
+ env_var = env_var.split('=', 1)
+ self.env_vars[env_var[0]] = None if len(env_var) == 1 else env_var[1]
+ self.included_tests = included_tests or []
+ self.excluded_tests = excluded_tests or []
+ self.module_name = os.path.splitext(os.path.basename(test_app))[0]
+ self.host_app_path = host_app_path
+
+ def fill_xctest_run(self, out_dir):
+ """Fills xctestrun file by egtests.
+
+ Args:
+ out_dir: (str) A path where xctestrun will store.
+
+ Returns:
+ A path to xctestrun file.
+ """
+ folder = os.path.abspath(os.path.join(out_dir, os.pardir))
+ if not os.path.exists(folder):
+ os.makedirs(folder)
+ xctestrun = os.path.join(folder, 'run_%d.xctestrun' % int(time.time()))
+ if not os.path.exists(xctestrun):
+ with open(xctestrun, 'w'):
+ pass
+ # Creates a dict with data about egtests to run - fill all required fields:
+ # egtests_module, egtest_app_path, egtests_xctest_path and
+ # filtered tests if filter is specified.
+ # Write data in temp xctest run file.
+ plistlib.writePlist(self.fill_xctestrun_node(), xctestrun)
+ return xctestrun
+
+ def fill_xctestrun_node(self):
+ """Fills only required nodes for egtests in xctestrun file.
+
+ Returns:
+ A node with filled required fields about egtests.
+ """
+ module = self.module_name + '_module'
+
+ # If --run-with-custom-webkit is passed as a test arg, set up
+ # DYLD_FRAMEWORK_PATH to load the custom webkit modules.
+ dyld_framework_path = self.project_path + ':'
+ if '--run-with-custom-webkit' in self.test_args:
+ if self.host_app_path:
+ webkit_path = os.path.join(self.host_app_path, 'WebKitFrameworks')
+ else:
+ webkit_path = os.path.join(self.test_app_path, 'WebKitFrameworks')
+ dyld_framework_path = dyld_framework_path + webkit_path + ':'
+
+ module_data = {
+ 'TestBundlePath': self.test_app_path,
+ 'TestHostPath': self.test_app_path,
+ 'TestingEnvironmentVariables': {
+ 'DYLD_LIBRARY_PATH':
+ '%s:__PLATFORMS__/iPhoneSimulator.platform/Developer/Library' %
+ self.project_path,
+ 'DYLD_FRAMEWORK_PATH':
+ '%s:__PLATFORMS__/iPhoneSimulator.platform/'
+ 'Developer/Library/Frameworks' % dyld_framework_path,
+ }
+ }
+
+ xctestrun_data = {module: module_data}
+ kif_filter = []
+ gtest_filter = []
+
+ if self.included_tests:
+ kif_filter = get_kif_test_filter(self.included_tests, invert=False)
+ gtest_filter = get_gtest_filter(self.included_tests, invert=False)
+ elif self.excluded_tests:
+ kif_filter = get_kif_test_filter(self.excluded_tests, invert=True)
+ gtest_filter = get_gtest_filter(self.excluded_tests, invert=True)
+
+ if kif_filter:
+ self.env_vars['GKIF_SCENARIO_FILTER'] = gtest_filter
+ if gtest_filter:
+ # Removed previous gtest-filter if exists.
+ self.test_args = [el for el in self.test_args
+ if not el.startswith('--gtest_filter=')]
+ self.test_args.append('--gtest_filter=%s' % gtest_filter)
+
+ if self.env_vars:
+ xctestrun_data[module].update({'EnvironmentVariables': self.env_vars})
+ if self.test_args:
+ xctestrun_data[module].update({'CommandLineArguments': self.test_args})
+
+ if self.excluded_tests:
+ xctestrun_data[module].update({
+ 'SkipTestIdentifiers': self.excluded_tests
+ })
+ if self.included_tests:
+ xctestrun_data[module].update({
+ 'OnlyTestIdentifiers': self.included_tests
+ })
+ return xctestrun_data
+
+ def command(self, out_dir, destination, shards):
+ """Returns the command that launches tests using xcodebuild.
+
+ Format of command:
+ xcodebuild test-without-building -xctestrun file.xctestrun \
+ -parallel-testing-enabled YES -parallel-testing-worker-count %d% \
+ [-destination "destination"] -resultBundlePath %output_path%
+
+ Args:
+ out_dir: (str) An output directory.
+ destination: (str) A destination of running simulator.
+ shards: (int) A number of shards.
+
+ Returns:
+ A list of strings forming the command to launch the test.
+ """
+ cmd = [
+ 'xcodebuild', 'test-without-building',
+ '-xctestrun', self.fill_xctest_run(out_dir),
+ '-destination', destination,
+ '-resultBundlePath', out_dir
+ ]
+ if shards > 1:
+ cmd += ['-parallel-testing-enabled', 'YES',
+ '-parallel-testing-worker-count', str(shards)]
+ return cmd
+
+
+class EgtestsApp(GTestsApp):
+ """Egtests to run.
+
+ Stores data about egtests:
+ egtests_app: full path to egtests app.
+ project_path: root project folder.
+ module_name: egtests module name.
+ included_tests: List of tests to run.
+ excluded_tests: List of tests not to run.
+ """
+
+ def __init__(self,
+ egtests_app,
+ included_tests=None,
+ excluded_tests=None,
+ test_args=None,
+ env_vars=None,
+ host_app_path=None):
+ """Initialize Egtests.
+
+ Args:
+ egtests_app: (str) full path to egtests app.
+ included_tests: (list) Specific tests to run
+ E.g.
+ [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
+ excluded_tests: (list) Specific tests not to run
+ E.g.
+ [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
+ test_args: List of strings to pass as arguments to the test when
+ launching.
+ env_vars: List of environment variables to pass to the test itself.
+ host_app_path: (str) full path to host app.
+
+ Raises:
+ AppNotFoundError: If the given app does not exist
+ """
+ super(EgtestsApp, self).__init__(egtests_app, included_tests,
+ excluded_tests, test_args, env_vars,
+ host_app_path)
+
+ def _xctest_path(self):
+ """Gets xctest-file from egtests/PlugIns folder.
+
+ Returns:
+ A path for xctest in the format of /PlugIns/file.xctest
+
+ Raises:
+ PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
+ XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
+ """
+ plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
+ if not os.path.exists(plugins_dir):
+ raise test_runner.PlugInsNotFoundError(plugins_dir)
+ plugin_xctest = None
+ if os.path.exists(plugins_dir):
+ for plugin in os.listdir(plugins_dir):
+ if plugin.endswith('.xctest'):
+ plugin_xctest = os.path.join(plugins_dir, plugin)
+ if not plugin_xctest:
+ raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
+ return plugin_xctest.replace(self.test_app_path, '')
+
+ def fill_xctestrun_node(self):
+ """Fills only required nodes for egtests in xctestrun file.
+
+ Returns:
+ A node with filled required fields about egtests.
+ """
+ xctestrun_data = super(EgtestsApp, self).fill_xctestrun_node()
+ module_data = xctestrun_data[self.module_name + '_module']
+
+ module_data['TestingEnvironmentVariables']['DYLD_INSERT_LIBRARIES'] = (
+ '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
+ 'usr/lib/libXCTestBundleInject.dylib')
+ module_data['TestBundlePath'] = '__TESTHOST__/%s' % self._xctest_path()
+ module_data['TestingEnvironmentVariables'][
+ 'XCInjectBundleInto'] = '__TESTHOST__/%s' % self.module_name
+
+ if self.host_app_path:
+ # Module data specific to EG2 tests
+ module_data['IsUITestBundle'] = True
+ module_data['IsXCTRunnerHostedTestBundle'] = True
+ module_data['UITargetAppPath'] = '%s' % self.host_app_path
+ # Special handling for Xcode10.2
+ dependent_products = [
+ module_data['UITargetAppPath'],
+ module_data['TestBundlePath'],
+ module_data['TestHostPath']
+ ]
+ module_data['DependentProductPaths'] = dependent_products
+ # Module data specific to EG1 tests
+ else:
+ module_data['IsAppHostedTestBundle'] = True
+
+ return xctestrun_data
diff --git a/ios/build/bots/scripts/test_apps_test.py b/ios/build/bots/scripts/test_apps_test.py
new file mode 100644
index 0000000..bd1acfa6
--- /dev/null
+++ b/ios/build/bots/scripts/test_apps_test.py
@@ -0,0 +1,56 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Unittests for test_apps.py."""
+
+import test_apps
+import test_runner_test
+
+
+class GetKIFTestFilterTest(test_runner_test.TestCase):
+ """Tests for test_runner.get_kif_test_filter."""
+
+ def test_correct(self):
+ """Ensures correctness of filter."""
+ tests = [
+ 'KIF.test1',
+ 'KIF.test2',
+ ]
+ expected = 'NAME:test1|test2'
+
+ self.assertEqual(expected, test_apps.get_kif_test_filter(tests))
+
+ def test_correct_inverted(self):
+ """Ensures correctness of inverted filter."""
+ tests = [
+ 'KIF.test1',
+ 'KIF.test2',
+ ]
+ expected = '-NAME:test1|test2'
+
+ self.assertEqual(expected,
+ test_apps.get_kif_test_filter(tests, invert=True))
+
+
+class GetGTestFilterTest(test_runner_test.TestCase):
+ """Tests for test_runner.get_gtest_filter."""
+
+ def test_correct(self):
+ """Ensures correctness of filter."""
+ tests = [
+ 'test.1',
+ 'test.2',
+ ]
+ expected = 'test.1:test.2'
+
+ self.assertEqual(test_apps.get_gtest_filter(tests), expected)
+
+ def test_correct_inverted(self):
+ """Ensures correctness of inverted filter."""
+ tests = [
+ 'test.1',
+ 'test.2',
+ ]
+ expected = '-test.1:test.2'
+
+ self.assertEqual(test_apps.get_gtest_filter(tests, invert=True), expected)
diff --git a/ios/build/bots/scripts/test_runner.py b/ios/build/bots/scripts/test_runner.py
index 1eb1d21..cdb53b0 100644
--- a/ios/build/bots/scripts/test_runner.py
+++ b/ios/build/bots/scripts/test_runner.py
@@ -10,21 +10,18 @@
import collections
import distutils.version
-import json
import logging
-from multiprocessing import pool
import os
-import plistlib
import psutil
import re
import shutil
import subprocess
-import tempfile
import threading
import time
import gtest_utils
import iossim_util
+import test_apps
import xctest_utils
LOGGER = logging.getLogger(__name__)
@@ -244,7 +241,7 @@
child process that may block its parent and for such cases
proc_name refers to the name of child process.
If proc_name is not specified, process name will be used to kill process.
- Parser: A parser.
+ parser: A parser.
timeout: A timeout(in seconds) to subprocess.stdout.readline method.
"""
out = []
@@ -275,46 +272,6 @@
return out
-def get_kif_test_filter(tests, invert=False):
- """Returns the KIF test filter to filter the given test cases.
-
- Args:
- tests: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will match
- everything except the given test cases.
-
- Returns:
- A string which can be supplied to GKIF_SCENARIO_FILTER.
- """
- # A pipe-separated list of test cases with the "KIF." prefix omitted.
- # e.g. NAME:a|b|c matches KIF.a, KIF.b, KIF.c.
- # e.g. -NAME:a|b|c matches everything except KIF.a, KIF.b, KIF.c.
- test_filter = '|'.join(test.split('KIF.', 1)[-1] for test in tests)
- if invert:
- return '-NAME:%s' % test_filter
- return 'NAME:%s' % test_filter
-
-
-def get_gtest_filter(tests, invert=False):
- """Returns the GTest filter to filter the given test cases.
-
- Args:
- tests: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will match
- everything except the given test cases.
-
- Returns:
- A string which can be supplied to --gtest_filter.
- """
- # A colon-separated list of tests cases.
- # e.g. a:b:c matches a, b, c.
- # e.g. -a:b:c matches everything except a, b, c.
- test_filter = ':'.join(test for test in tests)
- if invert:
- return '-%s' % test_filter
- return test_filter
-
-
def xcode_select(xcode_app_path):
"""Switch the default Xcode system-wide to `xcode_app_path`.
@@ -425,41 +382,6 @@
return test_pattern.findall(subprocess.check_output(cmd))
-def shard_xctest(object_path, shards, test_cases=None):
- """Gets EarlGrey test methods inside a test target and splits them into shards
-
- Args:
- object_path: Path of the test target bundle.
- shards: Number of shards to split tests.
- test_cases: Passed in test cases to run.
-
- Returns:
- A list of test shards.
- """
- test_names = get_test_names(object_path)
- # If test_cases are passed in, only shard the intersection of them and the
- # listed tests. Format of passed-in test_cases can be either 'testSuite' or
- # 'testSuite/testMethod'. The listed tests are tuples of ('testSuite',
- # 'testMethod'). The intersection includes both test suites and test methods.
- tests_set = set()
- if test_cases:
- for test in test_names:
- test_method = '%s/%s' % (test[0], test[1])
- if test[0] in test_cases or test_method in test_cases:
- tests_set.add(test_method)
- else:
- for test in test_names:
- # 'ChromeTestCase' is the parent class of all EarlGrey test classes. It
- # has no real tests.
- if 'ChromeTestCase' != test[0]:
- tests_set.add('%s/%s' % (test[0], test[1]))
-
- tests = sorted(tests_set)
- shard_len = len(tests)/shards + (len(tests) % shards > 0)
- test_shards=[tests[i:i + shard_len] for i in range(0, len(tests), shard_len)]
- return test_shards
-
-
class TestRunner(object):
"""Base class containing common functionality."""
@@ -551,13 +473,14 @@
if not os.path.exists(self.xctest_path):
raise XCTestPlugInNotFoundError(self.xctest_path)
- def get_launch_command(self, test_filter=None, invert=False):
+ def get_launch_command(self, test_app, out_dir, destination, shards=1):
"""Returns the command that can be used to launch the test app.
Args:
- test_filter: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will
- match everything except the given test cases.
+ test_app: An app that stores data about test required to run.
+ out_dir: (str) A path for results.
+ destination: (str) A destination of device/simulator.
+ shards: (int) How many shards the tests should be divided into.
Returns:
A list of strings forming the command to launch the test.
@@ -625,15 +548,14 @@
shutil.rmtree(DERIVED_DATA)
os.mkdir(DERIVED_DATA)
- def run_tests(self, test_shard=None):
+ def run_tests(self, cmd=None):
"""Runs passed-in tests.
Args:
- test_shard: Test cases to be included in the run.
+ cmd: Command to run tests.
Return:
out: (list) List of strings of subprocess's output.
- udid: (string) Name of the simulator device in the run.
returncode: (int) Return code of subprocess.
"""
raise NotImplementedError
@@ -679,39 +601,19 @@
else:
parser = gtest_utils.GTestLogParser()
- if shards > 1:
- test_shards = shard_xctest(
- os.path.join(self.app_path, self.app_name),
- shards,
- self.test_cases
- )
-
- thread_pool = pool.ThreadPool(processes=shards)
- for out, name, ret in thread_pool.imap_unordered(
- self.run_tests, test_shards):
- LOGGER.info('Simulator %s', name)
- for line in out:
- LOGGER.info(line)
- parser.ProcessLine(line)
- returncode = ret if ret else 0
- thread_pool.close()
- thread_pool.join()
- else:
- # TODO(crbug.com/812705): Implement test sharding for unit tests.
- # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
- proc = self.start_proc(cmd)
- old_handler = self.set_sigterm_handler(
- lambda _signum, _frame: self.handle_sigterm(proc))
- print_process_output(proc, 'xcodebuild', parser)
-
- LOGGER.info('Waiting for test process to terminate.')
- proc.wait()
- LOGGER.info('Test process terminated.')
- self.set_sigterm_handler(old_handler)
- sys.stdout.flush()
- LOGGER.debug('Stdout flushed after test process.')
-
- returncode = proc.returncode
+ # TODO(crbug.com/812705): Implement test sharding for unit tests.
+ # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
+ proc = self.start_proc(cmd)
+ old_handler = self.set_sigterm_handler(
+ lambda _signum, _frame: self.handle_sigterm(proc))
+ print_process_output(proc, 'xcodebuild', parser)
+ LOGGER.info('Waiting for test process to terminate.')
+ proc.wait()
+ LOGGER.info('Test process terminated.')
+ self.set_sigterm_handler(old_handler)
+ sys.stdout.flush()
+ LOGGER.debug('Stdout flushed after test process.')
+ returncode = proc.returncode
if self.xctest and parser.SystemAlertPresent():
raise SystemAlertPresentError()
@@ -729,7 +631,7 @@
LOGGER.info('%s returned %s\n', cmd[0], returncode)
- # iossim can return 5 if it exits noncleanly even if all tests passed.
+ # xcodebuild can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(returncode, parser.CompletedWithoutFailure())
return result
@@ -737,7 +639,21 @@
def launch(self):
"""Launches the test app."""
self.set_up()
- cmd = self.get_launch_command()
+ destination = 'id=%s' % self.udid
+ if self.xctest:
+ test_app = test_apps.EgtestsApp(
+ self.app_path,
+ included_tests=self.test_cases,
+ env_vars=self.env_vars,
+ test_args=self.test_args)
+ else:
+ test_app = test_apps.GTestsApp(
+ self.app_path,
+ included_tests=self.test_cases,
+ env_vars=self.env_vars,
+ test_args=self.test_args)
+ out_dir = os.path.join(self.out_dir, 'TestResults')
+ cmd = self.get_launch_command(test_app, out_dir, destination, self.shards)
try:
result = self._run(cmd=cmd, shards=self.shards or 1)
if result.crashed and not result.crashed_test:
@@ -745,6 +661,9 @@
# it crashed on startup. Try one more time.
self.shutdown_and_restart()
LOGGER.warning('Crashed on startup, retrying...\n')
+ out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup')
+ cmd = self.get_launch_command(test_app, out_dir, destination,
+ self.shards)
result = self._run(cmd)
if result.crashed and not result.crashed_test:
@@ -755,16 +674,19 @@
flaked = result.flaked_tests
try:
- # XCTests cannot currently be resumed at the next test case.
- while not self.xctest and result.crashed and result.crashed_test:
+ while result.crashed and result.crashed_test:
# If the app crashes during a specific test case, then resume at the
# next test case. This is achieved by filtering out every test case
# which has already run.
LOGGER.warning('Crashed during %s, resuming...\n',
result.crashed_test)
- result = self._run(self.get_launch_command(
- test_filter=passed + failed.keys() + flaked.keys(), invert=True,
- ))
+ test_app.excluded_tests = passed + failed.keys() + flaked.keys()
+ retry_out_dir = os.path.join(
+ self.out_dir, 'retry_after_crash_%d' % int(time.time()))
+ result = self._run(
+ self.get_launch_command(
+ test_app, os.path.join(retry_out_dir, str(int(time.time()))),
+ destination))
passed.extend(result.passed_tests)
failed.update(result.failed_tests)
flaked.update(result.flaked_tests)
@@ -776,14 +698,17 @@
# Retry failed test cases.
retry_results = {}
+ test_app.excluded_tests = []
if self.retries and failed:
LOGGER.warning('%s tests failed and will be retried.\n', len(failed))
for i in xrange(self.retries):
for test in failed.keys():
LOGGER.info('Retry #%s for %s.\n', i + 1, test)
- retry_result = self._run(self.get_launch_command(
- test_filter=[test]
- ))
+ test_app.included_tests = [test]
+ retry_out_dir = os.path.join(self.out_dir, test + '_failed',
+ 'retry_%d' % i)
+ retry_result = self._run(
+ self.get_launch_command(test_app, retry_out_dir, destination))
# If the test passed on retry, consider it flake instead of failure.
if test in retry_result.passed_tests:
flaked[test] = failed.pop(test)
@@ -982,7 +907,7 @@
report_time = report_name[len(self.app_name) + 1:].split('_')[0]
# The timestamp format in a crash report is big-endian and therefore
- # a staight string comparison works.
+ # a straight string comparison works.
if report_time > self.start_time:
with open(os.path.join(crash_reports_dir, crash_report)) as f:
self.logs['crash report (%s)' % report_time] = (
@@ -1007,32 +932,21 @@
self.homedir = ''
LOGGER.debug('End of tear_down.')
- def run_tests(self, test_shard=None):
+ def run_tests(self, cmd):
"""Runs passed-in tests. Builds a command and create a simulator to
run tests.
Args:
- test_shard: Test cases to be included in the run.
+ cmd: A running command.
Return:
out: (list) List of strings of subprocess's output.
- udid: (string) Name of the simulator device in the run.
returncode: (int) Return code of subprocess.
"""
- cmd = self.sharding_cmd[:]
- cmd.extend(['-u', self.udid])
- if test_shard:
- for test in test_shard:
- cmd.extend(['-t', test])
-
- cmd.append(self.app_path)
- if self.xctest_path:
- cmd.append(self.xctest_path)
-
proc = self.start_proc(cmd)
out = print_process_output(proc, 'xcodebuild',
xctest_utils.XCTestLogParser())
self.deleteSimulator(self.udid)
- return (out, self.udid, proc.returncode)
+ return (out, proc.returncode)
def getSimulator(self):
"""Gets a simulator or creates a new one by device types and runtimes.
@@ -1048,59 +962,19 @@
if udid:
iossim_util.delete_simulator_by_udid(udid)
- def get_launch_command(self, test_filter=None, invert=False, test_shard=None):
+ def get_launch_command(self, test_app, out_dir, destination, shards=1):
"""Returns the command that can be used to launch the test app.
Args:
- test_filter: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will
- match everything except the given test cases.
- test_shard: How many shards the tests should be divided into.
+ test_app: An app that stores data about test required to run.
+ out_dir: (str) A path for results.
+ destination: (str) A destination of device/simulator.
+ shards: (int) How many shards the tests should be divided into.
Returns:
A list of strings forming the command to launch the test.
"""
- cmd = [
- self.iossim_path,
- '-d', self.platform,
- '-s', self.version,
- ]
-
- for env_var in self.env_vars:
- cmd.extend(['-e', env_var])
-
- for test_arg in self.test_args:
- cmd.extend(['-c', test_arg])
-
- # If --run-with-custom-webkit is passed as a test arg, set
- # DYLD_FRAMEWORK_PATH to point to the embedded custom webkit frameworks.
- if test_arg == '--run-with-custom-webkit':
- cmd.extend(['-e', 'DYLD_FRAMEWORK_PATH=' +
- os.path.join(self.app_path, 'WebKitFrameworks')])
-
- if self.xctest_path:
- self.sharding_cmd = cmd[:]
- if test_filter:
- # iossim doesn't support inverted filters for XCTests.
- if not invert:
- for test in test_filter:
- cmd.extend(['-t', test])
- elif test_shard:
- for test in test_shard:
- cmd.extend(['-t', test])
- elif not invert:
- for test_case in self.test_cases:
- cmd.extend(['-t', test_case])
- elif test_filter:
- kif_filter = get_kif_test_filter(test_filter, invert=invert)
- gtest_filter = get_gtest_filter(test_filter, invert=invert)
- cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
- cmd.extend(['-c', '--gtest_filter=%s' % gtest_filter])
-
- cmd.append(self.app_path)
- if self.xctest_path:
- cmd.append(self.xctest_path)
- return cmd
+ return test_app.command(out_dir, destination, shards)
def get_launch_env(self):
"""Returns a dict of environment variables to use to launch the test app.
@@ -1179,27 +1053,6 @@
if xctest or is_iOS13:
if is_iOS13:
self.xctest_path = get_xctest_from_app(self.app_path)
- self.xctestrun_file = tempfile.mkstemp()[1]
- self.xctestrun_data = {
- 'TestTargetName': {
- 'IsAppHostedTestBundle': True,
- 'TestBundlePath': '%s' % self.xctest_path,
- 'TestHostPath': '%s' % self.app_path,
- 'TestingEnvironmentVariables': {
- 'DYLD_INSERT_LIBRARIES':
- '__PLATFORMS__/iPhoneOS.platform/Developer/usr/lib/'
- 'libXCTestBundleInject.dylib',
- 'DYLD_LIBRARY_PATH':
- '__PLATFORMS__/iPhoneOS.platform/Developer/Library',
- 'DYLD_FRAMEWORK_PATH':
- '__PLATFORMS__/iPhoneOS.platform/Developer/'
- 'Library/Frameworks',
- 'XCInjectBundleInto':
- '__TESTHOST__/%s' % self.app_name
- }
- }
- }
-
self.restart = restart
def uninstall_apps(self):
@@ -1280,71 +1133,40 @@
self.retrieve_crash_reports()
self.uninstall_apps()
- def set_xctest_filters(self, test_filter=None, invert=False):
- """Sets the tests be included in the test run."""
- if self.test_cases:
- filter = self.test_cases
- if test_filter:
- # If inverted, the filter should match tests in test_cases except the
- # ones in test_filter. Otherwise, the filter should be tests both in
- # test_cases and test_filter. test_filter is used for test retries, it
- # should be a subset of test_cases. If the intersection of test_cases
- # and test_filter fails, use test_filter.
- filter = (sorted(set(filter) - set(test_filter)) if invert
- else sorted(set(filter) & set(test_filter)) or test_filter)
- self.xctestrun_data['TestTargetName'].update(
- {'OnlyTestIdentifiers': filter})
- elif test_filter:
- if invert:
- self.xctestrun_data['TestTargetName'].update(
- {'SkipTestIdentifiers': test_filter})
- else:
- self.xctestrun_data['TestTargetName'].update(
- {'OnlyTestIdentifiers': test_filter})
-
def get_command_line_args_xctest_unittests(self, filtered_tests):
command_line_args = ['--enable-run-ios-unittests-with-xctest']
if filtered_tests:
command_line_args.append('--gtest_filter=%s' % filtered_tests)
return command_line_args
- def get_launch_command(self, test_filter=None, invert=False):
+ def get_launch_command(self, test_app, out_dir, destination, shards=1):
"""Returns the command that can be used to launch the test app.
Args:
- test_filter: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will
- match everything except the given test cases.
+ test_app: An app that stores data about test required to run.
+ out_dir: (str) A path for results.
+ destination: (str) A destination of device/simulator.
+ shards: (int) How many shards the tests should be divided into.
Returns:
A list of strings forming the command to launch the test.
"""
if self.xctest_path:
- if self.env_vars:
- self.xctestrun_data['TestTargetName'].update(
- {'EnvironmentVariables': self.env_vars})
+ command_line_args = test_app.test_args
- command_line_args = self.test_args
-
- if self.xctest:
- self.set_xctest_filters(test_filter, invert)
- else:
+ if not self.xctest:
filtered_tests = []
- if test_filter:
- filtered_tests = get_gtest_filter(test_filter, invert=invert)
+ if test_app.included_tests:
+ filtered_tests = test_apps.get_gtest_filter(
+ test_app.included_tests, invert=False)
+ elif test_app.excluded_tests:
+ filtered_tests = test_apps.get_gtest_filter(
+ test_app.excluded_tests, invert=True)
command_line_args.append(
self.get_command_line_args_xctest_unittests(filtered_tests))
-
if command_line_args:
- self.xctestrun_data['TestTargetName'].update(
- {'CommandLineArguments': command_line_args})
- plistlib.writePlist(self.xctestrun_data, self.xctestrun_file)
- return [
- 'xcodebuild',
- 'test-without-building',
- '-xctestrun', self.xctestrun_file,
- '-destination', 'id=%s' % self.udid,
- ]
+ test_app.test_args = command_line_args
+ return test_app.command(out_dir, destination, shards)
cmd = [
'idevice-app-runner',
@@ -1352,11 +1174,23 @@
'--start', self.cfbundleid,
]
args = []
+ gtest_filter = []
+ kif_filter = []
- if test_filter:
- kif_filter = get_kif_test_filter(test_filter, invert=invert)
- gtest_filter = get_gtest_filter(test_filter, invert=invert)
+ if test_app.included_tests:
+ kif_filter = test_apps.get_kif_test_filter(test_app.included_tests,
+ invert=False)
+ gtest_filter = test_apps.get_gtest_filter(test_app.included_tests,
+ invert=False)
+ elif test_app.excluded_tests:
+ kif_filter = test_apps.get_kif_test_filter(test_app.excluded_tests,
+ invert=True)
+ gtest_filter = test_apps.get_gtest_filter(test_app.excluded_tests,
+ invert=True)
+
+ if kif_filter:
cmd.extend(['-D', 'GKIF_SCENARIO_FILTER=%s' % kif_filter])
+ if gtest_filter:
args.append('--gtest_filter=%s' % gtest_filter)
for env_var in self.env_vars:
diff --git a/ios/build/bots/scripts/test_runner_test.py b/ios/build/bots/scripts/test_runner_test.py
index 12662219..fef4902d 100755
--- a/ios/build/bots/scripts/test_runner_test.py
+++ b/ios/build/bots/scripts/test_runner_test.py
@@ -6,16 +6,84 @@
"""Unittests for test_runner.py."""
import collections
-import glob
import logging
import mock
import os
-import subprocess
import tempfile
import unittest
+import iossim_util
import test_runner
+SIMULATORS_LIST = {
+ 'devices': {
+ 'com.apple.CoreSimulator.SimRuntime.iOS-11-4': [{
+ 'isAvailable': True,
+ 'name': 'iPhone 5s',
+ 'state': 'Shutdown',
+ 'udid': 'E4E66320-177A-450A-9BA1-488D85B7278E'
+ }],
+ 'com.apple.CoreSimulator.SimRuntime.iOS-13-2': [
+ {
+ 'isAvailable': True,
+ 'name': 'iPhone X',
+ 'state': 'Shutdown',
+ 'udid': 'E4E66321-177A-450A-9BA1-488D85B7278E'
+ },
+ {
+ 'isAvailable': True,
+ 'name': 'iPhone 11',
+ 'state': 'Shutdown',
+ 'udid': 'A4E66321-177A-450A-9BA1-488D85B7278E'
+ }
+ ]
+ },
+ 'devicetypes': [
+ {
+ 'name': 'iPhone 5s',
+ 'bundlePath': '/path/iPhone 4s/Content',
+ 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-5s'
+ },
+ {
+ 'name': 'iPhone X',
+ 'bundlePath': '/path/iPhone X/Content',
+ 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-X'
+ },
+ {
+ 'name': 'iPhone 11',
+ 'bundlePath': '/path/iPhone 11/Content',
+ 'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-11'
+ },
+ ],
+ 'pairs': [],
+ 'runtimes': [
+ {
+ "buildversion": "15F79",
+ "bundlePath": "/path/Runtimes/iOS 11.4.simruntime",
+ "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-11-4",
+ "isAvailable": True,
+ "name": "iOS 11.4",
+ "version": "11.4"
+ },
+ {
+ "buildversion": "17A844",
+ "bundlePath": "/path/Runtimes/iOS 13.1.simruntime",
+ "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-1",
+ "isAvailable": True,
+ "name": "iOS 13.1",
+ "version": "13.1"
+ },
+ {
+ "buildversion": "17B102",
+ "bundlePath": "/path/Runtimes/iOS.simruntime",
+ "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-2",
+ "isAvailable": True,
+ "name": "iOS 13.2",
+ "version": "13.2.2"
+ },
+ ]
+}
+
class TestCase(unittest.TestCase):
"""Test case which supports installing mocks. Uninstalls on tear down."""
@@ -49,56 +117,6 @@
setattr(obj, member, original_value)
-class GetKIFTestFilterTest(TestCase):
- """Tests for test_runner.get_kif_test_filter."""
-
- def test_correct(self):
- """Ensures correctness of filter."""
- tests = [
- 'KIF.test1',
- 'KIF.test2',
- ]
- expected = 'NAME:test1|test2'
-
- self.assertEqual(test_runner.get_kif_test_filter(tests), expected)
-
- def test_correct_inverted(self):
- """Ensures correctness of inverted filter."""
- tests = [
- 'KIF.test1',
- 'KIF.test2',
- ]
- expected = '-NAME:test1|test2'
-
- self.assertEqual(
- test_runner.get_kif_test_filter(tests, invert=True), expected)
-
-
-class GetGTestFilterTest(TestCase):
- """Tests for test_runner.get_gtest_filter."""
-
- def test_correct(self):
- """Ensures correctness of filter."""
- tests = [
- 'test.1',
- 'test.2',
- ]
- expected = 'test.1:test.2'
-
- self.assertEqual(test_runner.get_gtest_filter(tests), expected)
-
- def test_correct_inverted(self):
- """Ensures correctness of inverted filter."""
- tests = [
- 'test.1',
- 'test.2',
- ]
- expected = '-test.1:test.2'
-
- self.assertEqual(
- test_runner.get_gtest_filter(tests, invert=True), expected)
-
-
class InstallXcodeTest(TestCase):
"""Tests install_xcode."""
@@ -119,6 +137,7 @@
def setUp(self):
super(SimulatorTestRunnerTest, self).setUp()
+ self.mock(iossim_util, 'get_simulator_list', lambda: SIMULATORS_LIST)
def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
return True
@@ -158,8 +177,8 @@
test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -170,8 +189,8 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -199,8 +218,8 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -211,8 +230,6 @@
def test_run(self):
"""Ensures the _run method is correct with test sharding."""
- def shard_xctest(object_path, shards, test_cases=None):
- return [['a/1', 'b/2'], ['c/3', 'd/4'], ['e/5']]
def run_tests(self, test_shard=None):
out = []
@@ -227,14 +244,13 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
xctest=True,
)
- self.mock(test_runner, 'shard_xctest', shard_xctest)
self.mock(test_runner.SimulatorTestRunner, 'run_tests', run_tests)
tr.xctest_path = 'fake.xctest'
@@ -253,8 +269,8 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -270,8 +286,8 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -279,12 +295,12 @@
tr.xctest_path = 'fake.xctest'
# Cases test_filter is not empty, with empty/non-empty self.test_cases.
tr.test_cases = []
- cmd = tr.get_launch_command(['a'], invert=False)
+ cmd = tr.get_launch_command(['a'])
self.assertIn('-t', cmd)
self.assertIn('a', cmd)
tr.test_cases = ['a', 'b']
- cmd = tr.get_launch_command(['a'], invert=False)
+ cmd = tr.get_launch_command(['a'])
self.assertIn('-t', cmd)
self.assertIn('a', cmd)
self.assertNotIn('b', cmd)
@@ -344,8 +360,8 @@
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
- 'platform',
- 'os',
+ 'iPhone X',
+ '11.4',
'xcode-version',
'xcode-build',
'out-dir',
@@ -379,70 +395,6 @@
)
self.tr.xctestrun_data = {'TestTargetName':{}}
- def test_with_test_filter_without_test_cases(self):
- """Ensures tests in the run with test_filter and no test_cases."""
- self.tr.set_xctest_filters(['a', 'b'], invert=False)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
- ['a', 'b']
- )
-
- def test_invert_with_test_filter_without_test_cases(self):
- """Ensures tests in the run invert with test_filter and no test_cases."""
- self.tr.set_xctest_filters(['a', 'b'], invert=True)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['SkipTestIdentifiers'],
- ['a', 'b']
- )
-
- def test_with_test_filter_with_test_cases(self):
- """Ensures tests in the run with test_filter and test_cases."""
- self.tr.test_cases = ['a', 'b', 'c', 'd']
- self.tr.set_xctest_filters(['a', 'b', 'irrelevant test'], invert=False)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
- ['a', 'b']
- )
-
- def test_invert_with_test_filter_with_test_cases(self):
- """Ensures tests in the run invert with test_filter and test_cases."""
- self.tr.test_cases = ['a', 'b', 'c', 'd']
- self.tr.set_xctest_filters(['a', 'b', 'irrelevant test'], invert=True)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
- ['c', 'd']
- )
-
- def test_without_test_filter_without_test_cases(self):
- """Ensures tests in the run with no test_filter and no test_cases."""
- self.tr.set_xctest_filters(test_filter=None, invert=False)
- self.assertIsNone(
- self.tr.xctestrun_data['TestTargetName'].get('OnlyTestIdentifiers'))
-
- def test_invert_without_test_filter_without_test_cases(self):
- """Ensures tests in the run invert with no test_filter and no test_cases."""
- self.tr.set_xctest_filters(test_filter=None, invert=True)
- self.assertIsNone(
- self.tr.xctestrun_data['TestTargetName'].get('OnlyTestIdentifiers'))
-
- def test_without_test_filter_with_test_cases(self):
- """Ensures tests in the run with no test_filter but test_cases."""
- self.tr.test_cases = ['a', 'b', 'c', 'd']
- self.tr.set_xctest_filters(test_filter=None, invert=False)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
- ['a', 'b', 'c', 'd']
- )
-
- def test_invert_without_test_filter_with_test_cases(self):
- """Ensures tests in the run invert with no test_filter but test_cases."""
- self.tr.test_cases = ['a', 'b', 'c', 'd']
- self.tr.set_xctest_filters(test_filter=None, invert=True)
- self.assertEqual(
- self.tr.xctestrun_data['TestTargetName']['OnlyTestIdentifiers'],
- ['a', 'b', 'c', 'd']
- )
-
@mock.patch('subprocess.check_output', autospec=True)
def test_get_test_names(self, mock_subprocess):
otool_output = (
diff --git a/ios/build/bots/scripts/wpr_runner.py b/ios/build/bots/scripts/wpr_runner.py
index 8b14e76e..a529822 100644
--- a/ios/build/bots/scripts/wpr_runner.py
+++ b/ios/build/bots/scripts/wpr_runner.py
@@ -12,8 +12,8 @@
import sys
import gtest_utils
+import test_apps
import test_runner
-import xcodebuild_runner
import xctest_utils
LOGGER = logging.getLogger(__name__)
@@ -158,7 +158,7 @@
'--enable-features=AutofillShowTypePredictions',
'-autofillautomation=%s' % recipe_path,
]
- wpr_egtests_app = xcodebuild_runner.EgtestsApp(
+ wpr_egtests_app = test_apps.EgtestsApp(
self.app_path,
included_tests=["AutofillAutomationTestCase"],
env_vars=self.env_vars,
@@ -174,11 +174,7 @@
self.version, self.platform, test_name,
self.test_attempt_count[test_name])
out_dir = os.path.join(self.out_dir, destination_folder)
-
- launch_command = xcodebuild_runner.LaunchCommand(
- wpr_egtests_app, destination, self.shards, self.retries)
- return launch_command.command(wpr_egtests_app, out_dir, destination,
- self.shards)
+ return wpr_egtests_app.command(out_dir, destination, self.shards)
def get_launch_env(self):
"""Returns a dict of environment variables to use to launch the test app.
@@ -381,30 +377,37 @@
self.deleteSimulator(udid)
- # iossim can return 5 if it exits noncleanly even if all tests passed.
+ # xcodebuild can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
# NOTE: total_returncode is 0 OR the last non-zero return code from a test.
result.finalize(total_returncode, completed_without_failure)
return result
- def get_launch_command(self, test_filter=[], invert=False):
+ def get_launch_command(self, test_app=None, out_dir=None,
+ destination=None, shards=1):
"""Returns a config dict for the test, instead of the real launch command.
Normally this is passed into _run as the command it should use, but since
the WPR runner builds its own cmd, we use this to configure the function.
Args:
- test_filter: List of test cases to filter.
- invert: Whether to invert the filter or not. Inverted, the filter will
- match everything except the given test cases.
+ test_app: A test app needed to run.
+ out_dir: (str) A path for results.
+ destination: (str) A destination of device/simulator.
+ shards: (int) How many shards the tests should be divided into.
Returns:
A dict forming the configuration for the test.
"""
test_config = {}
- test_config['invert'] = invert
- test_config['test_filter'] = test_filter
+ if test_app:
+ if test_app.included_tests:
+ test_config['invert'] = False
+ test_config['test_filter'] = test_app.included_tests
+ elif test_app.excluded_tests:
+ test_config['invert'] = True
+ test_config['test_filter'] = test_app.excluded_tests
return test_config
def proxy_start(self):
diff --git a/ios/build/bots/scripts/wpr_runner_test.py b/ios/build/bots/scripts/wpr_runner_test.py
index e410d60..2284b1a 100644
--- a/ios/build/bots/scripts/wpr_runner_test.py
+++ b/ios/build/bots/scripts/wpr_runner_test.py
@@ -9,6 +9,7 @@
import subprocess
import unittest
+import iossim_util
import test_runner
import test_runner_test
import wpr_runner
@@ -38,6 +39,8 @@
lambda a, b: True)
self.mock(wpr_runner.WprProxySimulatorTestRunner,
'copy_trusted_certificate', lambda a: True)
+ self.mock(iossim_util, 'get_simulator',
+ lambda a, b: 'E4E66320-177A-450A-9BA1-488D85B7278E')
def test_app_not_found(self):
"""Ensures AppNotFoundError is raised."""
@@ -178,7 +181,7 @@
self.mock(subprocess, 'Popen', popen)
tr.xctest_path = 'fake.xctest'
- cmd = tr.get_launch_command(test_filter=test_filter, invert=invert)
+ cmd = {'invert': invert, 'test_filter': test_filter}
return tr._run(cmd=cmd, shards=1)
def test_run_no_filter(self):
diff --git a/ios/build/bots/scripts/xcodebuild_runner.py b/ios/build/bots/scripts/xcodebuild_runner.py
index f1df448..991a9fa2e 100644
--- a/ios/build/bots/scripts/xcodebuild_runner.py
+++ b/ios/build/bots/scripts/xcodebuild_runner.py
@@ -7,13 +7,13 @@
import collections
import distutils.version
import logging
-import multiprocessing
+from multiprocessing import pool
import os
-import plistlib
import subprocess
import time
import iossim_util
+import test_apps
import test_runner
import xcode_log_parser
@@ -102,135 +102,6 @@
LOGGER.info('Error while killing a process: %s' % ex)
-class EgtestsApp(object):
- """Egtests to run.
-
- Stores data about egtests:
- egtests_app: full path to egtests app.
- project_path: root project folder.
- module_name: egtests module name.
- included_tests: List of tests to run.
- excluded_tests: List of tests not to run.
- """
-
- def __init__(self, egtests_app, included_tests=None, excluded_tests=None,
- test_args=None, env_vars=None, host_app_path=None):
- """Initialize Egtests.
-
- Args:
- egtests_app: (str) full path to egtests app.
- included_tests: (list) Specific tests to run
- E.g.
- [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
- excluded_tests: (list) Specific tests not to run
- E.g.
- [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
- test_args: List of strings to pass as arguments to the test when
- launching.
- env_vars: List of environment variables to pass to the test itself.
- host_app_path: (str) full path to host app.
-
- Raises:
- AppNotFoundError: If the given app does not exist
- """
- if not os.path.exists(egtests_app):
- raise test_runner.AppNotFoundError(egtests_app)
- self.egtests_path = egtests_app
- self.project_path = os.path.dirname(self.egtests_path)
- self.module_name = os.path.splitext(os.path.basename(egtests_app))[0]
- self.included_tests = included_tests or []
- self.excluded_tests = excluded_tests or []
- self.test_args = test_args
- self.env_vars = env_vars
- self.host_app_path = host_app_path
-
- def _xctest_path(self):
- """Gets xctest-file from egtests/PlugIns folder.
-
- Returns:
- A path for xctest in the format of /PlugIns/file.xctest
-
- Raises:
- PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
- XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
- """
- plugins_dir = os.path.join(self.egtests_path, 'PlugIns')
- if not os.path.exists(plugins_dir):
- raise test_runner.PlugInsNotFoundError(plugins_dir)
- plugin_xctest = None
- if os.path.exists(plugins_dir):
- for plugin in os.listdir(plugins_dir):
- if plugin.endswith('.xctest'):
- plugin_xctest = os.path.join(plugins_dir, plugin)
- if not plugin_xctest:
- raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
- return plugin_xctest.replace(self.egtests_path, '')
-
- def xctestrun_node(self):
- """Fills only required nodes for egtests in xctestrun file.
-
- Returns:
- A node with filled required fields about egtests.
- """
- module = self.module_name + '_module'
-
- # If --run-with-custom-webkit is passed as a test arg, set up
- # DYLD_FRAMEWORK_PATH to load the custom webkit modules.
- dyld_framework_path = self.project_path + ':'
- if '--run-with-custom-webkit' in self.test_args:
- if self.host_app_path:
- webkit_path = os.path.join(self.host_app_path, 'WebKitFrameworks')
- else:
- webkit_path = os.path.join(self.egtests_path, 'WebKitFrameworks')
- dyld_framework_path = dyld_framework_path + webkit_path + ':'
-
- module_data = {
- 'TestBundlePath': '__TESTHOST__%s' % self._xctest_path(),
- 'TestHostPath': '%s' % self.egtests_path,
- 'TestingEnvironmentVariables': {
- 'DYLD_INSERT_LIBRARIES': (
- '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
- 'usr/lib/libXCTestBundleInject.dylib'),
- 'DYLD_LIBRARY_PATH': self.project_path,
- 'DYLD_FRAMEWORK_PATH': dyld_framework_path,
- 'XCInjectBundleInto': '__TESTHOST__/%s' % self.module_name
- }
- }
- # Add module data specific to EG2 or EG1 tests
- # EG2 tests
- if self.host_app_path:
- module_data['IsUITestBundle'] = True
- module_data['IsXCTRunnerHostedTestBundle'] = True
- module_data['UITargetAppPath'] = '%s' % self.host_app_path
- # Special handling for Xcode10.2
- dependent_products = [
- module_data['UITargetAppPath'],
- module_data['TestBundlePath'],
- module_data['TestHostPath']
- ]
- module_data['DependentProductPaths'] = dependent_products
- # EG1 tests
- else:
- module_data['IsAppHostedTestBundle'] = True
-
- xctestrun_data = {
- module: module_data
- }
- if self.excluded_tests:
- xctestrun_data[module].update(
- {'SkipTestIdentifiers': self.excluded_tests})
- if self.included_tests:
- xctestrun_data[module].update(
- {'OnlyTestIdentifiers': self.included_tests})
- if self.env_vars:
- xctestrun_data[module].update(
- {'EnvironmentVariables': self.env_vars})
- if self.test_args:
- xctestrun_data[module].update(
- {'CommandLineArguments': self.test_args})
- return xctestrun_data
-
-
class LaunchCommand(object):
"""Stores xcodebuild test launching command."""
@@ -255,7 +126,7 @@
Raises:
LaunchCommandCreationError: if one of parameters was not set properly.
"""
- if not isinstance(egtests_app, EgtestsApp):
+ if not isinstance(egtests_app, test_apps.EgtestsApp):
raise test_runner.AppNotFoundError(
'Parameter `egtests_app` is not EgtestsApp: %s' % egtests_app)
self.egtests_app = egtests_app
@@ -313,12 +184,12 @@
def launch(self):
"""Launches tests using xcodebuild."""
- cmd_list = []
self.test_results['attempts'] = []
cancelled_statuses = {'TESTS_DID_NOT_START', 'BUILD_INTERRUPTED'}
shards = self.shards
- running_tests = set(get_all_tests(self.egtests_app.egtests_path,
- self.egtests_app.included_tests))
+ running_tests = set(
+ get_all_tests(self.egtests_app.test_app_path,
+ self.egtests_app.included_tests))
# total number of attempts is self.retries+1
for attempt in range(self.retries + 1):
# Erase all simulators per each attempt
@@ -330,8 +201,8 @@
erase_all_simulators()
erase_all_simulators(XTDEVICE_FOLDER)
outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
- cmd_list = self.command(self.egtests_app, outdir_attempt,
- 'id=%s' % self.udid, shards)
+ cmd_list = self.egtests_app.command(outdir_attempt,
+ 'id=%s' % self.udid, shards)
# TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
LOGGER.info('Start test attempt #%d for command [%s]' % (
attempt, ' '.join(cmd_list)))
@@ -372,59 +243,6 @@
'logs': self.logs
}
- def fill_xctest_run(self, egtests_app):
- """Fills xctestrun file by egtests.
-
- Args:
- egtests_app: (EgtestsApp) An Egetsts_app to run.
-
- Returns:
- A path to xctestrun file.
-
- Raises:
- AppNotFoundError if egtests is empty.
- """
- if not egtests_app:
- raise test_runner.AppNotFoundError('Egtests is not found!')
- xctestrun = os.path.join(
- os.path.abspath(os.path.join(self.out_dir, os.pardir)),
- 'run_%d.xctestrun' % int(time.time()))
- if not os.path.exists(xctestrun):
- with open(xctestrun, 'w'):
- pass
- # Creates a dict with data about egtests to run - fill all required fields:
- # egtests_module, egtest_app_path, egtests_xctest_path and
- # filtered tests if filter is specified.
- # Write data in temp xctest run file.
- plistlib.writePlist(egtests_app.xctestrun_node(), xctestrun)
- return xctestrun
-
- def command(self, egtests_app, out_dir, destination, shards):
- """Returns the command that launches tests using xcodebuild.
-
- Format of command:
- xcodebuild test-without-building -xctestrun file.xctestrun \
- -parallel-testing-enabled YES -parallel-testing-worker-count %d% \
- [-destination "destination"] -resultBundlePath %output_path%
-
- Args:
- egtests_app: (EgtestsApp) An egetsts_app to run.
- out_dir: (str) An output directory.
- destination: (str) A destination of running simulator.
- shards: (int) A number of shards.
-
- Returns:
- A list of strings forming the command to launch the test.
- """
- cmd = ['xcodebuild', 'test-without-building',
- '-xctestrun', self.fill_xctest_run(egtests_app),
- '-destination', destination,
- '-resultBundlePath', out_dir]
- if shards > 1:
- cmd += ['-parallel-testing-enabled', 'YES',
- '-parallel-testing-worker-count', str(shards)]
- return cmd
-
class SimulatorParallelTestRunner(test_runner.SimulatorTestRunner):
"""Class for running simulator tests using xCode."""
@@ -531,23 +349,25 @@
"""Launches tests using xcodebuild."""
launch_commands = []
for params in self.sharding_data:
+ test_app = test_apps.EgtestsApp(
+ params['app'],
+ included_tests=params['test_cases'],
+ env_vars=self.env_vars,
+ test_args=self.test_args,
+ host_app_path=params['host'])
launch_commands.append(
LaunchCommand(
- EgtestsApp(
- params['app'],
- included_tests=params['test_cases'],
- env_vars=self.env_vars,
- test_args=self.test_args,
- host_app_path=params['host']),
+ test_app,
udid=params['udid'],
shards=params['shards'],
retries=self.retries,
out_dir=os.path.join(self.out_dir, params['udid']),
env=self.get_launch_env()))
- pool = multiprocessing.pool.ThreadPool(len(launch_commands))
+ thread_pool = pool.ThreadPool(len(launch_commands))
attempts_results = []
- for result in pool.imap_unordered(LaunchCommand.launch, launch_commands):
+ for result in thread_pool.imap_unordered(LaunchCommand.launch,
+ launch_commands):
attempts_results.append(result['test_results']['attempts'])
# Gets passed tests
diff --git a/ios/build/bots/scripts/xcodebuild_runner_test.py b/ios/build/bots/scripts/xcodebuild_runner_test.py
index f91c5b6..df2780f 100644
--- a/ios/build/bots/scripts/xcodebuild_runner_test.py
+++ b/ios/build/bots/scripts/xcodebuild_runner_test.py
@@ -7,9 +7,11 @@
import mock
import os
+import iossim_util
import plistlib
import shutil
import tempfile
+import test_apps
import test_runner
import test_runner_test
import xcode_log_parser
@@ -18,7 +20,7 @@
_ROOT_FOLDER_PATH = 'root/folder'
_XCODE_BUILD_VERSION = '10B61'
-_DESTINATION = 'platform=iOS Simulator,OS=12.0,name=iPhone X'
+_DESTINATION = 'A4E66321-177A-450A-9BA1-488D85B7278E'
_OUT_DIR = 'out/dir'
_XTEST_RUN = '/tmp/temp_file.xctestrun'
_EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH
@@ -36,6 +38,8 @@
self.mock(xcodebuild_runner, 'get_all_tests',
lambda _1, _2: ['Class1/passedTest1', 'Class1/passedTest2'])
self.tmpdir = tempfile.mkdtemp()
+ self.mock(iossim_util, 'get_simulator_list',
+ lambda: test_runner_test.SIMULATORS_LIST)
def tearDown(self):
shutil.rmtree(self.tmpdir, ignore_errors=True)
@@ -107,92 +111,54 @@
def testEgtests_not_found_egtests_app(self):
self.mock(os.path, 'exists', lambda _: False)
with self.assertRaises(test_runner.AppNotFoundError):
- xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+ test_apps.EgtestsApp(_EGTESTS_APP_PATH)
def testEgtests_not_found_plugins(self):
- egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+ egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
self.mock(os.path, 'exists', lambda _: False)
with self.assertRaises(test_runner.PlugInsNotFoundError):
egtests._xctest_path()
def testEgtests_found_xctest(self):
self.assertEqual('/PlugIns/any_egtests.xctest',
- xcodebuild_runner.EgtestsApp(
- _EGTESTS_APP_PATH)._xctest_path())
+ test_apps.EgtestsApp(_EGTESTS_APP_PATH)._xctest_path())
@mock.patch('os.listdir', autospec=True)
def testEgtests_not_found_xctest(self, mock_listdir):
mock_listdir.return_value = ['random_file']
- egtest = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+ egtest = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
with self.assertRaises(test_runner.XCTestPlugInNotFoundError):
egtest._xctest_path()
def testEgtests_xctestRunNode_without_filter(self):
- egtest_node = xcodebuild_runner.EgtestsApp(
- _EGTESTS_APP_PATH).xctestrun_node()['any_egtests_module']
+ egtest_node = test_apps.EgtestsApp(
+ _EGTESTS_APP_PATH).fill_xctestrun_node()['any_egtests_module']
self.assertNotIn('OnlyTestIdentifiers', egtest_node)
self.assertNotIn('SkipTestIdentifiers', egtest_node)
def testEgtests_xctestRunNode_with_filter_only_identifiers(self):
filtered_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
'TestCase2/testMethod1', 'TestCase1/testMethod2']
- egtest_node = xcodebuild_runner.EgtestsApp(
- _EGTESTS_APP_PATH, included_tests=filtered_tests).xctestrun_node()[
- 'any_egtests_module']
+ egtest_node = test_apps.EgtestsApp(
+ _EGTESTS_APP_PATH,
+ included_tests=filtered_tests).fill_xctestrun_node()['any_egtests_module']
self.assertEqual(filtered_tests, egtest_node['OnlyTestIdentifiers'])
self.assertNotIn('SkipTestIdentifiers', egtest_node)
def testEgtests_xctestRunNode_with_filter_skip_identifiers(self):
skipped_tests = ['TestCase1/testMethod1', 'TestCase1/testMethod2',
'TestCase2/testMethod1', 'TestCase1/testMethod2']
- egtest_node = xcodebuild_runner.EgtestsApp(
- _EGTESTS_APP_PATH, excluded_tests=skipped_tests
- ).xctestrun_node()['any_egtests_module']
+ egtest_node = test_apps.EgtestsApp(
+ _EGTESTS_APP_PATH,
+ excluded_tests=skipped_tests).fill_xctestrun_node()['any_egtests_module']
self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers'])
self.assertNotIn('OnlyTestIdentifiers', egtest_node)
- @mock.patch('xcodebuild_runner.LaunchCommand.fill_xctest_run', autospec=True)
- def testLaunchCommand_command(self, mock_fill_xctestrun):
- mock_fill_xctestrun.return_value = _XTEST_RUN
- mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
- type(mock_egtest).egtests_path = mock.PropertyMock(
- return_value=_EGTESTS_APP_PATH)
- cmd = xcodebuild_runner.LaunchCommand(
- mock_egtest, _DESTINATION, shards=3, retries=1, out_dir=_OUT_DIR)
- self.assertEqual(['xcodebuild', 'test-without-building',
- '-xctestrun', '/tmp/temp_file.xctestrun',
- '-destination',
- 'platform=iOS Simulator,OS=12.0,name=iPhone X',
- '-resultBundlePath', 'out/dir',
- '-parallel-testing-enabled', 'YES',
- '-parallel-testing-worker-count', '3'],
- cmd.command(egtests_app=mock_egtest,
- out_dir=_OUT_DIR,
- destination=_DESTINATION,
- shards=3))
-
- @mock.patch('plistlib.writePlist', autospec=True)
- @mock.patch('os.path.join', autospec=True)
- @mock.patch('test_runner.get_current_xcode_info', autospec=True)
- def testFill_xctest_run(self, xcode_version, mock_path_join, _):
- mock_path_join.return_value = _XTEST_RUN
- mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
- xcode_version.return_value = {'version': '10.2.1'}
- launch_command = xcodebuild_runner.LaunchCommand(
- mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR)
- self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest))
- self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
-
- def testFill_xctest_run_exception(self):
- with self.assertRaises(test_runner.AppNotFoundError):
- xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1,
- out_dir=_OUT_DIR).fill_xctest_run([])
-
@mock.patch('test_runner.get_current_xcode_info', autospec=True)
@mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results,
xcode_version):
- egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+ egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
xcode_version.return_value = {'version': '10.2.1'}
mock_collect_results.side_effect = [
{'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []},
@@ -211,7 +177,7 @@
@mock.patch('xcode_log_parser.XcodeLogParser.collect_test_results')
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results,
xcode_version):
- egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
+ egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
xcode_version.return_value = {'version': '10.2.1'}
mock_collect_results.side_effect = [
{'failed': {'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0'},
