ios/build/bots/scripts/test_runner_test.py

#!/usr/bin/python
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unittests for test_runner.py."""

import collections
import logging
import mock
import os
import tempfile
import unittest

import iossim_util
import test_runner

SIMULATORS_LIST = {
    'devices': {
        'com.apple.CoreSimulator.SimRuntime.iOS-11-4': [{
            'isAvailable': True,
            'name': 'iPhone 5s',
            'state': 'Shutdown',
            'udid': 'E4E66320-177A-450A-9BA1-488D85B7278E'
        }],
        'com.apple.CoreSimulator.SimRuntime.iOS-13-2': [
            {
                'isAvailable': True,
                'name': 'iPhone X',
                'state': 'Shutdown',
                'udid': 'E4E66321-177A-450A-9BA1-488D85B7278E'
            },
            {
                'isAvailable': True,
                'name': 'iPhone 11',
                'state': 'Shutdown',
                'udid': 'A4E66321-177A-450A-9BA1-488D85B7278E'
            }
        ]
    },
    'devicetypes': [
        {
            'name': 'iPhone 5s',
            'bundlePath': '/path/iPhone 4s/Content',
            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-5s'
        },
        {
            'name': 'iPhone X',
            'bundlePath': '/path/iPhone X/Content',
            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-X'
        },
        {
            'name': 'iPhone 11',
            'bundlePath': '/path/iPhone 11/Content',
            'identifier': 'com.apple.CoreSimulator.SimDeviceType.iPhone-11'
        },
    ],
    'pairs': [],
    'runtimes': [
        {
            "buildversion": "15F79",
            "bundlePath": "/path/Runtimes/iOS 11.4.simruntime",
            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-11-4",
            "isAvailable": True,
            "name": "iOS 11.4",
            "version": "11.4"
        },
        {
            "buildversion": "17A844",
            "bundlePath": "/path/Runtimes/iOS 13.1.simruntime",
            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-1",
            "isAvailable": True,
            "name": "iOS 13.1",
            "version": "13.1"
        },
        {
            "buildversion": "17B102",
            "bundlePath": "/path/Runtimes/iOS.simruntime",
            "identifier": "com.apple.CoreSimulator.SimRuntime.iOS-13-2",
            "isAvailable": True,
            "name": "iOS 13.2",
            "version": "13.2.2"
        },
    ]
}


class TestCase(unittest.TestCase):
  """Test case which supports installing mocks. Uninstalls on tear down."""

  def __init__(self, *args, **kwargs):
    """Initializes a new instance of this class."""
    super(TestCase, self).__init__(*args, **kwargs)

    # Maps object to a dict which maps names of mocked members to their
    # original values.
    self._mocks = collections.OrderedDict()

  def mock(self, obj, member, mock):
    """Installs mock in place of the named member of the given obj.

    Args:
      obj: Any object.
      member: String naming the attribute of the object to mock.
      mock: The mock to install.
    """
    self._mocks.setdefault(obj, collections.OrderedDict()).setdefault(
        member, getattr(obj, member))
    setattr(obj, member, mock)

  def tearDown(self, *args, **kwargs):
    """Uninstalls mocks."""
    super(TestCase, self).tearDown(*args, **kwargs)

    for obj in self._mocks:
      for member, original_value in self._mocks[obj].iteritems():
        setattr(obj, member, original_value)


class InstallXcodeTest(TestCase):
  """Tests install_xcode."""

  def setUp(self):
    super(InstallXcodeTest, self).setUp()
    self.mock(test_runner, 'xcode_select', lambda _: None)
    self.mock(os.path, 'exists', lambda _: True)

  def test_success(self):
    self.assertTrue(test_runner.install_xcode('test_build', 'true', 'path'))

  def test_failure(self):
    self.assertFalse(test_runner.install_xcode('test_build', 'false', 'path'))


class SimulatorTestRunnerTest(TestCase):
  """Tests for test_runner.SimulatorTestRunner."""

  def setUp(self):
    super(SimulatorTestRunnerTest, self).setUp()
    self.mock(iossim_util, 'get_simulator_list', lambda: SIMULATORS_LIST)

    def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
      return True

    self.mock(test_runner, 'get_current_xcode_info', lambda: {
        'version': 'test version', 'build': 'test build', 'path': 'test/path'})
    self.mock(test_runner, 'install_xcode', install_xcode)
    self.mock(test_runner.subprocess, 'check_output',
              lambda _: 'fake-bundle-id')
    self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
    self.mock(os.path, 'exists', lambda _: True)
    self.mock(test_runner.TestRunner, 'set_sigterm_handler',
      lambda self, handler: 0)
    self.mock(os, 'listdir', lambda _: [])

  def test_app_not_found(self):
    """Ensures AppNotFoundError is raised."""

    self.mock(os.path, 'exists', lambda p: not p.endswith('fake-app'))

    with self.assertRaises(test_runner.AppNotFoundError):
      test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'platform',
        'os',
        'xcode-version',
        '', # Empty xcode-build
        'out-dir',
      )

  def test_iossim_not_found(self):
    """Ensures SimulatorNotFoundError is raised."""
    self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim'))

    with self.assertRaises(test_runner.SimulatorNotFoundError):
      test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'iPhone X',
        '11.4',
        'xcode-version',
        'xcode-build',
        'out-dir',
      )

  def test_init(self):
    """Ensures instance is created."""
    tr = test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'iPhone X',
        '11.4',
        'xcode-version',
        'xcode-build',
        'out-dir',
    )

    self.assertTrue(tr)

  def test_startup_crash(self):
    """Ensures test is relaunched once on startup crash."""
    def set_up(self):
      return

    @staticmethod
    def _run(cmd, shards=None):
      return collections.namedtuple('result', ['crashed', 'crashed_test'])(
          crashed=True, crashed_test=None)

    def tear_down(self):
      return

    self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
    self.mock(test_runner.TestRunner, '_run', _run)
    self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)

    tr = test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'iPhone X',
        '11.4',
        'xcode-version',
        'xcode-build',
        'out-dir',
        xctest=True,
    )
    with self.assertRaises(test_runner.AppLaunchError):
      tr.launch()

  def test_run(self):
    """Ensures the _run method is correct with test sharding."""

    def run_tests(self, test_shard=None):
      out = []
      for test in test_shard:
        testname = test.split('/')
        out.append('Test Case \'-[%s %s]\' started.' %
                   (testname[0], testname[1]))
        out.append('Test Case \'-[%s %s]\' passed (0.1 seconds)' %
                   (testname[0], testname[1]))
      return (out, 0, 0)

    tr = test_runner.SimulatorTestRunner(
      'fake-app',
      'fake-iossim',
      'iPhone X',
      '11.4',
      'xcode-version',
      'xcode-build',
      'out-dir',
      xctest=True,
    )
    self.mock(test_runner.SimulatorTestRunner, 'run_tests', run_tests)

    tr.xctest_path = 'fake.xctest'
    cmd = tr.get_launch_command()
    result = tr._run(cmd=cmd, shards=3)
    self.assertIn('a/1', result.passed_tests)
    self.assertIn('b/2', result.passed_tests)
    self.assertIn('c/3', result.passed_tests)
    self.assertIn('d/4', result.passed_tests)
    self.assertIn('e/5', result.passed_tests)

  def test_run_with_system_alert(self):
    """Ensures SystemAlertPresentError is raised when warning 'System alert
      view is present, so skipping all tests' is in the output."""
    with self.assertRaises(test_runner.SystemAlertPresentError):
      tr = test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'iPhone X',
        '11.4',
        'xcode-version',
        'xcode-build',
        'out-dir',
        xctest=True,
      )
      tr.xctest_path = 'fake.xctest'
      cmd = ['echo', 'System alert view is present, so skipping all tests!']
      result = tr._run(cmd=cmd)

  def test_get_launch_command(self):
    """Ensures launch command is correct with test_filters, test sharding and
      test_cases."""
    tr = test_runner.SimulatorTestRunner(
      'fake-app',
      'fake-iossim',
      'iPhone X',
      '11.4',
      'xcode-version',
      'xcode-build',
      'out-dir',
    )
    tr.xctest_path = 'fake.xctest'
    # Cases test_filter is not empty, with empty/non-empty self.test_cases.
    tr.test_cases = []
    cmd = tr.get_launch_command(['a'])
    self.assertIn('-t', cmd)
    self.assertIn('a', cmd)

    tr.test_cases = ['a', 'b']
    cmd = tr.get_launch_command(['a'])
    self.assertIn('-t', cmd)
    self.assertIn('a', cmd)
    self.assertNotIn('b', cmd)

    # Cases test_filter is empty, with empty/non-empty self.test_cases.
    tr.test_cases = []
    cmd = tr.get_launch_command(test_filter=None, invert=False)
    self.assertNotIn('-t', cmd)

    tr.test_cases = ['a', 'b']
    cmd = tr.get_launch_command(test_filter=None, invert=False)
    self.assertIn('-t', cmd)
    self.assertIn('a', cmd)
    self.assertIn('b', cmd)

  def test_relaunch(self):
    """Ensures test is relaunched on test crash until tests complete."""
    def set_up(self):
      return

    @staticmethod
    def _run(cmd, shards=None):
      result = collections.namedtuple(
          'result', [
              'crashed',
              'crashed_test',
              'failed_tests',
              'flaked_tests',
              'passed_tests',
          ],
      )
      if '-e' not in cmd:
        # First run, has no test filter supplied. Mock a crash.
        return result(
            crashed=True,
            crashed_test='c',
            failed_tests={'b': ['b-out'], 'c': ['Did not complete.']},
            flaked_tests={'d': ['d-out']},
            passed_tests=['a'],
        )
      else:
        return result(
            crashed=False,
            crashed_test=None,
            failed_tests={},
            flaked_tests={},
            passed_tests=[],
        )

    def tear_down(self):
      return

    self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
    self.mock(test_runner.TestRunner, '_run', _run)
    self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)

    tr = test_runner.SimulatorTestRunner(
        'fake-app',
        'fake-iossim',
        'iPhone X',
        '11.4',
        'xcode-version',
        'xcode-build',
        'out-dir',
    )
    tr.launch()
    self.assertTrue(tr.logs)


class DeviceTestRunnerTest(TestCase):
  def setUp(self):
    super(DeviceTestRunnerTest, self).setUp()

    def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
      return True

    self.mock(test_runner, 'get_current_xcode_info', lambda: {
        'version': 'test version', 'build': 'test build', 'path': 'test/path'})
    self.mock(test_runner, 'install_xcode', install_xcode)
    self.mock(test_runner.subprocess, 'check_output',
              lambda _: 'fake-bundle-id')
    self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
    self.mock(os.path, 'exists', lambda _: True)
    self.mock(os, 'listdir', lambda _: [])
    self.mock(tempfile, 'mkstemp', lambda: '/tmp/tmp_file')

    self.tr = test_runner.DeviceTestRunner(
        'fake-app',
        'xcode-version',
        'xcode-build',
        'out-dir',
    )
    self.tr.xctestrun_data = {'TestTargetName':{}}


if __name__ == '__main__':
  logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
    level=logging.DEBUG, datefmt='%I:%M:%S')
  unittest.main()