Rework how isolates are generated in MB.
This patch changes how MB will generate the .isolate files
for a GN build:
First, we rename 'ninja_to_gn.pyl' to 'gn_isolate_map.pyl',
and change it from a straight compile_target -> gn_label map to
a map of compile_target -> { "label": gn_label, "type": string,
"args": optional list<string> }.
Valid values for "type" are
"windowed_test_launcher"
: the test is a gtest-based test that uses the
'brave-new-test-launcher' from //base/test:test_support and
needs to run under Xvfb if run on an X11-based platform
(i.e., if use_x11=true in GN).
"console_test_launcher"
: the test is a gtest-based test that uses the
'brave-new-test-launcher' from //base/test:test_support but does
not need Xvfb.
"raw"
: the test is just an executable and may take an optional list
of command-line arguments (specified in the 'args' key),
but does not need any extra files or other special processing.
"unknown"
: (the default), which indicates that we don't know what the command
line needs to be (this is a fatal error).
Second, we read the new file in, and build a lookup map of test
types to command lines. We will probably need to keep iterating
on the list of test types and command lines, and perhaps provide
some escape hatches so that we can handle the full variety of tests.
[email protected]
BUG=503942, 504079
Review URL: https://codereview.chromium.org/1239343003
Cr-Commit-Position: refs/heads/master@{#339804}
diff --git a/tools/mb/mb.py b/tools/mb/mb.py
index 026a194..509902f 100755
--- a/tools/mb/mb.py
+++ b/tools/mb/mb.py
@@ -336,14 +336,14 @@
# the compile targets to the matching GN labels.
contents = self.ReadFile(self.args.swarming_targets_file)
swarming_targets = contents.splitlines()
- ninja_targets_to_labels = ast.literal_eval(self.ReadFile(os.path.join(
- self.chromium_src_dir, 'testing', 'buildbot', 'ninja_to_gn.pyl')))
+ gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
+ self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
gn_labels = []
for target in swarming_targets:
- if not target in ninja_targets_to_labels:
+ if not target in gn_isolate_map:
raise MBErr('test target "%s" not found in %s' %
- (target, '//testing/buildbot/ninja_to_gn.pyl'))
- gn_labels.append(ninja_targets_to_labels[target])
+ (target, '//testing/buildbot/gn_isolate_map.pyl'))
+ gn_labels.append(gn_isolate_map[target]['label'])
gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
@@ -363,7 +363,8 @@
if not self.Exists(deps_path):
raise MBErr('did not generate %s' % deps_path)
- command, extra_files = self.GetIsolateCommand(target, vals)
+ command, extra_files = self.GetIsolateCommand(target, vals,
+ gn_isolate_map)
runtime_deps = self.ReadFile(deps_path).splitlines()
@@ -448,15 +449,55 @@
return ret
- def GetIsolateCommand(self, target, vals):
- extra_files = []
+ def RunGNIsolate(self, vals):
+ build_path = self.args.path[0]
+ inp = self.ReadInputJSON(['targets'])
+ if self.args.verbose:
+ self.Print()
+ self.Print('isolate input:')
+ self.PrintJSON(inp)
+ self.Print()
+ output_path = self.args.output_path[0]
- # TODO(dpranke): We should probably pull this from
- # the test list info in //testing/buildbot/*.json,
- # and assert that the test has can_use_on_swarming_builders: True,
- # but we hardcode it here for now.
- test_type = {}.get(target, 'gtest_test')
+ for target in inp['targets']:
+ runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
+ if not self.Exists(runtime_deps_path):
+ self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
+ output_path)
+
+ command, extra_files = self.GetIsolateCommand(target, vals, None)
+
+ runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
+
+
+ isolate_path = self.ToAbsPath(build_path, target + '.isolate')
+ self.WriteFile(isolate_path,
+ pprint.pformat({
+ 'variables': {
+ 'command': command,
+ 'files': sorted(runtime_deps + extra_files),
+ 'read_only': 1,
+ }
+ }) + '\n')
+
+ self.WriteJSON(
+ {
+ 'args': [
+ '--isolated',
+ self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
+ '--isolate',
+ self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
+ ],
+ 'dir': self.chromium_src_dir,
+ 'version': 1,
+ },
+ isolate_path + 'd.gen.json',
+ )
+
+ return 0
+
+ def GetIsolateCommand(self, target, vals, gn_isolate_map):
# This needs to mirror the settings in //build/config/ui.gni:
# use_x11 = is_linux && !use_ozone.
# TODO(dpranke): Figure out how to keep this in sync better.
@@ -470,41 +511,48 @@
executable_suffix = '.exe' if sys.platform == 'win32' else ''
- if test_type == 'gtest_test':
- extra_files.append('../../testing/test_env.py')
+ test_type = gn_isolate_map[target]['type']
+ cmdline = []
+ extra_files = []
- if use_x11:
- # TODO(dpranke): Figure out some way to figure out which
- # test steps really need xvfb.
- extra_files.append('xdisplaycheck')
- extra_files.append('../../testing/xvfb.py')
-
- cmdline = [
- '../../testing/xvfb.py',
- '.',
- './' + str(target),
- '--brave-new-test-launcher',
- '--test-launcher-bot-mode',
- '--asan=%d' % asan,
- '--msan=%d' % msan,
- '--tsan=%d' % tsan,
- ]
- else:
- cmdline = [
+ if use_x11 and test_type == 'windowed_test_launcher':
+ extra_files = [
+ 'xdisplaycheck',
'../../testing/test_env.py',
- '.',
+ '../../testing/xvfb.py',
+ ]
+ cmdline = [
+ '../../testing/xvfb.py',
+ '.',
+ './' + str(target),
+ '--brave-new-test-launcher',
+ '--test-launcher-bot-mode',
+ '--asan=%d' % asan,
+ '--msan=%d' % msan,
+ '--tsan=%d' % tsan,
+ ]
+ elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
+ extra_files = [
+ '../../testing/test_env.py'
+ ]
+ cmdline = [
+ '../../testing/test_env.py',
'./' + str(target) + executable_suffix,
'--brave-new-test-launcher',
'--test-launcher-bot-mode',
'--asan=%d' % asan,
'--msan=%d' % msan,
'--tsan=%d' % tsan,
- ]
- else:
- # TODO(dpranke): Handle script_tests and other types of swarmed tests.
- self.WriteFailureAndRaise('unknown test type "%s" for %s' %
- (test_type, target), output_path=None)
+ ]
+ elif test_type in ('raw'):
+ extra_files = []
+ cmdline = [
+ './' + str(target) + executable_suffix,
+ ] + gn_isolate_map[target].get('args')
+ else:
+ self.WriteFailureAndRaise('No command line for %s found (test type %s).'
+ % (target, test_type), output_path=None)
return cmdline, extra_files
