blob: 294e501c081b8372ac8a8c597968ada7740e00af [file] [log] [blame]
dprankefe4602312015-04-08 16:20:351#!/usr/bin/env python
2# Copyright 2015 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""MB - the Meta-Build wrapper around GYP and GN
7
8MB is a wrapper script for GYP and GN that can be used to generate build files
9for sets of canned configurations and analyze them.
10"""
11
12from __future__ import print_function
13
14import argparse
15import ast
dprankec3441d12015-06-23 23:01:3516import errno
dprankefe4602312015-04-08 16:20:3517import json
18import os
19import pipes
dpranked8113582015-06-05 20:08:2520import pprint
dprankefe4602312015-04-08 16:20:3521import shlex
22import shutil
23import sys
24import subprocess
dprankef61de2f2015-05-14 04:09:5625import tempfile
dprankefe4602312015-04-08 16:20:3526
27
28def main(args):
dprankeee5b51f62015-04-09 00:03:2229 mbw = MetaBuildWrapper()
30 mbw.ParseArgs(args)
31 return mbw.args.func()
dprankefe4602312015-04-08 16:20:3532
33
34class MetaBuildWrapper(object):
35 def __init__(self):
36 p = os.path
37 d = os.path.dirname
38 self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
39 self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
40 'mb_config.pyl')
dpranked1fba482015-04-14 20:54:5141 self.platform = sys.platform
dprankefe4602312015-04-08 16:20:3542 self.args = argparse.Namespace()
43 self.configs = {}
44 self.masters = {}
45 self.mixins = {}
46 self.private_configs = []
47 self.common_dev_configs = []
48 self.unsupported_configs = []
49
50 def ParseArgs(self, argv):
51 def AddCommonOptions(subp):
52 subp.add_argument('-b', '--builder',
53 help='builder name to look up config from')
54 subp.add_argument('-m', '--master',
55 help='master name to look up config from')
56 subp.add_argument('-c', '--config',
57 help='configuration to analyze')
58 subp.add_argument('-f', '--config-file', metavar='PATH',
59 default=self.default_config,
60 help='path to config file '
61 '(default is //tools/mb/mb_config.pyl)')
62 subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
63 help='path to goma directory (default is %(default)s).')
64 subp.add_argument('-n', '--dryrun', action='store_true',
65 help='Do a dry run (i.e., do nothing, just print '
66 'the commands that will run)')
67 subp.add_argument('-q', '--quiet', action='store_true',
dprankea5a77ca2015-07-16 23:24:1768 help='Do not print anything on success, '
69 'just return an exit code.')
dprankefe4602312015-04-08 16:20:3570 subp.add_argument('-v', '--verbose', action='count',
71 help='verbose logging (may specify multiple times).')
72
73 parser = argparse.ArgumentParser(prog='mb')
74 subps = parser.add_subparsers()
75
76 subp = subps.add_parser('analyze',
77 help='analyze whether changes to a set of files '
78 'will cause a set of binaries to be rebuilt.')
79 AddCommonOptions(subp)
tsergeantf061c992015-07-16 01:34:0080 subp.add_argument('--swarming-targets-file',
81 help='save runtime dependencies for targets listed '
82 'in file.')
dpranked8113582015-06-05 20:08:2583 subp.add_argument('path', nargs=1,
dprankefe4602312015-04-08 16:20:3584 help='path build was generated into.')
85 subp.add_argument('input_path', nargs=1,
86 help='path to a file containing the input arguments '
87 'as a JSON object.')
88 subp.add_argument('output_path', nargs=1,
89 help='path to a file containing the output arguments '
90 'as a JSON object.')
91 subp.set_defaults(func=self.CmdAnalyze)
92
93 subp = subps.add_parser('gen',
94 help='generate a new set of build files')
95 AddCommonOptions(subp)
dpranke74559b52015-06-10 21:20:3996 subp.add_argument('--swarming-targets-file',
97 help='save runtime dependencies for targets listed '
98 'in file.')
dpranked8113582015-06-05 20:08:2599 subp.add_argument('path', nargs=1,
dprankefe4602312015-04-08 16:20:35100 help='path to generate build into')
101 subp.set_defaults(func=self.CmdGen)
102
103 subp = subps.add_parser('lookup',
104 help='look up the command for a given config or '
105 'builder')
106 AddCommonOptions(subp)
107 subp.set_defaults(func=self.CmdLookup)
108
109 subp = subps.add_parser('validate',
110 help='validate the config file')
dprankea5a77ca2015-07-16 23:24:17111 subp.add_argument('-f', '--config-file', metavar='PATH',
112 default=self.default_config,
113 help='path to config file '
114 '(default is //tools/mb/mb_config.pyl)')
115 subp.add_argument('-q', '--quiet', action='store_true',
116 help='Do not print anything on success, '
117 'just return an exit code.')
dprankefe4602312015-04-08 16:20:35118 subp.set_defaults(func=self.CmdValidate)
119
120 subp = subps.add_parser('help',
121 help='Get help on a subcommand.')
122 subp.add_argument(nargs='?', action='store', dest='subcommand',
123 help='The command to get help for.')
124 subp.set_defaults(func=self.CmdHelp)
125
126 self.args = parser.parse_args(argv)
127
128 def CmdAnalyze(self):
129 vals = self.GetConfig()
130 if vals['type'] == 'gn':
131 return self.RunGNAnalyze(vals)
132 elif vals['type'] == 'gyp':
133 return self.RunGYPAnalyze(vals)
134 else:
135 raise MBErr('Unknown meta-build type "%s"' % vals['type'])
136
137 def CmdGen(self):
138 vals = self.GetConfig()
139 if vals['type'] == 'gn':
Dirk Pranke0fd41bcd2015-06-19 00:05:50140 return self.RunGNGen(vals)
dpranke08d2ab12015-04-24 21:54:20141 if vals['type'] == 'gyp':
Dirk Pranke0fd41bcd2015-06-19 00:05:50142 return self.RunGYPGen(vals)
dpranke08d2ab12015-04-24 21:54:20143
144 raise MBErr('Unknown meta-build type "%s"' % vals['type'])
dprankefe4602312015-04-08 16:20:35145
146 def CmdLookup(self):
147 vals = self.GetConfig()
148 if vals['type'] == 'gn':
dpranked1fba482015-04-14 20:54:51149 cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
dprankefe4602312015-04-08 16:20:35150 elif vals['type'] == 'gyp':
151 cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
152 else:
153 raise MBErr('Unknown meta-build type "%s"' % vals['type'])
154
155 self.PrintCmd(cmd)
156 return 0
157
158 def CmdHelp(self):
159 if self.args.subcommand:
160 self.ParseArgs([self.args.subcommand, '--help'])
161 else:
162 self.ParseArgs(['--help'])
163
164 def CmdValidate(self):
165 errs = []
166
167 # Read the file to make sure it parses.
168 self.ReadConfigFile()
169
170 # Figure out the whole list of configs and ensure that no config is
171 # listed in more than one category.
172 all_configs = {}
173 for config in self.common_dev_configs:
174 all_configs[config] = 'common_dev_configs'
175 for config in self.private_configs:
176 if config in all_configs:
177 errs.append('config "%s" listed in "private_configs" also '
178 'listed in "%s"' % (config, all_configs['config']))
179 else:
180 all_configs[config] = 'private_configs'
181 for config in self.unsupported_configs:
182 if config in all_configs:
183 errs.append('config "%s" listed in "unsupported_configs" also '
184 'listed in "%s"' % (config, all_configs['config']))
185 else:
186 all_configs[config] = 'unsupported_configs'
187
188 for master in self.masters:
189 for builder in self.masters[master]:
190 config = self.masters[master][builder]
191 if config in all_configs and all_configs[config] not in self.masters:
192 errs.append('Config "%s" used by a bot is also listed in "%s".' %
193 (config, all_configs[config]))
194 else:
195 all_configs[config] = master
196
197 # Check that every referenced config actually exists.
198 for config, loc in all_configs.items():
199 if not config in self.configs:
200 errs.append('Unknown config "%s" referenced from "%s".' %
201 (config, loc))
202
203 # Check that every actual config is actually referenced.
204 for config in self.configs:
205 if not config in all_configs:
206 errs.append('Unused config "%s".' % config)
207
208 # Figure out the whole list of mixins, and check that every mixin
209 # listed by a config or another mixin actually exists.
210 referenced_mixins = set()
211 for config, mixins in self.configs.items():
212 for mixin in mixins:
213 if not mixin in self.mixins:
214 errs.append('Unknown mixin "%s" referenced by config "%s".' %
215 (mixin, config))
216 referenced_mixins.add(mixin)
217
218 for mixin in self.mixins:
219 for sub_mixin in self.mixins[mixin].get('mixins', []):
220 if not sub_mixin in self.mixins:
221 errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
222 (sub_mixin, mixin))
223 referenced_mixins.add(sub_mixin)
224
225 # Check that every mixin defined is actually referenced somewhere.
226 for mixin in self.mixins:
227 if not mixin in referenced_mixins:
228 errs.append('Unreferenced mixin "%s".' % mixin)
229
230 if errs:
231 raise MBErr('mb config file %s has problems:\n ' + '\n '.join(errs))
232
233 if not self.args.quiet:
234 self.Print('mb config file %s looks ok.' % self.args.config_file)
235 return 0
236
237 def GetConfig(self):
238 self.ReadConfigFile()
239 config = self.ConfigFromArgs()
240 if not config in self.configs:
241 raise MBErr('Config "%s" not found in %s' %
242 (config, self.args.config_file))
243
244 return self.FlattenConfig(config)
245
246 def ReadConfigFile(self):
247 if not self.Exists(self.args.config_file):
248 raise MBErr('config file not found at %s' % self.args.config_file)
249
250 try:
251 contents = ast.literal_eval(self.ReadFile(self.args.config_file))
252 except SyntaxError as e:
253 raise MBErr('Failed to parse config file "%s": %s' %
254 (self.args.config_file, e))
255
256 self.common_dev_configs = contents['common_dev_configs']
257 self.configs = contents['configs']
258 self.masters = contents['masters']
259 self.mixins = contents['mixins']
260 self.private_configs = contents['private_configs']
261 self.unsupported_configs = contents['unsupported_configs']
262
263 def ConfigFromArgs(self):
264 if self.args.config:
265 if self.args.master or self.args.builder:
266 raise MBErr('Can not specific both -c/--config and -m/--master or '
267 '-b/--builder')
268
269 return self.args.config
270
271 if not self.args.master or not self.args.builder:
272 raise MBErr('Must specify either -c/--config or '
273 '(-m/--master and -b/--builder)')
274
275 if not self.args.master in self.masters:
276 raise MBErr('Master name "%s" not found in "%s"' %
277 (self.args.master, self.args.config_file))
278
279 if not self.args.builder in self.masters[self.args.master]:
280 raise MBErr('Builder name "%s" not found under masters[%s] in "%s"' %
281 (self.args.builder, self.args.master, self.args.config_file))
282
283 return self.masters[self.args.master][self.args.builder]
284
285 def FlattenConfig(self, config):
286 mixins = self.configs[config]
287 vals = {
288 'type': None,
289 'gn_args': [],
290 'gyp_config': [],
291 'gyp_defines': [],
292 }
293
294 visited = []
295 self.FlattenMixins(mixins, vals, visited)
296 return vals
297
298 def FlattenMixins(self, mixins, vals, visited):
299 for m in mixins:
300 if m not in self.mixins:
301 raise MBErr('Unknown mixin "%s"' % m)
dprankeee5b51f62015-04-09 00:03:22302
303 # TODO: check for cycles in mixins.
dprankefe4602312015-04-08 16:20:35304
305 visited.append(m)
306
307 mixin_vals = self.mixins[m]
308 if 'type' in mixin_vals:
309 vals['type'] = mixin_vals['type']
310 if 'gn_args' in mixin_vals:
311 if vals['gn_args']:
312 vals['gn_args'] += ' ' + mixin_vals['gn_args']
313 else:
314 vals['gn_args'] = mixin_vals['gn_args']
315 if 'gyp_config' in mixin_vals:
316 vals['gyp_config'] = mixin_vals['gyp_config']
317 if 'gyp_defines' in mixin_vals:
318 if vals['gyp_defines']:
319 vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
320 else:
321 vals['gyp_defines'] = mixin_vals['gyp_defines']
322 if 'mixins' in mixin_vals:
323 self.FlattenMixins(mixin_vals['mixins'], vals, visited)
324 return vals
325
Dirk Pranke0fd41bcd2015-06-19 00:05:50326 def RunGNGen(self, vals):
327 path = self.args.path[0]
328
dpranked1fba482015-04-14 20:54:51329 cmd = self.GNCmd('gen', path, vals['gn_args'])
dpranke74559b52015-06-10 21:20:39330
331 swarming_targets = []
332 if self.args.swarming_targets_file:
333 # We need GN to generate the list of runtime dependencies for
334 # the compile targets listed (one per line) in the file so
335 # we can run them via swarming. We use ninja_to_gn.pyl to convert
336 # the compile targets to the matching GN labels.
337 contents = self.ReadFile(self.args.swarming_targets_file)
338 swarming_targets = contents.splitlines()
339 ninja_targets_to_labels = ast.literal_eval(self.ReadFile(os.path.join(
340 self.chromium_src_dir, 'testing', 'buildbot', 'ninja_to_gn.pyl')))
341 gn_labels = []
342 for target in swarming_targets:
343 if not target in ninja_targets_to_labels:
344 raise MBErr('test target "%s" not found in %s' %
345 (target, '//testing/buildbot/ninja_to_gn.pyl'))
346 gn_labels.append(ninja_targets_to_labels[target])
347
348 gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
dprankec3441d12015-06-23 23:01:35349
350 # Since GN hasn't run yet, the build directory may not even exist.
351 self.MaybeMakeDirectory(self.ToAbsPath(path))
352
dpranke74559b52015-06-10 21:20:39353 self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
354 cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
355
dprankefe4602312015-04-08 16:20:35356 ret, _, _ = self.Run(cmd)
dpranke74559b52015-06-10 21:20:39357
358 for target in swarming_targets:
dpranke34bd39d2015-06-24 02:36:52359 if sys.platform == 'win32':
360 deps_path = self.ToAbsPath(path, target + '.exe.runtime_deps')
361 else:
362 deps_path = self.ToAbsPath(path, target + '.runtime_deps')
dpranke74559b52015-06-10 21:20:39363 if not self.Exists(deps_path):
364 raise MBErr('did not generate %s' % deps_path)
365
dpranked5b2b9432015-06-23 16:55:30366 command, extra_files = self.GetIsolateCommand(target, vals)
367
368 runtime_deps = self.ReadFile(deps_path).splitlines()
369
370 isolate_path = self.ToAbsPath(path, target + '.isolate')
371 self.WriteFile(isolate_path,
372 pprint.pformat({
373 'variables': {
374 'command': command,
375 'files': sorted(runtime_deps + extra_files),
376 'read_only': 1,
377 }
378 }) + '\n')
379
380 self.WriteJSON(
381 {
382 'args': [
383 '--isolated',
dpranke34bd39d2015-06-24 02:36:52384 self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
dpranked5b2b9432015-06-23 16:55:30385 '--isolate',
dpranke34bd39d2015-06-24 02:36:52386 self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
dpranked5b2b9432015-06-23 16:55:30387 ],
388 'dir': self.chromium_src_dir,
389 'version': 1,
390 },
391 isolate_path + 'd.gen.json',
392 )
393
394
dprankefe4602312015-04-08 16:20:35395 return ret
396
dpranked1fba482015-04-14 20:54:51397 def GNCmd(self, subcommand, path, gn_args=''):
398 if self.platform == 'linux2':
399 gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
400 'gn')
401 elif self.platform == 'darwin':
402 gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
403 'gn')
404 else:
405 gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
406 'gn.exe')
407
408 cmd = [gn_path, subcommand, path]
dprankeee5b51f62015-04-09 00:03:22409 gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
dprankefe4602312015-04-08 16:20:35410 if gn_args:
411 cmd.append('--args=%s' % gn_args)
412 return cmd
413
Dirk Pranke0fd41bcd2015-06-19 00:05:50414 def RunGYPGen(self, vals):
415 path = self.args.path[0]
416
dprankefe4602312015-04-08 16:20:35417 output_dir, gyp_config = self.ParseGYPConfigPath(path)
418 if gyp_config != vals['gyp_config']:
419 raise MBErr('The last component of the path (%s) must match the '
420 'GYP configuration specified in the config (%s), and '
421 'it does not.' % (gyp_config, vals['gyp_config']))
422 cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
423 ret, _, _ = self.Run(cmd)
424 return ret
425
426 def RunGYPAnalyze(self, vals):
427 output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
428 if gyp_config != vals['gyp_config']:
429 raise MBErr('The last component of the path (%s) must match the '
430 'GYP configuration specified in the config (%s), and '
431 'it does not.' % (gyp_config, vals['gyp_config']))
dprankecda00332015-04-11 04:18:32432 if self.args.verbose:
433 inp = self.GetAnalyzeInput()
434 self.Print()
435 self.Print('analyze input:')
436 self.PrintJSON(inp)
437 self.Print()
438
dprankefe4602312015-04-08 16:20:35439 cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
440 cmd.extend(['-G', 'config_path=%s' % self.args.input_path[0],
441 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
442 ret, _, _ = self.Run(cmd)
dprankecda00332015-04-11 04:18:32443 if not ret and self.args.verbose:
444 outp = json.loads(self.ReadFile(self.args.output_path[0]))
445 self.Print()
446 self.Print('analyze output:')
dpranke74559b52015-06-10 21:20:39447 self.PrintJSON(outp)
dprankecda00332015-04-11 04:18:32448 self.Print()
449
dprankefe4602312015-04-08 16:20:35450 return ret
451
dpranked8113582015-06-05 20:08:25452 def GetIsolateCommand(self, target, vals):
dpranked8113582015-06-05 20:08:25453 extra_files = []
454
455 # TODO(dpranke): We should probably pull this from
456 # the test list info in //testing/buildbot/*.json,
457 # and assert that the test has can_use_on_swarming_builders: True,
458 # but we hardcode it here for now.
459 test_type = {}.get(target, 'gtest_test')
460
461 # This needs to mirror the settings in //build/config/ui.gni:
462 # use_x11 = is_linux && !use_ozone.
463 # TODO(dpranke): Figure out how to keep this in sync better.
464 use_x11 = (sys.platform == 'linux2' and
465 not 'target_os="android"' in vals['gn_args'] and
466 not 'use_ozone=true' in vals['gn_args'])
467
468 asan = 'is_asan=true' in vals['gn_args']
469 msan = 'is_msan=true' in vals['gn_args']
470 tsan = 'is_tsan=true' in vals['gn_args']
471
472 executable_suffix = '.exe' if sys.platform == 'win32' else ''
473
474 if test_type == 'gtest_test':
475 extra_files.append('../../testing/test_env.py')
476
477 if use_x11:
478 # TODO(dpranke): Figure out some way to figure out which
479 # test steps really need xvfb.
480 extra_files.append('xdisplaycheck')
481 extra_files.append('../../testing/xvfb.py')
482
483 cmdline = [
484 '../../testing/xvfb.py',
485 '.',
486 './' + str(target),
487 '--brave-new-test-launcher',
488 '--test-launcher-bot-mode',
489 '--asan=%d' % asan,
490 '--msan=%d' % msan,
491 '--tsan=%d' % tsan,
492 ]
493 else:
494 cmdline = [
495 '../../testing/test_env.py',
496 '.',
497 './' + str(target) + executable_suffix,
498 '--brave-new-test-launcher',
499 '--test-launcher-bot-mode',
500 '--asan=%d' % asan,
501 '--msan=%d' % msan,
502 '--tsan=%d' % tsan,
503 ]
504 else:
dpranked5b2b9432015-06-23 16:55:30505 # TODO(dpranke): Handle script_tests and other types of swarmed tests.
dpranked8113582015-06-05 20:08:25506 self.WriteFailureAndRaise('unknown test type "%s" for %s' %
dpranked5b2b9432015-06-23 16:55:30507 (test_type, target), output_path=None)
dpranked8113582015-06-05 20:08:25508
509
510 return cmdline, extra_files
511
dpranke74559b52015-06-10 21:20:39512 def ToAbsPath(self, build_path, *comps):
dpranked8113582015-06-05 20:08:25513 return os.path.join(self.chromium_src_dir,
514 self.ToSrcRelPath(build_path),
dpranke74559b52015-06-10 21:20:39515 *comps)
dpranked8113582015-06-05 20:08:25516
dprankeee5b51f62015-04-09 00:03:22517 def ToSrcRelPath(self, path):
518 """Returns a relative path from the top of the repo."""
519 # TODO: Support normal paths in addition to source-absolute paths.
dprankefe4602312015-04-08 16:20:35520 assert(path.startswith('//'))
dpranke34bd39d2015-06-24 02:36:52521 return path[2:].replace('/', os.sep)
dprankefe4602312015-04-08 16:20:35522
523 def ParseGYPConfigPath(self, path):
dprankeee5b51f62015-04-09 00:03:22524 rpath = self.ToSrcRelPath(path)
525 output_dir, _, config = rpath.rpartition('/')
dprankefe4602312015-04-08 16:20:35526 self.CheckGYPConfigIsSupported(config, path)
527 return output_dir, config
528
529 def CheckGYPConfigIsSupported(self, config, path):
530 if config not in ('Debug', 'Release'):
531 if (sys.platform in ('win32', 'cygwin') and
532 config not in ('Debug_x64', 'Release_x64')):
533 raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
534 config, path)
535
536 def GYPCmd(self, output_dir, gyp_defines, config):
537 gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
538 cmd = [
539 sys.executable,
540 os.path.join('build', 'gyp_chromium'),
541 '-G',
542 'output_dir=' + output_dir,
543 '-G',
544 'config=' + config,
545 ]
546 for d in shlex.split(gyp_defines):
547 cmd += ['-D', d]
548 return cmd
549
Dirk Pranke0fd41bcd2015-06-19 00:05:50550 def RunGNAnalyze(self, vals):
551 # analyze runs before 'gn gen' now, so we need to run gn gen
552 # in order to ensure that we have a build directory.
553 ret = self.RunGNGen(vals)
554 if ret:
555 return ret
556
dpranked8113582015-06-05 20:08:25557 inp = self.ReadInputJSON(['files', 'targets'])
dprankecda00332015-04-11 04:18:32558 if self.args.verbose:
559 self.Print()
560 self.Print('analyze input:')
561 self.PrintJSON(inp)
562 self.Print()
563
564 output_path = self.args.output_path[0]
dprankefe4602312015-04-08 16:20:35565
566 # Bail out early if a GN file was modified, since 'gn refs' won't know
567 # what to do about it.
568 if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
Dirk Prankec965fa32015-04-14 23:46:29569 self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
dprankefe4602312015-04-08 16:20:35570 return 0
571
dprankef61de2f2015-05-14 04:09:56572 # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
573 if 'all' in inp['targets']:
dpranke76734662015-04-16 02:17:50574 self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
575 return 0
576
Dirk Pranke12ee2db2015-04-14 23:15:32577 ret = 0
dprankef61de2f2015-05-14 04:09:56578 response_file = self.TempFile()
579 response_file.write('\n'.join(inp['files']) + '\n')
580 response_file.close()
581
582 matching_targets = []
583 try:
dpranked1fba482015-04-14 20:54:51584 cmd = self.GNCmd('refs', self.args.path[0]) + [
dpranke067d0142015-05-14 22:52:45585 '@%s' % response_file.name, '--all', '--as=output']
dprankecda00332015-04-11 04:18:32586 ret, out, _ = self.Run(cmd)
dpranke0b3b7882015-04-24 03:38:12587 if ret and not 'The input matches no targets' in out:
dprankecda00332015-04-11 04:18:32588 self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
589 output_path)
dprankef61de2f2015-05-14 04:09:56590 build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
591 for output in out.splitlines():
592 build_output = output.replace(build_dir, '')
593 if build_output in inp['targets']:
594 matching_targets.append(build_output)
dpranke067d0142015-05-14 22:52:45595
596 cmd = self.GNCmd('refs', self.args.path[0]) + [
597 '@%s' % response_file.name, '--all']
598 ret, out, _ = self.Run(cmd)
599 if ret and not 'The input matches no targets' in out:
600 self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
601 output_path)
602 for label in out.splitlines():
603 build_target = label[2:]
604 # We want to accept 'chrome/android:chrome_shell_apk' and
605 # just 'chrome_shell_apk'. This may result in too many targets
606 # getting built, but we can adjust that later if need be.
607 for input_target in inp['targets']:
608 if (input_target == build_target or
609 build_target.endswith(':' + input_target)):
610 matching_targets.append(input_target)
dprankef61de2f2015-05-14 04:09:56611 finally:
612 self.RemoveFile(response_file.name)
dprankefe4602312015-04-08 16:20:35613
dprankef61de2f2015-05-14 04:09:56614 if matching_targets:
dprankefe4602312015-04-08 16:20:35615 # TODO: it could be that a target X might depend on a target Y
616 # and both would be listed in the input, but we would only need
617 # to specify target X as a build_target (whereas both X and Y are
618 # targets). I'm not sure if that optimization is generally worth it.
dprankef61de2f2015-05-14 04:09:56619 self.WriteJSON({'targets': sorted(matching_targets),
620 'build_targets': sorted(matching_targets),
dprankecda00332015-04-11 04:18:32621 'status': 'Found dependency'}, output_path)
dprankefe4602312015-04-08 16:20:35622 else:
623 self.WriteJSON({'targets': [],
624 'build_targets': [],
dprankecda00332015-04-11 04:18:32625 'status': 'No dependency'}, output_path)
626
627 if not ret and self.args.verbose:
628 outp = json.loads(self.ReadFile(output_path))
629 self.Print()
630 self.Print('analyze output:')
631 self.PrintJSON(outp)
632 self.Print()
dprankefe4602312015-04-08 16:20:35633
634 return 0
635
dpranked8113582015-06-05 20:08:25636 def ReadInputJSON(self, required_keys):
dprankefe4602312015-04-08 16:20:35637 path = self.args.input_path[0]
dprankecda00332015-04-11 04:18:32638 output_path = self.args.output_path[0]
dprankefe4602312015-04-08 16:20:35639 if not self.Exists(path):
dprankecda00332015-04-11 04:18:32640 self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
dprankefe4602312015-04-08 16:20:35641
642 try:
643 inp = json.loads(self.ReadFile(path))
644 except Exception as e:
645 self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
dprankecda00332015-04-11 04:18:32646 (path, e), output_path)
dpranked8113582015-06-05 20:08:25647
648 for k in required_keys:
649 if not k in inp:
650 self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
651 output_path)
dprankefe4602312015-04-08 16:20:35652
653 return inp
654
dpranked5b2b9432015-06-23 16:55:30655 def WriteFailureAndRaise(self, msg, output_path):
656 if output_path:
657 self.WriteJSON({'error': msg}, output_path)
dprankefe4602312015-04-08 16:20:35658 raise MBErr(msg)
659
dprankecda00332015-04-11 04:18:32660 def WriteJSON(self, obj, path):
661 try:
662 self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
663 except Exception as e:
664 raise MBErr('Error %s writing to the output path "%s"' %
665 (e, path))
dprankefe4602312015-04-08 16:20:35666
667 def PrintCmd(self, cmd):
668 if cmd[0] == sys.executable:
669 cmd = ['python'] + cmd[1:]
670 self.Print(*[pipes.quote(c) for c in cmd])
671
dprankecda00332015-04-11 04:18:32672 def PrintJSON(self, obj):
673 self.Print(json.dumps(obj, indent=2, sort_keys=True))
674
dprankefe4602312015-04-08 16:20:35675 def Print(self, *args, **kwargs):
676 # This function largely exists so it can be overridden for testing.
677 print(*args, **kwargs)
678
679 def Run(self, cmd):
680 # This function largely exists so it can be overridden for testing.
681 if self.args.dryrun or self.args.verbose:
682 self.PrintCmd(cmd)
683 if self.args.dryrun:
684 return 0, '', ''
685 ret, out, err = self.Call(cmd)
686 if self.args.verbose:
687 if out:
dprankeee5b51f62015-04-09 00:03:22688 self.Print(out, end='')
dprankefe4602312015-04-08 16:20:35689 if err:
dprankeee5b51f62015-04-09 00:03:22690 self.Print(err, end='', file=sys.stderr)
dprankefe4602312015-04-08 16:20:35691 return ret, out, err
692
693 def Call(self, cmd):
694 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
695 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
696 out, err = p.communicate()
697 return p.returncode, out, err
698
699 def ExpandUser(self, path):
700 # This function largely exists so it can be overridden for testing.
701 return os.path.expanduser(path)
702
703 def Exists(self, path):
704 # This function largely exists so it can be overridden for testing.
705 return os.path.exists(path)
706
dprankec3441d12015-06-23 23:01:35707 def MaybeMakeDirectory(self, path):
708 try:
709 os.makedirs(path)
710 except OSError, e:
711 if e.errno != errno.EEXIST:
712 raise
713
dprankefe4602312015-04-08 16:20:35714 def ReadFile(self, path):
715 # This function largely exists so it can be overriden for testing.
716 with open(path) as fp:
717 return fp.read()
718
dprankef61de2f2015-05-14 04:09:56719 def RemoveFile(self, path):
720 # This function largely exists so it can be overriden for testing.
721 os.remove(path)
722
723 def TempFile(self, mode='w'):
724 # This function largely exists so it can be overriden for testing.
725 return tempfile.NamedTemporaryFile(mode=mode, delete=False)
726
dprankefe4602312015-04-08 16:20:35727 def WriteFile(self, path, contents):
728 # This function largely exists so it can be overriden for testing.
dpranked5b2b9432015-06-23 16:55:30729 if self.args.dryrun or self.args.verbose:
730 self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
dprankefe4602312015-04-08 16:20:35731 with open(path, 'w') as fp:
732 return fp.write(contents)
733
dprankef61de2f2015-05-14 04:09:56734
dprankefe4602312015-04-08 16:20:35735class MBErr(Exception):
736 pass
737
738
739if __name__ == '__main__':
740 try:
741 sys.exit(main(sys.argv[1:]))
742 except MBErr as e:
743 print(e)
744 sys.exit(1)
745 except KeyboardInterrupt:
746 print("interrupted, exiting", stream=sys.stderr)
747 sys.exit(130)