blob: 51de8faf5a3d7e6151cc4ab124c94f0d802206e4 [file] [log] [blame]
[email protected]0a88a652012-03-09 00:34:451#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Sets environment variables needed to run a chromium unit test."""
7
Trent Aptedb1852432018-01-25 02:15:108import io
[email protected]0a88a652012-03-09 00:34:459import os
[email protected]76361be452012-08-30 22:48:1410import stat
[email protected]0a88a652012-03-09 00:34:4511import subprocess
12import sys
Trent Aptedb1852432018-01-25 02:15:1013import time
[email protected]0a88a652012-03-09 00:34:4514
15# This is hardcoded to be src/ relative to this script.
16ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
17
[email protected]76361be452012-08-30 22:48:1418CHROME_SANDBOX_ENV = 'CHROME_DEVEL_SANDBOX'
19CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
20
21
earthdok748e1352015-01-27 23:04:0822def get_sandbox_env(env):
23 """Returns the environment flags needed for the SUID sandbox to work."""
John Abd-El-Malek14ae0542014-10-15 17:52:3124 extra_env = {}
[email protected]76361be452012-08-30 22:48:1425 chrome_sandbox_path = env.get(CHROME_SANDBOX_ENV, CHROME_SANDBOX_PATH)
earthdok748e1352015-01-27 23:04:0826 # The above would silently disable the SUID sandbox if the env value were
27 # an empty string. We don't want to allow that. http://crbug.com/245376
28 # TODO(jln): Remove this check once it's no longer possible to disable the
29 # sandbox that way.
30 if not chrome_sandbox_path:
31 chrome_sandbox_path = CHROME_SANDBOX_PATH
32 extra_env[CHROME_SANDBOX_ENV] = chrome_sandbox_path
John Abd-El-Malek14ae0542014-10-15 17:52:3133
34 return extra_env
35
36
37def trim_cmd(cmd):
38 """Removes internal flags from cmd since they're just used to communicate from
39 the host machine to this script running on the swarm slaves."""
earthdokeeb065302015-02-04 18:18:0440 sanitizers = ['asan', 'lsan', 'msan', 'tsan']
41 internal_flags = frozenset('--%s=%d' % (name, value)
42 for name in sanitizers
43 for value in [0, 1])
John Abd-El-Malek14ae0542014-10-15 17:52:3144 return [i for i in cmd if i not in internal_flags]
[email protected]76361be452012-08-30 22:48:1445
[email protected]0a88a652012-03-09 00:34:4546
[email protected]4bf4d632012-05-31 15:50:3047def fix_python_path(cmd):
48 """Returns the fixed command line to call the right python executable."""
49 out = cmd[:]
50 if out[0] == 'python':
51 out[0] = sys.executable
52 elif out[0].endswith('.py'):
53 out.insert(0, sys.executable)
54 return out
55
56
pcc46233c22017-06-20 22:11:4157def get_sanitizer_env(cmd, asan, lsan, msan, tsan, cfi_diag):
earthdokeeb065302015-02-04 18:18:0458 """Returns the envirnoment flags needed for sanitizer tools."""
John Abd-El-Malek14ae0542014-10-15 17:52:3159
60 extra_env = {}
61
earthdokeeb065302015-02-04 18:18:0462 # Instruct GTK to use malloc while running sanitizer-instrumented tests.
earthdoke6c54a42014-10-16 18:02:3663 extra_env['G_SLICE'] = 'always-malloc'
John Abd-El-Malek14ae0542014-10-15 17:52:3164
65 extra_env['NSS_DISABLE_ARENA_FREE_LIST'] = '1'
66 extra_env['NSS_DISABLE_UNLOAD'] = '1'
67
68 # TODO(glider): remove the symbolizer path once
69 # https://code.google.com/p/address-sanitizer/issues/detail?id=134 is fixed.
Nico Weber19097312016-01-29 18:13:1570 symbolizer_path = os.path.join(ROOT_DIR,
71 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer')
John Abd-El-Malek14ae0542014-10-15 17:52:3172
earthdokeeb065302015-02-04 18:18:0473 if lsan or tsan:
John Abd-El-Malek14ae0542014-10-15 17:52:3174 # LSan is not sandbox-compatible, so we can use online symbolization. In
75 # fact, it needs symbolization to be able to apply suppressions.
76 symbolization_options = ['symbolize=1',
77 'external_symbolizer_path=%s' % symbolizer_path]
pcc46233c22017-06-20 22:11:4178 elif (asan or msan or cfi_diag) and sys.platform not in ['win32', 'cygwin']:
timurrrr064f9522015-02-12 15:46:3279 # ASan uses a script for offline symbolization, except on Windows.
John Abd-El-Malek14ae0542014-10-15 17:52:3180 # Important note: when running ASan with leak detection enabled, we must use
81 # the LSan symbolization options above.
82 symbolization_options = ['symbolize=0']
vadimsh5cf3c442014-10-20 10:22:3883 # Set the path to llvm-symbolizer to be used by asan_symbolize.py
84 extra_env['LLVM_SYMBOLIZER_PATH'] = symbolizer_path
timurrrr7e67df92015-02-12 21:50:5285 else:
86 symbolization_options = []
John Abd-El-Malek14ae0542014-10-15 17:52:3187
earthdokeeb065302015-02-04 18:18:0488 if asan:
89 asan_options = symbolization_options[:]
90 if lsan:
91 asan_options.append('detect_leaks=1')
John Abd-El-Malek14ae0542014-10-15 17:52:3192
timurrrr7e67df92015-02-12 21:50:5293 if asan_options:
94 extra_env['ASAN_OPTIONS'] = ' '.join(asan_options)
John Abd-El-Malek14ae0542014-10-15 17:52:3195
earthdokeeb065302015-02-04 18:18:0496 if sys.platform == 'darwin':
97 isolate_output_dir = os.path.abspath(os.path.dirname(cmd[0]))
98 # This is needed because the test binary has @executable_path embedded in
99 # it that the OS tries to resolve to the cache directory and not the
100 # mapped directory.
101 extra_env['DYLD_LIBRARY_PATH'] = str(isolate_output_dir)
102
103 if lsan:
104 if asan or msan:
105 lsan_options = []
106 else:
107 lsan_options = symbolization_options[:]
108 if sys.platform == 'linux2':
109 # Use the debug version of libstdc++ under LSan. If we don't, there will
110 # be a lot of incomplete stack traces in the reports.
111 extra_env['LD_LIBRARY_PATH'] = '/usr/lib/x86_64-linux-gnu/debug:'
112
earthdokeeb065302015-02-04 18:18:04113 extra_env['LSAN_OPTIONS'] = ' '.join(lsan_options)
114
115 if msan:
116 msan_options = symbolization_options[:]
117 if lsan:
118 msan_options.append('detect_leaks=1')
119 extra_env['MSAN_OPTIONS'] = ' '.join(msan_options)
120
121 if tsan:
122 tsan_options = symbolization_options[:]
123 extra_env['TSAN_OPTIONS'] = ' '.join(tsan_options)
John Abd-El-Malek14ae0542014-10-15 17:52:31124
pcc46233c22017-06-20 22:11:41125 # CFI uses the UBSan runtime to provide diagnostics.
126 if cfi_diag:
127 ubsan_options = symbolization_options[:] + ['print_stacktrace=1']
128 extra_env['UBSAN_OPTIONS'] = ' '.join(ubsan_options)
129
John Abd-El-Malek14ae0542014-10-15 17:52:31130 return extra_env
131
132
glider9d919342015-02-06 17:42:27133def get_sanitizer_symbolize_command(json_path=None, executable_path=None):
earthdok9bd5d582015-01-29 21:19:36134 """Construct the command to invoke offline symbolization script."""
Nico Weber19097312016-01-29 18:13:15135 script_path = os.path.join(
136 ROOT_DIR, 'tools', 'valgrind', 'asan', 'asan_symbolize.py')
earthdok9bd5d582015-01-29 21:19:36137 cmd = [sys.executable, script_path]
138 if json_path is not None:
139 cmd.append('--test-summary-json-file=%s' % json_path)
glider9d919342015-02-06 17:42:27140 if executable_path is not None:
141 cmd.append('--executable-path=%s' % executable_path)
earthdok9bd5d582015-01-29 21:19:36142 return cmd
143
144
145def get_json_path(cmd):
146 """Extract the JSON test summary path from a command line."""
147 json_path_flag = '--test-launcher-summary-output='
148 for arg in cmd:
149 if arg.startswith(json_path_flag):
150 return arg.split(json_path_flag).pop()
151 return None
152
153
154def symbolize_snippets_in_json(cmd, env):
155 """Symbolize output snippets inside the JSON test summary."""
156 json_path = get_json_path(cmd)
157 if json_path is None:
158 return
159
160 try:
glider9d919342015-02-06 17:42:27161 symbolize_command = get_sanitizer_symbolize_command(
162 json_path=json_path, executable_path=cmd[0])
earthdok9bd5d582015-01-29 21:19:36163 p = subprocess.Popen(symbolize_command, stderr=subprocess.PIPE, env=env)
164 (_, stderr) = p.communicate()
165 except OSError as e:
thakis45643b42016-01-29 21:53:42166 print >> sys.stderr, 'Exception while symbolizing snippets: %s' % e
167 raise
earthdok9bd5d582015-01-29 21:19:36168
169 if p.returncode != 0:
thakis45643b42016-01-29 21:53:42170 print >> sys.stderr, "Error: failed to symbolize snippets in JSON:\n"
171 print >> sys.stderr, stderr
172 raise subprocess.CalledProcessError(p.returncode, symbolize_command)
earthdok9bd5d582015-01-29 21:19:36173
174
Trent Aptedb1852432018-01-25 02:15:10175def run_command_with_output(argv, stdoutfile, env=None, cwd=None):
176 """ Run command and stream its stdout/stderr to the console & |stdoutfile|.
177 """
178 print('Running %r in %r (env: %r)' % (argv, cwd, env))
179 assert stdoutfile
180 with io.open(stdoutfile, 'w') as writer, io.open(stdoutfile, 'r', 1) as \
181 reader:
182 process = subprocess.Popen(argv, env=env, cwd=cwd, stdout=writer,
183 stderr=subprocess.STDOUT)
184 while process.poll() is None:
185 sys.stdout.write(reader.read())
186 time.sleep(0.1)
187 # Read the remaining.
188 sys.stdout.write(reader.read())
189 print('Command %r returned exit code %d' % (argv, process.returncode))
190 return process.returncode
191
192
193def run_executable(cmd, env, stdoutfile=None):
[email protected]0a88a652012-03-09 00:34:45194 """Runs an executable with:
Dirk Pranke50e557b2017-12-01 23:48:09195 - CHROME_HEADLESS set to indicate that the test is running on a
196 bot and shouldn't do anything interactive like show modal dialogs.
[email protected]3766ed1c2012-07-26 20:53:56197 - environment variable CR_SOURCE_ROOT set to the root directory.
[email protected]0a88a652012-03-09 00:34:45198 - environment variable LANGUAGE to en_US.UTF-8.
earthdok748e1352015-01-27 23:04:08199 - environment variable CHROME_DEVEL_SANDBOX set
[email protected]0a88a652012-03-09 00:34:45200 - Reuses sys.executable automatically.
201 """
Dirk Pranke50e557b2017-12-01 23:48:09202 extra_env = {
203 # Set to indicate that the executable is running non-interactively on
204 # a bot.
205 'CHROME_HEADLESS': '1',
206
207 # Many tests assume a English interface...
208 'LANG': 'en_US.UTF-8',
209 }
210
[email protected]3766ed1c2012-07-26 20:53:56211 # Used by base/base_paths_linux.cc as an override. Just make sure the default
212 # logic is used.
213 env.pop('CR_SOURCE_ROOT', None)
earthdok748e1352015-01-27 23:04:08214 extra_env.update(get_sandbox_env(env))
John Abd-El-Malek4569c422014-10-09 05:10:53215
216 # Copy logic from tools/build/scripts/slave/runtest.py.
217 asan = '--asan=1' in cmd
218 lsan = '--lsan=1' in cmd
earthdokeeb065302015-02-04 18:18:04219 msan = '--msan=1' in cmd
220 tsan = '--tsan=1' in cmd
pcc46233c22017-06-20 22:11:41221 cfi_diag = '--cfi-diag=1' in cmd
Trent Aptedb1852432018-01-25 02:15:10222 if stdoutfile or sys.platform in ['win32', 'cygwin']:
timurrrr064f9522015-02-12 15:46:32223 # Symbolization works in-process on Windows even when sandboxed.
224 use_symbolization_script = False
225 else:
226 # LSan doesn't support sandboxing yet, so we use the in-process symbolizer.
227 # Note that ASan and MSan can work together with LSan.
pcc46233c22017-06-20 22:11:41228 use_symbolization_script = (asan or msan or cfi_diag) and not lsan
John Abd-El-Malek4569c422014-10-09 05:10:53229
pcc46233c22017-06-20 22:11:41230 if asan or lsan or msan or tsan or cfi_diag:
231 extra_env.update(get_sanitizer_env(cmd, asan, lsan, msan, tsan, cfi_diag))
earthdokeeb065302015-02-04 18:18:04232
Timur Iskhodzhanovfdbfd4e12015-02-05 13:50:23233 if lsan or tsan:
234 # LSan and TSan are not sandbox-friendly.
235 cmd.append('--no-sandbox')
John Abd-El-Malek14ae0542014-10-15 17:52:31236
237 cmd = trim_cmd(cmd)
John Abd-El-Malek4569c422014-10-09 05:10:53238
[email protected]8ba98352012-05-23 20:43:59239 # Ensure paths are correctly separated on windows.
240 cmd[0] = cmd[0].replace('/', os.path.sep)
[email protected]4bf4d632012-05-31 15:50:30241 cmd = fix_python_path(cmd)
John Abd-El-Malek14ae0542014-10-15 17:52:31242
243 print('Additional test environment:\n%s\n'
244 'Command: %s\n' % (
245 '\n'.join(' %s=%s' %
246 (k, v) for k, v in sorted(extra_env.iteritems())),
247 ' '.join(cmd)))
maruelf00125f82016-11-19 00:01:14248 sys.stdout.flush()
John Abd-El-Malek14ae0542014-10-15 17:52:31249 env.update(extra_env or {})
[email protected]50ec9f232012-03-16 04:18:23250 try:
Trent Aptedb1852432018-01-25 02:15:10251 if stdoutfile:
252 # Write to stdoutfile and poll to produce terminal output.
253 return run_command_with_output(cmd, env=env, stdoutfile=stdoutfile)
254 elif use_symbolization_script:
255 # See above comment regarding offline symbolization.
John Abd-El-Malek4569c422014-10-09 05:10:53256 # Need to pipe to the symbolizer script.
257 p1 = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
258 stderr=sys.stdout)
glider9d919342015-02-06 17:42:27259 p2 = subprocess.Popen(
260 get_sanitizer_symbolize_command(executable_path=cmd[0]),
261 env=env, stdin=p1.stdout)
John Abd-El-Malek4569c422014-10-09 05:10:53262 p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
vadimsh1eaeb2982014-10-20 12:28:45263 p1.wait()
John Abd-El-Malek4569c422014-10-09 05:10:53264 p2.wait()
earthdok9bd5d582015-01-29 21:19:36265 # Also feed the out-of-band JSON output to the symbolizer script.
266 symbolize_snippets_in_json(cmd, env)
vadimsh1eaeb2982014-10-20 12:28:45267 return p1.returncode
John Abd-El-Malek4569c422014-10-09 05:10:53268 else:
269 return subprocess.call(cmd, env=env)
[email protected]50ec9f232012-03-16 04:18:23270 except OSError:
271 print >> sys.stderr, 'Failed to start %s' % cmd
272 raise
[email protected]0a88a652012-03-09 00:34:45273
274
275def main():
gliderb73f1872014-10-09 16:24:56276 return run_executable(sys.argv[1:], os.environ.copy())
[email protected]0a88a652012-03-09 00:34:45277
278
[email protected]ed763a72012-08-29 03:51:22279if __name__ == '__main__':
[email protected]0a88a652012-03-09 00:34:45280 sys.exit(main())