[email protected] | cb155a8 | 2011-11-29 17:25:34 | [diff] [blame] | 1 | #!/usr/bin/env python |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 2 | # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Snapshot Build Bisect Tool |
| 7 | |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 8 | This script bisects a snapshot archive using binary search. It starts at |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 9 | a bad revision (it will try to guess HEAD) and asks for a last known-good |
| 10 | revision. It will then binary search across this revision range by downloading, |
| 11 | unzipping, and opening Chromium for you. After testing the specific revision, |
| 12 | it will ask you whether it is good or bad before continuing the search. |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 13 | """ |
| 14 | |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 15 | from __future__ import print_function |
| 16 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 17 | # The base URL for stored build archives. |
| 18 | CHROMIUM_BASE_URL = ('http://commondatastorage.googleapis.com' |
| 19 | '/chromium-browser-snapshots') |
| 20 | WEBKIT_BASE_URL = ('http://commondatastorage.googleapis.com' |
| 21 | '/chromium-webkit-snapshots') |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 22 | ASAN_BASE_URL = ('http://commondatastorage.googleapis.com' |
| 23 | '/chromium-browser-asan') |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 24 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 25 | # URL template for viewing changelogs between revisions. |
pshenoy | 9ce271f | 2014-09-02 22:14:05 | [diff] [blame] | 26 | CHANGELOG_URL = ('https://chromium.googlesource.com/chromium/src/+log/%s..%s') |
| 27 | |
| 28 | # URL to convert SVN revision to git hash. |
pshenoy | 13cb79e0 | 2014-09-05 01:42:53 | [diff] [blame] | 29 | CRREV_URL = ('https://cr-rev.appspot.com/_ah/api/crrev/v1/redirect/') |
[email protected] | f6a71a7 | 2009-10-08 19:55:38 | [diff] [blame] | 30 | |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 31 | # DEPS file URL. |
Di Mu | 08c5968 | 2016-07-11 23:05:07 | [diff] [blame] | 32 | DEPS_FILE = ('https://chromium.googlesource.com/chromium/src/+/%s/DEPS') |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 33 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 34 | # Blink changelogs URL. |
| 35 | BLINK_CHANGELOG_URL = ('http://build.chromium.org' |
| 36 | '/f/chromium/perf/dashboard/ui/changelog_blink.html' |
| 37 | '?url=/trunk&range=%d%%3A%d') |
| 38 | |
| 39 | DONE_MESSAGE_GOOD_MIN = ('You are probably looking for a change made after %s (' |
| 40 | 'known good), but no later than %s (first known bad).') |
| 41 | DONE_MESSAGE_GOOD_MAX = ('You are probably looking for a change made after %s (' |
| 42 | 'known bad), but no later than %s (first known good).') |
[email protected] | 05ff3fd | 2012-04-17 23:24:06 | [diff] [blame] | 43 | |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 44 | CHROMIUM_GITHASH_TO_SVN_URL = ( |
| 45 | 'https://chromium.googlesource.com/chromium/src/+/%s?format=json') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 46 | |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 47 | BLINK_GITHASH_TO_SVN_URL = ( |
| 48 | 'https://chromium.googlesource.com/chromium/blink/+/%s?format=json') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 49 | |
| 50 | GITHASH_TO_SVN_URL = { |
| 51 | 'chromium': CHROMIUM_GITHASH_TO_SVN_URL, |
| 52 | 'blink': BLINK_GITHASH_TO_SVN_URL, |
| 53 | } |
| 54 | |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 55 | VERSION_HISTORY_URL = ('https://versionhistory.googleapis.com/v1/chrome' |
| 56 | '/platforms/win/channels/stable/versions/all/releases') |
| 57 | |
| 58 | OMAHA_REVISIONS_URL = ('https://omahaproxy.appspot.com/deps.json?version=%s') |
| 59 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 60 | # Search pattern to be matched in the JSON output from |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 61 | # CHROMIUM_GITHASH_TO_SVN_URL to get the chromium revision (svn revision). |
pshenoy | b23a145 | 2014-09-05 22:52:05 | [diff] [blame] | 62 | CHROMIUM_SEARCH_PATTERN_OLD = ( |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 63 | r'.*git-svn-id: svn://svn.chromium.org/chrome/trunk/src@(\d+) ') |
pshenoy | b23a145 | 2014-09-05 22:52:05 | [diff] [blame] | 64 | CHROMIUM_SEARCH_PATTERN = ( |
Josip Sokcevic | 2613bc6 | 2021-08-04 23:41:05 | [diff] [blame] | 65 | r'Cr-Commit-Position: refs/heads/(?:master|main)@{#(\d+)}') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 66 | |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 67 | # Search pattern to be matched in the json output from |
| 68 | # BLINK_GITHASH_TO_SVN_URL to get the blink revision (svn revision). |
| 69 | BLINK_SEARCH_PATTERN = ( |
| 70 | r'.*git-svn-id: svn://svn.chromium.org/blink/trunk@(\d+) ') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 71 | |
| 72 | SEARCH_PATTERN = { |
| 73 | 'chromium': CHROMIUM_SEARCH_PATTERN, |
| 74 | 'blink': BLINK_SEARCH_PATTERN, |
| 75 | } |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 76 | |
[email protected] | 48036978 | 2014-08-22 20:15:58 | [diff] [blame] | 77 | CREDENTIAL_ERROR_MESSAGE = ('You are attempting to access protected data with ' |
| 78 | 'no configured credentials') |
| 79 | |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 80 | ############################################################################### |
| 81 | |
Dominic Mazzoni | 215e80b | 2017-11-29 20:05:27 | [diff] [blame] | 82 | import glob |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 83 | import json |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 84 | import optparse |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 85 | import os |
| 86 | import re |
[email protected] | 61ea90a | 2013-09-26 10:17:34 | [diff] [blame] | 87 | import shlex |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 88 | import shutil |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 89 | import subprocess |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 90 | import sys |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 91 | import tempfile |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 92 | import threading |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 93 | from distutils.version import LooseVersion |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 94 | from xml.etree import ElementTree |
[email protected] | bd8dcb9 | 2010-03-31 01:05:24 | [diff] [blame] | 95 | import zipfile |
| 96 | |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 97 | if sys.version_info[0] == 3: |
| 98 | import urllib.request as urllib |
| 99 | else: |
| 100 | import urllib |
| 101 | |
[email protected] | cb155a8 | 2011-11-29 17:25:34 | [diff] [blame] | 102 | |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 103 | class PathContext(object): |
| 104 | """A PathContext is used to carry the information used to construct URLs and |
| 105 | paths when dealing with the storage server and archives.""" |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 106 | def __init__(self, base_url, platform, good_revision, bad_revision, |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 107 | is_asan, use_local_cache, flash_path = None): |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 108 | super(PathContext, self).__init__() |
| 109 | # Store off the input parameters. |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 110 | self.base_url = base_url |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 111 | self.platform = platform # What's passed in to the '-a/--archive' option. |
| 112 | self.good_revision = good_revision |
| 113 | self.bad_revision = bad_revision |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 114 | self.is_asan = is_asan |
| 115 | self.build_type = 'release' |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 116 | self.flash_path = flash_path |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 117 | # Dictionary which stores svn revision number as key and it's |
| 118 | # corresponding git hash as value. This data is populated in |
| 119 | # _FetchAndParse and used later in GetDownloadURL while downloading |
| 120 | # the build. |
| 121 | self.githash_svn_dict = {} |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 122 | # The name of the ZIP file in a revision directory on the server. |
| 123 | self.archive_name = None |
| 124 | |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 125 | # Whether to cache and use the list of known revisions in a local file to |
| 126 | # speed up the initialization of the script at the next run. |
| 127 | self.use_local_cache = use_local_cache |
| 128 | |
| 129 | # Locate the local checkout to speed up the script by using locally stored |
| 130 | # metadata. |
| 131 | abs_file_path = os.path.abspath(os.path.realpath(__file__)) |
| 132 | local_src_path = os.path.join(os.path.dirname(abs_file_path), '..') |
| 133 | if abs_file_path.endswith(os.path.join('tools', 'bisect-builds.py')) and\ |
| 134 | os.path.exists(os.path.join(local_src_path, '.git')): |
| 135 | self.local_src_path = os.path.normpath(local_src_path) |
| 136 | else: |
| 137 | self.local_src_path = None |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 138 | |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 139 | # Set some internal members: |
| 140 | # _listing_platform_dir = Directory that holds revisions. Ends with a '/'. |
| 141 | # _archive_extract_dir = Uncompressed directory in the archive_name file. |
| 142 | # _binary_name = The name of the executable to run. |
dmazzoni | 76e907d | 2015-01-22 08:14:49 | [diff] [blame] | 143 | if self.platform in ('linux', 'linux64', 'linux-arm', 'chromeos'): |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 144 | self._binary_name = 'chrome' |
Avi Drissman | a1149077 | 2021-05-18 00:27:33 | [diff] [blame] | 145 | elif self.platform in ('mac', 'mac64', 'mac-arm'): |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 146 | self.archive_name = 'chrome-mac.zip' |
| 147 | self._archive_extract_dir = 'chrome-mac' |
[email protected] | 48036978 | 2014-08-22 20:15:58 | [diff] [blame] | 148 | elif self.platform in ('win', 'win64'): |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 149 | # Note: changed at revision 591483; see GetDownloadURL and GetLaunchPath |
| 150 | # below where these are patched. |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 151 | self.archive_name = 'chrome-win32.zip' |
| 152 | self._archive_extract_dir = 'chrome-win32' |
| 153 | self._binary_name = 'chrome.exe' |
| 154 | else: |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 155 | raise Exception('Invalid platform: %s' % self.platform) |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 156 | |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 157 | if self.platform in ('linux', 'linux64', 'linux-arm', 'chromeos'): |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 158 | # Note: changed at revision 591483; see GetDownloadURL and GetLaunchPath |
| 159 | # below where these are patched. |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 160 | self.archive_name = 'chrome-linux.zip' |
| 161 | self._archive_extract_dir = 'chrome-linux' |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 162 | if self.platform == 'linux': |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 163 | self._listing_platform_dir = 'Linux/' |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 164 | elif self.platform == 'linux64': |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 165 | self._listing_platform_dir = 'Linux_x64/' |
| 166 | elif self.platform == 'linux-arm': |
| 167 | self._listing_platform_dir = 'Linux_ARM_Cross-Compile/' |
| 168 | elif self.platform == 'chromeos': |
| 169 | self._listing_platform_dir = 'Linux_ChromiumOS_Full/' |
| 170 | elif self.platform in ('mac', 'mac64'): |
| 171 | self._listing_platform_dir = 'Mac/' |
| 172 | self._binary_name = 'Chromium.app/Contents/MacOS/Chromium' |
Avi Drissman | a1149077 | 2021-05-18 00:27:33 | [diff] [blame] | 173 | elif self.platform in ('mac-arm'): |
| 174 | self._listing_platform_dir = 'Mac_Arm/' |
| 175 | self._binary_name = 'Chromium.app/Contents/MacOS/Chromium' |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 176 | elif self.platform == 'win': |
| 177 | self._listing_platform_dir = 'Win/' |
jiawei.shao | 734efbc9 | 2016-09-23 02:11:45 | [diff] [blame] | 178 | elif self.platform == 'win64': |
| 179 | self._listing_platform_dir = 'Win_x64/' |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 180 | |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 181 | def GetASANPlatformDir(self): |
| 182 | """ASAN builds are in directories like "linux-release", or have filenames |
| 183 | like "asan-win32-release-277079.zip". This aligns to our platform names |
| 184 | except in the case of Windows where they use "win32" instead of "win".""" |
| 185 | if self.platform == 'win': |
| 186 | return 'win32' |
| 187 | else: |
| 188 | return self.platform |
| 189 | |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 190 | def GetListingURL(self, marker=None): |
| 191 | """Returns the URL for a directory listing, with an optional marker.""" |
| 192 | marker_param = '' |
| 193 | if marker: |
| 194 | marker_param = '&marker=' + str(marker) |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 195 | if self.is_asan: |
| 196 | prefix = '%s-%s' % (self.GetASANPlatformDir(), self.build_type) |
| 197 | return self.base_url + '/?delimiter=&prefix=' + prefix + marker_param |
| 198 | else: |
| 199 | return (self.base_url + '/?delimiter=/&prefix=' + |
| 200 | self._listing_platform_dir + marker_param) |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 201 | |
| 202 | def GetDownloadURL(self, revision): |
| 203 | """Gets the download URL for a build archive of a specific revision.""" |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 204 | if self.is_asan: |
| 205 | return '%s/%s-%s/%s-%d.zip' % ( |
| 206 | ASAN_BASE_URL, self.GetASANPlatformDir(), self.build_type, |
| 207 | self.GetASANBaseName(), revision) |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 208 | if str(revision) in self.githash_svn_dict: |
| 209 | revision = self.githash_svn_dict[str(revision)] |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 210 | archive_name = self.archive_name |
| 211 | |
| 212 | # At revision 591483, the names of two of the archives changed |
| 213 | # due to: https://chromium-review.googlesource.com/#/q/1226086 |
| 214 | # See: http://crbug.com/789612 |
| 215 | if revision >= 591483: |
| 216 | if self.platform == 'chromeos': |
| 217 | archive_name = 'chrome-chromeos.zip' |
| 218 | elif self.platform in ('win', 'win64'): |
| 219 | archive_name = 'chrome-win.zip' |
| 220 | |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 221 | return '%s/%s%s/%s' % (self.base_url, self._listing_platform_dir, |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 222 | revision, archive_name) |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 223 | |
| 224 | def GetLastChangeURL(self): |
| 225 | """Returns a URL to the LAST_CHANGE file.""" |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 226 | return self.base_url + '/' + self._listing_platform_dir + 'LAST_CHANGE' |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 227 | |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 228 | def GetASANBaseName(self): |
| 229 | """Returns the base name of the ASAN zip file.""" |
| 230 | if 'linux' in self.platform: |
| 231 | return 'asan-symbolized-%s-%s' % (self.GetASANPlatformDir(), |
| 232 | self.build_type) |
| 233 | else: |
| 234 | return 'asan-%s-%s' % (self.GetASANPlatformDir(), self.build_type) |
| 235 | |
| 236 | def GetLaunchPath(self, revision): |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 237 | """Returns a relative path (presumably from the archive extraction location) |
| 238 | that is used to run the executable.""" |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 239 | if self.is_asan: |
| 240 | extract_dir = '%s-%d' % (self.GetASANBaseName(), revision) |
| 241 | else: |
| 242 | extract_dir = self._archive_extract_dir |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 243 | |
| 244 | # At revision 591483, the names of two of the archives changed |
| 245 | # due to: https://chromium-review.googlesource.com/#/q/1226086 |
| 246 | # See: http://crbug.com/789612 |
| 247 | if revision >= 591483: |
| 248 | if self.platform == 'chromeos': |
| 249 | extract_dir = 'chrome-chromeos' |
| 250 | elif self.platform in ('win', 'win64'): |
Lei Zhang | 1c8c6f7e | 2018-11-09 16:46:30 | [diff] [blame] | 251 | extract_dir = 'chrome-win' |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 252 | |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 253 | return os.path.join(extract_dir, self._binary_name) |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 254 | |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 255 | def ParseDirectoryIndex(self, last_known_rev): |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 256 | """Parses the Google Storage directory listing into a list of revision |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 257 | numbers.""" |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 258 | |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 259 | def _GetMarkerForRev(revision): |
| 260 | if self.is_asan: |
| 261 | return '%s-%s/%s-%d.zip' % ( |
| 262 | self.GetASANPlatformDir(), self.build_type, |
| 263 | self.GetASANBaseName(), revision) |
| 264 | return '%s%d' % (self._listing_platform_dir, revision) |
| 265 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 266 | def _FetchAndParse(url): |
| 267 | """Fetches a URL and returns a 2-Tuple of ([revisions], next-marker). If |
| 268 | next-marker is not None, then the listing is a partial listing and another |
| 269 | fetch should be performed with next-marker being the marker= GET |
| 270 | parameter.""" |
| 271 | handle = urllib.urlopen(url) |
| 272 | document = ElementTree.parse(handle) |
| 273 | |
| 274 | # All nodes in the tree are namespaced. Get the root's tag name to extract |
| 275 | # the namespace. Etree does namespaces as |{namespace}tag|. |
| 276 | root_tag = document.getroot().tag |
| 277 | end_ns_pos = root_tag.find('}') |
| 278 | if end_ns_pos == -1: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 279 | raise Exception('Could not locate end namespace for directory index') |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 280 | namespace = root_tag[:end_ns_pos + 1] |
| 281 | |
| 282 | # Find the prefix (_listing_platform_dir) and whether or not the list is |
| 283 | # truncated. |
| 284 | prefix_len = len(document.find(namespace + 'Prefix').text) |
| 285 | next_marker = None |
| 286 | is_truncated = document.find(namespace + 'IsTruncated') |
| 287 | if is_truncated is not None and is_truncated.text.lower() == 'true': |
| 288 | next_marker = document.find(namespace + 'NextMarker').text |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 289 | # Get a list of all the revisions. |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 290 | revisions = [] |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 291 | githash_svn_dict = {} |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 292 | if self.is_asan: |
| 293 | asan_regex = re.compile(r'.*%s-(\d+)\.zip$' % (self.GetASANBaseName())) |
| 294 | # Non ASAN builds are in a <revision> directory. The ASAN builds are |
| 295 | # flat |
| 296 | all_prefixes = document.findall(namespace + 'Contents/' + |
| 297 | namespace + 'Key') |
| 298 | for prefix in all_prefixes: |
| 299 | m = asan_regex.match(prefix.text) |
| 300 | if m: |
| 301 | try: |
| 302 | revisions.append(int(m.group(1))) |
| 303 | except ValueError: |
| 304 | pass |
| 305 | else: |
| 306 | all_prefixes = document.findall(namespace + 'CommonPrefixes/' + |
| 307 | namespace + 'Prefix') |
| 308 | # The <Prefix> nodes have content of the form of |
| 309 | # |_listing_platform_dir/revision/|. Strip off the platform dir and the |
| 310 | # trailing slash to just have a number. |
| 311 | for prefix in all_prefixes: |
| 312 | revnum = prefix.text[prefix_len:-1] |
| 313 | try: |
dimu | a1dfa0ce | 2016-03-31 01:08:45 | [diff] [blame] | 314 | revnum = int(revnum) |
| 315 | revisions.append(revnum) |
| 316 | # Notes: |
| 317 | # Ignore hash in chromium-browser-snapshots as they are invalid |
| 318 | # Resulting in 404 error in fetching pages: |
| 319 | # https://chromium.googlesource.com/chromium/src/+/[rev_hash] |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 320 | except ValueError: |
| 321 | pass |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 322 | return (revisions, next_marker, githash_svn_dict) |
[email protected] | 9639b00 | 2013-08-30 14:45:52 | [diff] [blame] | 323 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 324 | # Fetch the first list of revisions. |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 325 | if last_known_rev: |
| 326 | revisions = [] |
| 327 | # Optimization: Start paging at the last known revision (local cache). |
| 328 | next_marker = _GetMarkerForRev(last_known_rev) |
| 329 | # Optimization: Stop paging at the last known revision (remote). |
| 330 | last_change_rev = GetChromiumRevision(self, self.GetLastChangeURL()) |
| 331 | if last_known_rev == last_change_rev: |
| 332 | return [] |
| 333 | else: |
| 334 | (revisions, next_marker, new_dict) = _FetchAndParse(self.GetListingURL()) |
| 335 | self.githash_svn_dict.update(new_dict) |
| 336 | last_change_rev = None |
| 337 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 338 | # If the result list was truncated, refetch with the next marker. Do this |
| 339 | # until an entire directory listing is done. |
| 340 | while next_marker: |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 341 | sys.stdout.write('\rFetching revisions at marker %s' % next_marker) |
| 342 | sys.stdout.flush() |
| 343 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 344 | next_url = self.GetListingURL(next_marker) |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 345 | (new_revisions, next_marker, new_dict) = _FetchAndParse(next_url) |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 346 | revisions.extend(new_revisions) |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 347 | self.githash_svn_dict.update(new_dict) |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 348 | if last_change_rev and last_change_rev in new_revisions: |
| 349 | break |
| 350 | sys.stdout.write('\r') |
| 351 | sys.stdout.flush() |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 352 | return revisions |
| 353 | |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 354 | def _GetSVNRevisionFromGitHashWithoutGitCheckout(self, git_sha1, depot): |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 355 | json_url = GITHASH_TO_SVN_URL[depot] % git_sha1 |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 356 | response = urllib.urlopen(json_url) |
| 357 | if response.getcode() == 200: |
| 358 | try: |
| 359 | data = json.loads(response.read()[4:]) |
| 360 | except ValueError: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 361 | print('ValueError for JSON URL: %s' % json_url) |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 362 | raise ValueError |
| 363 | else: |
| 364 | raise ValueError |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 365 | if 'message' in data: |
| 366 | message = data['message'].split('\n') |
| 367 | message = [line for line in message if line.strip()] |
| 368 | search_pattern = re.compile(SEARCH_PATTERN[depot]) |
| 369 | result = search_pattern.search(message[len(message)-1]) |
| 370 | if result: |
| 371 | return result.group(1) |
pshenoy | b23a145 | 2014-09-05 22:52:05 | [diff] [blame] | 372 | else: |
| 373 | if depot == 'chromium': |
| 374 | result = re.search(CHROMIUM_SEARCH_PATTERN_OLD, |
| 375 | message[len(message)-1]) |
| 376 | if result: |
| 377 | return result.group(1) |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 378 | print('Failed to get svn revision number for %s' % git_sha1) |
[email protected] | 1f99f4d | 2014-07-23 16:44:14 | [diff] [blame] | 379 | raise ValueError |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 380 | |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 381 | def _GetSVNRevisionFromGitHashFromGitCheckout(self, git_sha1, depot): |
| 382 | def _RunGit(command, path): |
| 383 | command = ['git'] + command |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 384 | shell = sys.platform.startswith('win') |
| 385 | proc = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 386 | stderr=subprocess.PIPE, cwd=path) |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 387 | (output, _) = proc.communicate() |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 388 | return (output, proc.returncode) |
| 389 | |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 390 | path = self.local_src_path |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 391 | if depot == 'blink': |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 392 | path = os.path.join(self.local_src_path, 'third_party', 'WebKit') |
| 393 | revision = None |
| 394 | try: |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 395 | command = ['svn', 'find-rev', git_sha1] |
| 396 | (git_output, return_code) = _RunGit(command, path) |
| 397 | if not return_code: |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 398 | revision = git_output.strip('\n') |
| 399 | except ValueError: |
| 400 | pass |
| 401 | if not revision: |
| 402 | command = ['log', '-n1', '--format=%s', git_sha1] |
| 403 | (git_output, return_code) = _RunGit(command, path) |
| 404 | if not return_code: |
| 405 | revision = re.match('SVN changes up to revision ([0-9]+)', git_output) |
| 406 | revision = revision.group(1) if revision else None |
| 407 | if revision: |
| 408 | return revision |
| 409 | raise ValueError |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 410 | |
| 411 | def GetSVNRevisionFromGitHash(self, git_sha1, depot='chromium'): |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 412 | if not self.local_src_path: |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 413 | return self._GetSVNRevisionFromGitHashWithoutGitCheckout(git_sha1, depot) |
| 414 | else: |
| 415 | return self._GetSVNRevisionFromGitHashFromGitCheckout(git_sha1, depot) |
| 416 | |
Bruce Dawson | b590808 | 2020-11-09 23:01:11 | [diff] [blame] | 417 | def GetRevList(self, archive): |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 418 | """Gets the list of revision numbers between self.good_revision and |
| 419 | self.bad_revision.""" |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 420 | |
| 421 | cache = {} |
| 422 | # The cache is stored in the same directory as bisect-builds.py |
| 423 | cache_filename = os.path.join( |
| 424 | os.path.abspath(os.path.dirname(__file__)), |
| 425 | '.bisect-builds-cache.json') |
| 426 | cache_dict_key = self.GetListingURL() |
| 427 | |
| 428 | def _LoadBucketFromCache(): |
| 429 | if self.use_local_cache: |
| 430 | try: |
| 431 | with open(cache_filename) as cache_file: |
rob | 1c83605 | 2015-05-18 16:34:02 | [diff] [blame] | 432 | for (key, value) in json.load(cache_file).items(): |
| 433 | cache[key] = value |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 434 | revisions = cache.get(cache_dict_key, []) |
| 435 | githash_svn_dict = cache.get('githash_svn_dict', {}) |
| 436 | if revisions: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 437 | print('Loaded revisions %d-%d from %s' % |
| 438 | (revisions[0], revisions[-1], cache_filename)) |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 439 | return (revisions, githash_svn_dict) |
| 440 | except (EnvironmentError, ValueError): |
| 441 | pass |
| 442 | return ([], {}) |
| 443 | |
| 444 | def _SaveBucketToCache(): |
| 445 | """Save the list of revisions and the git-svn mappings to a file. |
| 446 | The list of revisions is assumed to be sorted.""" |
| 447 | if self.use_local_cache: |
| 448 | cache[cache_dict_key] = revlist_all |
| 449 | cache['githash_svn_dict'] = self.githash_svn_dict |
| 450 | try: |
| 451 | with open(cache_filename, 'w') as cache_file: |
| 452 | json.dump(cache, cache_file) |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 453 | print('Saved revisions %d-%d to %s' % |
| 454 | (revlist_all[0], revlist_all[-1], cache_filename)) |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 455 | except EnvironmentError: |
| 456 | pass |
| 457 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 458 | # Download the revlist and filter for just the range between good and bad. |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 459 | minrev = min(self.good_revision, self.bad_revision) |
| 460 | maxrev = max(self.good_revision, self.bad_revision) |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 461 | |
| 462 | (revlist_all, self.githash_svn_dict) = _LoadBucketFromCache() |
| 463 | last_known_rev = revlist_all[-1] if revlist_all else 0 |
| 464 | if last_known_rev < maxrev: |
| 465 | revlist_all.extend(map(int, self.ParseDirectoryIndex(last_known_rev))) |
| 466 | revlist_all = list(set(revlist_all)) |
| 467 | revlist_all.sort() |
| 468 | _SaveBucketToCache() |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 469 | |
| 470 | revlist = [x for x in revlist_all if x >= int(minrev) and x <= int(maxrev)] |
Bruce Dawson | b590808 | 2020-11-09 23:01:11 | [diff] [blame] | 471 | if len(revlist) < 2: # Don't have enough builds to bisect. |
| 472 | last_known_rev = revlist_all[-1] if revlist_all else 0 |
| 473 | first_known_rev = revlist_all[0] if revlist_all else 0 |
| 474 | # Check for specifying a number before the available range. |
| 475 | if maxrev < first_known_rev: |
| 476 | msg = ( |
| 477 | 'First available bisect revision for %s is %d. Be sure to specify revision ' |
| 478 | 'numbers, not branch numbers.' % (archive, first_known_rev)) |
| 479 | raise (RuntimeError(msg)) |
| 480 | |
| 481 | # Check for specifying a number beyond the available range. |
| 482 | if maxrev > last_known_rev: |
| 483 | # Check for the special case of linux where bisect builds stopped at |
| 484 | # revision 382086, around March 2016. |
| 485 | if archive == 'linux': |
| 486 | msg = 'Last available bisect revision for %s is %d. Try linux64 instead.' % ( |
| 487 | archive, last_known_rev) |
| 488 | else: |
| 489 | msg = 'Last available bisect revision for %s is %d. Try a different good/bad range.' % ( |
| 490 | archive, last_known_rev) |
| 491 | raise (RuntimeError(msg)) |
| 492 | |
| 493 | # Otherwise give a generic message. |
| 494 | msg = 'We don\'t have enough builds to bisect. revlist: %s' % revlist |
| 495 | raise RuntimeError(msg) |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 496 | |
| 497 | # Set good and bad revisions to be legit revisions. |
| 498 | if revlist: |
| 499 | if self.good_revision < self.bad_revision: |
| 500 | self.good_revision = revlist[0] |
| 501 | self.bad_revision = revlist[-1] |
| 502 | else: |
| 503 | self.bad_revision = revlist[0] |
| 504 | self.good_revision = revlist[-1] |
| 505 | |
| 506 | # Fix chromium rev so that the deps blink revision matches REVISIONS file. |
| 507 | if self.base_url == WEBKIT_BASE_URL: |
| 508 | revlist_all.sort() |
| 509 | self.good_revision = FixChromiumRevForBlink(revlist, |
| 510 | revlist_all, |
| 511 | self, |
| 512 | self.good_revision) |
| 513 | self.bad_revision = FixChromiumRevForBlink(revlist, |
| 514 | revlist_all, |
| 515 | self, |
| 516 | self.bad_revision) |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 517 | return revlist |
| 518 | |
prasadv | 2375e6d | 2017-03-20 19:23:23 | [diff] [blame] | 519 | |
| 520 | def IsMac(): |
| 521 | return sys.platform.startswith('darwin') |
| 522 | |
| 523 | |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 524 | def UnzipFilenameToDir(filename, directory): |
| 525 | """Unzip |filename| to |directory|.""" |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 526 | cwd = os.getcwd() |
| 527 | if not os.path.isabs(filename): |
| 528 | filename = os.path.join(cwd, filename) |
[email protected] | bd8dcb9 | 2010-03-31 01:05:24 | [diff] [blame] | 529 | # Make base. |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 530 | if not os.path.isdir(directory): |
| 531 | os.mkdir(directory) |
| 532 | os.chdir(directory) |
prasadv | 2375e6d | 2017-03-20 19:23:23 | [diff] [blame] | 533 | |
| 534 | # The Python ZipFile does not support symbolic links, which makes it |
| 535 | # unsuitable for Mac builds. so use ditto instead. |
| 536 | if IsMac(): |
| 537 | unzip_cmd = ['ditto', '-x', '-k', filename, '.'] |
| 538 | proc = subprocess.Popen(unzip_cmd, bufsize=0, stdout=subprocess.PIPE, |
| 539 | stderr=subprocess.PIPE) |
| 540 | proc.communicate() |
| 541 | os.chdir(cwd) |
| 542 | return |
| 543 | |
| 544 | zf = zipfile.ZipFile(filename) |
[email protected] | e29c08c | 2012-09-17 20:50:50 | [diff] [blame] | 545 | # Extract files. |
| 546 | for info in zf.infolist(): |
| 547 | name = info.filename |
| 548 | if name.endswith('/'): # dir |
| 549 | if not os.path.isdir(name): |
| 550 | os.makedirs(name) |
| 551 | else: # file |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 552 | directory = os.path.dirname(name) |
John Budorick | 06e5df1 | 2015-02-27 17:44:27 | [diff] [blame] | 553 | if not os.path.isdir(directory): |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 554 | os.makedirs(directory) |
[email protected] | e29c08c | 2012-09-17 20:50:50 | [diff] [blame] | 555 | out = open(name, 'wb') |
| 556 | out.write(zf.read(name)) |
| 557 | out.close() |
| 558 | # Set permissions. Permission info in external_attr is shifted 16 bits. |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 559 | os.chmod(name, info.external_attr >> 16) |
[email protected] | e29c08c | 2012-09-17 20:50:50 | [diff] [blame] | 560 | os.chdir(cwd) |
[email protected] | bd8dcb9 | 2010-03-31 01:05:24 | [diff] [blame] | 561 | |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 562 | |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 563 | def FetchRevision(context, rev, filename, quit_event=None, progress_event=None): |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 564 | """Downloads and unzips revision |rev|. |
| 565 | @param context A PathContext instance. |
| 566 | @param rev The Chromium revision number/tag to download. |
| 567 | @param filename The destination for the downloaded file. |
| 568 | @param quit_event A threading.Event which will be set by the master thread to |
| 569 | indicate that the download should be aborted. |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 570 | @param progress_event A threading.Event which will be set by the master thread |
| 571 | to indicate that the progress of the download should be |
| 572 | displayed. |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 573 | """ |
| 574 | def ReportHook(blocknum, blocksize, totalsize): |
[email protected] | 946be75 | 2011-10-25 23:34:21 | [diff] [blame] | 575 | if quit_event and quit_event.isSet(): |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 576 | raise RuntimeError('Aborting download of revision %s' % str(rev)) |
[email protected] | 946be75 | 2011-10-25 23:34:21 | [diff] [blame] | 577 | if progress_event and progress_event.isSet(): |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 578 | size = blocknum * blocksize |
| 579 | if totalsize == -1: # Total size not known. |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 580 | progress = 'Received %d bytes' % size |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 581 | else: |
| 582 | size = min(totalsize, size) |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 583 | progress = 'Received %d of %d bytes, %.2f%%' % ( |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 584 | size, totalsize, 100.0 * size / totalsize) |
| 585 | # Send a \r to let all progress messages use just one line of output. |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 586 | sys.stdout.write('\r' + progress) |
[email protected] | 468a977 | 2011-08-09 18:42:00 | [diff] [blame] | 587 | sys.stdout.flush() |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 588 | download_url = context.GetDownloadURL(rev) |
| 589 | try: |
John Budorick | 06e5df1 | 2015-02-27 17:44:27 | [diff] [blame] | 590 | urllib.urlretrieve(download_url, filename, ReportHook) |
[email protected] | 946be75 | 2011-10-25 23:34:21 | [diff] [blame] | 591 | if progress_event and progress_event.isSet(): |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 592 | print() |
mikecase | e2b6ce8 | 2015-02-06 18:22:39 | [diff] [blame] | 593 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 594 | except RuntimeError: |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 595 | pass |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 596 | |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 597 | |
Dominic Mazzoni | 215e80b | 2017-11-29 20:05:27 | [diff] [blame] | 598 | def CopyMissingFileFromCurrentSource(src_glob, dst): |
| 599 | """Work around missing files in archives. |
| 600 | This happens when archives of Chrome don't contain all of the files |
| 601 | needed to build it. In many cases we can work around this using |
| 602 | files from the current checkout. The source is in the form of a glob |
| 603 | so that it can try to look for possible sources of the file in |
| 604 | multiple locations, but we just arbitrarily try the first match. |
| 605 | |
| 606 | Silently fail if this doesn't work because we don't yet have clear |
| 607 | markers for builds that require certain files or a way to test |
| 608 | whether or not launching Chrome succeeded. |
| 609 | """ |
| 610 | if not os.path.exists(dst): |
| 611 | matches = glob.glob(src_glob) |
| 612 | if matches: |
| 613 | shutil.copy2(matches[0], dst) |
| 614 | |
| 615 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 616 | def RunRevision(context, revision, zip_file, profile, num_runs, command, args): |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 617 | """Given a zipped revision, unzip it and run the test.""" |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 618 | print('Trying revision %s...' % str(revision)) |
[email protected] | 3ff00b7 | 2011-07-20 21:34:47 | [diff] [blame] | 619 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 620 | # Create a temp directory and unzip the revision into it. |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 621 | cwd = os.getcwd() |
| 622 | tempdir = tempfile.mkdtemp(prefix='bisect_tmp') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 623 | UnzipFilenameToDir(zip_file, tempdir) |
dmazzoni | 76e907d | 2015-01-22 08:14:49 | [diff] [blame] | 624 | |
Dominic Mazzoni | 215e80b | 2017-11-29 20:05:27 | [diff] [blame] | 625 | # Hack: Some Chrome OS archives are missing some files; try to copy them |
| 626 | # from the local directory. |
Dominic Mazzoni | e84e40b | 2018-10-08 06:44:45 | [diff] [blame] | 627 | if context.platform == 'chromeos' and revision < 591483: |
Dominic Mazzoni | 215e80b | 2017-11-29 20:05:27 | [diff] [blame] | 628 | CopyMissingFileFromCurrentSource('third_party/icu/common/icudtl.dat', |
| 629 | '%s/chrome-linux/icudtl.dat' % tempdir) |
| 630 | CopyMissingFileFromCurrentSource('*out*/*/libminigbm.so', |
| 631 | '%s/chrome-linux/libminigbm.so' % tempdir) |
dmazzoni | 76e907d | 2015-01-22 08:14:49 | [diff] [blame] | 632 | |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 633 | os.chdir(tempdir) |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 634 | |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 635 | # Run the build as many times as specified. |
[email protected] | 4646a75 | 2013-07-19 22:14:34 | [diff] [blame] | 636 | testargs = ['--user-data-dir=%s' % profile] + args |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 637 | # The sandbox must be run as root on Official Chrome, so bypass it. |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 638 | if (context.flash_path and context.platform.startswith('linux')): |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 639 | testargs.append('--no-sandbox') |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 640 | if context.flash_path: |
| 641 | testargs.append('--ppapi-flash-path=%s' % context.flash_path) |
| 642 | # We have to pass a large enough Flash version, which currently needs not |
| 643 | # be correct. Instead of requiring the user of the script to figure out and |
| 644 | # pass the correct version we just spoof it. |
| 645 | testargs.append('--ppapi-flash-version=99.9.999.999') |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 646 | |
[email protected] | 4646a75 | 2013-07-19 22:14:34 | [diff] [blame] | 647 | runcommand = [] |
[email protected] | 61ea90a | 2013-09-26 10:17:34 | [diff] [blame] | 648 | for token in shlex.split(command): |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 649 | if token == '%a': |
[email protected] | 4646a75 | 2013-07-19 22:14:34 | [diff] [blame] | 650 | runcommand.extend(testargs) |
| 651 | else: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 652 | runcommand.append( |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 653 | token.replace('%p', os.path.abspath(context.GetLaunchPath(revision))). |
| 654 | replace('%s', ' '.join(testargs))) |
[email protected] | fb61fc3 | 2019-04-18 19:47:20 | [diff] [blame] | 655 | result = None |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 656 | try: |
[email protected] | fb61fc3 | 2019-04-18 19:47:20 | [diff] [blame] | 657 | for _ in range(num_runs): |
| 658 | subproc = subprocess.Popen( |
| 659 | runcommand, |
| 660 | bufsize=-1, |
| 661 | stdout=subprocess.PIPE, |
| 662 | stderr=subprocess.PIPE) |
| 663 | (stdout, stderr) = subproc.communicate() |
| 664 | result = (subproc.returncode, stdout, stderr) |
| 665 | if subproc.returncode: |
| 666 | break |
| 667 | return result |
| 668 | finally: |
| 669 | os.chdir(cwd) |
| 670 | try: |
| 671 | shutil.rmtree(tempdir, True) |
| 672 | except Exception: |
| 673 | pass |
[email protected] | 79f1474 | 2010-03-10 01:01:57 | [diff] [blame] | 674 | |
[email protected] | cb155a8 | 2011-11-29 17:25:34 | [diff] [blame] | 675 | |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 676 | # The arguments status, stdout and stderr are unused. |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 677 | # They are present here because this function is passed to Bisect which then |
| 678 | # calls it with 5 arguments. |
| 679 | # pylint: disable=W0613 |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 680 | def AskIsGoodBuild(rev, exit_status, stdout, stderr): |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 681 | """Asks the user whether build |rev| is good or bad.""" |
Fergal Daly | 0dd1953 | 2019-04-04 07:45:33 | [diff] [blame] | 682 | if exit_status: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 683 | print('Chrome exit_status: %d. Use s to see output' % exit_status) |
[email protected] | 79f1474 | 2010-03-10 01:01:57 | [diff] [blame] | 684 | # Loop until we get a response that we can parse. |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 685 | while True: |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 686 | prompt = ('Revision %s is ' |
| 687 | '[(g)ood/(b)ad/(r)etry/(u)nknown/(s)tdout/(q)uit]: ' % str(rev)) |
| 688 | if sys.version_info[0] == 3: |
| 689 | response = input(prompt) |
| 690 | else: |
| 691 | response = raw_input(prompt) |
wangxianzhu | d8c4c56 | 2015-12-15 23:39:51 | [diff] [blame] | 692 | if response in ('g', 'b', 'r', 'u'): |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 693 | return response |
wangxianzhu | d8c4c56 | 2015-12-15 23:39:51 | [diff] [blame] | 694 | if response == 'q': |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 695 | raise SystemExit() |
wangxianzhu | d8c4c56 | 2015-12-15 23:39:51 | [diff] [blame] | 696 | if response == 's': |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 697 | print(stdout) |
| 698 | print(stderr) |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 699 | |
[email protected] | cb155a8 | 2011-11-29 17:25:34 | [diff] [blame] | 700 | |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 701 | def IsGoodASANBuild(rev, exit_status, stdout, stderr): |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 702 | """Determine if an ASAN build |rev| is good or bad |
| 703 | |
| 704 | Will examine stderr looking for the error message emitted by ASAN. If not |
| 705 | found then will fallback to asking the user.""" |
| 706 | if stderr: |
| 707 | bad_count = 0 |
| 708 | for line in stderr.splitlines(): |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 709 | print(line) |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 710 | if line.find('ERROR: AddressSanitizer:') != -1: |
| 711 | bad_count += 1 |
| 712 | if bad_count > 0: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 713 | print('Revision %d determined to be bad.' % rev) |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 714 | return 'b' |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 715 | return AskIsGoodBuild(rev, exit_status, stdout, stderr) |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 716 | |
| 717 | |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 718 | def DidCommandSucceed(rev, exit_status, stdout, stderr): |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 719 | if exit_status: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 720 | print('Bad revision: %s' % rev) |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 721 | return 'b' |
| 722 | else: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 723 | print('Good revision: %s' % rev) |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 724 | return 'g' |
| 725 | |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 726 | |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 727 | class DownloadJob(object): |
| 728 | """DownloadJob represents a task to download a given Chromium revision.""" |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 729 | |
| 730 | def __init__(self, context, name, rev, zip_file): |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 731 | super(DownloadJob, self).__init__() |
| 732 | # Store off the input parameters. |
| 733 | self.context = context |
| 734 | self.name = name |
| 735 | self.rev = rev |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 736 | self.zip_file = zip_file |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 737 | self.quit_event = threading.Event() |
| 738 | self.progress_event = threading.Event() |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 739 | self.thread = None |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 740 | |
| 741 | def Start(self): |
| 742 | """Starts the download.""" |
| 743 | fetchargs = (self.context, |
| 744 | self.rev, |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 745 | self.zip_file, |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 746 | self.quit_event, |
| 747 | self.progress_event) |
| 748 | self.thread = threading.Thread(target=FetchRevision, |
| 749 | name=self.name, |
| 750 | args=fetchargs) |
| 751 | self.thread.start() |
| 752 | |
| 753 | def Stop(self): |
| 754 | """Stops the download which must have been started previously.""" |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 755 | assert self.thread, 'DownloadJob must be started before Stop is called.' |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 756 | self.quit_event.set() |
| 757 | self.thread.join() |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 758 | os.unlink(self.zip_file) |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 759 | |
| 760 | def WaitFor(self): |
| 761 | """Prints a message and waits for the download to complete. The download |
| 762 | must have been started previously.""" |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 763 | assert self.thread, 'DownloadJob must be started before WaitFor is called.' |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 764 | print('Downloading revision %s...' % str(self.rev)) |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 765 | self.progress_event.set() # Display progress of download. |
rob | 8a4543f | 2016-01-20 00:43:59 | [diff] [blame] | 766 | try: |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 767 | while self.thread.is_alive(): |
rob | 8a4543f | 2016-01-20 00:43:59 | [diff] [blame] | 768 | # The parameter to join is needed to keep the main thread responsive to |
| 769 | # signals. Without it, the program will not respond to interruptions. |
| 770 | self.thread.join(1) |
| 771 | except (KeyboardInterrupt, SystemExit): |
| 772 | self.Stop() |
| 773 | raise |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 774 | |
| 775 | |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 776 | def VerifyEndpoint(fetch, context, rev, profile, num_runs, command, try_args, |
| 777 | evaluate, expected_answer): |
| 778 | fetch.WaitFor() |
| 779 | try: |
Roman Sorokin | 760f06cd | 2019-12-24 08:35:41 | [diff] [blame] | 780 | answer = 'r' |
| 781 | # This is intended to allow evaluate() to return 'r' to retry RunRevision. |
| 782 | while answer == 'r': |
| 783 | (exit_status, stdout, stderr) = RunRevision( |
| 784 | context, rev, fetch.zip_file, profile, num_runs, command, try_args) |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 785 | answer = evaluate(rev, exit_status, stdout, stderr) |
| 786 | except Exception as e: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 787 | print(e, file=sys.stderr) |
Lei Zhang | 2fa7630 | 2018-11-09 20:16:31 | [diff] [blame] | 788 | raise SystemExit |
Roman Sorokin | 760f06cd | 2019-12-24 08:35:41 | [diff] [blame] | 789 | if (answer != expected_answer): |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 790 | print('Unexpected result at a range boundary! Your range is not correct.') |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 791 | raise SystemExit |
| 792 | |
| 793 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 794 | def Bisect(context, |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 795 | num_runs=1, |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 796 | command='%p %a', |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 797 | try_args=(), |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 798 | profile=None, |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 799 | evaluate=AskIsGoodBuild, |
Bruce Dawson | b590808 | 2020-11-09 23:01:11 | [diff] [blame] | 800 | verify_range=False, |
| 801 | archive=None): |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 802 | """Given known good and known bad revisions, run a binary search on all |
| 803 | archived revisions to determine the last known good revision. |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 804 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 805 | @param context PathContext object initialized with user provided parameters. |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 806 | @param num_runs Number of times to run each build for asking good/bad. |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 807 | @param try_args A tuple of arguments to pass to the test application. |
| 808 | @param profile The name of the user profile to run with. |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 809 | @param evaluate A function which returns 'g' if the argument build is good, |
| 810 | 'b' if it's bad or 'u' if unknown. |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 811 | @param verify_range If true, tests the first and last revisions in the range |
| 812 | before proceeding with the bisect. |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 813 | |
| 814 | Threading is used to fetch Chromium revisions in the background, speeding up |
| 815 | the user's experience. For example, suppose the bounds of the search are |
| 816 | good_rev=0, bad_rev=100. The first revision to be checked is 50. Depending on |
| 817 | whether revision 50 is good or bad, the next revision to check will be either |
| 818 | 25 or 75. So, while revision 50 is being checked, the script will download |
| 819 | revisions 25 and 75 in the background. Once the good/bad verdict on rev 50 is |
| 820 | known: |
| 821 | |
| 822 | - If rev 50 is good, the download of rev 25 is cancelled, and the next test |
| 823 | is run on rev 75. |
| 824 | |
| 825 | - If rev 50 is bad, the download of rev 75 is cancelled, and the next test |
| 826 | is run on rev 25. |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 827 | """ |
| 828 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 829 | if not profile: |
| 830 | profile = 'profile' |
| 831 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 832 | good_rev = context.good_revision |
| 833 | bad_rev = context.bad_revision |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 834 | cwd = os.getcwd() |
| 835 | |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 836 | print('Downloading list of known revisions...', end=' ') |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 837 | if not context.use_local_cache: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 838 | print('(use --use-local-cache to cache and re-use the list of revisions)') |
[email protected] | 28a3c12 | 2014-08-09 11:04:51 | [diff] [blame] | 839 | else: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 840 | print() |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 841 | _GetDownloadPath = lambda rev: os.path.join(cwd, |
| 842 | '%s-%s' % (str(rev), context.archive_name)) |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 843 | |
| 844 | # Get a list of revisions to bisect across. |
Bruce Dawson | b590808 | 2020-11-09 23:01:11 | [diff] [blame] | 845 | revlist = context.GetRevList(archive) |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 846 | |
| 847 | # Figure out our bookends and first pivot point; fetch the pivot revision. |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 848 | minrev = 0 |
| 849 | maxrev = len(revlist) - 1 |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 850 | pivot = int(maxrev / 2) |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 851 | rev = revlist[pivot] |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 852 | fetch = DownloadJob(context, 'initial_fetch', rev, _GetDownloadPath(rev)) |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 853 | fetch.Start() |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 854 | |
| 855 | if verify_range: |
| 856 | minrev_fetch = DownloadJob( |
| 857 | context, 'minrev_fetch', revlist[minrev], |
| 858 | _GetDownloadPath(revlist[minrev])) |
| 859 | maxrev_fetch = DownloadJob( |
| 860 | context, 'maxrev_fetch', revlist[maxrev], |
| 861 | _GetDownloadPath(revlist[maxrev])) |
| 862 | minrev_fetch.Start() |
| 863 | maxrev_fetch.Start() |
| 864 | try: |
| 865 | VerifyEndpoint(minrev_fetch, context, revlist[minrev], profile, num_runs, |
| 866 | command, try_args, evaluate, 'b' if bad_rev < good_rev else 'g') |
| 867 | VerifyEndpoint(maxrev_fetch, context, revlist[maxrev], profile, num_runs, |
| 868 | command, try_args, evaluate, 'g' if bad_rev < good_rev else 'b') |
| 869 | except (KeyboardInterrupt, SystemExit): |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 870 | print('Cleaning up...') |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 871 | fetch.Stop() |
| 872 | sys.exit(0) |
| 873 | finally: |
| 874 | minrev_fetch.Stop() |
| 875 | maxrev_fetch.Stop() |
| 876 | |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 877 | fetch.WaitFor() |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 878 | |
| 879 | # Binary search time! |
Bruce Dawson | 6225741 | 2020-01-17 17:39:53 | [diff] [blame] | 880 | prefetch_revisions = True |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 881 | while fetch and fetch.zip_file and maxrev - minrev > 1: |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 882 | if bad_rev < good_rev: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 883 | min_str, max_str = 'bad', 'good' |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 884 | else: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 885 | min_str, max_str = 'good', 'bad' |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 886 | print( |
| 887 | 'Bisecting range [%s (%s), %s (%s)], ' |
| 888 | 'roughly %d steps left.' % (revlist[minrev], min_str, revlist[maxrev], |
| 889 | max_str, int(maxrev - minrev).bit_length())) |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 890 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 891 | # Pre-fetch next two possible pivots |
| 892 | # - down_pivot is the next revision to check if the current revision turns |
| 893 | # out to be bad. |
| 894 | # - up_pivot is the next revision to check if the current revision turns |
| 895 | # out to be good. |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 896 | down_pivot = int((pivot - minrev) / 2) + minrev |
Bruce Dawson | 6225741 | 2020-01-17 17:39:53 | [diff] [blame] | 897 | if prefetch_revisions: |
| 898 | down_fetch = None |
| 899 | if down_pivot != pivot and down_pivot != minrev: |
| 900 | down_rev = revlist[down_pivot] |
| 901 | down_fetch = DownloadJob(context, 'down_fetch', down_rev, |
| 902 | _GetDownloadPath(down_rev)) |
| 903 | down_fetch.Start() |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 904 | |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 905 | up_pivot = int((maxrev - pivot) / 2) + pivot |
Bruce Dawson | 6225741 | 2020-01-17 17:39:53 | [diff] [blame] | 906 | if prefetch_revisions: |
| 907 | up_fetch = None |
| 908 | if up_pivot != pivot and up_pivot != maxrev: |
| 909 | up_rev = revlist[up_pivot] |
| 910 | up_fetch = DownloadJob(context, 'up_fetch', up_rev, |
| 911 | _GetDownloadPath(up_rev)) |
| 912 | up_fetch.Start() |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 913 | |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 914 | # Run test on the pivot revision. |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 915 | exit_status = None |
[email protected] | e29c08c | 2012-09-17 20:50:50 | [diff] [blame] | 916 | stdout = None |
| 917 | stderr = None |
| 918 | try: |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 919 | (exit_status, stdout, stderr) = RunRevision( |
| 920 | context, rev, fetch.zip_file, profile, num_runs, command, try_args) |
Bruce Dawson | 039777ef | 2020-10-26 20:34:47 | [diff] [blame] | 921 | except Exception as e: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 922 | print(e, file=sys.stderr) |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 923 | |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 924 | # Call the evaluate function to see if the current revision is good or bad. |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 925 | # On that basis, kill one of the background downloads and complete the |
| 926 | # other, as described in the comments above. |
| 927 | try: |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 928 | answer = evaluate(rev, exit_status, stdout, stderr) |
Bruce Dawson | 6225741 | 2020-01-17 17:39:53 | [diff] [blame] | 929 | prefetch_revisions = True |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 930 | if ((answer == 'g' and good_rev < bad_rev) |
| 931 | or (answer == 'b' and bad_rev < good_rev)): |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 932 | fetch.Stop() |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 933 | minrev = pivot |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 934 | if down_fetch: |
| 935 | down_fetch.Stop() # Kill the download of the older revision. |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 936 | fetch = None |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 937 | if up_fetch: |
| 938 | up_fetch.WaitFor() |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 939 | pivot = up_pivot |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 940 | fetch = up_fetch |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 941 | elif ((answer == 'b' and good_rev < bad_rev) |
| 942 | or (answer == 'g' and bad_rev < good_rev)): |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 943 | fetch.Stop() |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 944 | maxrev = pivot |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 945 | if up_fetch: |
| 946 | up_fetch.Stop() # Kill the download of the newer revision. |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 947 | fetch = None |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 948 | if down_fetch: |
| 949 | down_fetch.WaitFor() |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 950 | pivot = down_pivot |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 951 | fetch = down_fetch |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 952 | elif answer == 'r': |
Bruce Dawson | 6225741 | 2020-01-17 17:39:53 | [diff] [blame] | 953 | # Don't redundantly prefetch. |
| 954 | prefetch_revisions = False |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 955 | elif answer == 'u': |
| 956 | # Nuke the revision from the revlist and choose a new pivot. |
[email protected] | 1d4a0624 | 2013-08-20 22:53:12 | [diff] [blame] | 957 | fetch.Stop() |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 958 | revlist.pop(pivot) |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 959 | maxrev -= 1 # Assumes maxrev >= pivot. |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 960 | |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 961 | if maxrev - minrev > 1: |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 962 | # Alternate between using down_pivot or up_pivot for the new pivot |
| 963 | # point, without affecting the range. Do this instead of setting the |
| 964 | # pivot to the midpoint of the new range because adjacent revisions |
| 965 | # are likely affected by the same issue that caused the (u)nknown |
| 966 | # response. |
| 967 | if up_fetch and down_fetch: |
| 968 | fetch = [up_fetch, down_fetch][len(revlist) % 2] |
| 969 | elif up_fetch: |
| 970 | fetch = up_fetch |
| 971 | else: |
| 972 | fetch = down_fetch |
| 973 | fetch.WaitFor() |
| 974 | if fetch == up_fetch: |
| 975 | pivot = up_pivot - 1 # Subtracts 1 because revlist was resized. |
| 976 | else: |
| 977 | pivot = down_pivot |
[email protected] | 53bb634 | 2012-06-01 04:11:00 | [diff] [blame] | 978 | |
| 979 | if down_fetch and fetch != down_fetch: |
| 980 | down_fetch.Stop() |
| 981 | if up_fetch and fetch != up_fetch: |
| 982 | up_fetch.Stop() |
| 983 | else: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 984 | assert False, 'Unexpected return value from evaluate(): ' + answer |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 985 | except (KeyboardInterrupt, SystemExit): |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 986 | print('Cleaning up...') |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 987 | for f in [_GetDownloadPath(rev), |
| 988 | _GetDownloadPath(revlist[down_pivot]), |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 989 | _GetDownloadPath(revlist[up_pivot])]: |
[email protected] | afe3066 | 2011-07-30 01:05:52 | [diff] [blame] | 990 | try: |
| 991 | os.unlink(f) |
| 992 | except OSError: |
| 993 | pass |
| 994 | sys.exit(0) |
| 995 | |
| 996 | rev = revlist[pivot] |
| 997 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 998 | return (revlist[minrev], revlist[maxrev], context) |
[email protected] | 60ac66e3 | 2011-07-18 16:08:25 | [diff] [blame] | 999 | |
| 1000 | |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1001 | def GetBlinkDEPSRevisionForChromiumRevision(self, rev): |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 1002 | """Returns the blink revision that was in REVISIONS file at |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1003 | chromium revision |rev|.""" |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1004 | |
| 1005 | def _GetBlinkRev(url, blink_re): |
| 1006 | m = blink_re.search(url.read()) |
| 1007 | url.close() |
| 1008 | if m: |
fmalita | a898d22 | 2016-07-12 22:29:03 | [diff] [blame] | 1009 | return m.group(1) |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1010 | |
Di Mu | 08c5968 | 2016-07-11 23:05:07 | [diff] [blame] | 1011 | url = urllib.urlopen(DEPS_FILE % GetGitHashFromSVNRevision(rev)) |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1012 | if url.getcode() == 200: |
Di Mu | 08c5968 | 2016-07-11 23:05:07 | [diff] [blame] | 1013 | blink_re = re.compile(r'webkit_revision\D*\d+;\D*\d+;(\w+)') |
| 1014 | blink_git_sha = _GetBlinkRev(url, blink_re) |
| 1015 | return self.GetSVNRevisionFromGitHash(blink_git_sha, 'blink') |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1016 | raise Exception('Could not get Blink revision for Chromium rev %d' % rev) |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 1017 | |
| 1018 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1019 | def GetBlinkRevisionForChromiumRevision(context, rev): |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 1020 | """Returns the blink revision that was in REVISIONS file at |
| 1021 | chromium revision |rev|.""" |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 1022 | def _IsRevisionNumber(revision): |
| 1023 | if isinstance(revision, int): |
| 1024 | return True |
| 1025 | else: |
| 1026 | return revision.isdigit() |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1027 | if str(rev) in context.githash_svn_dict: |
| 1028 | rev = context.githash_svn_dict[str(rev)] |
| 1029 | file_url = '%s/%s%s/REVISIONS' % (context.base_url, |
| 1030 | context._listing_platform_dir, rev) |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 1031 | url = urllib.urlopen(file_url) |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1032 | if url.getcode() == 200: |
| 1033 | try: |
| 1034 | data = json.loads(url.read()) |
| 1035 | except ValueError: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1036 | print('ValueError for JSON URL: %s' % file_url) |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1037 | raise ValueError |
| 1038 | else: |
| 1039 | raise ValueError |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1040 | url.close() |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 1041 | if 'webkit_revision' in data: |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 1042 | blink_rev = data['webkit_revision'] |
| 1043 | if not _IsRevisionNumber(blink_rev): |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1044 | blink_rev = int(context.GetSVNRevisionFromGitHash(blink_rev, 'blink')) |
[email protected] | 3e7c8532 | 2014-06-27 20:27:36 | [diff] [blame] | 1045 | return blink_rev |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1046 | else: |
[email protected] | ff50d1c | 2013-04-17 18:49:36 | [diff] [blame] | 1047 | raise Exception('Could not get blink revision for cr rev %d' % rev) |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1048 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1049 | |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 1050 | def FixChromiumRevForBlink(revisions_final, revisions, self, rev): |
| 1051 | """Returns the chromium revision that has the correct blink revision |
| 1052 | for blink bisect, DEPS and REVISIONS file might not match since |
| 1053 | blink snapshots point to tip of tree blink. |
| 1054 | Note: The revisions_final variable might get modified to include |
| 1055 | additional revisions.""" |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1056 | blink_deps_rev = GetBlinkDEPSRevisionForChromiumRevision(self, rev) |
[email protected] | 37ed317 | 2013-09-24 23:49:30 | [diff] [blame] | 1057 | |
| 1058 | while (GetBlinkRevisionForChromiumRevision(self, rev) > blink_deps_rev): |
| 1059 | idx = revisions.index(rev) |
| 1060 | if idx > 0: |
| 1061 | rev = revisions[idx-1] |
| 1062 | if rev not in revisions_final: |
| 1063 | revisions_final.insert(0, rev) |
| 1064 | |
| 1065 | revisions_final.sort() |
| 1066 | return rev |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1067 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1068 | |
[email protected] | 5980b75 | 2014-07-02 00:34:40 | [diff] [blame] | 1069 | def GetChromiumRevision(context, url): |
[email protected] | 801fb65 | 2012-07-20 20:13:50 | [diff] [blame] | 1070 | """Returns the chromium revision read from given URL.""" |
| 1071 | try: |
| 1072 | # Location of the latest build revision number |
[email protected] | 5980b75 | 2014-07-02 00:34:40 | [diff] [blame] | 1073 | latest_revision = urllib.urlopen(url).read() |
| 1074 | if latest_revision.isdigit(): |
| 1075 | return int(latest_revision) |
| 1076 | return context.GetSVNRevisionFromGitHash(latest_revision) |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1077 | except Exception: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1078 | print('Could not determine latest revision. This could be bad...') |
[email protected] | 801fb65 | 2012-07-20 20:13:50 | [diff] [blame] | 1079 | return 999999999 |
| 1080 | |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1081 | |
| 1082 | def GetRevision(revision_text): |
| 1083 | """Translates from a text description of a revision to an integral revision |
| 1084 | number. Currently supported formats are a number (i.e.; '782793') or a |
| 1085 | milestone specifier (i.e.; 'M85') or a full version string |
| 1086 | (i.e. '85.0.4183.121').""" |
| 1087 | |
| 1088 | # Check if we already have a revision number, such as when -g or -b is |
| 1089 | # omitted. |
| 1090 | if type(revision_text) == type(0): |
| 1091 | return revision_text |
| 1092 | |
| 1093 | # Translate from stable milestone name to the latest version number released |
| 1094 | # for that milestone, i.e.; 'M85' to '85.0.4183.121'. |
| 1095 | if revision_text[:1].upper() == 'M': |
| 1096 | milestone = revision_text[1:] |
| 1097 | response = urllib.urlopen(VERSION_HISTORY_URL) |
| 1098 | version_history = json.loads(response.read()) |
| 1099 | version_matcher = re.compile( |
| 1100 | '.*versions/(\d*)\.(\d*)\.(\d*)\.(\d*)/releases.*') |
| 1101 | for version in version_history['releases']: |
| 1102 | match = version_matcher.match(version['name']) |
| 1103 | # There will be multiple versions of each milestone, but we just grab the |
| 1104 | # first one that we see which will be the most recent version. If you need |
| 1105 | # more granularity then specify a full version number or revision number. |
| 1106 | if match and match.groups()[0] == milestone: |
| 1107 | revision_text = '.'.join(match.groups()) |
| 1108 | break |
| 1109 | if revision_text[:1].upper() == 'M': |
| 1110 | raise Exception('No stable release matching %s found.' % revision_text) |
| 1111 | |
| 1112 | # Translate from version number to commit position, also known as revision |
| 1113 | # number. |
| 1114 | if len(revision_text.split('.')) == 4: |
| 1115 | response = urllib.urlopen(OMAHA_REVISIONS_URL % revision_text) |
| 1116 | revision_details = json.loads(response.read()) |
| 1117 | revision_text = revision_details['chromium_base_position'] |
| 1118 | |
David Sanders | 3ff471b | 2022-03-17 23:39:18 | [diff] [blame] | 1119 | if not revision_text: |
| 1120 | raise Exception( |
| 1121 | "No 'chromium_base_position' matching %s found." % chromium_base_position) |
| 1122 | |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1123 | # Translate from text commit position to integer commit position. |
| 1124 | return int(revision_text) |
| 1125 | |
| 1126 | |
pshenoy | cd6bd68 | 2014-09-10 20:50:22 | [diff] [blame] | 1127 | def GetGitHashFromSVNRevision(svn_revision): |
| 1128 | crrev_url = CRREV_URL + str(svn_revision) |
| 1129 | url = urllib.urlopen(crrev_url) |
| 1130 | if url.getcode() == 200: |
| 1131 | data = json.loads(url.read()) |
| 1132 | if 'git_sha' in data: |
| 1133 | return data['git_sha'] |
| 1134 | |
pshenoy | 9ce271f | 2014-09-02 22:14:05 | [diff] [blame] | 1135 | def PrintChangeLog(min_chromium_rev, max_chromium_rev): |
| 1136 | """Prints the changelog URL.""" |
| 1137 | |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1138 | print(' ' + CHANGELOG_URL % (GetGitHashFromSVNRevision(min_chromium_rev), |
| 1139 | GetGitHashFromSVNRevision(max_chromium_rev))) |
| 1140 | |
pshenoy | 9ce271f | 2014-09-02 22:14:05 | [diff] [blame] | 1141 | |
elawrence | 446bcc3 | 2017-04-14 17:18:51 | [diff] [blame] | 1142 | def error_internal_option(option, opt, value, parser): |
[email protected] | fb61fc3 | 2019-04-18 19:47:20 | [diff] [blame] | 1143 | raise optparse.OptionValueError( |
| 1144 | 'The -o and -r options are only\navailable in the internal version of ' |
| 1145 | 'this script. Google\nemployees should visit http://go/bisect-builds ' |
| 1146 | 'for\nconfiguration instructions.') |
[email protected] | 801fb65 | 2012-07-20 20:13:50 | [diff] [blame] | 1147 | |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 1148 | def main(): |
[email protected] | 2c1d273 | 2009-10-29 19:52:17 | [diff] [blame] | 1149 | usage = ('%prog [options] [-- chromium-options]\n' |
[email protected] | 887c918 | 2013-02-12 20:30:31 | [diff] [blame] | 1150 | 'Perform binary search on the snapshot builds to find a minimal\n' |
| 1151 | 'range of revisions where a behavior change happened. The\n' |
| 1152 | 'behaviors are described as "good" and "bad".\n' |
| 1153 | 'It is NOT assumed that the behavior of the later revision is\n' |
[email protected] | 09c58da | 2013-01-07 21:30:17 | [diff] [blame] | 1154 | 'the bad one.\n' |
[email protected] | 178aab7 | 2010-10-08 17:21:38 | [diff] [blame] | 1155 | '\n' |
[email protected] | 887c918 | 2013-02-12 20:30:31 | [diff] [blame] | 1156 | 'Revision numbers should use\n' |
[email protected] | 887c918 | 2013-02-12 20:30:31 | [diff] [blame] | 1157 | ' SVN revisions (e.g. 123456) for chromium builds, from trunk.\n' |
| 1158 | ' Use base_trunk_revision from http://omahaproxy.appspot.com/\n' |
| 1159 | ' for earlier revs.\n' |
| 1160 | ' Chrome\'s about: build number and omahaproxy branch_revision\n' |
| 1161 | ' are incorrect, they are from branches.\n' |
| 1162 | '\n' |
Bruce Dawson | e357305 | 2020-06-29 23:14:35 | [diff] [blame] | 1163 | 'Use "-- <args-to-pass-to-chromium>" to pass arbitrary extra \n' |
| 1164 | 'arguments to the test binaries.\n' |
| 1165 | 'E.g., add "-- --no-first-run" to bypass the first run prompts.') |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 1166 | parser = optparse.OptionParser(usage=usage) |
[email protected] | 1a45d22 | 2009-09-19 01:58:57 | [diff] [blame] | 1167 | # Strangely, the default help output doesn't include the choice list. |
Avi Drissman | a1149077 | 2021-05-18 00:27:33 | [diff] [blame] | 1168 | choices = ['mac', 'mac64', 'mac-arm', 'win', 'win64', 'linux', 'linux64', |
| 1169 | 'linux-arm', 'chromeos'] |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 1170 | parser.add_option('-a', '--archive', |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1171 | choices=choices, |
| 1172 | help='The buildbot archive to bisect [%s].' % |
| 1173 | '|'.join(choices)) |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1174 | parser.add_option('-b', |
| 1175 | '--bad', |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1176 | type='str', |
| 1177 | help='A bad revision to start bisection. ' |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1178 | 'May be earlier or later than the good revision. ' |
| 1179 | 'Default is HEAD. Can be a revision number, milestone ' |
| 1180 | 'name (eg. M85, matches the most recent stable release of ' |
| 1181 | 'that milestone) or version number (eg. 85.0.4183.121)') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1182 | parser.add_option('-f', '--flash_path', |
| 1183 | type='str', |
| 1184 | help='Absolute path to a recent Adobe Pepper Flash ' |
| 1185 | 'binary to be used in this bisection (e.g. ' |
| 1186 | 'on Windows C:\...\pepflashplayer.dll and on Linux ' |
| 1187 | '/opt/google/chrome/PepperFlash/' |
| 1188 | 'libpepflashplayer.so).') |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1189 | parser.add_option('-g', |
| 1190 | '--good', |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1191 | type='str', |
| 1192 | help='A good revision to start bisection. ' + |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1193 | 'May be earlier or later than the bad revision. ' + |
| 1194 | 'Default is 0. Can be a revision number, milestone ' |
| 1195 | 'name (eg. M85, matches the most recent stable release of ' |
| 1196 | 'that milestone) or version number (eg. 85.0.4183.121)') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1197 | parser.add_option('-p', '--profile', '--user-data-dir', |
| 1198 | type='str', |
| 1199 | default='profile', |
| 1200 | help='Profile to use; this will not reset every run. ' |
| 1201 | 'Defaults to a clean profile.') |
| 1202 | parser.add_option('-t', '--times', |
| 1203 | type='int', |
| 1204 | default=1, |
| 1205 | help='Number of times to run each build before asking ' |
| 1206 | 'if it\'s good or bad. Temporary profiles are reused.') |
Bruce Dawson | e357305 | 2020-06-29 23:14:35 | [diff] [blame] | 1207 | parser.add_option('-c', |
| 1208 | '--command', |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1209 | type='str', |
| 1210 | default='%p %a', |
| 1211 | help='Command to execute. %p and %a refer to Chrome ' |
Bruce Dawson | e357305 | 2020-06-29 23:14:35 | [diff] [blame] | 1212 | 'executable and specified extra arguments respectively. ' |
| 1213 | 'Use %s to specify all extra arguments as one string. ' |
| 1214 | 'Defaults to "%p %a". Note that any extra paths specified ' |
| 1215 | 'should be absolute. If you just need to append an ' |
| 1216 | 'argument to the Chrome command line use "-- ' |
| 1217 | '<args-to-pass-to-chromium>" instead.') |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1218 | parser.add_option('-l', '--blink', |
| 1219 | action='store_true', |
| 1220 | help='Use Blink bisect instead of Chromium. ') |
| 1221 | parser.add_option('', '--not-interactive', |
| 1222 | action='store_true', |
| 1223 | default=False, |
| 1224 | help='Use command exit code to tell good/bad revision.') |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 1225 | parser.add_option('--asan', |
| 1226 | dest='asan', |
| 1227 | action='store_true', |
| 1228 | default=False, |
| 1229 | help='Allow the script to bisect ASAN builds') |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 1230 | parser.add_option('--use-local-cache', |
| 1231 | dest='use_local_cache', |
[email protected] | 6a7a5d6 | 2014-07-09 04:45:50 | [diff] [blame] | 1232 | action='store_true', |
| 1233 | default=False, |
rob | 724c906 | 2015-01-22 00:26:42 | [diff] [blame] | 1234 | help='Use a local file in the current directory to cache ' |
| 1235 | 'a list of known revisions to speed up the ' |
| 1236 | 'initialization of this script.') |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 1237 | parser.add_option('--verify-range', |
| 1238 | dest='verify_range', |
| 1239 | action='store_true', |
| 1240 | default=False, |
| 1241 | help='Test the first and last revisions in the range ' + |
| 1242 | 'before proceeding with the bisect.') |
elawrence | 446bcc3 | 2017-04-14 17:18:51 | [diff] [blame] | 1243 | parser.add_option("-r", action="callback", callback=error_internal_option) |
| 1244 | parser.add_option("-o", action="callback", callback=error_internal_option) |
[email protected] | b3b2051 | 2013-08-26 18:51:04 | [diff] [blame] | 1245 | |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 1246 | (opts, args) = parser.parse_args() |
| 1247 | |
| 1248 | if opts.archive is None: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1249 | print('Error: missing required parameter: --archive') |
| 1250 | print() |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 1251 | parser.print_help() |
| 1252 | return 1 |
| 1253 | |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 1254 | if opts.asan: |
| 1255 | supported_platforms = ['linux', 'mac', 'win'] |
| 1256 | if opts.archive not in supported_platforms: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1257 | print('Error: ASAN bisecting only supported on these platforms: [%s].' % |
| 1258 | ('|'.join(supported_platforms))) |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 1259 | return 1 |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 1260 | |
| 1261 | if opts.asan: |
| 1262 | base_url = ASAN_BASE_URL |
| 1263 | elif opts.blink: |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 1264 | base_url = WEBKIT_BASE_URL |
| 1265 | else: |
| 1266 | base_url = CHROMIUM_BASE_URL |
| 1267 | |
[email protected] | 183706d9 | 2011-06-10 13:06:22 | [diff] [blame] | 1268 | # Create the context. Initialize 0 for the revisions as they are set below. |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1269 | context = PathContext(base_url, opts.archive, opts.good, opts.bad, |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 1270 | opts.asan, opts.use_local_cache, |
vitalybuka | 4d1e1e41 | 2015-07-06 17:21:06 | [diff] [blame] | 1271 | opts.flash_path) |
mikecase | a8cd284c | 2014-12-02 21:30:58 | [diff] [blame] | 1272 | |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 1273 | # Pick a starting point, try to get HEAD for this. |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1274 | if not opts.bad: |
| 1275 | context.bad_revision = '999.0.0.0' |
| 1276 | context.bad_revision = GetChromiumRevision( |
| 1277 | context, context.GetLastChangeURL()) |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 1278 | |
| 1279 | # Find out when we were good. |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1280 | if not opts.good: |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 1281 | context.good_revision = 0 |
[email protected] | 801fb65 | 2012-07-20 20:13:50 | [diff] [blame] | 1282 | |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 1283 | if opts.flash_path: |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1284 | msg = 'Could not find Flash binary at %s' % opts.flash_path |
| 1285 | assert os.path.exists(opts.flash_path), msg |
[email protected] | fc3702e | 2013-11-09 04:23:00 | [diff] [blame] | 1286 | |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1287 | context.good_revision = GetRevision(context.good_revision) |
| 1288 | context.bad_revision = GetRevision(context.bad_revision) |
[email protected] | 801fb65 | 2012-07-20 20:13:50 | [diff] [blame] | 1289 | |
[email protected] | 5e93cf16 | 2012-01-28 02:16:56 | [diff] [blame] | 1290 | if opts.times < 1: |
| 1291 | print('Number of times to run (%d) must be greater than or equal to 1.' % |
| 1292 | opts.times) |
| 1293 | parser.print_help() |
| 1294 | return 1 |
| 1295 | |
skobes | 21b5cdfb | 2016-03-21 23:13:02 | [diff] [blame] | 1296 | if opts.not_interactive: |
| 1297 | evaluator = DidCommandSucceed |
| 1298 | elif opts.asan: |
[email protected] | 01188669 | 2014-08-01 21:00:21 | [diff] [blame] | 1299 | evaluator = IsGoodASANBuild |
| 1300 | else: |
| 1301 | evaluator = AskIsGoodBuild |
| 1302 | |
[email protected] | 2e0f267 | 2014-08-13 20:32:58 | [diff] [blame] | 1303 | # Save these revision numbers to compare when showing the changelog URL |
| 1304 | # after the bisect. |
| 1305 | good_rev = context.good_revision |
| 1306 | bad_rev = context.bad_revision |
| 1307 | |
Bruce Dawson | cd63ea2 | 2020-10-26 16:37:41 | [diff] [blame] | 1308 | print('Scanning from %d to %d (%d revisions).' % |
| 1309 | (good_rev, bad_rev, abs(good_rev - bad_rev))) |
| 1310 | |
Bruce Dawson | b590808 | 2020-11-09 23:01:11 | [diff] [blame] | 1311 | (min_chromium_rev, max_chromium_rev, |
| 1312 | context) = Bisect(context, opts.times, opts.command, args, opts.profile, |
| 1313 | evaluator, opts.verify_range, opts.archive) |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 1314 | |
[email protected] | ff50d1c | 2013-04-17 18:49:36 | [diff] [blame] | 1315 | # Get corresponding blink revisions. |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1316 | try: |
[email protected] | 4c6fec6b | 2013-09-17 17:44:08 | [diff] [blame] | 1317 | min_blink_rev = GetBlinkRevisionForChromiumRevision(context, |
| 1318 | min_chromium_rev) |
| 1319 | max_blink_rev = GetBlinkRevisionForChromiumRevision(context, |
| 1320 | max_chromium_rev) |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1321 | except Exception: |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1322 | # Silently ignore the failure. |
[email protected] | ff50d1c | 2013-04-17 18:49:36 | [diff] [blame] | 1323 | min_blink_rev, max_blink_rev = 0, 0 |
[email protected] | b2fe7f2 | 2011-10-25 22:58:31 | [diff] [blame] | 1324 | |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1325 | if opts.blink: |
| 1326 | # We're done. Let the user know the results in an official manner. |
| 1327 | if good_rev > bad_rev: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1328 | print(DONE_MESSAGE_GOOD_MAX % (str(min_blink_rev), str(max_blink_rev))) |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1329 | else: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1330 | print(DONE_MESSAGE_GOOD_MIN % (str(min_blink_rev), str(max_blink_rev))) |
[email protected] | eadd95d | 2012-11-02 22:42:09 | [diff] [blame] | 1331 | |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1332 | print('BLINK CHANGELOG URL:') |
| 1333 | print(' ' + BLINK_CHANGELOG_URL % (max_blink_rev, min_blink_rev)) |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1334 | |
[email protected] | d0149c5c | 2012-05-29 21:12:11 | [diff] [blame] | 1335 | else: |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1336 | # We're done. Let the user know the results in an official manner. |
| 1337 | if good_rev > bad_rev: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1338 | print(DONE_MESSAGE_GOOD_MAX % (str(min_chromium_rev), |
| 1339 | str(max_chromium_rev))) |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1340 | else: |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1341 | print(DONE_MESSAGE_GOOD_MIN % (str(min_chromium_rev), |
| 1342 | str(max_chromium_rev))) |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1343 | if min_blink_rev != max_blink_rev: |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1344 | print ('NOTE: There is a Blink roll in the range, ' |
| 1345 | 'you might also want to do a Blink bisect.') |
[email protected] | 3bdaa475 | 2013-09-30 20:13:36 | [diff] [blame] | 1346 | |
Raul Tambre | 57e09d6 | 2019-09-22 17:18:52 | [diff] [blame] | 1347 | print('CHANGELOG URL:') |
Jason Kersey | 97bb027a | 2016-05-11 20:10:43 | [diff] [blame] | 1348 | PrintChangeLog(min_chromium_rev, max_chromium_rev) |
[email protected] | cb155a8 | 2011-11-29 17:25:34 | [diff] [blame] | 1349 | |
[email protected] | 4df583c | 2014-07-31 17:11:55 | [diff] [blame] | 1350 | |
[email protected] | 67e0bc6 | 2009-09-03 22:06:09 | [diff] [blame] | 1351 | if __name__ == '__main__': |
[email protected] | 7ad66a7 | 2009-09-04 17:52:33 | [diff] [blame] | 1352 | sys.exit(main()) |