blob: 3f13fe5d75259af01039fd3b107efa4c2c35c4b6 [file] [log] [blame]
[email protected]fb2b8eb2009-04-23 21:03:421#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A wrapper script to manage a set of client modules in different SCM.
18
19This script is intended to be used to help basic management of client
20program sources residing in one or more Subversion modules, along with
21other modules it depends on, also in Subversion, but possibly on
22multiple respositories, making a wrapper system apparently necessary.
23
24Files
25 .gclient : Current client configuration, written by 'config' command.
26 Format is a Python script defining 'solutions', a list whose
27 entries each are maps binding the strings "name" and "url"
28 to strings specifying the name and location of the client
29 module, as well as "custom_deps" to a map similar to the DEPS
30 file below.
31 .gclient_entries : A cache constructed by 'update' command. Format is a
32 Python script defining 'entries', a list of the names
33 of all modules in the client
34 <module>/DEPS : Python script defining var 'deps' as a map from each requisite
35 submodule name to a URL where it can be found (via one SCM)
36
37Hooks
38 .gclient and DEPS files may optionally contain a list named "hooks" to
39 allow custom actions to be performed based on files that have changed in the
40 working copy as a result of a "sync"/"update" or "revert" operation. Hooks
41 can also be run based on what files have been modified in the working copy
42 with the "runhooks" operation. If any of these operation are run with
43 --force, all known hooks will run regardless of the state of the working
44 copy.
45
46 Each item in a "hooks" list is a dict, containing these two keys:
47 "pattern" The associated value is a string containing a regular
48 expression. When a file whose pathname matches the expression
49 is checked out, updated, or reverted, the hook's "action" will
50 run.
51 "action" A list describing a command to run along with its arguments, if
52 any. An action command will run at most one time per gclient
53 invocation, regardless of how many files matched the pattern.
54 The action is executed in the same directory as the .gclient
55 file. If the first item in the list is the string "python",
56 the current Python interpreter (sys.executable) will be used
57 to run the command.
58
59 Example:
60 hooks = [
61 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
62 "action": ["python", "image_indexer.py", "--all"]},
63 ]
64"""
65
66__author__ = "[email protected] (Darin Fisher)"
[email protected]df7a3132009-05-12 17:49:4967__version__ = "0.3.2"
[email protected]fb2b8eb2009-04-23 21:03:4268
69import errno
70import optparse
71import os
72import re
73import stat
74import subprocess
75import sys
76import time
77import urlparse
78import xml.dom.minidom
79import urllib
80
[email protected]fb2b8eb2009-04-23 21:03:4281
82SVN_COMMAND = "svn"
83
84
85# default help text
86DEFAULT_USAGE_TEXT = (
87"""usage: %prog <subcommand> [options] [--] [svn options/args...]
88a wrapper for managing a set of client modules in svn.
89Version """ + __version__ + """
90
91subcommands:
92 cleanup
93 config
94 diff
95 revert
96 status
97 sync
98 update
99 runhooks
100 revinfo
101
102Options and extra arguments can be passed to invoked svn commands by
103appending them to the command line. Note that if the first such
104appended option starts with a dash (-) then the options must be
105preceded by -- to distinguish them from gclient options.
106
107For additional help on a subcommand or examples of usage, try
108 %prog help <subcommand>
109 %prog help files
110""")
111
112GENERIC_UPDATE_USAGE_TEXT = (
113 """Perform a checkout/update of the modules specified by the gclient
114configuration; see 'help config'. Unless --revision is specified,
115then the latest revision of the root solutions is checked out, with
116dependent submodule versions updated according to DEPS files.
117If --revision is specified, then the given revision is used in place
118of the latest, either for a single solution or for all solutions.
119Unless the --force option is provided, solutions and modules whose
120local revision matches the one to update (i.e., they have not changed
121in the repository) are *not* modified.
122This a synonym for 'gclient %(alias)s'
123
124usage: gclient %(cmd)s [options] [--] [svn update options/args]
125
126Valid options:
127 --force : force update even for unchanged modules
128 --revision REV : update/checkout all solutions with specified revision
129 --revision SOLUTION@REV : update given solution to specified revision
130 --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
131 --verbose : output additional diagnostics
[email protected]b8b6b872009-06-30 18:50:56132 --head : update to latest revision, instead of last good revision
[email protected]fb2b8eb2009-04-23 21:03:42133
134Examples:
135 gclient %(cmd)s
136 update files from SVN according to current configuration,
137 *for modules which have changed since last update or sync*
138 gclient %(cmd)s --force
139 update files from SVN according to current configuration, for
140 all modules (useful for recovering files deleted from local copy)
141""")
142
143COMMAND_USAGE_TEXT = {
144 "cleanup":
145 """Clean up all working copies, using 'svn cleanup' for each module.
146Additional options and args may be passed to 'svn cleanup'.
147
148usage: cleanup [options] [--] [svn cleanup args/options]
149
150Valid options:
151 --verbose : output additional diagnostics
152""",
153 "config": """Create a .gclient file in the current directory; this
154specifies the configuration for further commands. After update/sync,
155top-level DEPS files in each module are read to determine dependent
156modules to operate on as well. If optional [url] parameter is
157provided, then configuration is read from a specified Subversion server
158URL. Otherwise, a --spec option must be provided.
159
160usage: config [option | url] [safesync url]
161
162Valid options:
163 --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
164 *Note that due to Cygwin/Python brokenness, it
165 probably can't contain any newlines.*
166
167Examples:
168 gclient config https://gclient.googlecode.com/svn/trunk/gclient
169 configure a new client to check out gclient.py tool sources
170 gclient config --spec='solutions=[{"name":"gclient","""
171 '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
172 '"custom_deps":{}}]',
173 "diff": """Display the differences between two revisions of modules.
174(Does 'svn diff' for each checked out module and dependences.)
175Additional args and options to 'svn diff' can be passed after
176gclient options.
177
178usage: diff [options] [--] [svn args/options]
179
180Valid options:
181 --verbose : output additional diagnostics
182
183Examples:
184 gclient diff
185 simple 'svn diff' for configured client and dependences
186 gclient diff -- -x -b
187 use 'svn diff -x -b' to suppress whitespace-only differences
188 gclient diff -- -r HEAD -x -b
189 diff versus the latest version of each module
190""",
191 "revert":
192 """Revert every file in every managed directory in the client view.
193
194usage: revert
195""",
196 "status":
197 """Show the status of client and dependent modules, using 'svn diff'
198for each module. Additional options and args may be passed to 'svn diff'.
199
200usage: status [options] [--] [svn diff args/options]
201
202Valid options:
203 --verbose : output additional diagnostics
204""",
205 "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
206 "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
207 "help": """Describe the usage of this program or its subcommands.
208
209usage: help [options] [subcommand]
210
211Valid options:
212 --verbose : output additional diagnostics
213""",
214 "runhooks":
215 """Runs hooks for files that have been modified in the local working copy,
216according to 'svn status'.
217
218usage: runhooks [options]
219
220Valid options:
221 --force : runs all known hooks, regardless of the working
222 copy status
223 --verbose : output additional diagnostics
224""",
225 "revinfo":
226 """Outputs source path, server URL and revision information for every
227dependency in all solutions (no local checkout required).
228
229usage: revinfo [options]
230""",
231}
232
233# parameterized by (solution_name, solution_url, safesync_url)
234DEFAULT_CLIENT_FILE_TEXT = (
235 """
236# An element of this array (a \"solution\") describes a repository directory
237# that will be checked out into your working copy. Each solution may
238# optionally define additional dependencies (via its DEPS file) to be
239# checked out alongside the solution's directory. A solution may also
240# specify custom dependencies (via the \"custom_deps\" property) that
241# override or augment the dependencies specified by the DEPS file.
242# If a \"safesync_url\" is specified, it is assumed to reference the location of
243# a text file which contains nothing but the last known good SCM revision to
244# sync against. It is fetched if specified and used unless --head is passed
245solutions = [
246 { \"name\" : \"%s\",
247 \"url\" : \"%s\",
248 \"custom_deps\" : {
249 # To use the trunk of a component instead of what's in DEPS:
250 #\"component\": \"https://svnserver/component/trunk/\",
251 # To exclude a component from your working copy:
252 #\"data/really_large_component\": None,
253 },
254 \"safesync_url\": \"%s\"
255 }
256]
257""")
258
259
260## Generic utils
261
[email protected]e105d8d2009-04-30 17:58:25262def ParseXML(output):
263 try:
264 return xml.dom.minidom.parseString(output)
265 except xml.parsers.expat.ExpatError:
266 return None
267
268
[email protected]483b0082009-05-07 02:57:14269def GetNamedNodeText(node, node_name):
270 child_nodes = node.getElementsByTagName(node_name)
271 if not child_nodes:
272 return None
273 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
274 return child_nodes[0].firstChild.nodeValue
275
276
277def GetNodeNamedAttributeText(node, node_name, attribute_name):
278 child_nodes = node.getElementsByTagName(node_name)
279 if not child_nodes:
280 return None
281 assert len(child_nodes) == 1
282 return child_nodes[0].getAttribute(attribute_name)
283
284
[email protected]fb2b8eb2009-04-23 21:03:42285class Error(Exception):
286 """gclient exception class."""
287 pass
288
289class PrintableObject(object):
290 def __str__(self):
291 output = ''
292 for i in dir(self):
293 if i.startswith('__'):
294 continue
295 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
296 return output
297
298
299def FileRead(filename):
300 content = None
301 f = open(filename, "rU")
302 try:
303 content = f.read()
304 finally:
305 f.close()
306 return content
307
308
309def FileWrite(filename, content):
310 f = open(filename, "w")
311 try:
312 f.write(content)
313 finally:
314 f.close()
315
316
317def RemoveDirectory(*path):
318 """Recursively removes a directory, even if it's marked read-only.
319
320 Remove the directory located at *path, if it exists.
321
322 shutil.rmtree() doesn't work on Windows if any of the files or directories
323 are read-only, which svn repositories and some .svn files are. We need to
324 be able to force the files to be writable (i.e., deletable) as we traverse
325 the tree.
326
327 Even with all this, Windows still sometimes fails to delete a file, citing
328 a permission error (maybe something to do with antivirus scans or disk
329 indexing). The best suggestion any of the user forums had was to wait a
330 bit and try again, so we do that too. It's hand-waving, but sometimes it
331 works. :/
332
333 On POSIX systems, things are a little bit simpler. The modes of the files
334 to be deleted doesn't matter, only the modes of the directories containing
335 them are significant. As the directory tree is traversed, each directory
336 has its mode set appropriately before descending into it. This should
337 result in the entire tree being removed, with the possible exception of
338 *path itself, because nothing attempts to change the mode of its parent.
339 Doing so would be hazardous, as it's not a directory slated for removal.
340 In the ordinary case, this is not a problem: for our purposes, the user
341 will never lack write permission on *path's parent.
342 """
343 file_path = os.path.join(*path)
344 if not os.path.exists(file_path):
345 return
346
347 if os.path.islink(file_path) or not os.path.isdir(file_path):
348 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
349
350 has_win32api = False
351 if sys.platform == 'win32':
352 has_win32api = True
353 # Some people don't have the APIs installed. In that case we'll do without.
354 try:
355 win32api = __import__('win32api')
356 win32con = __import__('win32con')
357 except ImportError:
358 has_win32api = False
359 else:
360 # On POSIX systems, we need the x-bit set on the directory to access it,
361 # the r-bit to see its contents, and the w-bit to remove files from it.
362 # The actual modes of the files within the directory is irrelevant.
363 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
364 for fn in os.listdir(file_path):
365 fullpath = os.path.join(file_path, fn)
366
367 # If fullpath is a symbolic link that points to a directory, isdir will
368 # be True, but we don't want to descend into that as a directory, we just
369 # want to remove the link. Check islink and treat links as ordinary files
370 # would be treated regardless of what they reference.
371 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
372 if sys.platform == 'win32':
373 os.chmod(fullpath, stat.S_IWRITE)
374 if has_win32api:
375 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
376 try:
377 os.remove(fullpath)
378 except OSError, e:
379 if e.errno != errno.EACCES or sys.platform != 'win32':
380 raise
381 print 'Failed to delete %s: trying again' % fullpath
382 time.sleep(0.1)
383 os.remove(fullpath)
384 else:
385 RemoveDirectory(fullpath)
386
387 if sys.platform == 'win32':
388 os.chmod(file_path, stat.S_IWRITE)
389 if has_win32api:
390 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
391 try:
392 os.rmdir(file_path)
393 except OSError, e:
394 if e.errno != errno.EACCES or sys.platform != 'win32':
395 raise
396 print 'Failed to remove %s: trying again' % file_path
397 time.sleep(0.1)
398 os.rmdir(file_path)
399
400
[email protected]df7a3132009-05-12 17:49:49401def SubprocessCall(command, in_directory, fail_status=None):
[email protected]fb2b8eb2009-04-23 21:03:42402 """Runs command, a list, in directory in_directory.
403
404 This function wraps SubprocessCallAndCapture, but does not perform the
405 capturing functions. See that function for a more complete usage
406 description.
407 """
408 # Call subprocess and capture nothing:
[email protected]df7a3132009-05-12 17:49:49409 SubprocessCallAndCapture(command, in_directory, fail_status)
[email protected]fb2b8eb2009-04-23 21:03:42410
411
[email protected]df7a3132009-05-12 17:49:49412def SubprocessCallAndCapture(command, in_directory, fail_status=None,
[email protected]fb2b8eb2009-04-23 21:03:42413 pattern=None, capture_list=None):
414 """Runs command, a list, in directory in_directory.
415
416 A message indicating what is being done, as well as the command's stdout,
417 is printed to out.
418
419 If a pattern is specified, any line in the output matching pattern will have
420 its first match group appended to capture_list.
421
422 If the command fails, as indicated by a nonzero exit status, gclient will
423 exit with an exit status of fail_status. If fail_status is None (the
424 default), gclient will raise an Error exception.
425 """
426
[email protected]df7a3132009-05-12 17:49:49427 print("\n________ running \'%s\' in \'%s\'"
428 % (' '.join(command), in_directory))
[email protected]fb2b8eb2009-04-23 21:03:42429
430 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
431 # executable, but shell=True makes subprocess on Linux fail when it's called
432 # with a list because it only tries to execute the first item in the list.
433 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
434 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
435
436 if pattern:
437 compiled_pattern = re.compile(pattern)
438
439 # Also, we need to forward stdout to prevent weird re-ordering of output.
440 # This has to be done on a per byte basis to make sure it is not buffered:
441 # normally buffering is done for each line, but if svn requests input, no
442 # end-of-line character is output after the prompt and it would not show up.
443 in_byte = kid.stdout.read(1)
444 in_line = ""
445 while in_byte:
446 if in_byte != "\r":
[email protected]df7a3132009-05-12 17:49:49447 sys.stdout.write(in_byte)
[email protected]fb2b8eb2009-04-23 21:03:42448 in_line += in_byte
449 if in_byte == "\n" and pattern:
450 match = compiled_pattern.search(in_line[:-1])
451 if match:
452 capture_list.append(match.group(1))
453 in_line = ""
454 in_byte = kid.stdout.read(1)
455 rv = kid.wait()
456
457 if rv:
458 msg = "failed to run command: %s" % " ".join(command)
459
460 if fail_status != None:
461 print >>sys.stderr, msg
462 sys.exit(fail_status)
463
464 raise Error(msg)
465
466
467def IsUsingGit(root, paths):
468 """Returns True if we're using git to manage any of our checkouts.
469 |entries| is a list of paths to check."""
470 for path in paths:
471 if os.path.exists(os.path.join(root, path, '.git')):
472 return True
473 return False
474
475# -----------------------------------------------------------------------------
476# SVN utils:
477
478
[email protected]df7a3132009-05-12 17:49:49479def RunSVN(args, in_directory):
[email protected]fb2b8eb2009-04-23 21:03:42480 """Runs svn, sending output to stdout.
481
482 Args:
483 args: A sequence of command line parameters to be passed to svn.
484 in_directory: The directory where svn is to be run.
485
486 Raises:
487 Error: An error occurred while running the svn command.
488 """
489 c = [SVN_COMMAND]
490 c.extend(args)
491
[email protected]df7a3132009-05-12 17:49:49492 SubprocessCall(c, in_directory)
[email protected]fb2b8eb2009-04-23 21:03:42493
494
[email protected]5c3a2ff2009-05-12 19:28:55495def CaptureSVN(args, in_directory=None, print_error=True):
[email protected]fb2b8eb2009-04-23 21:03:42496 """Runs svn, capturing output sent to stdout as a string.
497
498 Args:
499 args: A sequence of command line parameters to be passed to svn.
500 in_directory: The directory where svn is to be run.
501
502 Returns:
503 The output sent to stdout as a string.
504 """
505 c = [SVN_COMMAND]
506 c.extend(args)
507
508 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
509 # the svn.exe executable, but shell=True makes subprocess on Linux fail
510 # when it's called with a list because it only tries to execute the
511 # first string ("svn").
[email protected]5c3a2ff2009-05-12 19:28:55512 stderr = None
[email protected]672343d2009-05-20 20:03:25513 if not print_error:
[email protected]5c3a2ff2009-05-12 19:28:55514 stderr = subprocess.PIPE
[email protected]df7a3132009-05-12 17:49:49515 return subprocess.Popen(c,
516 cwd=in_directory,
517 shell=(sys.platform == 'win32'),
[email protected]5c3a2ff2009-05-12 19:28:55518 stdout=subprocess.PIPE,
519 stderr=stderr).communicate()[0]
[email protected]fb2b8eb2009-04-23 21:03:42520
521
[email protected]df7a3132009-05-12 17:49:49522def RunSVNAndGetFileList(args, in_directory, file_list):
[email protected]fb2b8eb2009-04-23 21:03:42523 """Runs svn checkout, update, or status, output to stdout.
524
525 The first item in args must be either "checkout", "update", or "status".
526
527 svn's stdout is parsed to collect a list of files checked out or updated.
528 These files are appended to file_list. svn's stdout is also printed to
529 sys.stdout as in RunSVN.
530
531 Args:
532 args: A sequence of command line parameters to be passed to svn.
533 in_directory: The directory where svn is to be run.
534
535 Raises:
536 Error: An error occurred while running the svn command.
537 """
538 command = [SVN_COMMAND]
539 command.extend(args)
540
541 # svn update and svn checkout use the same pattern: the first three columns
542 # are for file status, property status, and lock status. This is followed
543 # by two spaces, and then the path to the file.
544 update_pattern = '^... (.*)$'
545
546 # The first three columns of svn status are the same as for svn update and
547 # svn checkout. The next three columns indicate addition-with-history,
548 # switch, and remote lock status. This is followed by one space, and then
549 # the path to the file.
550 status_pattern = '^...... (.*)$'
551
552 # args[0] must be a supported command. This will blow up if it's something
553 # else, which is good. Note that the patterns are only effective when
554 # these commands are used in their ordinary forms, the patterns are invalid
555 # for "svn status --show-updates", for example.
556 pattern = {
557 'checkout': update_pattern,
558 'status': status_pattern,
559 'update': update_pattern,
560 }[args[0]]
561
[email protected]df7a3132009-05-12 17:49:49562 SubprocessCallAndCapture(command,
563 in_directory,
564 pattern=pattern,
565 capture_list=file_list)
[email protected]fb2b8eb2009-04-23 21:03:42566
567
[email protected]5c3a2ff2009-05-12 19:28:55568def CaptureSVNInfo(relpath, in_directory=None, print_error=True):
[email protected]2dc8a4d2009-05-11 12:41:20569 """Returns a dictionary from the svn info output for the given file.
[email protected]fb2b8eb2009-04-23 21:03:42570
571 Args:
572 relpath: The directory where the working copy resides relative to
573 the directory given by in_directory.
574 in_directory: The directory where svn is to be run.
[email protected]fb2b8eb2009-04-23 21:03:42575 """
[email protected]5c3a2ff2009-05-12 19:28:55576 output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error)
[email protected]df7a3132009-05-12 17:49:49577 dom = ParseXML(output)
[email protected]2dc8a4d2009-05-11 12:41:20578 result = {}
[email protected]483b0082009-05-07 02:57:14579 if dom:
[email protected]2dc8a4d2009-05-11 12:41:20580 def C(item, f):
581 if item is not None: return f(item)
[email protected]483b0082009-05-07 02:57:14582 # /info/entry/
583 # url
584 # reposityory/(root|uuid)
585 # wc-info/(schedule|depth)
586 # commit/(author|date)
587 # str() the results because they may be returned as Unicode, which
588 # interferes with the higher layers matching up things in the deps
589 # dictionary.
[email protected]2dc8a4d2009-05-11 12:41:20590 # TODO(maruel): Fix at higher level instead (!)
591 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
592 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
593 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
594 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
595 int)
596 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
597 str)
598 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
599 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
600 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
601 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
[email protected]fb2b8eb2009-04-23 21:03:42602 return result
603
604
[email protected]df7a3132009-05-12 17:49:49605def CaptureSVNHeadRevision(url):
[email protected]fb2b8eb2009-04-23 21:03:42606 """Get the head revision of a SVN repository.
607
608 Returns:
609 Int head revision
610 """
[email protected]df7a3132009-05-12 17:49:49611 info = CaptureSVN(["info", "--xml", url], os.getcwd())
[email protected]fb2b8eb2009-04-23 21:03:42612 dom = xml.dom.minidom.parseString(info)
613 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
614
615
[email protected]4810a962009-05-12 21:03:34616def CaptureSVNStatus(files):
617 """Returns the svn 1.5 svn status emulated output.
[email protected]fb2b8eb2009-04-23 21:03:42618
[email protected]4810a962009-05-12 21:03:34619 @files can be a string (one file) or a list of files.
[email protected]fb2b8eb2009-04-23 21:03:42620
[email protected]4810a962009-05-12 21:03:34621 Returns an array of (status, file) tuples."""
622 command = ["status", "--xml"]
623 if not files:
624 pass
625 elif isinstance(files, basestring):
626 command.append(files)
627 else:
628 command.extend(files)
[email protected]fb2b8eb2009-04-23 21:03:42629
[email protected]4810a962009-05-12 21:03:34630 status_letter = {
631 None: ' ',
632 '': ' ',
633 'added': 'A',
634 'conflicted': 'C',
635 'deleted': 'D',
636 'external': 'X',
637 'ignored': 'I',
638 'incomplete': '!',
639 'merged': 'G',
640 'missing': '!',
641 'modified': 'M',
642 'none': ' ',
643 'normal': ' ',
644 'obstructed': '~',
645 'replaced': 'R',
646 'unversioned': '?',
647 }
648 dom = ParseXML(CaptureSVN(command))
[email protected]e105d8d2009-04-30 17:58:25649 results = []
650 if dom:
651 # /status/target/entry/(wc-status|commit|author|date)
652 for target in dom.getElementsByTagName('target'):
653 base_path = target.getAttribute('path')
654 for entry in target.getElementsByTagName('entry'):
655 file = entry.getAttribute('path')
656 wc_status = entry.getElementsByTagName('wc-status')
657 assert len(wc_status) == 1
658 # Emulate svn 1.5 status ouput...
659 statuses = [' ' for i in range(7)]
660 # Col 0
661 xml_item_status = wc_status[0].getAttribute('item')
[email protected]4810a962009-05-12 21:03:34662 if xml_item_status in status_letter:
663 statuses[0] = status_letter[xml_item_status]
[email protected]e105d8d2009-04-30 17:58:25664 else:
665 raise Exception('Unknown item status "%s"; please implement me!' %
666 xml_item_status)
667 # Col 1
668 xml_props_status = wc_status[0].getAttribute('props')
669 if xml_props_status == 'modified':
670 statuses[1] = 'M'
671 elif xml_props_status == 'conflicted':
672 statuses[1] = 'C'
673 elif (not xml_props_status or xml_props_status == 'none' or
674 xml_props_status == 'normal'):
675 pass
676 else:
677 raise Exception('Unknown props status "%s"; please implement me!' %
678 xml_props_status)
[email protected]edd27d12009-05-01 17:46:56679 # Col 2
680 if wc_status[0].getAttribute('wc-locked') == 'true':
681 statuses[2] = 'L'
[email protected]e105d8d2009-04-30 17:58:25682 # Col 3
683 if wc_status[0].getAttribute('copied') == 'true':
684 statuses[3] = '+'
[email protected]4810a962009-05-12 21:03:34685 item = (''.join(statuses), file)
[email protected]e105d8d2009-04-30 17:58:25686 results.append(item)
687 return results
[email protected]fb2b8eb2009-04-23 21:03:42688
689
690### SCM abstraction layer
691
692
693class SCMWrapper(object):
694 """Add necessary glue between all the supported SCM.
695
696 This is the abstraction layer to bind to different SCM. Since currently only
697 subversion is supported, a lot of subersionism remains. This can be sorted out
698 once another SCM is supported."""
699 def __init__(self, url=None, root_dir=None, relpath=None,
700 scm_name='svn'):
701 # TODO(maruel): Deduce the SCM from the url.
702 self.scm_name = scm_name
703 self.url = url
704 self._root_dir = root_dir
705 if self._root_dir:
[email protected]e105d8d2009-04-30 17:58:25706 self._root_dir = self._root_dir.replace('/', os.sep)
[email protected]fb2b8eb2009-04-23 21:03:42707 self.relpath = relpath
708 if self.relpath:
[email protected]e105d8d2009-04-30 17:58:25709 self.relpath = self.relpath.replace('/', os.sep)
[email protected]fb2b8eb2009-04-23 21:03:42710
711 def FullUrlForRelativeUrl(self, url):
712 # Find the forth '/' and strip from there. A bit hackish.
713 return '/'.join(self.url.split('/')[:4]) + url
714
715 def RunCommand(self, command, options, args, file_list=None):
716 # file_list will have all files that are modified appended to it.
717
718 if file_list == None:
719 file_list = []
720
721 commands = {
722 'cleanup': self.cleanup,
723 'update': self.update,
724 'revert': self.revert,
725 'status': self.status,
726 'diff': self.diff,
727 'runhooks': self.status,
728 }
729
730 if not command in commands:
731 raise Error('Unknown command %s' % command)
732
733 return commands[command](options, args, file_list)
734
735 def cleanup(self, options, args, file_list):
736 """Cleanup working copy."""
737 command = ['cleanup']
738 command.extend(args)
[email protected]df7a3132009-05-12 17:49:49739 RunSVN(command, os.path.join(self._root_dir, self.relpath))
[email protected]fb2b8eb2009-04-23 21:03:42740
741 def diff(self, options, args, file_list):
742 # NOTE: This function does not currently modify file_list.
743 command = ['diff']
744 command.extend(args)
[email protected]df7a3132009-05-12 17:49:49745 RunSVN(command, os.path.join(self._root_dir, self.relpath))
[email protected]fb2b8eb2009-04-23 21:03:42746
747 def update(self, options, args, file_list):
748 """Runs SCM to update or transparently checkout the working copy.
749
750 All updated files will be appended to file_list.
751
752 Raises:
753 Error: if can't get URL for relative path.
754 """
755 # Only update if git is not controlling the directory.
[email protected]8626ff72009-05-13 02:57:02756 checkout_path = os.path.join(self._root_dir, self.relpath)
[email protected]0329e672009-05-13 18:41:04757 git_path = os.path.join(self._root_dir, self.relpath, '.git')
758 if os.path.exists(git_path):
[email protected]df7a3132009-05-12 17:49:49759 print("________ found .git directory; skipping %s" % self.relpath)
[email protected]fb2b8eb2009-04-23 21:03:42760 return
761
762 if args:
763 raise Error("Unsupported argument(s): %s" % ",".join(args))
764
765 url = self.url
766 components = url.split("@")
767 revision = None
768 forced_revision = False
769 if options.revision:
770 # Override the revision number.
771 url = '%s@%s' % (components[0], str(options.revision))
772 revision = int(options.revision)
773 forced_revision = True
774 elif len(components) == 2:
775 revision = int(components[1])
776 forced_revision = True
777
778 rev_str = ""
779 if revision:
780 rev_str = ' at %d' % revision
781
[email protected]0329e672009-05-13 18:41:04782 if not os.path.exists(checkout_path):
[email protected]fb2b8eb2009-04-23 21:03:42783 # We need to checkout.
[email protected]8626ff72009-05-13 02:57:02784 command = ['checkout', url, checkout_path]
[email protected]edd27d12009-05-01 17:46:56785 if revision:
786 command.extend(['--revision', str(revision)])
[email protected]df7a3132009-05-12 17:49:49787 RunSVNAndGetFileList(command, self._root_dir, file_list)
[email protected]edd27d12009-05-01 17:46:56788 return
[email protected]fb2b8eb2009-04-23 21:03:42789
790 # Get the existing scm url and the revision number of the current checkout.
[email protected]8626ff72009-05-13 02:57:02791 from_info = CaptureSVNInfo(os.path.join(checkout_path, '.'), '.')
[email protected]1998c6d2009-05-15 12:38:12792 if not from_info:
793 raise Error("Can't update/checkout %r if an unversioned directory is "
794 "present. Delete the directory and try again." %
795 checkout_path)
[email protected]fb2b8eb2009-04-23 21:03:42796
797 if options.manually_grab_svn_rev:
798 # Retrieve the current HEAD version because svn is slow at null updates.
799 if not revision:
[email protected]df7a3132009-05-12 17:49:49800 from_info_live = CaptureSVNInfo(from_info['URL'], '.')
[email protected]2dc8a4d2009-05-11 12:41:20801 revision = int(from_info_live['Revision'])
[email protected]fb2b8eb2009-04-23 21:03:42802 rev_str = ' at %d' % revision
803
[email protected]2dc8a4d2009-05-11 12:41:20804 if from_info['URL'] != components[0]:
[email protected]df7a3132009-05-12 17:49:49805 to_info = CaptureSVNInfo(url, '.')
[email protected]8626ff72009-05-13 02:57:02806 can_switch = ((from_info['Repository Root'] != to_info['Repository Root'])
807 and (from_info['UUID'] == to_info['UUID']))
808 if can_switch:
809 print("\n_____ relocating %s to a new checkout" % self.relpath)
[email protected]fb2b8eb2009-04-23 21:03:42810 # We have different roots, so check if we can switch --relocate.
811 # Subversion only permits this if the repository UUIDs match.
[email protected]fb2b8eb2009-04-23 21:03:42812 # Perform the switch --relocate, then rewrite the from_url
813 # to reflect where we "are now." (This is the same way that
814 # Subversion itself handles the metadata when switch --relocate
815 # is used.) This makes the checks below for whether we
816 # can update to a revision or have to switch to a different
817 # branch work as expected.
818 # TODO(maruel): TEST ME !
[email protected]2dc8a4d2009-05-11 12:41:20819 command = ["switch", "--relocate",
820 from_info['Repository Root'],
821 to_info['Repository Root'],
[email protected]fb2b8eb2009-04-23 21:03:42822 self.relpath]
[email protected]df7a3132009-05-12 17:49:49823 RunSVN(command, self._root_dir)
[email protected]2dc8a4d2009-05-11 12:41:20824 from_info['URL'] = from_info['URL'].replace(
825 from_info['Repository Root'],
826 to_info['Repository Root'])
[email protected]8626ff72009-05-13 02:57:02827 else:
828 if CaptureSVNStatus(checkout_path):
829 raise Error("Can't switch the checkout to %s; UUID don't match and "
830 "there is local changes in %s. Delete the directory and "
831 "try again." % (url, checkout_path))
832 # Ok delete it.
833 print("\n_____ switching %s to a new checkout" % self.relpath)
834 RemoveDirectory(checkout_path)
835 # We need to checkout.
836 command = ['checkout', url, checkout_path]
837 if revision:
838 command.extend(['--revision', str(revision)])
839 RunSVNAndGetFileList(command, self._root_dir, file_list)
840 return
841
[email protected]fb2b8eb2009-04-23 21:03:42842
843 # If the provided url has a revision number that matches the revision
844 # number of the existing directory, then we don't need to bother updating.
[email protected]2dc8a4d2009-05-11 12:41:20845 if not options.force and from_info['Revision'] == revision:
[email protected]fb2b8eb2009-04-23 21:03:42846 if options.verbose or not forced_revision:
[email protected]df7a3132009-05-12 17:49:49847 print("\n_____ %s%s" % (self.relpath, rev_str))
[email protected]fb2b8eb2009-04-23 21:03:42848 return
849
[email protected]8626ff72009-05-13 02:57:02850 command = ["update", checkout_path]
[email protected]fb2b8eb2009-04-23 21:03:42851 if revision:
852 command.extend(['--revision', str(revision)])
[email protected]df7a3132009-05-12 17:49:49853 RunSVNAndGetFileList(command, self._root_dir, file_list)
[email protected]fb2b8eb2009-04-23 21:03:42854
855 def revert(self, options, args, file_list):
856 """Reverts local modifications. Subversion specific.
857
858 All reverted files will be appended to file_list, even if Subversion
859 doesn't know about them.
860 """
861 path = os.path.join(self._root_dir, self.relpath)
862 if not os.path.isdir(path):
[email protected]edd27d12009-05-01 17:46:56863 # svn revert won't work if the directory doesn't exist. It needs to
864 # checkout instead.
[email protected]df7a3132009-05-12 17:49:49865 print("\n_____ %s is missing, synching instead" % self.relpath)
[email protected]edd27d12009-05-01 17:46:56866 # Don't reuse the args.
867 return self.update(options, [], file_list)
[email protected]fb2b8eb2009-04-23 21:03:42868
[email protected]df7a3132009-05-12 17:49:49869 files = CaptureSVNStatus(path)
[email protected]fb2b8eb2009-04-23 21:03:42870 # Batch the command.
871 files_to_revert = []
872 for file in files:
[email protected]4810a962009-05-12 21:03:34873 file_path = os.path.join(path, file[1])
[email protected]df7a3132009-05-12 17:49:49874 print(file_path)
[email protected]fb2b8eb2009-04-23 21:03:42875 # Unversioned file or unexpected unversioned file.
[email protected]4810a962009-05-12 21:03:34876 if file[0][0] in ('?', '~'):
[email protected]fb2b8eb2009-04-23 21:03:42877 # Remove extraneous file. Also remove unexpected unversioned
878 # directories. svn won't touch them but we want to delete these.
879 file_list.append(file_path)
880 try:
881 os.remove(file_path)
882 except EnvironmentError:
883 RemoveDirectory(file_path)
884
[email protected]4810a962009-05-12 21:03:34885 if file[0][0] != '?':
[email protected]fb2b8eb2009-04-23 21:03:42886 # For any other status, svn revert will work.
887 file_list.append(file_path)
[email protected]4810a962009-05-12 21:03:34888 files_to_revert.append(file[1])
[email protected]fb2b8eb2009-04-23 21:03:42889
890 # Revert them all at once.
891 if files_to_revert:
892 accumulated_paths = []
893 accumulated_length = 0
894 command = ['revert']
895 for p in files_to_revert:
896 # Some shell have issues with command lines too long.
897 if accumulated_length and accumulated_length + len(p) > 3072:
[email protected]df7a3132009-05-12 17:49:49898 RunSVN(command + accumulated_paths,
[email protected]fb2b8eb2009-04-23 21:03:42899 os.path.join(self._root_dir, self.relpath))
900 accumulated_paths = []
901 accumulated_length = 0
902 else:
903 accumulated_paths.append(p)
904 accumulated_length += len(p)
905 if accumulated_paths:
[email protected]df7a3132009-05-12 17:49:49906 RunSVN(command + accumulated_paths,
[email protected]fb2b8eb2009-04-23 21:03:42907 os.path.join(self._root_dir, self.relpath))
908
909 def status(self, options, args, file_list):
910 """Display status information."""
[email protected]edd27d12009-05-01 17:46:56911 path = os.path.join(self._root_dir, self.relpath)
[email protected]fb2b8eb2009-04-23 21:03:42912 command = ['status']
913 command.extend(args)
[email protected]edd27d12009-05-01 17:46:56914 if not os.path.isdir(path):
915 # svn status won't work if the directory doesn't exist.
[email protected]df7a3132009-05-12 17:49:49916 print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
917 "does not exist."
918 % (' '.join(command), path))
[email protected]edd27d12009-05-01 17:46:56919 # There's no file list to retrieve.
920 else:
[email protected]df7a3132009-05-12 17:49:49921 RunSVNAndGetFileList(command, path, file_list)
[email protected]fb2b8eb2009-04-23 21:03:42922
923
924## GClient implementation.
925
926
927class GClient(object):
928 """Object that represent a gclient checkout."""
929
930 supported_commands = [
931 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
932 ]
933
934 def __init__(self, root_dir, options):
935 self._root_dir = root_dir
936 self._options = options
937 self._config_content = None
938 self._config_dict = {}
939 self._deps_hooks = []
940
941 def SetConfig(self, content):
942 self._config_dict = {}
943 self._config_content = content
[email protected]df0032c2009-05-29 10:43:56944 try:
945 exec(content, self._config_dict)
946 except SyntaxError, e:
947 try:
948 # Try to construct a human readable error message
949 error_message = [
950 'There is a syntax error in your configuration file.',
951 'Line #%s, character %s:' % (e.lineno, e.offset),
952 '"%s"' % re.sub(r'[\r\n]*$', '', e.text) ]
953 except:
954 # Something went wrong, re-raise the original exception
955 raise e
956 else:
957 # Raise a new exception with the human readable message:
958 raise Error('\n'.join(error_message))
[email protected]fb2b8eb2009-04-23 21:03:42959
960 def SaveConfig(self):
961 FileWrite(os.path.join(self._root_dir, self._options.config_filename),
962 self._config_content)
963
964 def _LoadConfig(self):
965 client_source = FileRead(os.path.join(self._root_dir,
966 self._options.config_filename))
967 self.SetConfig(client_source)
968
969 def ConfigContent(self):
970 return self._config_content
971
972 def GetVar(self, key, default=None):
973 return self._config_dict.get(key, default)
974
975 @staticmethod
976 def LoadCurrentConfig(options, from_dir=None):
977 """Searches for and loads a .gclient file relative to the current working
978 dir.
979
980 Returns:
981 A dict representing the contents of the .gclient file or an empty dict if
982 the .gclient file doesn't exist.
983 """
984 if not from_dir:
985 from_dir = os.curdir
986 path = os.path.realpath(from_dir)
[email protected]0329e672009-05-13 18:41:04987 while not os.path.exists(os.path.join(path, options.config_filename)):
[email protected]fb2b8eb2009-04-23 21:03:42988 next = os.path.split(path)
989 if not next[1]:
990 return None
991 path = next[0]
[email protected]2806acc2009-05-15 12:33:34992 client = GClient(path, options)
[email protected]fb2b8eb2009-04-23 21:03:42993 client._LoadConfig()
994 return client
995
996 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
997 self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
998 solution_name, solution_url, safesync_url
999 ))
1000
1001 def _SaveEntries(self, entries):
1002 """Creates a .gclient_entries file to record the list of unique checkouts.
1003
1004 The .gclient_entries file lives in the same directory as .gclient.
1005
1006 Args:
1007 entries: A sequence of solution names.
1008 """
1009 text = "entries = [\n"
1010 for entry in entries:
1011 text += " \"%s\",\n" % entry
1012 text += "]\n"
1013 FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
1014 text)
1015
1016 def _ReadEntries(self):
1017 """Read the .gclient_entries file for the given client.
1018
1019 Args:
1020 client: The client for which the entries file should be read.
1021
1022 Returns:
1023 A sequence of solution names, which will be empty if there is the
1024 entries file hasn't been created yet.
1025 """
1026 scope = {}
1027 filename = os.path.join(self._root_dir, self._options.entries_filename)
[email protected]0329e672009-05-13 18:41:041028 if not os.path.exists(filename):
[email protected]fb2b8eb2009-04-23 21:03:421029 return []
1030 exec(FileRead(filename), scope)
1031 return scope["entries"]
1032
1033 class FromImpl:
1034 """Used to implement the From syntax."""
1035
1036 def __init__(self, module_name):
1037 self.module_name = module_name
1038
1039 def __str__(self):
1040 return 'From("%s")' % self.module_name
1041
1042 class _VarImpl:
1043 def __init__(self, custom_vars, local_scope):
1044 self._custom_vars = custom_vars
1045 self._local_scope = local_scope
1046
1047 def Lookup(self, var_name):
1048 """Implements the Var syntax."""
1049 if var_name in self._custom_vars:
1050 return self._custom_vars[var_name]
1051 elif var_name in self._local_scope.get("vars", {}):
1052 return self._local_scope["vars"][var_name]
1053 raise Error("Var is not defined: %s" % var_name)
1054
1055 def _ParseSolutionDeps(self, solution_name, solution_deps_content,
1056 custom_vars):
1057 """Parses the DEPS file for the specified solution.
1058
1059 Args:
1060 solution_name: The name of the solution to query.
1061 solution_deps_content: Content of the DEPS file for the solution
1062 custom_vars: A dict of vars to override any vars defined in the DEPS file.
1063
1064 Returns:
1065 A dict mapping module names (as relative paths) to URLs or an empty
1066 dict if the solution does not have a DEPS file.
1067 """
1068 # Skip empty
1069 if not solution_deps_content:
1070 return {}
1071 # Eval the content
1072 local_scope = {}
1073 var = self._VarImpl(custom_vars, local_scope)
1074 global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
1075 exec(solution_deps_content, global_scope, local_scope)
1076 deps = local_scope.get("deps", {})
1077
1078 # load os specific dependencies if defined. these dependencies may
1079 # override or extend the values defined by the 'deps' member.
1080 if "deps_os" in local_scope:
1081 deps_os_choices = {
1082 "win32": "win",
1083 "win": "win",
1084 "cygwin": "win",
1085 "darwin": "mac",
1086 "mac": "mac",
1087 "unix": "unix",
1088 "linux": "unix",
1089 "linux2": "unix",
1090 }
1091
1092 if self._options.deps_os is not None:
1093 deps_to_include = self._options.deps_os.split(",")
1094 if "all" in deps_to_include:
1095 deps_to_include = deps_os_choices.values()
1096 else:
1097 deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
1098
1099 deps_to_include = set(deps_to_include)
1100 for deps_os_key in deps_to_include:
1101 os_deps = local_scope["deps_os"].get(deps_os_key, {})
1102 if len(deps_to_include) > 1:
1103 # Ignore any overrides when including deps for more than one
1104 # platform, so we collect the broadest set of dependencies available.
1105 # We may end up with the wrong revision of something for our
1106 # platform, but this is the best we can do.
1107 deps.update([x for x in os_deps.items() if not x[0] in deps])
1108 else:
1109 deps.update(os_deps)
1110
1111 if 'hooks' in local_scope:
1112 self._deps_hooks.extend(local_scope['hooks'])
1113
1114 # If use_relative_paths is set in the DEPS file, regenerate
1115 # the dictionary using paths relative to the directory containing
1116 # the DEPS file.
1117 if local_scope.get('use_relative_paths'):
1118 rel_deps = {}
1119 for d, url in deps.items():
1120 # normpath is required to allow DEPS to use .. in their
1121 # dependency local path.
1122 rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
1123 return rel_deps
1124 else:
1125 return deps
1126
1127 def _ParseAllDeps(self, solution_urls, solution_deps_content):
1128 """Parse the complete list of dependencies for the client.
1129
1130 Args:
1131 solution_urls: A dict mapping module names (as relative paths) to URLs
1132 corresponding to the solutions specified by the client. This parameter
1133 is passed as an optimization.
1134 solution_deps_content: A dict mapping module names to the content
1135 of their DEPS files
1136
1137 Returns:
1138 A dict mapping module names (as relative paths) to URLs corresponding
1139 to the entire set of dependencies to checkout for the given client.
1140
1141 Raises:
1142 Error: If a dependency conflicts with another dependency or of a solution.
1143 """
1144 deps = {}
1145 for solution in self.GetVar("solutions"):
1146 custom_vars = solution.get("custom_vars", {})
1147 solution_deps = self._ParseSolutionDeps(
1148 solution["name"],
1149 solution_deps_content[solution["name"]],
1150 custom_vars)
1151
1152 # If a line is in custom_deps, but not in the solution, we want to append
1153 # this line to the solution.
1154 if "custom_deps" in solution:
1155 for d in solution["custom_deps"]:
1156 if d not in solution_deps:
1157 solution_deps[d] = solution["custom_deps"][d]
1158
1159 for d in solution_deps:
1160 if "custom_deps" in solution and d in solution["custom_deps"]:
1161 # Dependency is overriden.
1162 url = solution["custom_deps"][d]
1163 if url is None:
1164 continue
1165 else:
1166 url = solution_deps[d]
1167 # if we have a From reference dependent on another solution, then
1168 # just skip the From reference. When we pull deps for the solution,
1169 # we will take care of this dependency.
1170 #
1171 # If multiple solutions all have the same From reference, then we
1172 # should only add one to our list of dependencies.
1173 if type(url) != str:
1174 if url.module_name in solution_urls:
1175 # Already parsed.
1176 continue
1177 if d in deps and type(deps[d]) != str:
1178 if url.module_name == deps[d].module_name:
1179 continue
1180 else:
1181 parsed_url = urlparse.urlparse(url)
1182 scheme = parsed_url[0]
1183 if not scheme:
1184 # A relative url. Fetch the real base.
1185 path = parsed_url[2]
1186 if path[0] != "/":
1187 raise Error(
1188 "relative DEPS entry \"%s\" must begin with a slash" % d)
1189 # Create a scm just to query the full url.
[email protected]2806acc2009-05-15 12:33:341190 scm = SCMWrapper(solution["url"], self._root_dir, None)
[email protected]fb2b8eb2009-04-23 21:03:421191 url = scm.FullUrlForRelativeUrl(url)
1192 if d in deps and deps[d] != url:
1193 raise Error(
1194 "Solutions have conflicting versions of dependency \"%s\"" % d)
1195 if d in solution_urls and solution_urls[d] != url:
1196 raise Error(
1197 "Dependency \"%s\" conflicts with specified solution" % d)
1198 # Grab the dependency.
1199 deps[d] = url
1200 return deps
1201
1202 def _RunHookAction(self, hook_dict):
1203 """Runs the action from a single hook.
1204 """
1205 command = hook_dict['action'][:]
1206 if command[0] == 'python':
1207 # If the hook specified "python" as the first item, the action is a
1208 # Python script. Run it by starting a new copy of the same
1209 # interpreter.
1210 command[0] = sys.executable
1211
1212 # Use a discrete exit status code of 2 to indicate that a hook action
1213 # failed. Users of this script may wish to treat hook action failures
1214 # differently from VC failures.
[email protected]df7a3132009-05-12 17:49:491215 SubprocessCall(command, self._root_dir, fail_status=2)
[email protected]fb2b8eb2009-04-23 21:03:421216
1217 def _RunHooks(self, command, file_list, is_using_git):
1218 """Evaluates all hooks, running actions as needed.
1219 """
1220 # Hooks only run for these command types.
1221 if not command in ('update', 'revert', 'runhooks'):
1222 return
1223
1224 # Get any hooks from the .gclient file.
1225 hooks = self.GetVar("hooks", [])
1226 # Add any hooks found in DEPS files.
1227 hooks.extend(self._deps_hooks)
1228
1229 # If "--force" was specified, run all hooks regardless of what files have
1230 # changed. If the user is using git, then we don't know what files have
1231 # changed so we always run all hooks.
1232 if self._options.force or is_using_git:
1233 for hook_dict in hooks:
1234 self._RunHookAction(hook_dict)
1235 return
1236
1237 # Run hooks on the basis of whether the files from the gclient operation
1238 # match each hook's pattern.
1239 for hook_dict in hooks:
1240 pattern = re.compile(hook_dict['pattern'])
1241 for file in file_list:
1242 if not pattern.search(file):
1243 continue
1244
1245 self._RunHookAction(hook_dict)
1246
1247 # The hook's action only runs once. Don't bother looking for any
1248 # more matches.
1249 break
1250
1251 def RunOnDeps(self, command, args):
1252 """Runs a command on each dependency in a client and its dependencies.
1253
1254 The module's dependencies are specified in its top-level DEPS files.
1255
1256 Args:
1257 command: The command to use (e.g., 'status' or 'diff')
1258 args: list of str - extra arguments to add to the command line.
1259
1260 Raises:
1261 Error: If the client has conflicting entries.
1262 """
1263 if not command in self.supported_commands:
1264 raise Error("'%s' is an unsupported command" % command)
1265
1266 # Check for revision overrides.
1267 revision_overrides = {}
1268 for revision in self._options.revisions:
1269 if revision.find("@") == -1:
1270 raise Error(
1271 "Specify the full dependency when specifying a revision number.")
1272 revision_elem = revision.split("@")
1273 # Disallow conflicting revs
1274 if revision_overrides.has_key(revision_elem[0]) and \
1275 revision_overrides[revision_elem[0]] != revision_elem[1]:
1276 raise Error(
1277 "Conflicting revision numbers specified.")
1278 revision_overrides[revision_elem[0]] = revision_elem[1]
1279
1280 solutions = self.GetVar("solutions")
1281 if not solutions:
1282 raise Error("No solution specified")
1283
1284 # When running runhooks --force, there's no need to consult the SCM.
1285 # All known hooks are expected to run unconditionally regardless of working
1286 # copy state, so skip the SCM status check.
1287 run_scm = not (command == 'runhooks' and self._options.force)
1288
1289 entries = {}
1290 entries_deps_content = {}
1291 file_list = []
1292 # Run on the base solutions first.
1293 for solution in solutions:
1294 name = solution["name"]
1295 if name in entries:
1296 raise Error("solution %s specified more than once" % name)
1297 url = solution["url"]
1298 entries[name] = url
1299 if run_scm:
1300 self._options.revision = revision_overrides.get(name)
[email protected]2806acc2009-05-15 12:33:341301 scm = SCMWrapper(url, self._root_dir, name)
[email protected]fb2b8eb2009-04-23 21:03:421302 scm.RunCommand(command, self._options, args, file_list)
1303 self._options.revision = None
1304 try:
1305 deps_content = FileRead(os.path.join(self._root_dir, name,
1306 self._options.deps_file))
1307 except IOError, e:
1308 if e.errno != errno.ENOENT:
1309 raise
1310 deps_content = ""
1311 entries_deps_content[name] = deps_content
1312
1313 # Process the dependencies next (sort alphanumerically to ensure that
1314 # containing directories get populated first and for readability)
1315 deps = self._ParseAllDeps(entries, entries_deps_content)
1316 deps_to_process = deps.keys()
1317 deps_to_process.sort()
1318
1319 # First pass for direct dependencies.
1320 for d in deps_to_process:
1321 if type(deps[d]) == str:
1322 url = deps[d]
1323 entries[d] = url
1324 if run_scm:
1325 self._options.revision = revision_overrides.get(d)
[email protected]2806acc2009-05-15 12:33:341326 scm = SCMWrapper(url, self._root_dir, d)
[email protected]fb2b8eb2009-04-23 21:03:421327 scm.RunCommand(command, self._options, args, file_list)
1328 self._options.revision = None
1329
1330 # Second pass for inherited deps (via the From keyword)
1331 for d in deps_to_process:
1332 if type(deps[d]) != str:
1333 sub_deps = self._ParseSolutionDeps(
1334 deps[d].module_name,
1335 FileRead(os.path.join(self._root_dir,
1336 deps[d].module_name,
1337 self._options.deps_file)),
1338 {})
1339 url = sub_deps[d]
1340 entries[d] = url
1341 if run_scm:
1342 self._options.revision = revision_overrides.get(d)
[email protected]2806acc2009-05-15 12:33:341343 scm = SCMWrapper(url, self._root_dir, d)
[email protected]fb2b8eb2009-04-23 21:03:421344 scm.RunCommand(command, self._options, args, file_list)
1345 self._options.revision = None
1346
1347 is_using_git = IsUsingGit(self._root_dir, entries.keys())
1348 self._RunHooks(command, file_list, is_using_git)
1349
1350 if command == 'update':
[email protected]cdcee802009-06-23 15:30:421351 # Notify the user if there is an orphaned entry in their working copy.
1352 # Only delete the directory if there are no changes in it, and
1353 # delete_unversioned_trees is set to true.
[email protected]fb2b8eb2009-04-23 21:03:421354 prev_entries = self._ReadEntries()
1355 for entry in prev_entries:
1356 e_dir = os.path.join(self._root_dir, entry)
[email protected]0329e672009-05-13 18:41:041357 if entry not in entries and os.path.exists(e_dir):
[email protected]8399dc02009-06-23 21:36:251358 if not self._options.delete_unversioned_trees or \
1359 CaptureSVNStatus(e_dir):
[email protected]fb2b8eb2009-04-23 21:03:421360 # There are modified files in this entry
1361 entries[entry] = None # Keep warning until removed.
[email protected]df7a3132009-05-12 17:49:491362 print("\nWARNING: \"%s\" is no longer part of this client. "
1363 "It is recommended that you manually remove it.\n") % entry
[email protected]fb2b8eb2009-04-23 21:03:421364 else:
1365 # Delete the entry
[email protected]df7a3132009-05-12 17:49:491366 print("\n________ deleting \'%s\' " +
1367 "in \'%s\'") % (entry, self._root_dir)
[email protected]fb2b8eb2009-04-23 21:03:421368 RemoveDirectory(e_dir)
1369 # record the current list of entries for next time
1370 self._SaveEntries(entries)
1371
1372 def PrintRevInfo(self):
1373 """Output revision info mapping for the client and its dependencies. This
1374 allows the capture of a overall "revision" for the source tree that can
1375 be used to reproduce the same tree in the future. The actual output
1376 contains enough information (source paths, svn server urls and revisions)
1377 that it can be used either to generate external svn commands (without
1378 gclient) or as input to gclient's --rev option (with some massaging of
1379 the data).
1380
1381 NOTE: Unlike RunOnDeps this does not require a local checkout and is run
1382 on the Pulse master. It MUST NOT execute hooks.
1383
1384 Raises:
1385 Error: If the client has conflicting entries.
1386 """
1387 # Check for revision overrides.
1388 revision_overrides = {}
1389 for revision in self._options.revisions:
1390 if revision.find("@") < 0:
1391 raise Error(
1392 "Specify the full dependency when specifying a revision number.")
1393 revision_elem = revision.split("@")
1394 # Disallow conflicting revs
1395 if revision_overrides.has_key(revision_elem[0]) and \
1396 revision_overrides[revision_elem[0]] != revision_elem[1]:
1397 raise Error(
1398 "Conflicting revision numbers specified.")
1399 revision_overrides[revision_elem[0]] = revision_elem[1]
1400
1401 solutions = self.GetVar("solutions")
1402 if not solutions:
1403 raise Error("No solution specified")
1404
1405 entries = {}
1406 entries_deps_content = {}
1407
1408 # Inner helper to generate base url and rev tuple (including honoring
1409 # |revision_overrides|)
1410 def GetURLAndRev(name, original_url):
1411 if original_url.find("@") < 0:
1412 if revision_overrides.has_key(name):
1413 return (original_url, int(revision_overrides[name]))
1414 else:
1415 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
[email protected]df7a3132009-05-12 17:49:491416 return (original_url, CaptureSVNHeadRevision(original_url))
[email protected]fb2b8eb2009-04-23 21:03:421417 else:
1418 url_components = original_url.split("@")
1419 if revision_overrides.has_key(name):
1420 return (url_components[0], int(revision_overrides[name]))
1421 else:
1422 return (url_components[0], int(url_components[1]))
1423
1424 # Run on the base solutions first.
1425 for solution in solutions:
1426 name = solution["name"]
1427 if name in entries:
1428 raise Error("solution %s specified more than once" % name)
1429 (url, rev) = GetURLAndRev(name, solution["url"])
1430 entries[name] = "%s@%d" % (url, rev)
1431 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1432 entries_deps_content[name] = CaptureSVN(
[email protected]fb2b8eb2009-04-23 21:03:421433 ["cat",
1434 "%s/%s@%d" % (url,
1435 self._options.deps_file,
1436 rev)],
1437 os.getcwd())
1438
1439 # Process the dependencies next (sort alphanumerically to ensure that
1440 # containing directories get populated first and for readability)
1441 deps = self._ParseAllDeps(entries, entries_deps_content)
1442 deps_to_process = deps.keys()
1443 deps_to_process.sort()
1444
1445 # First pass for direct dependencies.
1446 for d in deps_to_process:
1447 if type(deps[d]) == str:
1448 (url, rev) = GetURLAndRev(d, deps[d])
1449 entries[d] = "%s@%d" % (url, rev)
1450
1451 # Second pass for inherited deps (via the From keyword)
1452 for d in deps_to_process:
1453 if type(deps[d]) != str:
1454 deps_parent_url = entries[deps[d].module_name]
1455 if deps_parent_url.find("@") < 0:
1456 raise Error("From %s missing revisioned url" % deps[d].module_name)
1457 deps_parent_url_components = deps_parent_url.split("@")
1458 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1459 deps_parent_content = CaptureSVN(
[email protected]fb2b8eb2009-04-23 21:03:421460 ["cat",
1461 "%s/%s@%s" % (deps_parent_url_components[0],
1462 self._options.deps_file,
1463 deps_parent_url_components[1])],
1464 os.getcwd())
1465 sub_deps = self._ParseSolutionDeps(
1466 deps[d].module_name,
1467 FileRead(os.path.join(self._root_dir,
1468 deps[d].module_name,
1469 self._options.deps_file)),
1470 {})
1471 (url, rev) = GetURLAndRev(d, sub_deps[d])
1472 entries[d] = "%s@%d" % (url, rev)
1473
[email protected]df7a3132009-05-12 17:49:491474 print(";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())]))
[email protected]fb2b8eb2009-04-23 21:03:421475
1476
1477## gclient commands.
1478
1479
1480def DoCleanup(options, args):
1481 """Handle the cleanup subcommand.
1482
1483 Raises:
1484 Error: if client isn't configured properly.
1485 """
[email protected]2806acc2009-05-15 12:33:341486 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421487 if not client:
1488 raise Error("client not configured; see 'gclient config'")
1489 if options.verbose:
1490 # Print out the .gclient file. This is longer than if we just printed the
1491 # client dict, but more legible, and it might contain helpful comments.
[email protected]df7a3132009-05-12 17:49:491492 print(client.ConfigContent())
[email protected]fb2b8eb2009-04-23 21:03:421493 options.verbose = True
1494 return client.RunOnDeps('cleanup', args)
1495
1496
1497def DoConfig(options, args):
1498 """Handle the config subcommand.
1499
1500 Args:
1501 options: If options.spec set, a string providing contents of config file.
1502 args: The command line args. If spec is not set,
1503 then args[0] is a string URL to get for config file.
1504
1505 Raises:
1506 Error: on usage error
1507 """
1508 if len(args) < 1 and not options.spec:
1509 raise Error("required argument missing; see 'gclient help config'")
[email protected]0329e672009-05-13 18:41:041510 if os.path.exists(options.config_filename):
[email protected]fb2b8eb2009-04-23 21:03:421511 raise Error("%s file already exists in the current directory" %
1512 options.config_filename)
[email protected]2806acc2009-05-15 12:33:341513 client = GClient('.', options)
[email protected]fb2b8eb2009-04-23 21:03:421514 if options.spec:
1515 client.SetConfig(options.spec)
1516 else:
1517 # TODO(darin): it would be nice to be able to specify an alternate relpath
1518 # for the given URL.
[email protected]1ab7ffc2009-06-03 17:21:371519 base_url = args[0].rstrip('/')
1520 name = base_url.split("/")[-1]
[email protected]fb2b8eb2009-04-23 21:03:421521 safesync_url = ""
1522 if len(args) > 1:
1523 safesync_url = args[1]
1524 client.SetDefaultConfig(name, base_url, safesync_url)
1525 client.SaveConfig()
1526
1527
1528def DoHelp(options, args):
1529 """Handle the help subcommand giving help for another subcommand.
1530
1531 Raises:
1532 Error: if the command is unknown.
1533 """
1534 if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
[email protected]df7a3132009-05-12 17:49:491535 print(COMMAND_USAGE_TEXT[args[0]])
[email protected]fb2b8eb2009-04-23 21:03:421536 else:
1537 raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
1538
1539
1540def DoStatus(options, args):
1541 """Handle the status subcommand.
1542
1543 Raises:
1544 Error: if client isn't configured properly.
1545 """
[email protected]2806acc2009-05-15 12:33:341546 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421547 if not client:
1548 raise Error("client not configured; see 'gclient config'")
1549 if options.verbose:
1550 # Print out the .gclient file. This is longer than if we just printed the
1551 # client dict, but more legible, and it might contain helpful comments.
[email protected]df7a3132009-05-12 17:49:491552 print(client.ConfigContent())
[email protected]fb2b8eb2009-04-23 21:03:421553 options.verbose = True
1554 return client.RunOnDeps('status', args)
1555
1556
1557def DoUpdate(options, args):
1558 """Handle the update and sync subcommands.
1559
1560 Raises:
1561 Error: if client isn't configured properly.
1562 """
[email protected]2806acc2009-05-15 12:33:341563 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421564
1565 if not client:
1566 raise Error("client not configured; see 'gclient config'")
1567
1568 if not options.head:
1569 solutions = client.GetVar('solutions')
1570 if solutions:
1571 for s in solutions:
1572 if s.get('safesync_url', ''):
1573 # rip through revisions and make sure we're not over-riding
1574 # something that was explicitly passed
1575 has_key = False
1576 for r in options.revisions:
1577 if r.split('@')[0] == s['name']:
1578 has_key = True
1579 break
1580
1581 if not has_key:
1582 handle = urllib.urlopen(s['safesync_url'])
1583 rev = handle.read().strip()
1584 handle.close()
1585 if len(rev):
1586 options.revisions.append(s['name']+'@'+rev)
1587
1588 if options.verbose:
1589 # Print out the .gclient file. This is longer than if we just printed the
1590 # client dict, but more legible, and it might contain helpful comments.
[email protected]df7a3132009-05-12 17:49:491591 print(client.ConfigContent())
[email protected]fb2b8eb2009-04-23 21:03:421592 return client.RunOnDeps('update', args)
1593
1594
1595def DoDiff(options, args):
1596 """Handle the diff subcommand.
1597
1598 Raises:
1599 Error: if client isn't configured properly.
1600 """
[email protected]2806acc2009-05-15 12:33:341601 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421602 if not client:
1603 raise Error("client not configured; see 'gclient config'")
1604 if options.verbose:
1605 # Print out the .gclient file. This is longer than if we just printed the
1606 # client dict, but more legible, and it might contain helpful comments.
[email protected]df7a3132009-05-12 17:49:491607 print(client.ConfigContent())
[email protected]fb2b8eb2009-04-23 21:03:421608 options.verbose = True
1609 return client.RunOnDeps('diff', args)
1610
1611
1612def DoRevert(options, args):
1613 """Handle the revert subcommand.
1614
1615 Raises:
1616 Error: if client isn't configured properly.
1617 """
[email protected]2806acc2009-05-15 12:33:341618 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421619 if not client:
1620 raise Error("client not configured; see 'gclient config'")
1621 return client.RunOnDeps('revert', args)
1622
1623
1624def DoRunHooks(options, args):
1625 """Handle the runhooks subcommand.
1626
1627 Raises:
1628 Error: if client isn't configured properly.
1629 """
[email protected]2806acc2009-05-15 12:33:341630 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421631 if not client:
1632 raise Error("client not configured; see 'gclient config'")
1633 if options.verbose:
1634 # Print out the .gclient file. This is longer than if we just printed the
1635 # client dict, but more legible, and it might contain helpful comments.
[email protected]df7a3132009-05-12 17:49:491636 print(client.ConfigContent())
[email protected]fb2b8eb2009-04-23 21:03:421637 return client.RunOnDeps('runhooks', args)
1638
1639
1640def DoRevInfo(options, args):
1641 """Handle the revinfo subcommand.
1642
1643 Raises:
1644 Error: if client isn't configured properly.
1645 """
[email protected]2806acc2009-05-15 12:33:341646 client = GClient.LoadCurrentConfig(options)
[email protected]fb2b8eb2009-04-23 21:03:421647 if not client:
1648 raise Error("client not configured; see 'gclient config'")
1649 client.PrintRevInfo()
1650
1651
1652gclient_command_map = {
1653 "cleanup": DoCleanup,
1654 "config": DoConfig,
1655 "diff": DoDiff,
1656 "help": DoHelp,
1657 "status": DoStatus,
1658 "sync": DoUpdate,
1659 "update": DoUpdate,
1660 "revert": DoRevert,
1661 "runhooks": DoRunHooks,
1662 "revinfo" : DoRevInfo,
1663}
1664
1665
1666def DispatchCommand(command, options, args, command_map=None):
1667 """Dispatches the appropriate subcommand based on command line arguments."""
1668 if command_map is None:
1669 command_map = gclient_command_map
1670
1671 if command in command_map:
1672 return command_map[command](options, args)
1673 else:
1674 raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
1675
1676
1677def Main(argv):
1678 """Parse command line arguments and dispatch command."""
1679
1680 option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
1681 version=__version__)
1682 option_parser.disable_interspersed_args()
1683 option_parser.add_option("", "--force", action="store_true", default=False,
1684 help=("(update/sync only) force update even "
1685 "for modules which haven't changed"))
1686 option_parser.add_option("", "--revision", action="append", dest="revisions",
1687 metavar="REV", default=[],
1688 help=("(update/sync only) sync to a specific "
1689 "revision, can be used multiple times for "
1690 "each solution, e.g. --revision=src@123, "
1691 "--revision=internal@32"))
1692 option_parser.add_option("", "--deps", default=None, dest="deps_os",
1693 metavar="OS_LIST",
1694 help=("(update/sync only) sync deps for the "
1695 "specified (comma-separated) platform(s); "
1696 "'all' will sync all platforms"))
1697 option_parser.add_option("", "--spec", default=None,
1698 help=("(config only) create a gclient file "
1699 "containing the provided string"))
1700 option_parser.add_option("", "--verbose", action="store_true", default=False,
1701 help="produce additional output for diagnostics")
1702 option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
1703 default=False,
1704 help="Skip svn up whenever possible by requesting "
1705 "actual HEAD revision from the repository")
1706 option_parser.add_option("", "--head", action="store_true", default=False,
1707 help=("skips any safesync_urls specified in "
1708 "configured solutions"))
[email protected]cdcee802009-06-23 15:30:421709 option_parser.add_option("", "--delete_unversioned_trees",
1710 action="store_true", default=False,
1711 help=("on update, delete any unexpected "
1712 "unversioned trees that are in the checkout"))
[email protected]fb2b8eb2009-04-23 21:03:421713
1714 if len(argv) < 2:
1715 # Users don't need to be told to use the 'help' command.
1716 option_parser.print_help()
1717 return 1
1718 # Add manual support for --version as first argument.
1719 if argv[1] == '--version':
1720 option_parser.print_version()
1721 return 0
1722
1723 # Add manual support for --help as first argument.
1724 if argv[1] == '--help':
1725 argv[1] = 'help'
1726
1727 command = argv[1]
1728 options, args = option_parser.parse_args(argv[2:])
1729
1730 if len(argv) < 3 and command == "help":
1731 option_parser.print_help()
1732 return 0
1733
1734 # Files used for configuration and state saving.
1735 options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
1736 options.entries_filename = ".gclient_entries"
1737 options.deps_file = "DEPS"
1738
[email protected]fb2b8eb2009-04-23 21:03:421739 options.platform = sys.platform
1740 return DispatchCommand(command, options, args)
1741
1742
1743if "__main__" == __name__:
1744 try:
1745 result = Main(sys.argv)
1746 except Error, e:
[email protected]df7a3132009-05-12 17:49:491747 print >> sys.stderr, "Error: %s" % str(e)
[email protected]fb2b8eb2009-04-23 21:03:421748 result = 1
1749 sys.exit(result)
1750
1751# vim: ts=2:sw=2:tw=80:et: