scm.py

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""SCM-specific utility classes."""

import cStringIO
import glob
import logging
import os
import re
import sys
import tempfile
import time
from xml.etree import ElementTree

import gclient_utils
import subprocess2


def ValidateEmail(email):
  return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
          is not None)


def GetCasedPath(path):
  """Elcheapos way to get the real path case on Windows."""
  if sys.platform.startswith('win') and os.path.exists(path):
    # Reconstruct the path.
    path = os.path.abspath(path)
    paths = path.split('\\')
    for i in range(len(paths)):
      if i == 0:
        # Skip drive letter.
        continue
      subpath = '\\'.join(paths[:i+1])
      prev = len('\\'.join(paths[:i]))
      # glob.glob will return the cased path for the last item only. This is why
      # we are calling it in a loop. Extract the data we want and put it back
      # into the list.
      paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
    path = '\\'.join(paths)
  return path


def GenFakeDiff(filename):
  """Generates a fake diff from a file."""
  file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True)
  filename = filename.replace(os.sep, '/')
  nb_lines = len(file_content)
  # We need to use / since patch on unix will fail otherwise.
  data = cStringIO.StringIO()
  data.write("Index: %s\n" % filename)
  data.write('=' * 67 + '\n')
  # Note: Should we use /dev/null instead?
  data.write("--- %s\n" % filename)
  data.write("+++ %s\n" % filename)
  data.write("@@ -0,0 +1,%d @@\n" % nb_lines)
  # Prepend '+' to every lines.
  for line in file_content:
    data.write('+')
    data.write(line)
  result = data.getvalue()
  data.close()
  return result


def determine_scm(root):
  """Similar to upload.py's version but much simpler.

  Returns 'svn', 'git' or None.
  """
  if os.path.isdir(os.path.join(root, '.svn')):
    return 'svn'
  elif os.path.isdir(os.path.join(root, '.git')):
    return 'git'
  else:
    try:
      subprocess2.check_call(
          ['git', 'rev-parse', '--show-cdup'],
          stdout=subprocess2.VOID,
          stderr=subprocess2.VOID,
          cwd=root)
      return 'git'
    except (OSError, subprocess2.CalledProcessError):
      return None


def only_int(val):
  if val.isdigit():
    return int(val)
  else:
    return 0


class GIT(object):
  current_version = None

  @staticmethod
  def ApplyEnvVars(kwargs):
    env = kwargs.pop('env', None) or os.environ.copy()
    # Don't prompt for passwords; just fail quickly and noisily.
    # By default, git will use an interactive terminal prompt when a username/
    # password is needed.  That shouldn't happen in the chromium workflow,
    # and if it does, then gclient may hide the prompt in the midst of a flood
    # of terminal spew.  The only indication that something has gone wrong
    # will be when gclient hangs unresponsively.  Instead, we disable the
    # password prompt and simply allow git to fail noisily.  The error
    # message produced by git will be copied to gclient's output.
    env.setdefault('GIT_ASKPASS', 'true')
    env.setdefault('SSH_ASKPASS', 'true')
    # 'cat' is a magical git string that disables pagers on all platforms.
    env.setdefault('GIT_PAGER', 'cat')
    return env

  @staticmethod
  def Capture(args, cwd, strip_out=True, **kwargs):
    env = GIT.ApplyEnvVars(kwargs)
    output = subprocess2.check_output(
        ['git'] + args,
        cwd=cwd, stderr=subprocess2.PIPE, env=env, **kwargs)
    return output.strip() if strip_out else output

  @staticmethod
  def CaptureStatus(files, cwd, upstream_branch):
    """Returns git status.

    @files can be a string (one file) or a list of files.

    Returns an array of (status, file) tuples."""
    if upstream_branch is None:
      upstream_branch = GIT.GetUpstreamBranch(cwd)
      if upstream_branch is None:
        raise gclient_utils.Error('Cannot determine upstream branch')
    command = ['diff', '--name-status', '--no-renames',
               '-r', '%s...' % upstream_branch]
    if not files:
      pass
    elif isinstance(files, basestring):
      command.append(files)
    else:
      command.extend(files)
    status = GIT.Capture(command, cwd)
    results = []
    if status:
      for statusline in status.splitlines():
        # 3-way merges can cause the status can be 'MMM' instead of 'M'. This
        # can happen when the user has 2 local branches and he diffs between
        # these 2 branches instead diffing to upstream.
        m = re.match('^(\w)+\t(.+)$', statusline)
        if not m:
          raise gclient_utils.Error(
              'status currently unsupported: %s' % statusline)
        # Only grab the first letter.
        results.append(('%s      ' % m.group(1)[0], m.group(2)))
    return results

  @staticmethod
  def IsWorkTreeDirty(cwd):
    return GIT.Capture(['status', '-s'], cwd=cwd) != ''

  @staticmethod
  def GetEmail(cwd):
    """Retrieves the user email address if known."""
    # We could want to look at the svn cred when it has a svn remote but it
    # should be fine for now, users should simply configure their git settings.
    try:
      return GIT.Capture(['config', 'user.email'], cwd=cwd)
    except subprocess2.CalledProcessError:
      return ''

  @staticmethod
  def ShortBranchName(branch):
    """Converts a name like 'refs/heads/foo' to just 'foo'."""
    return branch.replace('refs/heads/', '')

  @staticmethod
  def GetBranchRef(cwd):
    """Returns the full branch reference, e.g. 'refs/heads/master'."""
    return GIT.Capture(['symbolic-ref', 'HEAD'], cwd=cwd)

  @staticmethod
  def GetBranch(cwd):
    """Returns the short branch name, e.g. 'master'."""
    return GIT.ShortBranchName(GIT.GetBranchRef(cwd))

  @staticmethod
  def IsGitSvn(cwd):
    """Returns True if this repo looks like it's using git-svn."""
    # If you have any "svn-remote.*" config keys, we think you're using svn.
    try:
      GIT.Capture(['config', '--local', '--get-regexp', r'^svn-remote\.'],
                  cwd=cwd)
      return True
    except subprocess2.CalledProcessError:
      return False

  @staticmethod
  def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards):
    """Return the corresponding git ref if |base_url| together with |glob_spec|
    matches the full |url|.

    If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below).
    """
    fetch_suburl, as_ref = glob_spec.split(':')
    if allow_wildcards:
      glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl)
      if glob_match:
        # Parse specs like "branches/*/src:refs/remotes/svn/*" or
        # "branches/{472,597,648}/src:refs/remotes/svn/*".
        branch_re = re.escape(base_url)
        if glob_match.group(1):
          branch_re += '/' + re.escape(glob_match.group(1))
        wildcard = glob_match.group(2)
        if wildcard == '*':
          branch_re += '([^/]*)'
        else:
          # Escape and replace surrounding braces with parentheses and commas
          # with pipe symbols.
          wildcard = re.escape(wildcard)
          wildcard = re.sub('^\\\\{', '(', wildcard)
          wildcard = re.sub('\\\\,', '|', wildcard)
          wildcard = re.sub('\\\\}$', ')', wildcard)
          branch_re += wildcard
        if glob_match.group(3):
          branch_re += re.escape(glob_match.group(3))
        match = re.match(branch_re, url)
        if match:
          return re.sub('\*$', match.group(1), as_ref)

    # Parse specs like "trunk/src:refs/remotes/origin/trunk".
    if fetch_suburl:
      full_url = base_url + '/' + fetch_suburl
    else:
      full_url = base_url
    if full_url == url:
      return as_ref
    return None

  @staticmethod
  def GetSVNBranch(cwd):
    """Returns the svn branch name if found."""
    # Try to figure out which remote branch we're based on.
    # Strategy:
    # 1) iterate through our branch history and find the svn URL.
    # 2) find the svn-remote that fetches from the URL.

    # regexp matching the git-svn line that contains the URL.
    git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)

    # We don't want to go through all of history, so read a line from the
    # pipe at a time.
    # The -100 is an arbitrary limit so we don't search forever.
    cmd = ['git', 'log', '-100', '--pretty=medium']
    proc = subprocess2.Popen(cmd, cwd=cwd, stdout=subprocess2.PIPE)
    url = None
    for line in proc.stdout:
      match = git_svn_re.match(line)
      if match:
        url = match.group(1)
        proc.stdout.close()  # Cut pipe.
        break

    if url:
      svn_remote_re = re.compile(r'^svn-remote\.([^.]+)\.url (.*)$')
      remotes = GIT.Capture(
          ['config', '--local', '--get-regexp', r'^svn-remote\..*\.url'],
          cwd=cwd).splitlines()
      for remote in remotes:
        match = svn_remote_re.match(remote)
        if match:
          remote = match.group(1)
          base_url = match.group(2)
          try:
            fetch_spec = GIT.Capture(
                ['config', '--local', 'svn-remote.%s.fetch' % remote],
                cwd=cwd)
            branch = GIT.MatchSvnGlob(url, base_url, fetch_spec, False)
          except subprocess2.CalledProcessError:
            branch = None
          if branch:
            return branch
          try:
            branch_spec = GIT.Capture(
                ['config', '--local', 'svn-remote.%s.branches' % remote],
                cwd=cwd)
            branch = GIT.MatchSvnGlob(url, base_url, branch_spec, True)
          except subprocess2.CalledProcessError:
            branch = None
          if branch:
            return branch
          try:
            tag_spec = GIT.Capture(
                ['config', '--local', 'svn-remote.%s.tags' % remote],
                cwd=cwd)
            branch = GIT.MatchSvnGlob(url, base_url, tag_spec, True)
          except subprocess2.CalledProcessError:
            branch = None
          if branch:
            return branch

  @staticmethod
  def FetchUpstreamTuple(cwd):
    """Returns a tuple containg remote and remote ref,
       e.g. 'origin', 'refs/heads/master'
       Tries to be intelligent and understand git-svn.
    """
    remote = '.'
    branch = GIT.GetBranch(cwd)
    try:
      upstream_branch = GIT.Capture(
          ['config', '--local', 'branch.%s.merge' % branch], cwd=cwd)
    except subprocess2.CalledProcessError:
      upstream_branch = None
    if upstream_branch:
      try:
        remote = GIT.Capture(
            ['config', '--local', 'branch.%s.remote' % branch], cwd=cwd)
      except subprocess2.CalledProcessError:
        pass
    else:
      try:
        upstream_branch = GIT.Capture(
            ['config', '--local', 'rietveld.upstream-branch'], cwd=cwd)
      except subprocess2.CalledProcessError:
        upstream_branch = None
      if upstream_branch:
        try:
          remote = GIT.Capture(
              ['config', '--local', 'rietveld.upstream-remote'], cwd=cwd)
        except subprocess2.CalledProcessError:
          pass
      else:
        # Fall back on trying a git-svn upstream branch.
        if GIT.IsGitSvn(cwd):
          upstream_branch = GIT.GetSVNBranch(cwd)
        else:
          # Else, try to guess the origin remote.
          remote_branches = GIT.Capture(['branch', '-r'], cwd=cwd).split()
          if 'origin/master' in remote_branches:
            # Fall back on origin/master if it exits.
            remote = 'origin'
            upstream_branch = 'refs/heads/master'
          elif 'origin/trunk' in remote_branches:
            # Fall back on origin/trunk if it exists. Generally a shared
            # git-svn clone
            remote = 'origin'
            upstream_branch = 'refs/heads/trunk'
          else:
            # Give up.
            remote = None
            upstream_branch = None
    return remote, upstream_branch

  @staticmethod
  def RefToRemoteRef(ref, remote=None):
    """Convert a checkout ref to the equivalent remote ref.

    Returns:
      A tuple of the remote ref's (common prefix, unique suffix), or None if it
      doesn't appear to refer to a remote ref (e.g. it's a commit hash).
    """
    # TODO(mmoss): This is just a brute-force mapping based of the expected git
    # config. It's a bit better than the even more brute-force replace('heads',
    # ...), but could still be smarter (like maybe actually using values gleaned
    # from the git config).
    m = re.match('^(refs/(remotes/)?)?branch-heads/', ref or '')
    if m:
      return ('refs/remotes/branch-heads/', ref.replace(m.group(0), ''))
    if remote:
      m = re.match('^((refs/)?remotes/)?%s/|(refs/)?heads/' % remote, ref or '')
      if m:
        return ('refs/remotes/%s/' % remote, ref.replace(m.group(0), ''))
    return None

  @staticmethod
  def GetUpstreamBranch(cwd):
    """Gets the current branch's upstream branch."""
    remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
    if remote != '.' and upstream_branch:
      remote_ref = GIT.RefToRemoteRef(upstream_branch, remote)
      if remote_ref:
        upstream_branch = ''.join(remote_ref)
    return upstream_branch

  @staticmethod
  def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
                   files=None):
    """Diffs against the upstream branch or optionally another branch.

    full_move means that move or copy operations should completely recreate the
    files, usually in the prospect to apply the patch for a try job."""
    if not branch:
      branch = GIT.GetUpstreamBranch(cwd)
    command = ['diff', '-p', '--no-color', '--no-prefix', '--no-ext-diff',
               branch + "..." + branch_head]
    if full_move:
      command.append('--no-renames')
    else:
      command.append('-C')
    # TODO(maruel): --binary support.
    if files:
      command.append('--')
      command.extend(files)
    diff = GIT.Capture(command, cwd=cwd, strip_out=False).splitlines(True)
    for i in range(len(diff)):
      # In the case of added files, replace /dev/null with the path to the
      # file being added.
      if diff[i].startswith('--- /dev/null'):
        diff[i] = '--- %s' % diff[i+1][4:]
    return ''.join(diff)

  @staticmethod
  def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
    """Returns the list of modified files between two branches."""
    if not branch:
      branch = GIT.GetUpstreamBranch(cwd)
    command = ['diff', '--name-only', branch + "..." + branch_head]
    return GIT.Capture(command, cwd=cwd).splitlines(False)

  @staticmethod
  def GetPatchName(cwd):
    """Constructs a name for this patch."""
    short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd=cwd)
    return "%s#%s" % (GIT.GetBranch(cwd), short_sha)

  @staticmethod
  def GetCheckoutRoot(cwd):
    """Returns the top level directory of a git checkout as an absolute path.
    """
    root = GIT.Capture(['rev-parse', '--show-cdup'], cwd=cwd)
    return os.path.abspath(os.path.join(cwd, root))

  @staticmethod
  def GetGitDir(cwd):
    return os.path.abspath(GIT.Capture(['rev-parse', '--git-dir'], cwd=cwd))

  @staticmethod
  def IsInsideWorkTree(cwd):
    try:
      return GIT.Capture(['rev-parse', '--is-inside-work-tree'], cwd=cwd)
    except (OSError, subprocess2.CalledProcessError):
      return False

  @staticmethod
  def IsDirectoryVersioned(cwd, relative_dir):
    """Checks whether the given |relative_dir| is part of cwd's repo."""
    return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd))

  @staticmethod
  def CleanupDir(cwd, relative_dir):
    """Cleans up untracked file inside |relative_dir|."""
    return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd))

  @staticmethod
  def GetGitSvnHeadRev(cwd):
    """Gets the most recently pulled git-svn revision."""
    try:
      output = GIT.Capture(['svn', 'info'], cwd=cwd)
      match = re.search(r'^Revision: ([0-9]+)$', output, re.MULTILINE)
      return int(match.group(1)) if match else None
    except (subprocess2.CalledProcessError, ValueError):
      return None

  @staticmethod
  def ParseGitSvnSha1(output):
    """Parses git-svn output for the first sha1."""
    match = re.search(r'[0-9a-fA-F]{40}', output)
    return match.group(0) if match else None

  @staticmethod
  def GetSha1ForSvnRev(cwd, rev):
    """Returns a corresponding git sha1 for a SVN revision."""
    if not GIT.IsGitSvn(cwd=cwd):
      return None
    try:
      output = GIT.Capture(['svn', 'find-rev', 'r' + str(rev)], cwd=cwd)
      return GIT.ParseGitSvnSha1(output)
    except subprocess2.CalledProcessError:
      return None

  @staticmethod
  def GetBlessedSha1ForSvnRev(cwd, rev):
    """Returns a git commit hash from the master branch history that has
    accurate .DEPS.git and git submodules.  To understand why this is more
    complicated than a simple call to `git svn find-rev`, refer to:

    http://www.chromium.org/developers/how-tos/git-repo
    """
    git_svn_rev = GIT.GetSha1ForSvnRev(cwd, rev)
    if not git_svn_rev:
      return None
    try:
      output = GIT.Capture(
          ['rev-list', '--ancestry-path', '--reverse',
          '--grep', 'SVN changes up to revision [0-9]*',
          '%s..refs/remotes/origin/master' % git_svn_rev], cwd=cwd)
      if not output:
        return None
      sha1 = output.splitlines()[0]
      if not sha1:
        return None
      output = GIT.Capture(['rev-list', '-n', '1', '%s^1' % sha1], cwd=cwd)
      if git_svn_rev != output.rstrip():
        raise gclient_utils.Error(sha1)
      return sha1
    except subprocess2.CalledProcessError:
      return None

  @staticmethod
  def IsValidRevision(cwd, rev, sha_only=False):
    """Verifies the revision is a proper git revision.

    sha_only: Fail unless rev is a sha hash.
    """
    # 'git rev-parse foo' where foo is *any* 40 character hex string will return
    # the string and return code 0. So strip one character to force 'git
    # rev-parse' to do a hash table look-up and returns 128 if the hash is not
    # present.
    lookup_rev = rev
    if re.match(r'^[0-9a-fA-F]{40}$', rev):
      lookup_rev = rev[:-1]
    try:
      sha = GIT.Capture(['rev-parse', lookup_rev], cwd=cwd).lower()
      if lookup_rev != rev:
        # Make sure we get the original 40 chars back.
        return rev.lower() == sha
      if sha_only:
        return sha.startswith(rev.lower())
      return True
    except subprocess2.CalledProcessError:
      return False

  @classmethod
  def AssertVersion(cls, min_version):
    """Asserts git's version is at least min_version."""
    if cls.current_version is None:
      current_version = cls.Capture(['--version'], '.')
      matched = re.search(r'version ([0-9\.]+)', current_version)
      cls.current_version = matched.group(1)
    current_version_list = map(only_int, cls.current_version.split('.'))
    for min_ver in map(int, min_version.split('.')):
      ver = current_version_list.pop(0)
      if ver < min_ver:
        return (False, cls.current_version)
      elif ver > min_ver:
        return (True, cls.current_version)
    return (True, cls.current_version)