build/locale_tool.py

#!/usr/bin/env vpython
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Helper script used to manage locale-related files in Chromium.

This script is used to check, and potentially fix, many locale-related files
in your Chromium workspace, such as:

  - GRIT input files (.grd) and the corresponding translations (.xtb).

  - BUILD.gn files listing Android localized resource string resource .xml
    generated by GRIT for all supported Chrome locales. These correspond to
    <output> elements that use the type="android" attribute.

The --scan-dir <dir> option can be used to check for all files under a specific
directory, and the --fix-inplace option can be used to try fixing any file
that doesn't pass the check.

This can be very handy to avoid tedious and repetitive work when adding new
translations / locales to the Chrome code base, since this script can update
said input files for you.

Important note: checks and fix may fail on some input files. For example
remoting/resources/remoting_strings.grd contains an in-line comment element
inside its <outputs> section that breaks the script. The check will fail, and
trying to fix it too, but at least the file will not be modified.
"""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import unittest

# Assume this script is under build/
_SCRIPT_DIR = os.path.dirname(__file__)
_SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
_TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')

# Need to import android/gyp/util/resource_utils.py here.
sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))

from util import build_utils
from util import resource_utils


# This locale is the default and doesn't have translations.
_DEFAULT_LOCALE = 'en-US'

# Misc terminal codes to provide human friendly progress output.
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
_CONSOLE_CODE_ERASE_LINE = '\x1b[K'
_CONSOLE_START_LINE = (
    _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)

##########################################################################
##########################################################################
#####
#####    G E N E R I C   H E L P E R   F U N C T I O N S
#####
##########################################################################
##########################################################################

def _FixChromiumLangAttribute(lang):
  """Map XML "lang" attribute values to Chromium locale names."""
  _CHROMIUM_LANG_FIXES = {
      'en': 'en-US',  # For now, Chromium doesn't have an 'en' locale.
      'iw': 'he',  # 'iw' is the obsolete form of ISO 639-1 for Hebrew
      'no': 'nb',  # 'no' is used by the Translation Console for Norwegian (nb).
  }
  return _CHROMIUM_LANG_FIXES.get(lang, lang)


def _FixTranslationConsoleLocaleName(locale):
  _FIXES = {
      'nb': 'no',  # Norwegian.
      'he': 'iw',  # Hebrew
  }
  return _FIXES.get(locale, locale)


def _CompareLocaleLists(list_a, list_expected, list_name):
  """Compare two lists of locale names. Print errors if they differ.

  Args:
    list_a: First list of locales.
    list_expected: Second list of locales, as expected.
    list_name: Name of list printed in error messages.
  Returns:
    On success, return False. On error, print error messages and return True.
  """
  errors = []
  missing_locales = sorted(set(list_a) - set(list_expected))
  if missing_locales:
    errors.append('Missing locales: %s' % missing_locales)

  extra_locales = sorted(set(list_expected) - set(list_a))
  if extra_locales:
    errors.append('Unexpected locales: %s' % extra_locales)

  if errors:
    print 'Errors in %s definition:' % list_name
    for error in errors:
      print '  %s\n' % error
    return True

  return False


def _BuildIntervalList(input_list, predicate):
  """Find ranges of contiguous list items that pass a given predicate.

  Args:
    input_list: An input list of items of any type.
    predicate: A function that takes a list item and return True if it
      passes a given test.
  Returns:
    A list of (start_pos, end_pos) tuples, where all items in
    [start_pos, end_pos) pass the predicate.
  """
  result = []
  size = len(input_list)
  start = 0
  while True:
    # Find first item in list that passes the predicate.
    while start < size and not predicate(input_list[start]):
      start += 1

    if start >= size:
      return result

    # Find first item in the rest of the list that does not pass the
    # predicate.
    end = start + 1
    while end < size and predicate(input_list[end]):
      end += 1

    result.append((start, end))
    start = end + 1


def _SortListSubRange(input_list, start, end, key_func):
  """Sort an input list's sub-range according to a specific key function.

  Args:
    input_list: An input list.
    start: Sub-range starting position in list.
    end: Sub-range limit position in list.
    key_func: A function that extracts a sort key from a line.
  Returns:
    A copy of |input_list|, with all items in [|start|, |end|) sorted
    according to |key_func|.
  """
  result = input_list[:start]
  inputs = []
  for pos in xrange(start, end):
    line = input_list[pos]
    key = key_func(line)
    inputs.append((key, line))

  for _, line in sorted(inputs):
    result.append(line)

  result += input_list[end:]
  return result


def _SortElementsRanges(lines, element_predicate, element_key):
  """Sort all elements of a given type in a list of lines by a given key.

  Args:
    lines: input lines.
    element_predicate: predicate function to select elements to sort.
    element_key: lambda returning a comparison key for each element that
      passes the predicate.
  Returns:
    A new list of input lines, with lines [start..end) sorted.
  """
  intervals = _BuildIntervalList(lines, element_predicate)
  for start, end in intervals:
    lines = _SortListSubRange(lines, start, end, element_key)

  return lines


def _ProcessFile(input_file, locales, check_func, fix_func):
  """Process a given input file, potentially fixing it.

  Args:
    input_file: Input file path.
    locales: List of Chrome locales to consider / expect.
    check_func: A lambda called to check the input file lines with
      (input_lines, locales) argument. It must return an list of error
      messages, or None on success.
    fix_func: None, or a lambda called to fix the input file lines with
      (input_lines, locales). It must return the new list of lines for
      the input file, and may raise an Exception in case of error.
  Returns:
    True at the moment.
  """
  print '%sProcessing %s...' % (_CONSOLE_START_LINE, input_file),
  sys.stdout.flush()
  with open(input_file) as f:
    input_lines = f.readlines()
  errors = check_func(input_file, input_lines, locales)
  if errors:
    print '\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors))
    if fix_func:
      try:
        input_lines = fix_func(input_file, input_lines, locales)
        output = ''.join(input_lines)
        with open(input_file, 'wt') as f:
          f.write(output)
        print 'Fixed %s.' % input_file
      except Exception as e:  # pylint: disable=broad-except
        print 'Skipped %s: %s' % (input_file, e)

  return True


def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
  """Scan a directory for files that match a given predicate.

  Args:
    scan_dir: A list of top-level directories to start scan in.
    file_predicate: lambda function which is passed the file's base name
      and returns True if its full path, relative to |scan_dir|, should be
      passed in the result.
  Returns:
    A list of file full paths.
  """
  result = []
  for src_dir in scan_dirs:
    for root, _, files in os.walk(src_dir):
      result.extend(os.path.join(root, f) for f in files if file_predicate(f))
  return result


def _WriteFile(file_path, file_data):
  """Write |file_data| to |file_path|."""
  with open(file_path, 'w') as f:
    f.write(file_data)


def _FindGnExecutable():
  """Locate the real GN executable used by this Chromium checkout.

  This is needed because the depot_tools 'gn' wrapper script will look
  for .gclient and other things we really don't need here.

  Returns:
    Path of real host GN executable from current Chromium src/ checkout.
  """
  # Simply scan buildtools/*/gn and return the first one found so we don't
  # have to guess the platform-specific sub-directory name (e.g. 'linux64'
  # for 64-bit Linux machines).
  buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
  for subdir in os.listdir(buildtools_dir):
    subdir_path = os.path.join(buildtools_dir, subdir)
    if not os.path.isdir(subdir_path):
      continue
    gn_path = os.path.join(subdir_path, 'gn')
    if os.path.exists(gn_path):
      return gn_path
  return None


def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
  result = []
  input_str = ', '.join(input_list)
  while len(input_str) > available_width:
    pos = input_str.rfind(',', 0, available_width)
    result.append(input_str[:pos + 1])
    input_str = input_str[pos + 1:].lstrip()
  if trailing_comma and input_str:
    input_str += ','
  result.append(input_str)
  return result


class _PrettyPrintListAsLinesTest(unittest.TestCase):

  def test_empty_list(self):
    self.assertListEqual([''], _PrettyPrintListAsLines([], 10))

  def test_wrapping(self):
    input_list = ['foo', 'bar', 'zoo', 'tool']
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 8),
        ['foo,', 'bar,', 'zoo,', 'tool'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])

  def test_trailing_comma(self):
    input_list = ['foo', 'bar', 'zoo', 'tool']
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
        ['foo,', 'bar,', 'zoo,', 'tool,'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
        ['foo, bar,', 'zoo, tool,'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
        ['foo, bar, zoo, tool,'])


##########################################################################
##########################################################################
#####
#####    L O C A L E S   L I S T S
#####
##########################################################################
##########################################################################

# Various list of locales that will be extracted from build/config/locales.gni
# Do not use these directly, use ChromeLocales(), AndroidOmittedLocales() and
# IosUnsupportedLocales() instead to access these lists.
_INTERNAL_CHROME_LOCALES = []
_INTERNAL_ANDROID_OMITTED_LOCALES = []
_INTERNAL_IOS_UNSUPPORTED_LOCALES = []


def ChromeLocales():
  """Return the list of all locales supported by Chrome."""
  if not _INTERNAL_CHROME_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_CHROME_LOCALES


def AndroidOmittedLocales():
  """Reutrn the list of locales omitted from Android APKs."""
  if not _INTERNAL_ANDROID_OMITTED_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_ANDROID_OMITTED_LOCALES


def IosUnsupportedLocales():
  """Return the list of locales that are unsupported on iOS."""
  if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_IOS_UNSUPPORTED_LOCALES


def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
  """Populate an empty directory with a tiny set of working GN config files.

  This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
  to generate files containing the locales list. This takes about 300ms on
  a decent machine, instead of more than 5 seconds when running the equivalent
  commands from a real Chromium workspace, which requires regenerating more
  than 23k targets.

  Args:
    work_dir: target working directory.
    out_subdir_name: Name of output sub-directory.
  Returns:
    Full path of output directory created inside |work_dir|.
  """
  # Create top-level .gn file that must point to the BUILDCONFIG.gn.
  _WriteFile(os.path.join(work_dir, '.gn'),
             'buildconfig = "//BUILDCONFIG.gn"\n')
  # Create BUILDCONFIG.gn which must set a default toolchain. Also add
  # all variables that may be used in locales.gni in a declare_args() block.
  _WriteFile(os.path.join(work_dir, 'BUILDCONFIG.gn'),
             r'''set_default_toolchain("toolchain")
declare_args () {
  is_ios = false
}
''')

  # Create fake toolchain required by BUILDCONFIG.gn.
  os.mkdir(os.path.join(work_dir, 'toolchain'))
  _WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
             r'''toolchain("toolchain") {
  tool("stamp") {
    command = "touch {{output}}"  # Required by action()
  }
}
''')

  # Create top-level BUILD.gn, GN requires at least one target to build so do
  # that with a fake action which will never be invoked. Also write the locales
  # to misc files in the output directory.
  _WriteFile(os.path.join(work_dir, 'BUILD.gn'),
             r'''import("//locales.gni")

action("create_foo") {   # fake action to avoid GN complaints.
  script = "//build/create_foo.py"
  inputs = []
  outputs = [ "$target_out_dir/$target_name" ]
}

# Write the locales lists to files in the output directory.
_filename = root_build_dir + "/foo"
write_file(_filename + ".locales", locales, "json")
write_file(_filename + ".android_omitted_locales",
            android_chrome_omitted_locales,
            "json")
write_file(_filename + ".ios_unsupported_locales",
            ios_unsupported_locales,
            "json")
''')

  # Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
  shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
                  os.path.join(work_dir, 'locales.gni'))

  # Create output directory.
  out_path = os.path.join(work_dir, out_subdir_name)
  os.mkdir(out_path)

  # And ... we're good.
  return out_path


# Set this global variable to the path of a given temporary directory
# before calling _ExtractAllChromeLocalesLists() if you want to debug
# the locales list extraction process.
_DEBUG_LOCALES_WORK_DIR = None


def _ReadJsonList(file_path):
  """Read a JSON file that must contain a list, and return it."""
  with open(file_path) as f:
    data = json.load(f)
    assert isinstance(data, list), "JSON file %s is not a list!" % file_path
  return [item.encode('utf8') for item in data]


def _ExtractAllChromeLocalesLists():
  with build_utils.TempDir() as tmp_path:
    if _DEBUG_LOCALES_WORK_DIR:
      tmp_path = _DEBUG_LOCALES_WORK_DIR
      build_utils.DeleteDirectory(tmp_path)
      build_utils.MakeDirectory(tmp_path)

    out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')

    # NOTE: The file suffixes used here should be kept in sync with
    # build/config/locales.gni
    gn_executable = _FindGnExecutable()
    subprocess.check_output(
        [gn_executable, 'gen', out_path, '--root=' + tmp_path])

    global _INTERNAL_CHROME_LOCALES
    _INTERNAL_CHROME_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.locales'))

    global _INTERNAL_ANDROID_OMITTED_LOCALES
    _INTERNAL_ANDROID_OMITTED_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.android_omitted_locales'))

    global _INTERNAL_IOS_UNSUPPORTED_LOCALES
    _INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.ios_unsupported_locales'))


##########################################################################
##########################################################################
#####
#####    G R D   H E L P E R   F U N C T I O N S
#####
##########################################################################
##########################################################################

# Technical note:
#
# Even though .grd files are XML, an xml parser library is not used in order
# to preserve the original file's structure after modification. ElementTree
# tends to re-order attributes in each element when re-writing an XML
# document tree, which is undesirable here.
#
# Thus simple line-based regular expression matching is used instead.
#

# Misc regular expressions used to match elements and their attributes.
_RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
_RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
_RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
_RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
_RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
_RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')



def _IsGritInputFile(input_file):
  """Returns True iff this is a GRIT input file."""
  return input_file.endswith('.grd')


def _GetXmlLangAttribute(xml_line):
  """Extract the lang attribute value from an XML input line."""
  m = _RE_LANG_ATTRIBUTE.search(xml_line)
  if not m:
    return None
  return m.group(1)


class _GetXmlLangAttributeTest(unittest.TestCase):
  TEST_DATA = {
      '': None,
      'foo': None,
      'lang=foo': None,
      'lang="foo"': 'foo',
      '<something lang="foo bar" />': 'foo bar',
      '<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
  }

  def test_GetXmlLangAttribute(self):
    for test_line, expected in self.TEST_DATA.iteritems():
      self.assertEquals(_GetXmlLangAttribute(test_line), expected)


def _SortGrdElementsRanges(grd_lines, element_predicate):
  """Sort all .grd elements of a given type by their lang attribute."""
  return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)


def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
  """Check the element 'lang' attributes in specific .grd lines range.

  This really checks the following:
    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  locales = set()
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      errors.append('%d: Missing "lang" attribute in <output> element' % pos +
                    1)
      continue
    cr_locale = _FixChromiumLangAttribute(lang)
    if cr_locale in locales:
      errors.append(
          '%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
    locales.add(cr_locale)

  extra_locales = locales.difference(wanted_locales)
  if extra_locales:
    errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
                                                      sorted(extra_locales)))

  missing_locales = wanted_locales.difference(locales)
  if missing_locales:
    errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
                                                  sorted(missing_locales)))

  return errors


##########################################################################
##########################################################################
#####
#####    G R D   A N D R O I D   O U T P U T S
#####
##########################################################################
##########################################################################

def _IsGrdAndroidOutputLine(line):
  """Returns True iff this is an Android-specific <output> line."""
  m = _RE_OUTPUT_ELEMENT.search(line)
  if m:
    return 'type="android"' in m.group(1)
  return False

assert _IsGrdAndroidOutputLine('  <output type="android"/>')

# Many of the functions below have unused arguments due to genericity.
# pylint: disable=unused-argument

def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
                                               wanted_locales):
  """Check all <output> elements in specific input .grd lines range.

  This really checks the following:
    - Filenames exist for each listed locale.
    - Filenames are well-formed.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      continue
    cr_locale = _FixChromiumLangAttribute(lang)

    m = _RE_FILENAME_ATTRIBUTE.search(line)
    if not m:
      errors.append('%d: Missing filename attribute in <output> element' % pos +
                    1)
    else:
      filename = m.group(1)
      if not filename.endswith('.xml'):
        errors.append(
            '%d: Filename should end with ".xml": %s' % (pos + 1, filename))

      dirname = os.path.basename(os.path.dirname(filename))
      prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
                if cr_locale != _DEFAULT_LOCALE else 'values')
      if dirname != prefix:
        errors.append(
            '%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))

  return errors


def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
  """Check all <output> elements related to Android.

  Args:
    grd_file: Input .grd file path.
    grd_lines: List of input .grd lines.
    wanted_locales: set of wanted Chromium locale names.
  Returns:
    List of error message strings. Empty on success.
  """
  intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
    errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
                                                         wanted_locales)
  return errors


def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
  """Fix an input .grd line by adding missing Android outputs.

  Args:
    grd_file: Input .grd file path.
    grd_lines: Input .grd line list.
    wanted_locales: set of Chromium locale names.
  Returns:
    A new list of .grd lines, containing new <output> elements when needed
    for locales from |wanted_locales| that were not part of the input.
  """
  intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  for start, end in reversed(intervals):
    locales = set()
    for pos in xrange(start, end):
      lang = _GetXmlLangAttribute(grd_lines[pos])
      locale = _FixChromiumLangAttribute(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'bg'
    src_lang_attribute = 'lang="%s"' % src_locale
    src_line = None
    for pos in xrange(start, end):
      if src_lang_attribute in grd_lines[pos]:
        src_line = grd_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find <output> element with "%s" lang attribute' % src_locale)

    line_count = end - 1
    for locale in missing_locales:
      android_locale = resource_utils.ToAndroidLocaleName(locale)
      dst_line = src_line.replace(
          'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
              'values-%s/' % src_locale, 'values-%s/' % android_locale)
      grd_lines.insert(line_count, dst_line)
      line_count += 1

  # Sort the new <output> elements.
  return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)


##########################################################################
##########################################################################
#####
#####    G R D   T R A N S L A T I O N S
#####
##########################################################################
##########################################################################


def _IsTranslationGrdOutputLine(line):
  """Returns True iff this is an output .xtb <file> element."""
  m = _RE_TRANSLATION_ELEMENT.search(line)
  return m is not None


class _IsTranslationGrdOutputLineTest(unittest.TestCase):

  def test_GrdTranslationOutputLines(self):
    _VALID_INPUT_LINES = [
        '<file path="foo/bar.xtb" />',
        '<file path="foo/bar.xtb"/>',
        '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
        '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
        '  <file path="translations/aw_strings_ar.xtb" lang="ar" />',
    ]
    _INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']

    for line in _VALID_INPUT_LINES:
      self.assertTrue(
          _IsTranslationGrdOutputLine(line),
          '_IsTranslationGrdOutputLine() returned False for [%s]' % line)

    for line in _INVALID_INPUT_LINES:
      self.assertFalse(
          _IsTranslationGrdOutputLine(line),
          '_IsTranslationGrdOutputLine() returned True for [%s]' % line)


def _CheckGrdTranslationElementRange(grd_lines, start, end,
                                     wanted_locales):
  """Check all <translations> sub-elements in specific input .grd lines range.

  This really checks the following:
    - Each item has a 'path' attribute.
    - Each such path value ends up with '.xtb'.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      continue
    m = _RE_PATH_ATTRIBUTE.search(line)
    if not m:
      errors.append('%d: Missing path attribute in <file> element' % pos +
                    1)
    else:
      filename = m.group(1)
      if not filename.endswith('.xtb'):
        errors.append(
            '%d: Path should end with ".xtb": %s' % (pos + 1, filename))

  return errors


def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
  """Check all <file> elements that correspond to an .xtb output file.

  Args:
    grd_file: Input .grd file path.
    grd_lines: List of input .grd lines.
    wanted_locales: set of wanted Chromium locale names.
  Returns:
    List of error message strings. Empty on success.
  """
  wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
    errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
                                              wanted_locales)
  return errors


# Regular expression used to replace the lang attribute inside .xtb files.
_RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')


def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
  """Create a fake .xtb file.

  Args:
    src_xtb_path: Path to source .xtb file to copy from.
    dst_xtb_path: Path to destination .xtb file to write to.
    dst_locale: Destination locale, the lang attribute in the source file
      will be substituted with this value before its lines are written
      to the destination file.
  """
  with open(src_xtb_path) as f:
    src_xtb_lines = f.readlines()

  def replace_xtb_lang_attribute(line):
    m = _RE_TRANSLATIONBUNDLE.search(line)
    if not m:
      return line
    return line[:m.start(1)] + dst_locale + line[m.end(1):]

  dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
  with build_utils.AtomicOutput(dst_xtb_path) as tmp:
    tmp.writelines(dst_xtb_lines)


def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
  """Fix an input .grd line by adding missing Android outputs.

  This also creates fake .xtb files from the one provided for 'en-GB'.

  Args:
    grd_file: Input .grd file path.
    grd_lines: Input .grd line list.
    wanted_locales: set of Chromium locale names.
  Returns:
    A new list of .grd lines, containing new <output> elements when needed
    for locales from |wanted_locales| that were not part of the input.
  """
  wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  for start, end in reversed(intervals):
    locales = set()
    for pos in xrange(start, end):
      lang = _GetXmlLangAttribute(grd_lines[pos])
      locale = _FixChromiumLangAttribute(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'en-GB'
    src_lang_attribute = 'lang="%s"' % src_locale
    src_line = None
    for pos in xrange(start, end):
      if src_lang_attribute in grd_lines[pos]:
        src_line = grd_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find <file> element with "%s" lang attribute' % src_locale)

    src_path = os.path.join(
        os.path.dirname(grd_file),
        _RE_PATH_ATTRIBUTE.search(src_line).group(1))

    line_count = end - 1
    for locale in missing_locales:
      dst_line = src_line.replace(
          'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
              '_%s.xtb' % src_locale, '_%s.xtb' % locale)
      grd_lines.insert(line_count, dst_line)
      line_count += 1

      dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
      _CreateFakeXtbFileFrom(src_path, dst_path, locale)


  # Sort the new <output> elements.
  return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)


##########################################################################
##########################################################################
#####
#####    G N   A N D R O I D   O U T P U T S
#####
##########################################################################
##########################################################################

_RE_GN_VALUES_LIST_LINE = re.compile(
    r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')

def _IsBuildGnInputFile(input_file):
  """Returns True iff this is a BUILD.gn file."""
  return os.path.basename(input_file) == 'BUILD.gn'


def _GetAndroidGnOutputLocale(line):
  """Check a GN list, and return its Android locale if it is an output .xml"""
  m = _RE_GN_VALUES_LIST_LINE.match(line)
  if not m:
    return None

  if m.group(1):  # First group is optional and contains group 2.
    return m.group(2)

  return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)


def _IsAndroidGnOutputLine(line):
  """Returns True iff this is an Android-specific localized .xml output."""
  return _GetAndroidGnOutputLocale(line) != None


def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
  """Check that a range of GN lines corresponds to localized strings.

  Special case: Some BUILD.gn files list several non-localized .xml files
  that should be ignored by this function, e.g. in
  components/cronet/android/BUILD.gn, the following appears:

    inputs = [
      ...
      "sample/res/layout/activity_main.xml",
      "sample/res/layout/dialog_url.xml",
      "sample/res/values/dimens.xml",
      "sample/res/values/strings.xml",
      ...
    ]

  These are non-localized strings, and should be ignored. This function is
  used to detect them quickly.
  """
  for pos in xrange(start, end):
    if not 'values/' in gn_lines[pos]:
      return True
  return False


def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
  if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
    return []

  errors = []
  locales = set()
  for pos in xrange(start, end):
    line = gn_lines[pos]
    android_locale = _GetAndroidGnOutputLocale(line)
    assert android_locale != None
    cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
    if cr_locale in locales:
      errors.append('%s: Redefinition of output for "%s" locale' %
                    (pos + 1, android_locale))
    locales.add(cr_locale)

  extra_locales = locales.difference(wanted_locales)
  if extra_locales:
    errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
                                                sorted(extra_locales)))

  missing_locales = wanted_locales.difference(locales)
  if missing_locales:
    errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
                                                  sorted(missing_locales)))

  return errors


def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
  return errors


def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  # NOTE: Since this may insert new lines to each interval, process the
  # list in reverse order to maintain valid (start,end) positions during
  # the iteration.
  for start, end in reversed(intervals):
    if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
      continue

    locales = set()
    for pos in xrange(start, end):
      lang = _GetAndroidGnOutputLocale(gn_lines[pos])
      locale = resource_utils.ToChromiumLocaleName(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'bg'
    src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
    src_line = None
    for pos in xrange(start, end):
      if src_values in gn_lines[pos]:
        src_line = gn_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find output list item with "%s" locale' % src_locale)

    line_count = end - 1
    for locale in missing_locales:
      if locale == _DEFAULT_LOCALE:
        dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
      else:
        dst_line = src_line.replace(
            'values-%s/' % src_locale,
            'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
      gn_lines.insert(line_count, dst_line)
      line_count += 1

    gn_lines = _SortListSubRange(
        gn_lines, start, line_count,
        lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))

  return gn_lines


##########################################################################
##########################################################################
#####
#####    T R A N S L A T I O N   E X P E C T A T I O N S
#####
##########################################################################
##########################################################################

_EXPECTATIONS_FILENAME = 'translation_expectations.pyl'

# Technical note: the format of translation_expectations.pyl
# is a 'Python literal', which defines a python dictionary, so should
# be easy to parse. However, when modifying it, care should be taken
# to respect the line comments and the order of keys within the text
# file.


def _ReadPythonLiteralFile(pyl_path):
  """Read a .pyl file into a Python data structure."""
  with open(pyl_path) as f:
    pyl_content = f.read()
  # Evaluate as a Python data structure, use an empty global
  # and local dictionary.
  return eval(pyl_content, dict(), dict())


def _UpdateLocalesInExpectationLines(pyl_lines,
                                     wanted_locales,
                                     available_width=79):
  """Update the locales list(s) found in an expectations file.

  Args:
    pyl_lines: Iterable of input lines from the file.
    wanted_locales: Set or list of new locale names.
    available_width: Optional, number of character colums used
      to word-wrap the new list items.
  Returns:
    New list of updated lines.
  """
  locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
  result = []
  line_count = len(pyl_lines)
  line_num = 0
  DICT_START = '"languages": ['
  while line_num < line_count:
    line = pyl_lines[line_num]
    line_num += 1
    result.append(line)
    # Look for start of "languages" dictionary.
    pos = line.find(DICT_START)
    if pos < 0:
      continue

    start_margin = pos
    start_line = line_num
    # Skip over all lines from the list.
    while (line_num < line_count and
           not pyl_lines[line_num].rstrip().endswith('],')):
      line_num += 1
      continue

    if line_num == line_count:
      raise Exception('%d: Missing list termination!' % start_line)

    # Format the new list according to the new margin.
    locale_width = available_width - (start_margin + 2)
    locale_lines = _PrettyPrintListAsLines(
        locales_list, locale_width, trailing_comma=True)
    for locale_line in locale_lines:
      result.append(' ' * (start_margin + 2) + locale_line)
    result.append(' ' * start_margin + '],')
    line_num += 1

  return result


class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):

  def test_simple(self):
    self.maxDiff = 1000
    input_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
  "android_grd": {
    "languages": [
      "aa", "bb", "cc", "dd", "ee",
      "ff", "gg", "hh", "ii", "jj",
      "kk"],
  },
  # Example with bad indentation in input.
  "another_grd": {
         "languages": [
  "aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
      ],
  },
}
'''
    expected_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
  "android_grd": {
    "languages": [
      "A2", "AA", "BB", "CC", "DD",
      "E2", "EE", "FF", "GG", "HH",
      "I2", "II", "JJ", "KK",
    ],
  },
  # Example with bad indentation in input.
  "another_grd": {
         "languages": [
           "A2", "AA", "BB", "CC", "DD",
           "E2", "EE", "FF", "GG", "HH",
           "I2", "II", "JJ", "KK",
         ],
  },
}
'''
    input_lines = input_text.splitlines()
    test_locales = ([
        'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
        'E2', 'I2'
    ])
    expected_lines = expected_text.splitlines()
    self.assertListEqual(
        _UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
        expected_lines)

  def test_missing_list_termination(self):
    input_lines = r'''
  "languages": ['
    "aa", "bb", "cc", "dd"
'''.splitlines()
    with self.assertRaises(Exception) as cm:
      _UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)

    self.assertEqual(str(cm.exception), '2: Missing list termination!')


def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
  """Update all locales listed in a given expectations file.

  Args:
    pyl_path: Path to .pyl file to update.
    wanted_locales: List of locales that need to be written to
      the file.
  """
  tc_locales = {
      _FixTranslationConsoleLocaleName(locale)
      for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
  }

  with open(pyl_path) as f:
    input_lines = [l.rstrip() for l in f.readlines()]

  updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
  with build_utils.AtomicOutput(pyl_path) as f:
    f.writelines('\n'.join(updated_lines) + '\n')


##########################################################################
##########################################################################
#####
#####    C H E C K   E V E R Y T H I N G
#####
##########################################################################
##########################################################################

# pylint: enable=unused-argument


def _IsAllInputFile(input_file):
  return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)


def _CheckAllFiles(input_file, input_lines, wanted_locales):
  errors = []
  if _IsGritInputFile(input_file):
    errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
    errors += _CheckGrdAndroidOutputElements(
        input_file, input_lines, wanted_locales)
  elif _IsBuildGnInputFile(input_file):
    errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
  return errors


def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
  if _IsGritInputFile(input_file):
    lines = _AddMissingLocalesInGrdTranslations(
        input_file, input_lines, wanted_locales)
    lines = _AddMissingLocalesInGrdAndroidOutputs(
        input_file, lines, wanted_locales)
  elif _IsBuildGnInputFile(input_file):
    lines = _AddMissingLocalesInGnAndroidOutputs(
        input_file, input_lines, wanted_locales)
  return lines


##########################################################################
##########################################################################
#####
#####    C O M M A N D   H A N D L I N G
#####
##########################################################################
##########################################################################

class _Command(object):
  """A base class for all commands recognized by this script.

  Usage is the following:
    1) Derived classes must re-define the following class-based fields:
       - name: Command name (e.g. 'list-locales')
       - description: Command short description.
       - long_description: Optional. Command long description.
         NOTE: As a convenience, if the first character is a newline,
         it will be omitted in the help output.

    2) Derived classes for commands that take arguments should override
       RegisterExtraArgs(), which receives a corresponding argparse
       sub-parser as argument.

    3) Derived classes should implement a Run() command, which can read
       the current arguments from self.args.
  """
  name = None
  description = None
  long_description = None

  def __init__(self):
    self._parser = None
    self.args = None

  def RegisterExtraArgs(self, subparser):
    pass

  def RegisterArgs(self, parser):
    subp = parser.add_parser(
        self.name, help=self.description,
        description=self.long_description or self.description,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    self._parser = subp
    subp.set_defaults(command=self)
    group = subp.add_argument_group('%s arguments' % self.name)
    self.RegisterExtraArgs(group)

  def ProcessArgs(self, args):
    self.args = args


class _ListLocalesCommand(_Command):
  """Implement the 'list-locales' command to list locale lists of interest."""
  name = 'list-locales'
  description = 'List supported Chrome locales'
  long_description = r'''
List locales of interest, by default this prints all locales supported by
Chrome, but `--type=android_omitted` can be used to print the list of locales
omitted from Android APKs (but not app bundles), and `--type=ios_unsupported`
for the list of locales unsupported on iOS.

These values are extracted directly from build/config/locales.gni.

Additionally, use the --as-json argument to print the list as a JSON list,
instead of the default format (which is a space-separated list of locale names).
'''

  # Maps type argument to a function returning the corresponding locales list.
  TYPE_MAP = {
    'all': ChromeLocales,
    'android_omitted': AndroidOmittedLocales,
    'ios_unsupported': IosUnsupportedLocales,
  }

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '--as-json',
        action='store_true',
        help='Output as JSON list.')
    group.add_argument(
        '--type',
        choices=tuple(self.TYPE_MAP.viewkeys()),
        default='all',
        help='Select type of locale list to print.')

  def Run(self):
    locale_list = self.TYPE_MAP[self.args.type]()
    if self.args.as_json:
      print '[%s]' % ", ".join("'%s'" % loc for loc in locale_list)
    else:
      print ' '.join(locale_list)


class _CheckInputFileBaseCommand(_Command):
  """Used as a base for other _Command subclasses that check input files.

  Subclasses should also define the following class-level variables:

  - select_file_func:
      A predicate that receives a file name (not path) and return True if it
      should be selected for inspection. Used when scanning directories with
      '--scan-dir <dir>'.

  - check_func:
  - fix_func:
      Two functions passed as parameters to _ProcessFile(), see relevant
      documentation in this function's definition.
  """
  select_file_func = None
  check_func = None
  fix_func = None

  def RegisterExtraArgs(self, group):
    group.add_argument(
      '--scan-dir',
      action='append',
      help='Optional directory to scan for input files recursively.')
    group.add_argument(
      'input',
      nargs='*',
      help='Input file(s) to check.')
    group.add_argument(
      '--fix-inplace',
      action='store_true',
      help='Try to fix the files in-place too.')
    group.add_argument(
      '--add-locales',
      help='Space-separated list of additional locales to use')

  def Run(self):
    args = self.args
    input_files = []
    if args.input:
      input_files = args.input
    if args.scan_dir:
      input_files.extend(_ScanDirectoriesForFiles(
          args.scan_dir, self.select_file_func.__func__))
    locales = ChromeLocales()
    if args.add_locales:
      locales.extend(args.add_locales.split(' '))

    locales = set(locales)

    for input_file in input_files:
      _ProcessFile(input_file,
                   locales,
                   self.check_func.__func__,
                   self.fix_func.__func__ if args.fix_inplace else None)
    print '%sDone.' % (_CONSOLE_START_LINE)


class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
  name = 'check-grd-android-outputs'
  description = (
      'Check the Android resource (.xml) files outputs in GRIT input files.')
  long_description = r'''
Check the Android .xml files outputs in one or more input GRIT (.grd) files
for the following conditions:

    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.
    - Filenames exist for each listed locale.
    - Filenames are well-formed.
'''
  select_file_func = _IsGritInputFile
  check_func = _CheckGrdAndroidOutputElements
  fix_func = _AddMissingLocalesInGrdAndroidOutputs


class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
  name = 'check-grd-translations'
  description = (
      'Check the translation (.xtb) files outputted by .grd input files.')
  long_description = r'''
Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
for the following conditions:

    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.
    - Each item has a 'path' attribute.
    - Each such path value ends up with '.xtb'.
'''
  select_file_func = _IsGritInputFile
  check_func = _CheckGrdTranslations
  fix_func = _AddMissingLocalesInGrdTranslations


class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
  name = 'check-gn-android-outputs'
  description = 'Check the Android .xml file lists in GN build files.'
  long_description = r'''
Check one or more BUILD.gn file, looking for lists of Android resource .xml
files, and checking that:

  - There are no duplicated output files in the list.
  - Each output file belongs to a wanted Chromium locale.
  - There are no output files for unwanted Chromium locales.
'''
  select_file_func = _IsBuildGnInputFile
  check_func = _CheckGnAndroidOutputs
  fix_func = _AddMissingLocalesInGnAndroidOutputs


class _CheckAllCommand(_CheckInputFileBaseCommand):
  name = 'check-all'
  description = 'Check everything.'
  long_description = 'Equivalent to calling all other check-xxx commands.'
  select_file_func = _IsAllInputFile
  check_func = _CheckAllFiles
  fix_func = _AddMissingLocalesInAllFiles


class _UpdateExpectationsCommand(_Command):
  name = 'update-expectations'
  description = 'Update translation expectations file.'
  long_description = r'''
Update %s files to match the current list of locales supported by Chromium.
This is especially useful to add new locales before updating any GRIT or GN
input file with the --add-locales option.
''' % _EXPECTATIONS_FILENAME

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '--add-locales',
        help='Space-separated list of additional locales to use.')

  def Run(self):
    locales = ChromeLocales()
    add_locales = self.args.add_locales
    if add_locales:
      locales.extend(add_locales.split(' '))

    expectation_paths = [
        'tools/gritsettings/translation_expectations.pyl',
        'clank/tools/translation_expectations.pyl',
    ]
    missing_expectation_files = []
    for path in enumerate(expectation_paths):
      file_path = os.path.join(_TOP_SRC_DIR, path)
      if not os.path.exists(file_path):
        missing_expectation_files.append(file_path)
        continue
      _UpdateLocalesInExpectationFile(file_path, locales)

    if missing_expectation_files:
      sys.stderr.write('WARNING: Missing file(s): %s\n' %
                       (', '.join(missing_expectation_files)))


class _UnitTestsCommand(_Command):
  name = 'unit-tests'
  description = 'Run internal unit-tests for this script'

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '-v', '--verbose', action='count', help='Increase test verbosity.')
    group.add_argument('args', nargs=argparse.REMAINDER)

  def Run(self):
    argv = [_SCRIPT_NAME] + self.args.args
    unittest.main(argv=argv, verbosity=self.args.verbose)


# List of all commands supported by this script.
_COMMANDS = [
    _ListLocalesCommand,
    _CheckGrdAndroidOutputsCommand,
    _CheckGrdTranslationsCommand,
    _CheckGnAndroidOutputsCommand,
    _CheckAllCommand,
    _UpdateExpectationsCommand,
    _UnitTestsCommand,
]


def main(argv):
  parser = argparse.ArgumentParser(
      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)

  subparsers = parser.add_subparsers()
  commands = [clazz() for clazz in _COMMANDS]
  for command in commands:
    command.RegisterArgs(subparsers)

  if not argv:
    argv = ['--help']

  args = parser.parse_args(argv)
  args.command.ProcessArgs(args)
  args.command.Run()


if __name__ == "__main__":
  main(sys.argv[1:])