google/appengine/tools/devappserver2/application_configuration.py

#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Stores application configuration taken from e.g. app.yaml, queues.yaml."""

# TODO: Support more than just app.yaml.


import errno
import logging
import os
import os.path
import random
import string
import threading
import types

from google.appengine.api import appinfo
from google.appengine.api import appinfo_includes
from google.appengine.api import backendinfo
from google.appengine.api import dispatchinfo
from google.appengine.tools import yaml_translator
from google.appengine.tools.devappserver2 import errors

# Constants passed to functions registered with
# ModuleConfiguration.add_change_callback.
NORMALIZED_LIBRARIES_CHANGED = 1
SKIP_FILES_CHANGED = 2
HANDLERS_CHANGED = 3
INBOUND_SERVICES_CHANGED = 4
ENV_VARIABLES_CHANGED = 5
ERROR_HANDLERS_CHANGED = 6
NOBUILD_FILES_CHANGED = 7


def java_supported():
  """True if this SDK supports running Java apps in the dev appserver."""
  java_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'java')
  return os.path.isdir(java_dir)


class ModuleConfiguration(object):
  """Stores module configuration information.

  Most configuration options are mutable and may change any time
  check_for_updates is called. Client code must be able to cope with these
  changes.

  Other properties are immutable (see _IMMUTABLE_PROPERTIES) and are guaranteed
  to be constant for the lifetime of the instance.
  """

  _IMMUTABLE_PROPERTIES = [
      ('application', 'application'),
      ('version', 'major_version'),
      ('runtime', 'runtime'),
      ('threadsafe', 'threadsafe'),
      ('module', 'module_name'),
      ('basic_scaling', 'basic_scaling'),
      ('manual_scaling', 'manual_scaling'),
      ('automatic_scaling', 'automatic_scaling')]

  def __init__(self, config_path):
    """Initializer for ModuleConfiguration.

    Args:
      config_path: A string containing the full path of the yaml or xml file
          containing the configuration for this module.
    """
    self._config_path = config_path
    root = os.path.dirname(config_path)
    self._is_java = os.path.normpath(config_path).endswith(
        os.sep + 'WEB-INF' + os.sep + 'appengine-web.xml')
    if self._is_java:
      # We assume Java's XML-based config files only if config_path is
      # something like /foo/bar/WEB-INF/appengine-web.xml. In this case,
      # the application root is /foo/bar. Other apps, configured with YAML,
      # have something like /foo/bar/app.yaml, with application root /foo/bar.
      root = os.path.dirname(root)
    self._application_root = os.path.realpath(root)
    self._last_failure_message = None

    self._app_info_external, files_to_check = self._parse_configuration(
        self._config_path)
    self._mtimes = self._get_mtimes(files_to_check)
    self._application = '%s~%s' % (self.partition,
                                   self.application_external_name)
    self._api_version = self._app_info_external.api_version
    self._module_name = self._app_info_external.module
    self._version = self._app_info_external.version
    self._threadsafe = self._app_info_external.threadsafe
    self._basic_scaling = self._app_info_external.basic_scaling
    self._manual_scaling = self._app_info_external.manual_scaling
    self._automatic_scaling = self._app_info_external.automatic_scaling
    self._runtime = self._app_info_external.runtime
    if self._runtime == 'python':
      logging.warning(
          'The "python" runtime specified in "%s" is not supported - the '
          '"python27" runtime will be used instead. A description of the '
          'differences between the two can be found here:\n'
          'https://developers.google.com/appengine/docs/python/python25/diff27',
          self._config_path)
    self._minor_version_id = ''.join(random.choice(string.digits) for _ in
                                     range(18))

  @property
  def application_root(self):
    """The directory containing the application e.g. "/home/user/myapp"."""
    return self._application_root

  @property
  def application(self):
    return self._application

  @property
  def partition(self):
    return 'dev'

  @property
  def application_external_name(self):
    return self._app_info_external.application

  @property
  def api_version(self):
    return self._api_version

  @property
  def module_name(self):
    return self._module_name or appinfo.DEFAULT_MODULE

  @property
  def major_version(self):
    return self._version

  @property
  def minor_version(self):
    return self._minor_version_id

  @property
  def version_id(self):
    if self.module_name == appinfo.DEFAULT_MODULE:
      return '%s.%s' % (
          self.major_version,
          self._minor_version_id)
    else:
      return '%s:%s.%s' % (
          self.module_name,
          self.major_version,
          self._minor_version_id)

  @property
  def runtime(self):
    return self._runtime

  @property
  def effective_runtime(self):
    return self._app_info_external.GetEffectiveRuntime()

  @property
  def threadsafe(self):
    return self._threadsafe

  @property
  def basic_scaling(self):
    return self._basic_scaling

  @property
  def manual_scaling(self):
    return self._manual_scaling

  @property
  def automatic_scaling(self):
    return self._automatic_scaling

  @property
  def normalized_libraries(self):
    return self._app_info_external.GetNormalizedLibraries()

  @property
  def skip_files(self):
    return self._app_info_external.skip_files

  @property
  def nobuild_files(self):
    return self._app_info_external.nobuild_files

  @property
  def error_handlers(self):
    return self._app_info_external.error_handlers

  @property
  def handlers(self):
    return self._app_info_external.handlers

  @property
  def inbound_services(self):
    return self._app_info_external.inbound_services

  @property
  def env_variables(self):
    return self._app_info_external.env_variables

  @property
  def is_backend(self):
    return False

  def check_for_updates(self):
    """Return any configuration changes since the last check_for_updates call.

    Returns:
      A set containing the changes that occured. See the *_CHANGED module
      constants.
    """
    new_mtimes = self._get_mtimes(self._mtimes.keys())
    if new_mtimes == self._mtimes:
      return set()

    try:
      app_info_external, files_to_check = self._parse_configuration(
          self._config_path)
    except Exception, e:
      failure_message = str(e)
      if failure_message != self._last_failure_message:
        logging.error('Configuration is not valid: %s', failure_message)
      self._last_failure_message = failure_message
      return set()
    self._last_failure_message = None

    self._mtimes = self._get_mtimes(files_to_check)

    for app_info_attribute, self_attribute in self._IMMUTABLE_PROPERTIES:
      app_info_value = getattr(app_info_external, app_info_attribute)
      self_value = getattr(self, self_attribute)
      if (app_info_value == self_value or
          app_info_value == getattr(self._app_info_external,
                                    app_info_attribute)):
        # Only generate a warning if the value is both different from the
        # immutable value *and* different from the last loaded value.
        continue

      if isinstance(app_info_value, types.StringTypes):
        logging.warning('Restart the development module to see updates to "%s" '
                        '["%s" => "%s"]',
                        app_info_attribute,
                        self_value,
                        app_info_value)
      else:
        logging.warning('Restart the development module to see updates to "%s"',
                        app_info_attribute)

    changes = set()
    if (app_info_external.GetNormalizedLibraries() !=
        self.normalized_libraries):
      changes.add(NORMALIZED_LIBRARIES_CHANGED)
    if app_info_external.skip_files != self.skip_files:
      changes.add(SKIP_FILES_CHANGED)
    if app_info_external.nobuild_files != self.nobuild_files:
      changes.add(NOBUILD_FILES_CHANGED)
    if app_info_external.handlers != self.handlers:
      changes.add(HANDLERS_CHANGED)
    if app_info_external.inbound_services != self.inbound_services:
      changes.add(INBOUND_SERVICES_CHANGED)
    if app_info_external.env_variables != self.env_variables:
      changes.add(ENV_VARIABLES_CHANGED)
    if app_info_external.error_handlers != self.error_handlers:
      changes.add(ERROR_HANDLERS_CHANGED)

    self._app_info_external = app_info_external
    if changes:
      self._minor_version_id = ''.join(random.choice(string.digits) for _ in
                                       range(18))
    return changes

  @staticmethod
  def _get_mtimes(filenames):
    filename_to_mtime = {}
    for filename in filenames:
      try:
        filename_to_mtime[filename] = os.path.getmtime(filename)
      except OSError as e:
        # Ignore deleted includes.
        if e.errno != errno.ENOENT:
          raise
    return filename_to_mtime

  def _parse_configuration(self, configuration_path):
    """Parse a configuration file (like app.yaml or appengine-web.xml).

    Args:
      configuration_path: A string containing the full path of the yaml file
          containing the configuration for this module.

    Returns:
      A tuple where the first element is the parsed appinfo.AppInfoExternal
      object and the second element is a list of the paths of the files that
      were used to produce it, namely the input configuration_path and any
      other file that was included from that one.
    """
    if self._is_java:
      config, files = self._parse_java_configuration(configuration_path)
    else:
      with open(configuration_path) as f:
        config, files = appinfo_includes.ParseAndReturnIncludePaths(f)
    return config, [configuration_path] + files

  @staticmethod
  def _parse_java_configuration(app_engine_web_xml_path):
    """Parse appengine-web.xml and web.xml.

    Args:
      app_engine_web_xml_path: A string containing the full path of the
          .../WEB-INF/appengine-web.xml file. The corresponding
          .../WEB-INF/web.xml file must also be present.

    Returns:
      A tuple where the first element is the parsed appinfo.AppInfoExternal
      object and the second element is a list of the paths of the files that
      were used to produce it, namely the input appengine-web.xml file and the
      corresponding web.xml file.
    """
    with open(app_engine_web_xml_path) as f:
      app_engine_web_xml_str = f.read()
    web_inf_dir = os.path.dirname(app_engine_web_xml_path)
    web_xml_path = os.path.join(web_inf_dir, 'web.xml')
    with open(web_xml_path) as f:
      web_xml_str = f.read()
    static_files = []
    # TODO: need to enumerate static files here
    app_yaml_str = yaml_translator.TranslateXmlToYaml(
        app_engine_web_xml_str, web_xml_str, static_files)
    config = appinfo.LoadSingleAppInfo(app_yaml_str)
    return config, [app_engine_web_xml_path, web_xml_path]


class BackendsConfiguration(object):
  """Stores configuration information for a backends.yaml file."""

  def __init__(self, app_config_path, backend_config_path):
    """Initializer for BackendsConfiguration.

    Args:
      app_config_path: A string containing the full path of the yaml file
          containing the configuration for this module.
      backend_config_path: A string containing the full path of the
          backends.yaml file containing the configuration for backends.
    """
    self._update_lock = threading.RLock()
    self._base_module_configuration = ModuleConfiguration(app_config_path)
    backend_info_external = self._parse_configuration(
        backend_config_path)

    self._backends_name_to_backend_entry = {}
    for backend in backend_info_external.backends or []:
      self._backends_name_to_backend_entry[backend.name] = backend
    self._changes = dict(
        (backend_name, set())
        for backend_name in self._backends_name_to_backend_entry)

  @staticmethod
  def _parse_configuration(configuration_path):
    # TODO: It probably makes sense to catch the exception raised
    # by Parse() and re-raise it using a module-specific exception.
    with open(configuration_path) as f:
      return backendinfo.LoadBackendInfo(f)

  def get_backend_configurations(self):
    return [BackendConfiguration(self._base_module_configuration, self, entry)
            for entry in self._backends_name_to_backend_entry.values()]

  def check_for_updates(self, backend_name):
    """Return any configuration changes since the last check_for_updates call.

    Args:
      backend_name: A str containing the name of the backend to be checked for
          updates.

    Returns:
      A set containing the changes that occured. See the *_CHANGED module
      constants.
    """
    with self._update_lock:
      module_changes = self._base_module_configuration.check_for_updates()
      if module_changes:
        for backend_changes in self._changes.values():
          backend_changes.update(module_changes)
      changes = self._changes[backend_name]
      self._changes[backend_name] = set()
    return changes


class BackendConfiguration(object):
  """Stores backend configuration information.

  This interface is and must remain identical to ModuleConfiguration.
  """

  def __init__(self, module_configuration, backends_configuration,
               backend_entry):
    """Initializer for BackendConfiguration.

    Args:
      module_configuration: A ModuleConfiguration to use.
      backends_configuration: The BackendsConfiguration that tracks updates for
          this BackendConfiguration.
      backend_entry: A backendinfo.BackendEntry containing the backend
          configuration.
    """
    self._module_configuration = module_configuration
    self._backends_configuration = backends_configuration
    self._backend_entry = backend_entry

    if backend_entry.dynamic:
      self._basic_scaling = appinfo.BasicScaling(
          max_instances=backend_entry.instances or 1)
      self._manual_scaling = None
    else:
      self._basic_scaling = None
      self._manual_scaling = appinfo.ManualScaling(
          instances=backend_entry.instances or 1)
    self._minor_version_id = ''.join(random.choice(string.digits) for _ in
                                     range(18))

  @property
  def application_root(self):
    """The directory containing the application e.g. "/home/user/myapp"."""
    return self._module_configuration.application_root

  @property
  def application(self):
    return self._module_configuration.application

  @property
  def partition(self):
    return self._module_configuration.partition

  @property
  def application_external_name(self):
    return self._module_configuration.application_external_name

  @property
  def api_version(self):
    return self._module_configuration.api_version

  @property
  def module_name(self):
    return self._backend_entry.name

  @property
  def major_version(self):
    return self._module_configuration.major_version

  @property
  def minor_version(self):
    return self._minor_version_id

  @property
  def version_id(self):
    return '%s:%s.%s' % (
        self.module_name,
        self.major_version,
        self._minor_version_id)

  @property
  def runtime(self):
    return self._module_configuration.runtime

  @property
  def effective_runtime(self):
    return self._module_configuration.effective_runtime

  @property
  def threadsafe(self):
    return self._module_configuration.threadsafe

  @property
  def basic_scaling(self):
    return self._basic_scaling

  @property
  def manual_scaling(self):
    return self._manual_scaling

  @property
  def automatic_scaling(self):
    return None

  @property
  def normalized_libraries(self):
    return self._module_configuration.normalized_libraries

  @property
  def skip_files(self):
    return self._module_configuration.skip_files

  @property
  def nobuild_files(self):
    return self._module_configuration.nobuild_files

  @property
  def error_handlers(self):
    return self._module_configuration.error_handlers

  @property
  def handlers(self):
    if self._backend_entry.start:
      return [appinfo.URLMap(
          url='/_ah/start',
          script=self._backend_entry.start,
          login='admin')] + self._module_configuration.handlers
    return self._module_configuration.handlers

  @property
  def inbound_services(self):
    return self._module_configuration.inbound_services

  @property
  def env_variables(self):
    return self._module_configuration.env_variables

  @property
  def is_backend(self):
    return True

  def check_for_updates(self):
    """Return any configuration changes since the last check_for_updates call.

    Returns:
      A set containing the changes that occured. See the *_CHANGED module
      constants.
    """
    changes = self._backends_configuration.check_for_updates(
        self._backend_entry.name)
    if changes:
      self._minor_version_id = ''.join(random.choice(string.digits) for _ in
                                       range(18))
    return changes


class DispatchConfiguration(object):
  """Stores dispatcher configuration information."""

  def __init__(self, config_path):
    self._config_path = config_path
    self._mtime = os.path.getmtime(self._config_path)
    self._process_dispatch_entries(self._parse_configuration(self._config_path))

  @staticmethod
  def _parse_configuration(configuration_path):
    # TODO: It probably makes sense to catch the exception raised
    # by LoadSingleDispatch() and re-raise it using a module-specific exception.
    with open(configuration_path) as f:
      return dispatchinfo.LoadSingleDispatch(f)

  def check_for_updates(self):
    mtime = os.path.getmtime(self._config_path)
    if mtime > self._mtime:
      self._mtime = mtime
      try:
        dispatch_info_external = self._parse_configuration(self._config_path)
      except Exception, e:
        failure_message = str(e)
        logging.error('Configuration is not valid: %s', failure_message)
        return
      self._process_dispatch_entries(dispatch_info_external)

  def _process_dispatch_entries(self, dispatch_info_external):
    path_only_entries = []
    hostname_entries = []
    for entry in dispatch_info_external.dispatch:
      parsed_url = dispatchinfo.ParsedURL(entry.url)
      if parsed_url.host:
        hostname_entries.append(entry)
      else:
        path_only_entries.append((parsed_url, entry.module))
    if hostname_entries:
      logging.warning(
          'Hostname routing is not supported by the development server. The '
          'following dispatch entries will not match any requests:\n%s',
          '\n\t'.join(str(entry) for entry in hostname_entries))
    self._entries = path_only_entries

  @property
  def dispatch(self):
    return self._entries


class ApplicationConfiguration(object):
  """Stores application configuration information."""

  def __init__(self, config_paths):
    """Initializer for ApplicationConfiguration.

    Args:
      config_paths: A list of strings containing the paths to yaml files,
          or to directories containing them.
    """
    self.modules = []
    self.dispatch = None
    # It's really easy to add a test case that passes in a string rather than
    # a list of strings, so guard against that.
    assert not isinstance(config_paths, basestring)
    config_paths = self._config_files_from_paths(config_paths)
    for config_path in config_paths:
      # TODO: add support for backends.xml and dispatch.xml here
      if (config_path.endswith('backends.yaml') or
          config_path.endswith('backends.yml')):
        # TODO: Reuse the ModuleConfiguration created for the app.yaml
        # instead of creating another one for the same file.
        self.modules.extend(
            BackendsConfiguration(config_path.replace('backends.y', 'app.y'),
                                  config_path).get_backend_configurations())
      elif (config_path.endswith('dispatch.yaml') or
            config_path.endswith('dispatch.yml')):
        if self.dispatch:
          raise errors.InvalidAppConfigError(
              'Multiple dispatch.yaml files specified')
        self.dispatch = DispatchConfiguration(config_path)
      else:
        module_configuration = ModuleConfiguration(config_path)
        self.modules.append(module_configuration)
    application_ids = set(module.application
                          for module in self.modules)
    if len(application_ids) > 1:
      raise errors.InvalidAppConfigError(
          'More than one application ID found: %s' %
          ', '.join(sorted(application_ids)))

    self._app_id = application_ids.pop()
    module_names = set()
    for module in self.modules:
      if module.module_name in module_names:
        raise errors.InvalidAppConfigError('Duplicate module: %s' %
                                           module.module_name)
      module_names.add(module.module_name)
    if self.dispatch:
      if appinfo.DEFAULT_MODULE not in module_names:
        raise errors.InvalidAppConfigError(
            'A default module must be specified.')
      missing_modules = (
          set(module_name for _, module_name in self.dispatch.dispatch) -
          module_names)
      if missing_modules:
        raise errors.InvalidAppConfigError(
            'Modules %s specified in dispatch.yaml are not defined by a yaml '
            'file.' % sorted(missing_modules))

  def _config_files_from_paths(self, config_paths):
    """Return a list of the configuration files found in the given paths.

    For any path that is a directory, the returned list will contain the
    configuration files (app.yaml and optionally backends.yaml) found in that
    directory. If the directory is a Java app (contains a subdirectory
    WEB-INF with web.xml and application-web.xml files), then the returned
    list will contain the path to the application-web.xml file, which is treated
    as if it included web.xml. Paths that are not directories are added to the
    returned list as is.

    Args:
      config_paths: a list of strings that are file or directory paths.

    Returns:
      A list of strings that are file paths.
    """
    config_files = []
    for path in config_paths:
      config_files += (
          self._config_files_from_dir(path) if os.path.isdir(path) else [path])
    return config_files

  def _config_files_from_dir(self, dir_path):
    """Return a list of the configuration files found in the given directory.

    If the directory contains a subdirectory WEB-INF then we expect to find
    web.xml and application-web.xml in that subdirectory. The returned list
    will consist of the path to application-web.xml, which we treat as if it
    included xml.

    Otherwise, we expect to find an app.yaml and optionally a backends.yaml,
    and we return those in the list.

    Args:
      dir_path: a string that is the path to a directory.

    Returns:
      A list of strings that are file paths.
    """
    web_inf = os.path.join(dir_path, 'WEB-INF')
    if java_supported() and os.path.isdir(web_inf):
      return self._config_files_from_web_inf_dir(web_inf)
    app_yamls = self._files_in_dir_matching(dir_path, ['app.yaml', 'app.yml'])
    if not app_yamls:
      or_web_inf = ' or a WEB-INF subdirectory' if java_supported() else ''
      raise errors.AppConfigNotFoundError(
          '"%s" is a directory but does not contain app.yaml or app.yml%s' %
          (dir_path, or_web_inf))
    backend_yamls = self._files_in_dir_matching(
        dir_path, ['backends.yaml', 'backends.yml'])
    return app_yamls + backend_yamls

  def _config_files_from_web_inf_dir(self, web_inf):
    required = ['appengine-web.xml', 'web.xml']
    missing = [f for f in required
               if not os.path.exists(os.path.join(web_inf, f))]
    if missing:
      raise errors.AppConfigNotFoundError(
          'The "%s" subdirectory exists but is missing %s' %
          (web_inf, ' and '.join(missing)))
    return [os.path.join(web_inf, required[0])]

  @staticmethod
  def _files_in_dir_matching(dir_path, names):
    abs_names = [os.path.join(dir_path, name) for name in names]
    files = [f for f in abs_names if os.path.exists(f)]
    if len(files) > 1:
      raise errors.InvalidAppConfigError(
          'Directory "%s" contains %s' % (dir_path, ' and '.join(names)))
    return files

  @property
  def app_id(self):
    return self._app_id


def get_app_error_file(module_configuration):
  """Returns application specific file to handle errors.

  Dev AppServer only supports 'default' error code.

  Args:
    module_configuration: ModuleConfiguration.

  Returns:
      A string containing full path to error handler file or
      None if no 'default' error handler is specified.
  """
  for error_handler in module_configuration.error_handlers or []:
    if not error_handler.error_code or error_handler.error_code == 'default':
      return os.path.join(module_configuration.application_root,
                          error_handler.file)
  return None