#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Build NativeClient toolchain packages."""

import logging
import optparse
import os
import sys
import textwrap

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import pynacl.file_tools
import pynacl.gsd_storage
import pynacl.log_tools
import pynacl.local_storage_cache

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
NACL_DIR = os.path.dirname(SCRIPT_DIR)
ROOT_DIR = os.path.dirname(NACL_DIR)
BUILD_DIR = os.path.join(NACL_DIR, 'build')
PKG_VER_DIR = os.path.join(BUILD_DIR, 'package_version')
sys.path.append(PKG_VER_DIR)
import archive_info
import package_info

import once
import command_options

DEFAULT_CACHE_DIR = os.path.join(SCRIPT_DIR, 'cache')
DEFAULT_GIT_CACHE_DIR = os.path.join(SCRIPT_DIR, 'git_cache')
DEFAULT_SRC_DIR = os.path.join(SCRIPT_DIR, 'src')
DEFAULT_OUT_DIR = os.path.join(SCRIPT_DIR, 'out')


def PrintAnnotatorURL(cloud_item):
  """Print an URL in buildbot annotator form.

  Args:
    cloud_item: once.CloudStorageItem representing a memoized item in the cloud.
  """
  if cloud_item.dir_item:
    url = cloud_item.dir_item.url
    pynacl.log_tools.WriteAnnotatorLine('@@@STEP_LINK@download@%s@@@' % url)

    if cloud_item.log_url:
      log_url = cloud_item.log_url
      pynacl.log_tools.WriteAnnotatorLine('@@@STEP_LINK@log@%s@@@' % log_url)


class PackageBuilder(object):
  """Module to build a setup of packages."""

  def __init__(self, packages, package_targets, args):
    """Constructor.

    Args:
      packages: A dictionary with the following format. There are two types of
                packages: source and build (described below).
        {
          '<package name>': {
            'type': 'source',
                # Source packages are for sources; in particular remote sources
                # where it is not known whether they have changed until they are
                # synced (it can also or for tarballs which need to be
                # unpacked). Source package commands are run unconditionally
                # unless sync is skipped via the command-line option. Source
                # package contents are not memoized.
            'dependencies':  # optional
              [<list of package depdenencies>],
            'output_dirname': # optional
              '<directory name>', # Name of the directory to checkout sources
              # into (a subdirectory of the global source directory); defaults
              # to the package name.
            'commands':
              [<list of command.Runnable objects to run>],
            'inputs': # optional
              {<mapping whose keys are names, and whose values are files or
                directories (e.g. checked-in tarballs) used as input. Since
                source targets are unconditional, this is only useful as a
                convenience for commands, which may refer to the inputs by their
                key name>},
          },
          '<package name>': {
            'type': 'build',
                # Build packages are memoized, and will build only if their
                # inputs have changed. Their inputs consist of the output of
                # their package dependencies plus any file or directory inputs
                # given by their 'inputs' member
            'dependencies':  # optional
              [<list of package depdenencies>],
            'inputs': # optional
              {<mapping whose keys are names, and whose values are files or
                directories (e.g. checked-in tarballs) used as input>},
            'output_subdir': # optional
              '<directory name>', # Name of a subdir to be created in the output
               # directory, into which all output will be placed. If not present
               # output will go into the root of the output directory.
            'commands':
              [<list of command.Command objects to run>],
          },
          '<package name>': {
            'type': 'work',
              # Work packages have the same keys as build packages. However,
              # they are intended to be intermediate targets, and are not
              # memoized or included for package_version.py. Therefore they will
              # always run, regardless of whether their inputs have changed or
              # of whether source syncing is skipped via the command line.
            <same keys as build-type packages>
          },
        }
      package_targets: A dictionary with the following format. This is a
                       description of output package targets the packages are
                       built for. Each output package should contain a list of
                       <package_name> referenced in the previous "packages"
                       dictionary. This list of targets is expected to stay
                       the same from build to build, so it should include
                       package names even if they aren't being built. A package
                       target is usually the platform, such as "$OS_$ARCH",
                       while the output package is usually the toolchain name,
                       such as "nacl_arm_newlib".
        {
          '<package_target>': {
            '<output_package>':
              [<list of package names included in output package>]
          }
        }
      args: sys.argv[1:] or equivalent.
    """
    self._packages = packages
    self._package_targets = package_targets
    self.DecodeArgs(packages, args)
    self._build_once = once.Once(
        use_cached_results=self._options.use_cached_results,
        cache_results=self._options.cache_results,
        print_url=PrintAnnotatorURL,
        storage=self.CreateStorage(),
        extra_paths=self.ExtraSubstitutionPaths())
    self._signature_file = None
    if self._options.emit_signatures is not None:
      if self._options.emit_signatures == '-':
        self._signature_file = sys.stdout
      else:
        self._signature_file = open(self._options.emit_signatures, 'w')

  def Main(self):
    """Main entry point."""
    pynacl.file_tools.MakeDirectoryIfAbsent(self._options.source)
    pynacl.file_tools.MakeDirectoryIfAbsent(self._options.output)

    pynacl.log_tools.SetupLogging(
        verbose=self._options.verbose,
        log_file=self._options.log_file,
        quiet=self._options.quiet,
        no_annotator=self._options.no_annotator)
    self.BuildAll()
    self.OutputPackagesInformation()

  def GetOutputDir(self, package, use_subdir):
    # The output dir of source packages is in the source directory, and can be
    # overridden.
    if self._packages[package]['type'] == 'source':
      dirname = self._packages[package].get('output_dirname', package)
      return os.path.join(self._options.source, dirname)
    else:
      root = os.path.join(self._options.output, package + '_install')
      if use_subdir and 'output_subdir' in self._packages[package]:
        return os.path.join(root, self._packages[package]['output_subdir'])
      return root

  def BuildPackage(self, package):
    """Build a single package.

    Assumes dependencies of the package have been built.
    Args:
      package: Package to build.
    """

    package_info = self._packages[package]

    # Validate the package description.
    if 'type' not in package_info:
      raise Exception('package %s does not have a type' % package)
    type_text = package_info['type']
    if type_text not in ('source', 'build', 'work'):
      raise Exception('package %s has unrecognized type: %s' %
                      (package, type_text))
    is_source_target = type_text == 'source'
    is_build_target = type_text == 'build'

    if 'commands' not in package_info:
      raise Exception('package %s does not have any commands' % package)

    # Source targets are the only ones to run when doing sync-only.
    if not is_source_target and self._options.sync_sources_only:
      logging.debug('Build skipped: not running commands for %s' % package)
      return

    # Source targets do not run when skipping sync.
    if is_source_target and not (
        self._options.sync_sources or self._options.sync_sources_only):
      logging.debug('Sync skipped: not running commands for %s' % package)
      return

    pynacl.log_tools.WriteAnnotatorLine(
        '@@@BUILD_STEP %s (%s)@@@' % (package, type_text))
    logging.debug('Building %s package %s' % (type_text, package))

    dependencies = package_info.get('dependencies', [])

    # Collect a dict of all the inputs.
    inputs = {}
    # Add in explicit inputs.
    if 'inputs' in package_info:
      for key, value in package_info['inputs'].iteritems():
        if key in dependencies:
          raise Exception('key "%s" found in both dependencies and inputs of '
                          'package "%s"' % (key, package))
        inputs[key] = value
    elif type_text != 'source':
      # Non-source packages default to a particular input directory.
      inputs['src'] = os.path.join(self._options.source, package)
    # Add in each dependency by package name.
    for dependency in dependencies:
      inputs[dependency] = self.GetOutputDir(dependency, True)

    # Each package generates intermediate into output/<PACKAGE>_work.
    # Clobbered here explicitly.
    work_dir = os.path.join(self._options.output, package + '_work')
    if self._options.clobber:
      logging.debug('Clobbering working directory %s' % work_dir)
      pynacl.file_tools.RemoveDirectoryIfPresent(work_dir)
    pynacl.file_tools.MakeDirectoryIfAbsent(work_dir)

    output = self.GetOutputDir(package, False)
    output_subdir = self.GetOutputDir(package, True)

    if not is_source_target or self._options.clobber_source:
      logging.debug('Clobbering output directory %s' % output)
      pynacl.file_tools.RemoveDirectoryIfPresent(output)
      os.makedirs(output_subdir)

    commands = package_info.get('commands', [])

    # Create a command option object specifying current build.
    cmd_options = command_options.CommandOptions(
        work_dir=work_dir,
        clobber_working=self._options.clobber,
        clobber_source=self._options.clobber_source,
        trybot=self._options.trybot,
        buildbot=self._options.buildbot)

    # Do it.
    self._build_once.Run(
        package, inputs, output,
        commands=commands,
        cmd_options=cmd_options,
        working_dir=work_dir,
        memoize=is_build_target,
        signature_file=self._signature_file,
        subdir=output_subdir)

    if not is_source_target and self._options.install:
      install = pynacl.platform.CygPath(self._options.install)
      logging.debug('Installing output to %s' % install)
      pynacl.file_tools.CopyTree(output, install)

  def BuildOrder(self, targets):
    """Find what needs to be built in what order to build all targets.

    Args:
      targets: A list of target packages to build.
    Returns:
      A topologically sorted list of the targets plus their transitive
      dependencies, in an order that will allow things to be built.
    """
    order = []
    order_set = set()
    if self._options.ignore_dependencies:
      return targets
    def Add(target, target_path):
      if target in order_set:
        return
      if target not in self._packages:
        raise Exception('Unknown package %s' % target)
      next_target_path = target_path + [target]
      if target in target_path:
        raise Exception('Dependency cycle: %s' % ' -> '.join(next_target_path))
      for dependency in self._packages[target].get('dependencies', []):
        Add(dependency, next_target_path)
      order.append(target)
      order_set.add(target)
    for target in targets:
      Add(target, [])
    return order

  def BuildAll(self):
    """Build all packages selected and their dependencies."""
    for target in self._targets:
      self.BuildPackage(target)

  def OutputPackagesInformation(self):
    """Outputs packages information for the built data."""
    packages_dir = os.path.join(self._options.output, 'packages')
    pynacl.file_tools.RemoveDirectoryIfPresent(packages_dir)
    os.makedirs(packages_dir)

    built_packages = []
    for target, target_dict in self._package_targets.iteritems():
      target_dir = os.path.join(packages_dir, target)
      pynacl.file_tools.MakeDirectoryIfAbsent(target_dir)
      for output_package, components in target_dict.iteritems():
        package_desc = package_info.PackageInfo()

        include_package = False
        for component in components:
          if '.' in component:
            archive_name = component
          else:
            archive_name = component + '.tgz'
          cache_item = self._build_once.GetCachedCloudItemForPackage(component)
          if cache_item is None:
            archive_desc = archive_info.ArchiveInfo(name=archive_name)
          else:
            if cache_item.dir_item:
              include_package = True
              archive_desc = archive_info.ArchiveInfo(
                  name=archive_name,
                  hash=cache_item.dir_item.hash,
                  url=cache_item.dir_item.url,
                  log_url=cache_item.log_url)

          package_desc.AppendArchive(archive_desc)

        # Only output package file if an archive was actually included.
        if include_package:
          package_file = os.path.join(target_dir, output_package + '.json')
          package_desc.SavePackageFile(package_file)

          built_packages.append(package_file)

    if self._options.packages_file:
      packages_file = pynacl.platform.CygPath(self._options.packages_file)
      pynacl.file_tools.MakeParentDirectoryIfAbsent(packages_file)
      with open(packages_file, 'wt') as f:
        f.write('\n'.join(built_packages))

  def DecodeArgs(self, packages, args):
    """Decode command line arguments to this build.

    Populated self._options and self._targets.
    Args:
      packages: A list of package names to build.
      args: sys.argv[1:] or equivalent.
    """
    package_list = sorted(packages.keys())
    parser = optparse.OptionParser(
        usage='USAGE: %prog [options] [targets...]\n\n'
              'Available targets:\n' +
              '\n'.join(textwrap.wrap(' '.join(package_list))))
    parser.add_option(
        '-v', '--verbose', dest='verbose',
        default=False, action='store_true',
        help='Produce more output.')
    parser.add_option(
        '-q', '--quiet', dest='quiet',
        default=False, action='store_true',
        help='Produce no output.')
    parser.add_option(
        '-c', '--clobber', dest='clobber',
        default=False, action='store_true',
        help='Clobber working directories before building.')
    parser.add_option(
        '--cache', dest='cache',
        default=DEFAULT_CACHE_DIR,
        help='Select directory containing local storage cache.')
    parser.add_option(
        '-s', '--source', dest='source',
        default=DEFAULT_SRC_DIR,
        help='Select directory containing source checkouts.')
    parser.add_option(
        '--git-cache', dest='git_cache',
        default=DEFAULT_GIT_CACHE_DIR,
        help='Select directory containing the git cache for syncing.')
    parser.add_option(
        '-o', '--output', dest='output',
        default=DEFAULT_OUT_DIR,
        help='Select directory containing build output.')
    parser.add_option(
        '--packages-file', dest='packages_file',
        default=None,
        help='Output packages file describing list of package files built.')
    parser.add_option(
        '--no-use-cached-results', dest='use_cached_results',
        default=True, action='store_false',
        help='Do not rely on cached results.')
    parser.add_option(
        '--no-use-remote-cache', dest='use_remote_cache',
        default=True, action='store_false',
        help='Do not rely on non-local cached results.')
    parser.add_option(
        '--no-cache-results', dest='cache_results',
        default=True, action='store_false',
        help='Do not cache results.')
    parser.add_option(
        '--no-pinned', dest='pinned',
        default=True, action='store_false',
        help='Do not use pinned revisions.')
    parser.add_option(
        '--no-annotator', dest='no_annotator',
        default=False, action='store_true',
        help='Do not print annotator headings.')
    parser.add_option(
        '--trybot', dest='trybot',
        default=False, action='store_true',
        help='Clean source dirs, run and cache as if on trybot.')
    parser.add_option(
        '--buildbot', dest='buildbot',
        default=False, action='store_true',
        help='Clean source dirs, run and cache as if on a non-trybot buildbot.')
    parser.add_option(
        '--clobber-source', dest='clobber_source',
        default=False, action='store_true',
        help='Clobber source directories before building')
    parser.add_option(
        '-y', '--sync', dest='sync_sources',
        default=False, action='store_true',
        help='Run source target commands')
    parser.add_option(
        '--sync-only', dest='sync_sources_only',
        default=False, action='store_true',
        help='Run source target commands only')
    parser.add_option(
        '--disable-git-cache', dest='disable_git_cache',
        default=False, action='store_true',
        help='Disable git cache when syncing sources')
    parser.add_option(
        '--emit-signatures', dest='emit_signatures',
        help='Write human readable build signature for each step to FILE.',
        metavar='FILE')
    parser.add_option(
        '-i', '--ignore-dependencies', dest='ignore_dependencies',
        default=False, action='store_true',
        help='Ignore target dependencies and build only the specified target.')
    parser.add_option('--install', dest='install',
                      help='After building, copy contents of build packages' +
                      ' to the specified directory')
    parser.add_option('--log-file', dest='log_file',
                      default=None, action='store',
                      help='Log all logging into a log file.')
    options, targets = parser.parse_args(args)
    if options.trybot and options.buildbot:
      print >>sys.stderr, (
          'ERROR: Tried to run with both --trybot and --buildbot.')
      sys.exit(1)
    if options.trybot or options.buildbot:
      options.verbose = True
      options.quiet = False
      options.no_annotator = False
      options.sync_sources = True
      options.clobber = True
      options.emit_signatures = '-'
    self._options = options
    if not targets:
      if self._options.ignore_dependencies:
        print >>sys.stderr, (
            'ERROR: A target must be specified if ignoring target dependencies')
        sys.exit(1)
      targets = sorted(packages.keys())
    targets = self.BuildOrder(targets)
    self._targets = targets

  def CreateStorage(self):
    """Create a storage object for this build.

    Returns:
      A storage object (GSDStorage).
    """
    if self._options.buildbot:
      return pynacl.gsd_storage.GSDStorage(
          write_bucket='nativeclient-once',
          read_buckets=['nativeclient-once'])
    elif self._options.trybot:
      return pynacl.gsd_storage.GSDStorage(
          write_bucket='nativeclient-once-try',
          read_buckets=['nativeclient-once', 'nativeclient-once-try'])
    else:
      read_buckets = []
      if self._options.use_remote_cache:
        read_buckets += ['nativeclient-once']
      return pynacl.local_storage_cache.LocalStorageCache(
          cache_path=self._options.cache,
          storage=pynacl.gsd_storage.GSDStorage(
              write_bucket=None,
              read_buckets=read_buckets))

  def ExtraSubstitutionPaths(self):
    """Returns a dictionary of extra substitution paths allowed for commands."""
    if self._options.disable_git_cache:
      git_cache_dir = ''
    else:
      git_cache_dir = self._options.git_cache

    return {
        'top_srcdir': NACL_DIR,
        'git_cache_dir': git_cache_dir,
        }