#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipes for NativeClient toolchain packages.

The real entry plumbing is in toolchain_main.py.
"""

import collections
import fnmatch
import platform
import os
import re
import sys

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import pynacl.gsd_storage
import pynacl.platform

import command
import toolchain_main


SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
NACL_DIR = os.path.dirname(SCRIPT_DIR)


# See command.GenerateGitPatches for the schema of entries in this dict.
# Additionally, each may contain a 'repo' key whose value is the name
# to use in place of the package name when calling GitUrl (below).
GIT_REVISIONS = {
    'binutils': {
        'rev': 'b08b9f0894e43f0bb966f3ad9094a4405ce6f570',
        'upstream-branch': 'upstream/binutils-2_24-branch',
        'upstream-name': 'binutils-2.24',
        # This is tag binutils-2_24, but Gerrit won't let us push
        # non-annotated tags, and the upstream tag is not annotated.
        'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
        },
    'gcc': {
        'rev': 'b23dd79950a5453d3b3b5a0030d7a1894cafcffe',
        'upstream-branch': 'upstream/gcc-4_9-branch',
        'upstream-name': 'gcc-4.9.2',
         # Upstream tag gcc-4_9_2-release:
        'upstream-base': 'c1283af40b65f1ad862cf5b27e2d9ed10b2076b6',
        },
    'newlib': {
        'rev': 'bf66148d14c7fca26b9198dd5dc81e743893bb66',
        'upstream-branch': 'upstream/master',
        'upstream-name': 'newlib-2.1.0',
        # Upstream tag newlib_2_1_0:
        'upstream-base': '99fc6c167467b41466ec90e8260e9c49cbe3d13c',
        },
    'gdb': {
        'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8',
        'repo': 'binutils',
        'upstream-branch': 'upstream/gdb-7.7-branch',
        'upstream-name': 'gdb-7.7.1',
        # Upstream tag gdb-7.7-release:
        'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b',
        },
    }

TAR_FILES = {
    'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'),
    'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
    'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
    'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
    'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
    'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'),
    }

GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client'

ALT_GIT_BASE_URL = 'https://chromium.googlesource.com/a/native_client'

KNOWN_MIRRORS = [('http://git.chromium.org/native_client', GIT_BASE_URL)]
PUSH_MIRRORS = [('http://git.chromium.org/native_client', GIT_PUSH_URL),
                (ALT_GIT_BASE_URL, GIT_PUSH_URL),
                (GIT_BASE_URL, GIT_PUSH_URL)]


def GitUrl(package, push_url=False):
  repo = GIT_REVISIONS[package].get('repo', package)
  if push_url:
    base_url = GIT_PUSH_URL
  else:
    base_url = GIT_BASE_URL

  return '%s/nacl-%s.git' % (base_url, repo)


def CollectSources():
  sources = {}

  for package in TAR_FILES:
    tar_file = TAR_FILES[package]
    if fnmatch.fnmatch(tar_file, '*.bz2'):
      extract = EXTRACT_STRIP_TBZ2
    elif fnmatch.fnmatch(tar_file, '*.gz'):
      extract = EXTRACT_STRIP_TGZ
    else:
      raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
    sources[package] = {
        'type': 'source',
        'commands': [
            command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
                                                         '..', 'third_party',
                                                         tar_file)],
                            cwd='%(output)s'),
            ],
        }

  patch_packages = []
  patch_commands = []
  for package, info in GIT_REVISIONS.iteritems():
    sources[package] = {
        'type': 'source',
        'commands': command.SyncGitRepoCmds(GitUrl(package), '%(output)s',
                                            info['rev'],
                                            git_cache='%(git_cache_dir)s',
                                            push_url=GitUrl(package, True),
                                            known_mirrors=KNOWN_MIRRORS,
                                            push_mirrors=PUSH_MIRRORS),
        }
    patch_packages.append(package)
    patch_info = {'name': package}
    patch_info.update(info)
    patch_commands.append(
        command.GenerateGitPatches('%(' + package + ')s/.git', patch_info))

  sources['patches'] = {
      'type': 'build',
      'dependencies': patch_packages,
      'commands': patch_commands,
      }

  # The gcc_libs component gets the whole GCC source tree.
  sources['gcc_libs'] = sources['gcc']

  # The gcc component omits all the source directories that are used solely
  # for building target libraries.  We don't want those included in the
  # input hash calculation so that we don't rebuild the compiler when the
  # the only things that have changed are target libraries.
  sources['gcc'] = {
        'type': 'source',
        'dependencies': ['gcc_libs'],
        'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
            'boehm-gc',
            'libada',
            'libatomic',
            'libffi',
            'libgcc',
            'libgfortran',
            'libgo',
            'libgomp',
            'libitm',
            'libjava',
            'libmudflap',
            'libobjc',
            'libquadmath',
            'libsanitizer',
            'libssp',
            'libstdc++-v3',
            ])]
      }

  # We have to populate the newlib source tree with the "exported" form of
  # some headers from the native_client source tree.  The newlib build
  # needs these to be in the expected place.  By doing this in the source
  # target, these files will be part of the input hash and so we don't need
  # to do anything else to keep track of when they might have changed in
  # the native_client source tree.
  newlib_sys_nacl = command.path.join('%(output)s',
                                      'newlib', 'libc', 'sys', 'nacl')
  newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
                                                             dirname))
                   for dirname in ['bits', 'sys', 'machine']]
  newlib_unpack.append(command.Command([
      'python',
      command.path.join('%(top_srcdir)s', 'src',
                        'trusted', 'service_runtime', 'export_header.py'),
      command.path.join('%(top_srcdir)s', 'src',
                        'trusted', 'service_runtime', 'include'),
      newlib_sys_nacl,
      ]))
  sources['newlib']['commands'] += newlib_unpack

  return sources


# Canonical tuples we use for hosts.
WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-32')
MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')

# Map of native host tuple to extra tuples that it cross-builds for.
EXTRA_HOSTS_MAP = {
    LINUX_X86_64_TUPLE: [
        LINUX_X86_32_TUPLE,
        ARM_HOST_TUPLE,
        WINDOWS_HOST_TUPLE,
        ],
    }

# Map of native host tuple to host tuples that are "native enough".
# For these hosts, we will do a native-style build even though it's
# not the native tuple, just passing some extra compiler flags.
NATIVE_ENOUGH_MAP = {
    LINUX_X86_64_TUPLE: {
        LINUX_X86_32_TUPLE: ['-m32'],
        },
    }

# The list of targets to build toolchains for.
TARGET_LIST = ['arm', 'i686']

# List upload targets for each host we want to upload packages for.
TARGET = collections.namedtuple('TARGET', ['name', 'pkg_prefix'])
HOST_TARGET = collections.namedtuple('HOST_TARGET',
                                     ['os', 'arch', 'differ3264', 'targets'])

STANDARD_TARGETS = [TARGET('arm', '')]
LINUX_X86_64_TARGETS = [TARGET('arm', ''), TARGET('i686', 'ng_')]

UPLOAD_HOST_TARGETS = [
    HOST_TARGET('win', 'x86-32', False, STANDARD_TARGETS),
    HOST_TARGET('darwin', 'x86-64', False, STANDARD_TARGETS),
    HOST_TARGET('linux', 'arm', False, STANDARD_TARGETS),
    HOST_TARGET('linux', 'x86-32', False, STANDARD_TARGETS),
    HOST_TARGET('linux', 'x86-64', True, LINUX_X86_64_TARGETS),
    ]

# GDB is built by toolchain_build but injected into package targets built by
# other means. List out what package targets, packages, and the tar file we are
# injecting on top of here.
GDB_INJECT_HOSTS = [
  ('win', 'x86-32'),
  ('darwin', 'x86-64'),
  ('linux', 'x86-32'),
  ]

GDB_INJECT_PACKAGES = [
  ('nacl_x86_newlib', ['core_sdk.tgz', 'naclsdk.tgz']),
  ('nacl_x86_glibc', ['core_sdk.tar.bz2', 'toolchain.tar.bz2']),
  ('nacl_x86_newlib_raw', ['naclsdk.tgz']),
  ('nacl_x86_glibc_raw', ['toolchain.tar.bz2']),
  ]

# These are extra arguments to pass gcc's configure that vary by target.
TARGET_GCC_CONFIG = {
    'arm': ['--with-tune=cortex-a15'],
    }

PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
BUG_URL = 'http://gonacl.com/reportissue'

TAR_XV = ['tar', '-x', '-v']
EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
CONFIGURE_CMD = ['sh', '%(src)s/configure']
MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']

# This file gets installed by multiple packages' install steps, but it is
# never useful when installed in isolation.  So we remove it from the
# installation directories before packaging up.
REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
                                                   'share', 'info', 'dir'))

def ConfigureHostArch(host):
  configure_args = []

  is_cross = CrossCompiling(host)

  if is_cross:
    extra_cc_args = []
    configure_args.append('--host=' + host)
  else:
    extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
    if extra_cc_args:
      # The host we've chosen is "native enough", such as x86-32 on x86-64.
      # But it's not what config.guess will yield, so we need to supply
      # a --build switch to ensure things build correctly.
      configure_args.append('--build=' + host)

  extra_cxx_args = list(extra_cc_args)
  if fnmatch.fnmatch(host, '*-linux*'):
    # Avoid shipping binaries with a runtime dependency on
    # a particular version of the libstdc++ shared library.
    # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
    extra_cxx_args.append('-static-libstdc++')

  if extra_cc_args:
    # These are the defaults when there is no setting, but we will add
    # additional switches, so we must supply the command name too.
    if is_cross:
      cc = host + '-gcc'
    else:
      cc = 'gcc'
    configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))

  if extra_cxx_args:
    # These are the defaults when there is no setting, but we will add
    # additional switches, so we must supply the command name too.
    if is_cross:
      cxx = host + '-g++'
    else:
      cxx = 'g++'
    configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))

  if HostIsWindows(host):
    # The i18n support brings in runtime dependencies on MinGW DLLs
    # that we don't want to have to distribute alongside our binaries.
    # So just disable it, and compiler messages will always be in US English.
    configure_args.append('--disable-nls')

  return configure_args


def ConfigureHostCommon(host):
  return ConfigureHostArch(host) + [
      '--prefix=',
      '--disable-silent-rules',
      '--without-gcc-arch',
      ]


def ConfigureHostLib(host):
  return ConfigureHostCommon(host) + [
      '--disable-shared',
      ]


def ConfigureHostTool(host):
  return ConfigureHostCommon(host) + [
      '--with-pkgversion=' + PACKAGE_NAME,
      '--with-bugurl=' + BUG_URL,
      '--without-zlib',
      ]


def MakeCommand(host, extra_args=[]):
  if HostIsWindows(host):
    # There appears to be nothing we can pass at top-level configure time
    # that will prevent the configure scripts from finding MinGW's libiconv
    # and using it.  We have to force this variable into the environment
    # of the sub-configure runs, which are run via make.
    make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
  else:
    make_command = MAKE_PARALLEL_CMD
  return make_command + extra_args


# Return the 'make check' command to run.
# When cross-compiling, don't try to run test suites.
def MakeCheckCommand(host):
  if CrossCompiling(host):
    return ['true']
  return MAKE_CHECK_CMD


def InstallDocFiles(subdir, files):
  doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
  dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
                     for file in files]))
  commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
              [command.Copy(command.path.join('%(' + subdir + ')s', file),
                            command.path.join(doc_dir, file))
               for file in files])
  return commands


def NewlibLibcScript(arch, elfclass_x86_64='elf32'):
  template = """/*
 * This is a linker script that gets installed as libc.a for the
 * newlib-based NaCl toolchain.  It brings in the constituent
 * libraries that make up what -lc means semantically.
 */
OUTPUT_FORMAT(%s)
GROUP ( libnacl.a libcrt_common.a )
"""
  if arch == 'arm':
    # Listing three formats instead of one makes -EL/-EB switches work
    # for the endian-switchable ARM backend.
    format_list = ['elf32-littlearm-nacl',
                   'elf32-bigarm-nacl',
                   'elf32-littlearm-nacl']
  elif arch == 'i686':
    format_list = ['elf32-i386-nacl']
  elif arch == 'x86_64':
    format_list = ['%s-x86-64-nacl' % elfclass_x86_64]
  else:
    raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
  return template % ', '.join(['"' + fmt + '"' for fmt in format_list])


# The default strip behavior removes debugging and symbol table
# sections, but it leaves the .comment section.  This contains the
# compiler version string, and so it changes when the compiler changes
# even if the actual machine code it produces is completely identical.
# Hence, the target library packages will always change when the
# compiler changes unless these sections are removed.  Doing this
# requires somehow teaching the makefile rules to pass the
# --remove-section=.comment switch to TARGET-strip.  For the GCC
# target libraries, setting STRIP_FOR_TARGET is sufficient.  But
# quoting nightmares make it difficult to pass a command with a space
# in it as the STRIP_FOR_TARGET value.  So the build writes a little
# script that can be invoked with a simple name.
#
# Though the gcc target libraries' makefiles are smart enough to obey
# STRIP_FOR_TARGET for library files, the newlib makefiles just
# blindly use $(INSTALL_DATA) for both header (text) files and library
# files.  Hence it's necessary to override its INSTALL_DATA setting to
# one that will do stripping using this script, and thus the script
# must silently do nothing to non-binary files.
def ConfigureTargetPrep(arch):
  script_file = 'strip_for_target'

  config_target = arch + '-nacl'
  script_contents = """\
#!/bin/sh
mode=--strip-all
for arg; do
  case "$arg" in
  -*) ;;
  *)
    type=`file --brief --mime-type "$arg"`
    case "$type" in
      application/x-executable|application/x-sharedlib) ;;
      application/x-archive|application/x-object) mode=--strip-debug ;;
      *) exit 0 ;;
    esac
    ;;
  esac
done
exec %s-strip $mode --remove-section=.comment "$@"
""" % config_target

  return [
      command.WriteData(script_contents, script_file),
      command.Command(['chmod', '+x', script_file]),
      ]


def ConfigureTargetArgs(arch):
  config_target = arch + '-nacl'
  return [
      '--target=' + config_target,
      '--with-sysroot=/' + config_target,
      'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
      ]


def CommandsInBuild(command_lines):
  return [
      command.RemoveDirectory('build'),
      command.Mkdir('build'),
      ] + [command.Command(cmd, cwd='build')
           for cmd in command_lines]


def PopulateDeps(dep_dirs):
  commands = [command.RemoveDirectory('all_deps'),
              command.Mkdir('all_deps')]
  commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
               for dirname in dep_dirs]
  return commands


def WithDepsOptions(options, component=None):
  if component is None:
    directory = command.path.join('%(cwd)s', 'all_deps')
  else:
    directory = '%(abs_' + component + ')s'
  return ['--with-' + option + '=' + directory
          for option in options]


# Return the component name we'll use for a base component name and
# a host tuple.  The component names cannot contain dashes or other
# non-identifier characters, because the names of the files uploaded
# to Google Storage are constrained.  GNU configuration tuples contain
# dashes, which we translate to underscores.
def ForHost(component_name, host):
  return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)


# These are libraries that go into building the compiler itself.
def HostGccLibs(host):
  def H(component_name):
    return ForHost(component_name, host)
  host_gcc_libs = {
      H('gmp'): {
          'type': 'build',
          'dependencies': ['gmp'],
          'commands': [
              command.Command(ConfigureCommand('gmp') +
                              ConfigureHostLib(host) + [
                                  '--with-sysroot=%(abs_output)s',
                                  '--enable-cxx',
                                  # Without this, the built library will
                                  # assume the instruction set details
                                  # available on the build machine.  With
                                  # this, it dynamically chooses what code
                                  # to use based on the details of the
                                  # actual host CPU at runtime.
                                  '--enable-fat',
                                  ]),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              ],
          },
      H('mpfr'): {
          'type': 'build',
          'dependencies': ['mpfr', H('gmp')],
          'commands': [
              command.Command(ConfigureCommand('mpfr') +
                              ConfigureHostLib(host) +
                              WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              ],
          },
      H('mpc'): {
          'type': 'build',
          'dependencies': ['mpc', H('gmp'), H('mpfr')],
          'commands': PopulateDeps(['%(' + H('gmp') + ')s',
                                    '%(' + H('mpfr') + ')s']) + [
              command.Command(ConfigureCommand('mpc') +
                              ConfigureHostLib(host) +
                              WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              ],
          },
      H('isl'): {
          'type': 'build',
          'dependencies': ['isl', H('gmp')],
          'commands': [
              command.Command(ConfigureCommand('isl') +
                              ConfigureHostLib(host) +
                              WithDepsOptions(['sysroot', 'gmp-prefix'],
                                              H('gmp'))),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              # The .pc files wind up containing some absolute paths
              # that make the output depend on the build directory name.
              # The dependents' configure scripts don't need them anyway.
              command.RemoveDirectory(command.path.join(
                  '%(output)s', 'lib', 'pkgconfig')),
              ],
          },
      H('cloog'): {
          'type': 'build',
          'dependencies': ['cloog', H('gmp'), H('isl')],
          'commands': PopulateDeps(['%(' + H('gmp') + ')s',
                                    '%(' + H('isl') + ')s']) + [
              command.Command(ConfigureCommand('cloog') +
                              ConfigureHostLib(host) + [
                                  '--with-bits=gmp',
                                  '--with-isl=system',
                                  ] + WithDepsOptions(['sysroot',
                                                       'gmp-prefix',
                                                       'isl-prefix'])),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              # The .pc files wind up containing some absolute paths
              # that make the output depend on the build directory name.
              # The dependents' configure scripts don't need them anyway.
              command.RemoveDirectory(command.path.join(
                  '%(output)s', 'lib', 'pkgconfig')),
              ],
          },
      H('expat'): {
          'type': 'build',
          'dependencies': ['expat'],
          'commands': [
              command.Command(ConfigureCommand('expat') +
                              ConfigureHostLib(host)),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + [
                  # expat does not support the install-strip target.
                  'installlib',
                  'INSTALL=%(expat)s/conftools/install-sh -c -s',
                  'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
                  ]),
              ],
          },
      }
  return host_gcc_libs


HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']

def HostGccLibsDeps(host):
  return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]


def SDKLibs(host, target):
  def H(component_name):
    return ForHost(component_name, host)
  components = ['newlib_%s' % target,
                'gcc_libs_%s' % target,
                H('binutils_%s' % target),
                H('gcc_%s' % target),
                ]
  sdk_compiler = {
    H('sdk_compiler_%s' % target): {
        'type': 'work',
        'dependencies': components,
        'commands': [command.CopyRecursive('%(' + item + ')s', '%(output)s')
                     for item in components],
    },
  }
  sdk_libs = {
    'sdk_libs_%s' % target: {
        'type': 'build',
        'dependencies': [H('sdk_compiler_%s' % target)],
        'inputs': {
            'src_untrusted': os.path.join(NACL_DIR, 'src', 'untrusted'),
            'src_include': os.path.join(NACL_DIR, 'src', 'include'),
            'scons.py': os.path.join(NACL_DIR, 'scons.py'),
            'site_scons': os.path.join(NACL_DIR, 'site_scons'),
            'prep_nacl_sdk':
                os.path.join(NACL_DIR, 'build', 'prep_nacl_sdk.py'),
        },
        'commands': [
          command.Command(
              [sys.executable, '%(scons.py)s',
               '--verbose', 'MODE=nacl', '-j%(cores)s', 'naclsdk_validate=0',
               'platform=%s' % target,
               'nacl_newlib_dir=%(abs_' + H('sdk_compiler_%s' % target) + ')s',
               'DESTINATION_ROOT=%(work_dir)s',
               'includedir=' + command.path.join('%(output)s',
                                                 target + '-nacl', 'include'),
               'libdir=' + command.path.join('%(output)s',
                                             target + '-nacl', 'lib'),
               'install'],
              cwd=NACL_DIR),
        ],
    },
  }

  return dict(sdk_compiler.items() + sdk_libs.items())


def ConfigureCommand(source_component):
  return [command % {'src': '%(' + source_component + ')s'}
          for command in CONFIGURE_CMD]


# When doing a Canadian cross, we need native-hosted cross components
# to do the GCC build.
def GccDeps(host, target):
  components = ['binutils_' + target]
  if CrossCompiling(host):
    components.append('gcc_' + target)
    host = NATIVE_TUPLE
  return [ForHost(component, host) for component in components]


def GccCommand(host, target, cmd):
  components_for_path = GccDeps(host, target)
  return command.Command(
      cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
                      for component in components_for_path])


def ConfigureGccCommand(source_component, host, target, extra_args=[]):
  return GccCommand(
      host,
      target,
      ConfigureCommand(source_component) +
      ConfigureHostTool(host) +
      ConfigureTargetArgs(target) +
      TARGET_GCC_CONFIG.get(target, []) + [
          '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
          '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
          '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
          '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
          '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
          '--enable-cloog-backend=isl',
          '--disable-dlopen',
          '--disable-shared',
          '--with-newlib',
          '--with-linker-hash-style=gnu',
          '--enable-linker-build-id',
          '--enable-languages=c,c++,lto',
          ] + extra_args)



def HostTools(host, target):
  def H(component_name):
    return ForHost(component_name, host)

  def WindowsAlternate(if_windows, if_not_windows, if_mac=None):
    if if_mac is not None and HostIsMac(host):
      return if_mac
    elif HostIsWindows(host):
      return if_windows
    else:
      return if_not_windows

  # Return the file name with the appropriate suffix for an executable file.
  def Exe(file):
    return file + WindowsAlternate('.exe', '')

  tools = {
      H('binutils_' + target): {
          'type': 'build',
          'dependencies': ['binutils'],
          'commands': ConfigureTargetPrep(target) + [
              command.Command(
                  ConfigureCommand('binutils') +
                  ConfigureHostTool(host) +
                  ConfigureTargetArgs(target) + [
                      '--enable-deterministic-archives',
                      '--enable-gold',
                      ] + WindowsAlternate([], ['--enable-plugins'])),
              command.Command(MakeCommand(host)),
              command.Command(MakeCheckCommand(host)),
              command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
              REMOVE_INFO_DIR,
              ] + InstallDocFiles('binutils',
                                  ['COPYING3'] +
                                  [command.path.join(subdir, 'NEWS')
                                   for subdir in
                                   ['binutils', 'gas', 'ld', 'gold']]) +
              # The top-level lib* directories contain host libraries
              # that we don't want to include in the distribution.
              [command.RemoveDirectory(command.path.join('%(output)s', name))
               for name in ['lib', 'lib32', 'lib64']],
          },

      H('gcc_' + target): {
          'type': 'build',
          'dependencies': (['gcc'] + HostGccLibsDeps(host) +
                           GccDeps(host, target)),
          'commands': ConfigureTargetPrep(target) + [
              ConfigureGccCommand('gcc', host, target),
              # GCC's configure step writes configargs.h with some strings
              # including the configure command line, which get embedded
              # into the gcc driver binary.  The build only works if we use
              # absolute paths in some of the configure switches, but
              # embedding those paths makes the output differ in repeated
              # builds done in different directories, which we do not want.
              # So force the generation of that file early and then edit it
              # in place to replace the absolute paths with something that
              # never varies.  Note that the 'configure-gcc' target will
              # actually build some components before running gcc/configure.
              GccCommand(host, target,
                         MakeCommand(host, ['configure-gcc'])),
              command.Command(['sed', '-i', '-e',
                               ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
                                         (component, component)
                                         for component in
                                         HostGccLibsDeps(host)] +
                                        ['s@%(cwd)s@...@g']),
                               command.path.join('gcc', 'configargs.h')]),
              # gcc/Makefile's install rules ordinarily look at the
              # installed include directory for a limits.h to decide
              # whether the lib/gcc/.../include-fixed/limits.h header
              # should be made to expect a libc-supplied limits.h or not.
              # Since we're doing this build in a clean environment without
              # any libc installed, we need to force its hand here.
              GccCommand(host, target,
                         MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
              # gcc/Makefile's install targets populate this directory
              # only if it already exists.
              command.Mkdir(command.path.join('%(output)s',
                                              target + '-nacl', 'bin'),
                            True),
              GccCommand(host, target,
                         MAKE_DESTDIR_CMD + ['install-strip-gcc']),
              REMOVE_INFO_DIR,
              # Note we include COPYING.RUNTIME here and not with gcc_libs.
              ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
          },

      # GDB can support all the targets in one host tool.
      H('gdb'): {
          'type': 'build',
          'dependencies': ['gdb', H('expat')],
          'commands': [
              command.Command(
                  ConfigureCommand('gdb') +
                  ConfigureHostTool(host) + [
                      '--target=x86_64-nacl',
                      '--enable-targets=arm-none-eabi-nacl',
                      '--with-expat',
                      # Windows (MinGW) is missing ncurses; we need to
                      # build one here and link it in statically for
                      # --enable-tui.  See issue nativeclient:3911.
                      '--%s-tui' % WindowsAlternate('disable', 'enable'),
                      'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
                      'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
                      ] +
                  # TODO(mcgrathr): Should use --with-python to ensure
                  # we have it on Linux/Mac.
                  WindowsAlternate(['--without-python'], []) +
                  # TODO(mcgrathr): The default -Werror only breaks because
                  # the OSX default compiler is an old front-end that does
                  # not understand all the GCC options.  Maybe switch to
                  # using clang (system or Chromium-supplied) on Mac.
                  (['--disable-werror'] if HostIsMac(host) else [])),
              command.Command(MakeCommand(host) + ['all-gdb']),
              command.Command(MAKE_DESTDIR_CMD + [
                  '-C', 'gdb', 'install-strip',
                  ]),
              REMOVE_INFO_DIR,
              ] + [command.Command(['ln', '-f',
                                    command.path.join('%(abs_output)s',
                                                      'bin',
                                                      Exe('x86_64-nacl-gdb')),
                                    command.path.join('%(abs_output)s',
                                                      'bin',
                                                      Exe(arch + '-nacl-gdb'))])
                   for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [
                       'COPYING3',
                       command.path.join('gdb', 'NEWS'),
                       ]),
          },
      }

  # TODO(mcgrathr): The ARM cross environment does not supply a termcap
  # library, so it cannot build GDB.
  if host.startswith('arm') and CrossCompiling(host):
    del tools[H('gdb')]

  return tools

def TargetCommands(host, target, command_list):
  # First we have to copy the host tools into a common directory.
  # We can't just have both directories in our PATH, because the
  # compiler looks for the assembler and linker relative to itself.
  commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
                           '%(' + ForHost('gcc_' + target, host) + ')s'])
  bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
  commands += [command.Command(cmd, path_dirs=[bindir])
               for cmd in command_list]
  return commands


def TargetLibs(host, target):
  lib_deps = [ForHost(component + '_' + target, host)
              for component in ['binutils', 'gcc']]

  def NewlibFile(subdir, name):
    return command.path.join('%(output)s', target + '-nacl', subdir, name)

  newlib_sysroot = '%(abs_newlib_' + target + ')s'
  newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)

  # See the comment at ConfigureTargetPrep, above.
  newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
                                  '%(abs_newlib)s/install-sh',
                                  '-c', '-s', '-m', '644'])

  iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
  newlib_configure_args = [
      '--disable-libgloss',
      '--enable-newlib-iconv',
      '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
      '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
      '--enable-newlib-io-long-long',
      '--enable-newlib-io-long-double',
      '--enable-newlib-io-c99-formats',
      '--enable-newlib-mb',
      'CFLAGS=-O2',
      'INSTALL_DATA=' + newlib_install_data,
      ]

  newlib_post_install = [
      command.Rename(NewlibFile('lib', 'libc.a'),
                     NewlibFile('lib', 'libcrt_common.a')),
      command.WriteData(NewlibLibcScript(target),
                        NewlibFile('lib', 'libc.a')),
      ] + [
      command.Copy(
          command.path.join('%(pthread_headers)s', header),
          NewlibFile('include', header))
      for header in ('pthread.h', 'semaphore.h')
      ]


  libs = {
      'newlib_' + target: {
          'type': 'build',
          'dependencies': ['newlib'] + lib_deps,
          'inputs': { 'pthread_headers':
                      os.path.join(NACL_DIR, 'src', 'untrusted',
                                   'pthread') },
          'commands': (ConfigureTargetPrep(target) +
                       TargetCommands(host, target, [
                           ConfigureCommand('newlib') +
                           ConfigureHostTool(host) +
                           ConfigureTargetArgs(target) +
                           newlib_configure_args,
                           MakeCommand(host),
                           MAKE_DESTDIR_CMD + ['install-strip'],
                           ]) +
                       newlib_post_install +
                       InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
          },

      'gcc_libs_' + target: {
          'type': 'build',
          'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
                           HostGccLibsDeps(host)),
          # This actually builds the compiler again and uses that compiler
          # to build the target libraries.  That's by far the easiest thing
          # to get going given the interdependencies of the target
          # libraries (especially libgcc) on the gcc subdirectory, and
          # building the compiler doesn't really take all that long in the
          # grand scheme of things.
          # TODO(mcgrathr): If upstream ever cleans up all their
          # interdependencies better, unpack the compiler, configure with
          # --disable-gcc, and just build all-target.
          'commands': ConfigureTargetPrep(target) + [
              ConfigureGccCommand('gcc_libs', host, target, [
                  '--with-build-sysroot=' + newlib_sysroot,
                  ]),
              GccCommand(host, target,
                         MakeCommand(host) + [
                             'build_tooldir=' + newlib_tooldir,
                             'all-target',
                             ]),
              GccCommand(host, target,
                         MAKE_DESTDIR_CMD + ['install-strip-target']),
              REMOVE_INFO_DIR,
              ],
          },
      }
  return libs

# Compute it once.
NATIVE_TUPLE = pynacl.platform.PlatformTriple()


# For our purposes, "cross-compiling" means not literally that we are
# targetting a host that does not match NATIVE_TUPLE, but that we are
# targetting a host whose binaries we cannot run locally.  So x86-32
# on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
def CrossCompiling(host):
  return (host != NATIVE_TUPLE and
          host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))


def HostIsWindows(host):
  return host == WINDOWS_HOST_TUPLE


def HostIsMac(host):
  return host == MAC_HOST_TUPLE


# We build target libraries only on Linux for two reasons:
# 1. We only need to build them once.
# 2. Linux is the fastest to build.
# TODO(mcgrathr): In future set up some scheme whereby non-Linux
# bots can build target libraries but not archive them, only verifying
# that the results came out the same as the ones archived by the
# official builder bot.  That will serve as a test of the host tools
# on the other host platforms.
def BuildTargetLibsOn(host):
  return host == LINUX_X86_64_TUPLE


def GetPackageTargets():
  """Package Targets describes all the final package targets.

  This build can be built among many build bots, but eventually all things
  will be combined together. This package target dictionary describes the final
  output of the entire build.
  """
  package_targets = {}

  # Add in standard upload targets.
  for host_target in UPLOAD_HOST_TARGETS:
    for target in host_target.targets:
      target_arch = target.name
      package_prefix = target.pkg_prefix

      # Each package target contains non-platform specific newlib and gcc libs.
      # These packages are added inside of TargetLibs(host, target).
      newlib_package = 'newlib_%s' % target_arch
      gcc_lib_package = 'gcc_libs_%s' % target_arch
      sdk_lib_packages = ['sdk_libs_%s' % target_arch]
      shared_packages = [newlib_package, gcc_lib_package]

      # Each package target contains arm binutils and gcc.
      # These packages are added inside of HostTools(host, target).
      platform_triple = pynacl.platform.PlatformTriple(host_target.os,
                                                       host_target.arch)
      binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
      gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
      gdb_package = ForHost('gdb', platform_triple)

      # Create a list of packages for a target.
      platform_packages = [binutils_package, gcc_package, gdb_package]
      raw_packages = shared_packages + platform_packages
      all_packages = raw_packages + sdk_lib_packages

      os_name = pynacl.platform.GetOS(host_target.os)
      if host_target.differ3264:
        arch_name = pynacl.platform.GetArch3264(host_target.arch)
      else:
        arch_name = pynacl.platform.GetArch(host_target.arch)
      package_target = '%s_%s' % (os_name, arch_name)
      package_name = '%snacl_%s_newlib' % (package_prefix,
                                           pynacl.platform.GetArch(target_arch))
      raw_package_name = package_name + '_raw'

      # Toolchains by default are "raw" unless they include the Core SDK
      package_target_dict = package_targets.setdefault(package_target, {})
      package_target_dict.setdefault(raw_package_name, []).extend(raw_packages)
      package_target_dict.setdefault(package_name, []).extend(all_packages)

  # GDB is a special and shared, we will inject it into various other packages.
  for platform, arch in GDB_INJECT_HOSTS:
    platform_triple = pynacl.platform.PlatformTriple(platform, arch)
    os_name = pynacl.platform.GetOS(platform)
    arch_name = pynacl.platform.GetArch(arch)

    gdb_packages = [ForHost('gdb', platform_triple)]
    package_target = '%s_%s' % (os_name, arch_name)

    for package_name, package_archives in GDB_INJECT_PACKAGES:
      combined_packages = package_archives + gdb_packages
      package_target_dict = package_targets.setdefault(package_target, {})
      package_target_dict.setdefault(package_name, []).extend(combined_packages)

  return dict(package_targets)


def CollectPackagesForHost(host, targets):
  packages = HostGccLibs(host).copy()
  for target in targets:
    packages.update(HostTools(host, target))
    if BuildTargetLibsOn(host):
      packages.update(TargetLibs(host, target))
      packages.update(SDKLibs(host, target))
  return packages


def CollectPackages(targets):
  packages = CollectSources()

  packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))

  for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
    packages.update(CollectPackagesForHost(host, targets))

  return packages


PACKAGES = CollectPackages(TARGET_LIST)
PACKAGE_TARGETS = GetPackageTargets()


if __name__ == '__main__':
  tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
  # TODO(mcgrathr): The bot ought to run some native_client tests
  # using the new toolchain, like the old x86 toolchain bots do.
  tb.Main()