#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Fetch the source.

:author:       Michael Mulich
:copyright:    2010 by Penn State University
:organization: WebLion Group, Penn State University
:license:      GPL, see LICENSE for more detail
"""
import os
import sys
import logging
import urllib2
import tempfile
import ConfigParser
import shutil
from optparse import OptionParser
assert sys.version_info >= (2, 7), "Python >= 2.7 is required"
from subprocess import Popen, PIPE

try:
    import pkg_resources
    from setuptools.package_index import PackageIndex
    requirement_parser = pkg_resources.Requirement.parse
except ImportError:
    raise RuntimeError("This installation script and the resulting "
                       "application require Setuptools to be installed "
                       "prior to the build. Please install Setuptools or "
                       "Distribute before continuing.")

__version__ = '0.1.0'
HERE = os.path.abspath(os.path.dirname(__file__))
RUN_LOCATION = os.path.abspath(os.curdir)
PYPI = 'http://pypi.python.org/simple'
INDEXES = [PYPI] # ! list required for concatenation in Pip code !
USAGE = "usage: %prog [options] requirement [...]"

# Set up logging
logger = logging.getLogger("Fetch")
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.ERROR)

from pip.req import InstallRequirement, Link
from pip.index import PackageFinder
from pip.log import logger as pip_logger
# For debugging purposes:
if int(os.environ.get('DEBUG', '0')):
    pip_logger.consumers.append((25, sys.stdout,))
    logger.setLevel(logging.DEBUG)

# #################################### #
#   Requirements Gathering Functions   #
# #################################### #

def get_versions_config(location, record_to):
    """Locate the versions.cfg file. Parse the versions.cfg and return it.
    Optionally, we can record it to the file system and use that acquire
    the configuration at a later time."""
    logger.info("Obtaining the versions configuration at %s." % location)

    if not os.path.exists(record_to):
        # Generate a single .cfg from one location that may or may not
        # be extended by others elsewhere on the filesystem or net.
        gen_version_cfg_proc = Popen([sys.executable,
                                      os.path.join(HERE, 'gen_versions_cfg.py'),
                                      location],
                                     stdin=PIPE, stdout=PIPE)
        output, error = gen_version_cfg_proc.communicate()
 
        if gen_version_cfg_proc.returncode > 0:
            raise RuntimeError("Failed while attempting to generate the "
                               "versions.cfg file. The following error was "
                               "given: \n\n" + error)
        # Write the file out for recording purposes, but we never use it
        # outside the context of this fetching process.
        with open(record_to, 'w') as f:
            f.write(output)

    # Read in the version.cfg for pinning the requirements later in this
    # script.
    versions = ConfigParser.RawConfigParser()
    with open(record_to, 'r') as f:
        versions.readfp(f)
    if not versions.has_section('versions'):
        raise RuntimeError("We failed to obtain a useable versions.cfg. "
                           "Check the file for proper formatting. Location is "
                           "%s." % location)

    # Clean up the version values
    for name, value in versions.items('versions'):
        versions.set('versions', name, value.split('#')[0].strip())

    return dict(versions.items('versions'))

def find_source(req, src_dir, finder):
    loc = req.build_location(src_dir)
    # If we don't already have the distribution, go download it.
    if not os.path.exists(os.path.join(loc, 'setup.py')):
        if req.url is None:
            link = finder.find_requirement(req, upgrade=False)
        else:
            link = Link(req.url)
        assert link
        # We are now going to download the distribution's source using
        # the Link object created above.
        from pip.download import (is_vcs_url, is_file_url,
                                  unpack_vcs_link, unpack_file_url,
                                  unpack_http_url)
        try:
            # ---------------------------
            if is_vcs_url(link):
                unpack_vcs_link(link, loc)
            elif is_file_url(link):
                unpack_file_url(link, loc)
            else:
                unpack_http_url(link, loc, None, False)
            # ---------------------------
        except urllib2.HTTPError, e:
            raise Exception('111')
    return loc

def get_specs(pkg_requirement, versions):
    """Pin a version to the name if the name appears in the versions
    configuration file."""
    if not isinstance(pkg_requirement, pkg_resources.Requirement):
        raise TypeError("Expected a pkg_resources.Requirement instance. Got a "
                        "%s." % type(pkg_requirement))

    specs = []
    name = pkg_requirement.project_name.lower()

    if versions.get(name, False):
        if pkg_requirement.specs:
            logger.warn("%s has a version specifier: %s. We are overriding "
                        "this specifier with the one from versions.cfg."
                        % (pkg_requirement.project_name, pkg_requirement.specs))
        specs = [('==', versions[name],)]
    else:
        logger.debug("Couldn't find a known good version for %s." % name)

    return specs or pkg_requirement.specs

def get_pkg_requirement(requirement_line, versions):
    """Parse the requirement line and pin the version. Returns a pkg_resources
    Requirement instance."""
    try:
        pkg_requirement = pkg_resources.Requirement.parse(requirement_line)
    except ValueError, e:
        raise Exception("Requirement not parsable. (%s)" % requirement_line)
    pkg_requirement.specs = get_specs(pkg_requirement, versions)
    return pkg_requirement

def process_extra_requirement(pkg_requirement, comes_from_req):
    """Takes a pkg_resources Requirement instance and Pip InstallRequirement.
    Delivers the parameters necessary to process an extra requirment.
    See http://peak.telecommunity.com/DevCenter/setuptools#declaring-extras-optional-features-with-their-own-dependencies
    for more information about requirement extras."""
    if not isinstance(pkg_requirement, pkg_resources.Requirement):
        raise TypeError("Expected a pkg_resources.Requirement instance. Got a "
                        "%s." % type(pkg_requirement))
    elif not isinstance(comes_from_req, InstallRequirement):
        raise TypeError("Expected a pip.req.InstallRequirement instance. Got a "
                        "%s." % type(comes_from_req))        

    parsed_extras = [ extra.replace('_', '-')
                      for extra in pkg_requirement.extras ]
    return (pkg_requirement.project_name, parsed_extras, comes_from_req,)

def fetch(requirements, indexes, versions, src, logger):
    """The following code has been extracted from
    pip.req.RequirementSet.prepare_files. It has been greatly simplified for
    this fetch case. Additionally, I have modified it to only acquire
    distribution information and the distribution source itself."""
    # requirement_extras are Setuptools extra requirement delarations. We store
    # these are a three value tuple: 1) the requirement name 2) the extra names
    # and 3) where the extra requirements came from
    requirement_extras = []
    # satisfied requirements are those requirements that have been processed.
    # We store them in a dictionary for later use. There is no use for them
    # at this time, but there may be in the future.
    satisfied_reqs = {}

    # Setup a package finder, which will search the index for the requirement
    finder = PackageFinder([], indexes)

    while requirements or requirement_extras:
        has_extra_requirements = False
        if requirement_extras:
            req_name, extra_names, came_from_req = requirement_extras.pop(0)
            if req_name in satisfied_reqs:
                logger.info("Reprocessing %s to satisfy the '%s' extra "
                            "requirement(s)." % (req_name, extra_names))
            else:
                logger.info("Including %s's extra requirement(s): %s"
                            % (req_name, extra_names))
            pkg_requirement = get_pkg_requirement(req_name, versions)
            req = InstallRequirement(str(pkg_requirement), came_from_req)
            has_extra_requirements = True
        else:
            req = requirements.pop(0)

        if req.name in satisfied_reqs and not has_extra_requirements:
            continue

        # Download and unpack the requirement.
        logger.info("Downloading and unpacking:  %s" % req.name)

        req.source_dir = find_source(req, src, finder)
        req.run_egg_info()
        finder.add_dependency_links(req.dependency_links)

        # Roll through the regular requirements.
        for req_req in req.requirements():
            pkg_requirement = get_pkg_requirement(req_req, versions)

            if pkg_requirement.extras:
                requirement_extras.append(process_extra_requirement(pkg_requirement, req))
            else:
                new_req = InstallRequirement(str(pkg_requirement), req)
                requirements.append(new_req)

        # Only attempt to roll through the extra requirements if they are some.
        if has_extra_requirements:
            for extra in extra_names:
                section = "[%s]" % extra
                should_include_line = False
                for req_line in req.egg_info_lines('requires.txt'):
                    if req_line == section:
                        should_include_line = True
                        continue
                    elif req_line.startswith('[') and should_include_line:
                        # The next section was found
                        break
                    elif should_include_line:
                        pkg_requirement = get_pkg_requirement(req_line, versions)
                        if pkg_requirement.extras:
                            requirement_extras.append(process_extra_requirement(pkg_requirement, req))
                        else:
                            new_req = InstallRequirement(str(pkg_requirement), req)
                            requirements.append(new_req)

        satisfied_reqs[req.name] = req
    return satisfied_reqs

# ######################## #
#   Option parsing logic   #
# ######################## #

def store_abs_path(option, opt_str, value, parser):
    setattr(parser.values, option.dest, os.path.abspath(value))

def separation_parser(option, opt_str, value, parser,
                      sep_char, should_lower=False):
    new_value = [should_lower and v.lower() or v
                 for v in value.split(sep_char)
                 if v]
    setattr(parser.values, option.dest, tuple(new_value))

parser = OptionParser(usage=USAGE)
parser.add_option('-s', '--source-dir', dest='source_dir',
                  metavar='SOURCEDIR',
                  type='string', nargs=1,
                  action='callback', callback=store_abs_path,
                  default=os.path.join(RUN_LOCATION, 'source'),
                  help="Source directory")
parser.add_option('-c', '--config-dir', dest='config_dir',
                  metavar='CONFIGDIR',
                  type='string', nargs=1,
                  action='callback', callback=store_abs_path,
                  default=os.path.join(RUN_LOCATION, 'configuration'),
                  help="Configuration directory")                  
parser.add_option('--versions-cfg-url', dest='versions_cfg_url',
                  metavar='URL',
                  type='string', nargs=1,
                  help="Versions configuration file (versions.cfg) URL.")
parser.add_option('-i', '--index', dest='indexes',
                  metavar='INDEX',
                  type='string', nargs=1,
                  action='callback',
                  callback=separation_parser, callback_args=(';',),
                  default=tuple(),
                  help="List of index URLs separated by semicolons (';'). "
                       "PyPI is included by default. This index list takes "
                       "precedence over PyPI.")
setuptools_help_info = "(Setuptools variants are automatically excluded " \
                       "from the build.)"
parser.add_option('--exclude-singles', dest='single_exclusions',
                  metavar='DIST_NAME',
                  type='string', nargs=1,
                  action='callback',
                  callback=separation_parser, callback_args=(':', True),
                  default=tuple(),
                  # Unfortunately, we have to do a colon semparated list.
                  # This is due to optparse inablity to handle n+/- args and
                  # spaces are valid in distribution names.
                  help="Distributions to exclude from the build. A colon "
                       "separated string of distribution names. " + \
		       setuptools_help_info)
parser.add_option('--exclude', dest='exclusions',
                  metavar='DIST_NAME',
                  type='string', nargs=1,
                  action='callback',
                  callback=separation_parser, callback_args=(':', False),
                  default=tuple(),
                  help="Distributions and their dependents to exclude from "
                       "the build. A colon separated string of distribution "
                       "names. " + setuptools_help_info)


def main():
    options, args = parser.parse_args()
    # *. Initialize variables
    configs = options.config_dir
    src = options.source_dir
    copyright_filename = 'COPYRIGHT'
    ##versions_filename = 'versions.cfg'
    indexes = list(options.indexes) + INDEXES

    for directory in (configs, src,):
        if not os.path.exists(directory):
            os.makedirs(directory)

    # 1. Version Specification
    if options.versions_cfg_url:
        versions_filename = os.path.join(configs, 'versions.cfg')
        versions = get_versions_config(options.versions_cfg_url,
                                       record_to=versions_filename)
    else:
        versions = {}

    # 2. Obtain the Source
    # Download the distribution source and verify we have aquired all
    # the dependencies.

    # Initialize the base requires from the arguments
    parse_reqs = lambda r: [InstallRequirement.from_line(line) for line in r]
    requirements = parse_reqs(args)
    exclusion_requirements = parse_reqs(options.exclusions)

    satisfied_reqs = fetch(requirements, indexes, versions, src, logger)
    if exclusion_requirements:
        exclusion_reqs = fetch(exclusion_requirements, indexes, versions, src,
                               logger)
    else:
        exclusion_reqs = {}

    # 3. Clean Up
    single_reqs_to_remove = dict([(name,req)
                                  for name,req in satisfied_reqs.iteritems()
                                  if name.lower() in options.single_exclusions
                                  ])
    exclusion_reqs.update(single_reqs_to_remove)
    for name, req in exclusion_reqs.iteritems():
        logger.info("Excluding %s" % name)
        shutil.rmtree(req.source_dir)
        del satisfied_reqs[req.name]

    # 4. Produce copyright information
    with open(copyright_filename, 'w') as f:
        line_template = "%s by %s is licensed under %s.\n"
        pkg_info_defaults = {'author': 'Unknown', 'license': 'Unknown'}
        for req in satisfied_reqs.values():
            pkg_info = dict([(k.lower(),v) for k,v in req.pkg_info().items()])
            [pkg_info.setdefault(k,v) for k,v in pkg_info_defaults.items()]
            f.write(line_template % (pkg_info['name'], pkg_info['author'],
                                     pkg_info['license']))

if __name__ == '__main__':
    main()