#!/usr/bin/env python # -*- coding: utf-8 -*- """ Fetch the source. :author: Michael Mulich :copyright: 2010 by Penn State University :organization: WebLion Group, Penn State University :license: GPL, see LICENSE for more detail """ import os import sys import logging import urllib2 import tempfile import ConfigParser import shutil from optparse import OptionParser assert sys.version_info >= (2, 7), "Python >= 2.7 is required" from subprocess import Popen, PIPE try: import pkg_resources from setuptools.package_index import PackageIndex requirement_parser = pkg_resources.Requirement.parse except ImportError: raise RuntimeError("This installation script and the resulting " "application require Setuptools to be installed " "prior to the build. Please install Setuptools or " "Distribute before continuing.") __version__ = '0.1.0' HERE = os.path.abspath(os.path.dirname(__file__)) RUN_LOCATION = os.path.abspath(os.curdir) PYPI = 'http://pypi.python.org/simple' INDEXES = [PYPI] # ! list required for concatenation in Pip code ! USAGE = "usage: %prog [options] requirement [...]" # Set up logging logger = logging.getLogger("Fetch") logger.addHandler(logging.StreamHandler(sys.stdout)) logger.setLevel(logging.ERROR) from pip.req import InstallRequirement, Link from pip.index import PackageFinder from pip.log import logger as pip_logger # For debugging purposes: if int(os.environ.get('DEBUG', '0')): pip_logger.consumers.append((25, sys.stdout,)) logger.setLevel(logging.DEBUG) # #################################### # # Requirements Gathering Functions # # #################################### # def get_versions_config(location, record_to): """Locate the versions.cfg file. Parse the versions.cfg and return it. Optionally, we can record it to the file system and use that acquire the configuration at a later time.""" logger.info("Obtaining the versions configuration at %s." % location) if not os.path.exists(record_to): # Generate a single .cfg from one location that may or may not # be extended by others elsewhere on the filesystem or net. gen_version_cfg_proc = Popen([sys.executable, os.path.join(HERE, 'gen_versions_cfg.py'), location], stdin=PIPE, stdout=PIPE) output, error = gen_version_cfg_proc.communicate() if gen_version_cfg_proc.returncode > 0: raise RuntimeError("Failed while attempting to generate the " "versions.cfg file. The following error was " "given: \n\n" + error) # Write the file out for recording purposes, but we never use it # outside the context of this fetching process. with open(record_to, 'w') as f: f.write(output) # Read in the version.cfg for pinning the requirements later in this # script. versions = ConfigParser.RawConfigParser() with open(record_to, 'r') as f: versions.readfp(f) if not versions.has_section('versions'): raise RuntimeError("We failed to obtain a useable versions.cfg. " "Check the file for proper formatting. Location is " "%s." % location) # Clean up the version values for name, value in versions.items('versions'): versions.set('versions', name, value.split('#')[0].strip()) return dict(versions.items('versions')) def find_source(req, src_dir, finder): loc = req.build_location(src_dir) # If we don't already have the distribution, go download it. if not os.path.exists(os.path.join(loc, 'setup.py')): if req.url is None: link = finder.find_requirement(req, upgrade=False) else: link = Link(req.url) assert link # We are now going to download the distribution's source using # the Link object created above. from pip.download import (is_vcs_url, is_file_url, unpack_vcs_link, unpack_file_url, unpack_http_url) try: # --------------------------- if is_vcs_url(link): unpack_vcs_link(link, loc) elif is_file_url(link): unpack_file_url(link, loc) else: unpack_http_url(link, loc, None, False) # --------------------------- except urllib2.HTTPError, e: raise Exception('111') return loc def get_specs(pkg_requirement, versions): """Pin a version to the name if the name appears in the versions configuration file.""" if not isinstance(pkg_requirement, pkg_resources.Requirement): raise TypeError("Expected a pkg_resources.Requirement instance. Got a " "%s." % type(pkg_requirement)) specs = [] name = pkg_requirement.project_name.lower() if versions.get(name, False): if pkg_requirement.specs: logger.warn("%s has a version specifier: %s. We are overriding " "this specifier with the one from versions.cfg." % (pkg_requirement.project_name, pkg_requirement.specs)) specs = [('==', versions[name],)] else: logger.debug("Couldn't find a known good version for %s." % name) return specs or pkg_requirement.specs def get_pkg_requirement(requirement_line, versions): """Parse the requirement line and pin the version. Returns a pkg_resources Requirement instance.""" try: pkg_requirement = pkg_resources.Requirement.parse(requirement_line) except ValueError, e: raise Exception("Requirement not parsable. (%s)" % requirement_line) pkg_requirement.specs = get_specs(pkg_requirement, versions) return pkg_requirement def process_extra_requirement(pkg_requirement, comes_from_req): """Takes a pkg_resources Requirement instance and Pip InstallRequirement. Delivers the parameters necessary to process an extra requirment. See http://peak.telecommunity.com/DevCenter/setuptools#declaring-extras-optional-features-with-their-own-dependencies for more information about requirement extras.""" if not isinstance(pkg_requirement, pkg_resources.Requirement): raise TypeError("Expected a pkg_resources.Requirement instance. Got a " "%s." % type(pkg_requirement)) elif not isinstance(comes_from_req, InstallRequirement): raise TypeError("Expected a pip.req.InstallRequirement instance. Got a " "%s." % type(comes_from_req)) parsed_extras = [ extra.replace('_', '-') for extra in pkg_requirement.extras ] return (pkg_requirement.project_name, parsed_extras, comes_from_req,) def fetch(requirements, indexes, versions, src, logger): """The following code has been extracted from pip.req.RequirementSet.prepare_files. It has been greatly simplified for this fetch case. Additionally, I have modified it to only acquire distribution information and the distribution source itself.""" # requirement_extras are Setuptools extra requirement delarations. We store # these are a three value tuple: 1) the requirement name 2) the extra names # and 3) where the extra requirements came from requirement_extras = [] # satisfied requirements are those requirements that have been processed. # We store them in a dictionary for later use. There is no use for them # at this time, but there may be in the future. satisfied_reqs = {} # Setup a package finder, which will search the index for the requirement finder = PackageFinder([], indexes) while requirements or requirement_extras: has_extra_requirements = False if requirement_extras: req_name, extra_names, came_from_req = requirement_extras.pop(0) if req_name in satisfied_reqs: logger.info("Reprocessing %s to satisfy the '%s' extra " "requirement(s)." % (req_name, extra_names)) else: logger.info("Including %s's extra requirement(s): %s" % (req_name, extra_names)) pkg_requirement = get_pkg_requirement(req_name, versions) req = InstallRequirement(str(pkg_requirement), came_from_req) has_extra_requirements = True else: req = requirements.pop(0) if req.name in satisfied_reqs and not has_extra_requirements: continue # Download and unpack the requirement. logger.info("Downloading and unpacking: %s" % req.name) req.source_dir = find_source(req, src, finder) req.run_egg_info() finder.add_dependency_links(req.dependency_links) # Roll through the regular requirements. for req_req in req.requirements(): pkg_requirement = get_pkg_requirement(req_req, versions) if pkg_requirement.extras: requirement_extras.append(process_extra_requirement(pkg_requirement, req)) else: new_req = InstallRequirement(str(pkg_requirement), req) requirements.append(new_req) # Only attempt to roll through the extra requirements if they are some. if has_extra_requirements: for extra in extra_names: section = "[%s]" % extra should_include_line = False for req_line in req.egg_info_lines('requires.txt'): if req_line == section: should_include_line = True continue elif req_line.startswith('[') and should_include_line: # The next section was found break elif should_include_line: pkg_requirement = get_pkg_requirement(req_line, versions) if pkg_requirement.extras: requirement_extras.append(process_extra_requirement(pkg_requirement, req)) else: new_req = InstallRequirement(str(pkg_requirement), req) requirements.append(new_req) satisfied_reqs[req.name] = req return satisfied_reqs # ######################## # # Option parsing logic # # ######################## # def store_abs_path(option, opt_str, value, parser): setattr(parser.values, option.dest, os.path.abspath(value)) def separation_parser(option, opt_str, value, parser, sep_char, should_lower=False): new_value = [should_lower and v.lower() or v for v in value.split(sep_char) if v] setattr(parser.values, option.dest, tuple(new_value)) parser = OptionParser(usage=USAGE) parser.add_option('-s', '--source-dir', dest='source_dir', metavar='SOURCEDIR', type='string', nargs=1, action='callback', callback=store_abs_path, default=os.path.join(RUN_LOCATION, 'source'), help="Source directory") parser.add_option('-c', '--config-dir', dest='config_dir', metavar='CONFIGDIR', type='string', nargs=1, action='callback', callback=store_abs_path, default=os.path.join(RUN_LOCATION, 'configuration'), help="Configuration directory") parser.add_option('--versions-cfg-url', dest='versions_cfg_url', metavar='URL', type='string', nargs=1, help="Versions configuration file (versions.cfg) URL.") parser.add_option('-i', '--index', dest='indexes', metavar='INDEX', type='string', nargs=1, action='callback', callback=separation_parser, callback_args=(';',), default=tuple(), help="List of index URLs separated by semicolons (';'). " "PyPI is included by default. This index list takes " "precedence over PyPI.") setuptools_help_info = "(Setuptools variants are automatically excluded " \ "from the build.)" parser.add_option('--exclude-singles', dest='single_exclusions', metavar='DIST_NAME', type='string', nargs=1, action='callback', callback=separation_parser, callback_args=(':', True), default=tuple(), # Unfortunately, we have to do a colon semparated list. # This is due to optparse inablity to handle n+/- args and # spaces are valid in distribution names. help="Distributions to exclude from the build. A colon " "separated string of distribution names. " + \ setuptools_help_info) parser.add_option('--exclude', dest='exclusions', metavar='DIST_NAME', type='string', nargs=1, action='callback', callback=separation_parser, callback_args=(':', False), default=tuple(), help="Distributions and their dependents to exclude from " "the build. A colon separated string of distribution " "names. " + setuptools_help_info) def main(): options, args = parser.parse_args() # *. Initialize variables configs = options.config_dir src = options.source_dir copyright_filename = 'COPYRIGHT' ##versions_filename = 'versions.cfg' indexes = list(options.indexes) + INDEXES for directory in (configs, src,): if not os.path.exists(directory): os.makedirs(directory) # 1. Version Specification if options.versions_cfg_url: versions_filename = os.path.join(configs, 'versions.cfg') versions = get_versions_config(options.versions_cfg_url, record_to=versions_filename) else: versions = {} # 2. Obtain the Source # Download the distribution source and verify we have aquired all # the dependencies. # Initialize the base requires from the arguments parse_reqs = lambda r: [InstallRequirement.from_line(line) for line in r] requirements = parse_reqs(args) exclusion_requirements = parse_reqs(options.exclusions) satisfied_reqs = fetch(requirements, indexes, versions, src, logger) if exclusion_requirements: exclusion_reqs = fetch(exclusion_requirements, indexes, versions, src, logger) else: exclusion_reqs = {} # 3. Clean Up single_reqs_to_remove = dict([(name,req) for name,req in satisfied_reqs.iteritems() if name.lower() in options.single_exclusions ]) exclusion_reqs.update(single_reqs_to_remove) for name, req in exclusion_reqs.iteritems(): logger.info("Excluding %s" % name) shutil.rmtree(req.source_dir) del satisfied_reqs[req.name] # 4. Produce copyright information with open(copyright_filename, 'w') as f: line_template = "%s by %s is licensed under %s.\n" pkg_info_defaults = {'author': 'Unknown', 'license': 'Unknown'} for req in satisfied_reqs.values(): pkg_info = dict([(k.lower(),v) for k,v in req.pkg_info().items()]) [pkg_info.setdefault(k,v) for k,v in pkg_info_defaults.items()] f.write(line_template % (pkg_info['name'], pkg_info['author'], pkg_info['license'])) if __name__ == '__main__': main()