File: mxSetup.py

package info (click to toggle)
egenix-mx-base 3.2.8-1
links: PTS
area: main
in suites: jessie, jessie-kfreebsd
size: 8,420 kB
ctags: 6,208
sloc: ansic: 22,304; python: 18,124; sh: 137; makefile: 121
file content (5945 lines) | stat: -rwxr-xr-x 207,619 bytes
#!/usr/local/bin/python

""" Distutils Extensions needed for the mx Extensions.

    Copyright (c) 1997-2000, Marc-Andre Lemburg; mailto:mal@lemburg.com
    Copyright (c) 2000-2014, eGenix.com Software GmbH; mailto:info@egenix.com
    See the documentation for further information on copyrights,
    or contact the author. All Rights Reserved.
"""
#
# History:
# 3.9.1: Added mx_upload which allows uploading single dist files
# 3.9.0: Added sdist_web format for web installers
# 3.8.0: Changed to prebuilt filename version 2
# 3.7.0: Added mx_register command with option to add a hash tag
#        to the download_url (--with-hash-tag)
# 3.6.8: Removed mx_customize_compiler(). We now always use distutils'
#        version. See #943.
# 3.6.7: Fix for clang override needed for Mac OS X Lion (see #943)
# 3.6.6: Fix for moved customize_compiler() function (see #861)
# 3.6.5: added install option --force-non-pure
# 3.6.4: import commands and base classes directly from setuptools,
#        if present to fixes compatibility with pip (see #741)
# 3.6.3: added verify_package_version()
# 3.6.2: added fix-up for distutils.util.change_root()
# 3.6.1: added --exclude-files to bdist_prebuilt
# 3.6.0: added support for Python 2.7's distutils version
# 3.5.1: added support to have prebuilt archives detect version and
#        and platform mismatches before installing a package
# 3.5.0: added own non-setuptools version of bdist_egg to build eggs
#        straight from the available packages rather than relying on
#        setuptools to do this
# 3.4.2: added support for using embedded .dylibs
# 3.4.1: only include top-level documentation directories in prebuilt archives
# 3.4.0: made the inclusion of the data source files optional in bdist_prebuilt,
#        fixed the prebuilt file search mechanism to always convert to platform
#        convention
# 3.3.1: have mx_version()'s snapshot default to true if prerelease is 'dev'
# 3.3.0: use our own mx_get_platform() version in order to support prebuilt
#        installers on platforms that have slightly different platform strings
# 3.2.1: py_version() will no longer add the Unicode info on Windows
# 3.2.0: Add support for Python 2.6 Windows builds
# 3.1.1: Allow Unicode values for meta data
# 3.1.0: changed the build pickle name used by prebuilt archives
# 3.0.0: added mx_build_data, removed the need to use "build --skip"
#        when installing prebuilt archives, uninstall now also works
#        for prebuilt archives
# 2.9.0: added support for (orig, dest) data_files definitions
# 2.8.0: added optional setuptools support
# 2.7.0: added mx_bdist_prebuilt
# 2.6.0: added mx_bdist_msi that allows setting the product name
# 2.5.1: add support for distutils 2.5.0 which broke the MSVCCompiler
#        compile options hack
# 2.5.0: patch install_lib to not produce weird byte-code filenames;
#        add work-around for change in bdist_wininst introduced in
#        Python 2.4 that broke the py_version() approach
# 2.4.0: remove monkey patch and make use of py_version() instead
# 2.3.0: monkey patch get_platform() to include the Unicode width info
# 2.2.0: replaced .announce() and .warn() usage with log object
# 2.1.2: added wide Unicode support
# 2.1.1: added support for Python 2.3 way of finding MSVC paths
# 2.1.0: added bdist_zope, support for classifiers
# 2.0.0: reworked the include and lib path logic; factored out the
#        compiler preparation
# 1.9.0: added new include and lib path logic; added bdist_zope command
# Older revisions are not documented.
#
import types, glob, os, sys, re, cPickle, copy, imp, shutil
import urllib2, urlparse

### Globals

# Module version
__version__ = '3.9.0'

# Generate debug output for mxSetup.py ?
_debug = int(os.environ.get('EGENIX_MXSETUP_DEBUG', 0))

# Python version running this module
python_version = '%i.%i' % sys.version_info[:2]

# Prebuilt archive marker file
PREBUILT_MARKER = 'PREBUILT'

# Allowed configuration value types; all other configuration entries
# are removed by run_setup()
ALLOWED_SETUP_TYPES = (types.StringType,
                       types.ListType,
                       types.TupleType,
                       types.IntType,
                       types.FloatType)
if python_version >= '2.6':
    ALLOWED_SETUP_TYPES += (types.UnicodeType,)

# Some Python distutils versions don't support all setup keywords we
# use
UNSUPPORTED_SETUP_KEYWORDS = ()
if python_version < '2.3':
    UNSUPPORTED_SETUP_KEYWORDS = UNSUPPORTED_SETUP_KEYWORDS + (
        'classifiers',
        'download_url',
        )

# Filename suffixes used for Python modules on this platform
PY_SUFFIXES = [suffix for (suffix, mode, order) in imp.get_suffixes()]
if '.pyc' not in PY_SUFFIXES:
    PY_SUFFIXES.append('.pyc')
if '.pyo' not in PY_SUFFIXES:
    PY_SUFFIXES.append('.pyo')

# Namespace __init__.py file content required by setuptools namespace
# packages
SETUPTOOLS_NAMESPACE_INIT = """\
__import__('pkg_resources').declare_namespace(__name__)
"""
    
### Python compatibility support

if 1:
    # Patch True/False into builtins for those versions of Python that
    # don't support it
    try:
        True
    except NameError:
        __builtins__['True'] = 1
        __builtins__['False'] = 0

    # StringTypes into types module for those versions of Python that
    # don't support it
    try:
        types.StringTypes
    except AttributeError:
        types.StringTypes = (types.StringType, types.UnicodeType)

    # Patch isinstance() to support tuple arguments
    try:
        isinstance('', types.StringTypes)
    except TypeError:
        def py22_isinstance(obj, classes,
                            orig_isinstance=isinstance):
            if type(classes) is not types.TupleType:
                return orig_isinstance(obj, classes)
            for classobj in classes:
                if orig_isinstance(obj, classobj):
                    return True
            return False
        __builtins__['isinstance'] = py22_isinstance

    # UnicodeDecodeError is new in Python 2.3
    try:
        UnicodeDecodeError
    except NameError:
        UnicodeDecodeError = UnicodeError

if python_version < '2.2':
    def module_loaded(name):
        return sys.modules.has_key(name)
    def has_substring(substr, text):
        return text.find(substr) >= 0
else:
    def module_loaded(name):
        return name in sys.modules
    def has_substring(substr, text):
        return substr in text

### distutils fix-ups

# distutils.util.change_root() has a bug on nt and os2: it fails with
# an IndexError in case pathname is empty. We fix this by
# monkey-patching distutils.
        
import distutils.util

orig_change_root = distutils.util.change_root

def change_root(new_root, pathname):
    if os.name == 'nt':
        (drive, path) = os.path.splitdrive(pathname)
        if path and path[0] == '\\':
            path = path[1:]
        return os.path.join(new_root, path)
    elif os.name == 'os2':
        (drive, path) = os.path.splitdrive(pathname)
        if path and path[0] == os.sep:
            path = path[1:]
        return os.path.join(new_root, path)
    else:
        return orig_change_root(new_root, pathname)

distutils.util.change_root = change_root

### Setuptools support

# Let setuptools monkey-patch distutils, if a command-line option
# --use-setuptools is given and enable the setuptools work-arounds
# if it was already loaded (see ticket #547).
if module_loaded('setuptools'):
    import setuptools
elif '--use-setuptools' in sys.argv:
    sys.argv.remove('--use-setuptools')
    try:
        import setuptools
        print 'running mxSetup.py with setuptools patched distutils'
    except ImportError:
        print 'could not import setuptools; ignoring --use-setuptools'
        setuptools = None
else:
    setuptools = None

### Distutils platform support

import distutils.util

if not hasattr(distutils.util, 'set_platform'):
    # Replace the distutils get_platform() function with our own, since we
    # will in some cases need to adjust its return value, e.g. for
    # pre-built archives.
    orig_get_platform = distutils.util.get_platform

    # Global platform string
    PLATFORM = orig_get_platform()

    def mx_get_platform():

        """ Return the platform string that distutils uses through-out
            the system.

        """
        return PLATFORM

    # Replace distutils' own get_platform() function
    distutils.util.get_platform = mx_get_platform

    def mx_set_platform(platform):

        """ Adjust the platform string that distutils uses to platform.

            This is needed e.g. when installing pre-built setups, since
            the target system platform string may well be different from
            the build system one, e.g. due to OS version differences or
            Mac OS X fat binaries that get installed on i386/ppc systems.

        """
        global PLATFORM
        if PLATFORM != platform:
            log.info('adjusting distutils platform string to %r' % platform)
            PLATFORM = platform

else:
    # For Python 2.7+ and 3.2+ we don't need to monkey-patch
    # distutils, since it now has a set_platform() API.
    #
    # XXX Turns out that this useful functionality was removed again,
    #     before 2.7 and 3.2 were released. See the discussion on
    #     http://bugs.python.org/issue13994 and #861. Leaving the code
    #     here in case it gets added again.

    def mx_get_platform():

        """ Return the platform string that distutils uses through-out
            the system.

        """
        return distutils.util.get_platform()

    def mx_set_platform(platform):

        """ Adjust the platform string that distutils uses to platform.

            This is needed e.g. when installing pre-built setups, since
            the target system platform string may well be different from
            the build system one, e.g. due to OS version differences or
            Mac OS X fat binaries that get installed on i386/ppc systems.

        """
        if platform != mx_get_platform():
            log.info('adjusting distutils platform string to %r' % platform)
            distutils.util.set_platform(platform)

### Load distutils

# This has to be done after importing setuptools, since it heavily
# monkey-patches distutils with its black magic...

from distutils.errors import \
     DistutilsError, DistutilsExecError, CompileError, CCompilerError, \
     DistutilsSetupError
if setuptools is not None:
    from setuptools import setup, Extension, Command
    from setuptools import Distribution
    from setuptools.command.install import install
else:
    from distutils.core import setup, Extension, Command
    from distutils.dist import Distribution
    from distutils.command.install import install
from distutils.msvccompiler import MSVCCompiler
from distutils.util import execute
from distutils.version import StrictVersion
from distutils.dir_util import remove_tree, mkpath, create_tree
from distutils.spawn import spawn, find_executable
from distutils.command.config import config
from distutils.command.build import build
from distutils.command.build_ext import build_ext
from distutils.command.build_clib import build_clib
from distutils.command.build_py import build_py
from distutils.command.bdist import bdist
from distutils.command.bdist_rpm import bdist_rpm
from distutils.command.bdist_dumb import bdist_dumb
from distutils.command.bdist_wininst import bdist_wininst
from distutils.command.install_data import install_data
from distutils.command.install_lib import install_lib
from distutils.command.sdist import sdist
from distutils.command.register import register
from distutils.command.clean import clean
import distutils.archive_util

# Upload command is only available in Python 2.5+
if python_version >= '2.5':
    from distutils.command.upload import upload
else:
    upload = None

# distutils changed a lot in Python 2.7/3.2 due to many
# distutils.sysconfig APIs having been moved to the new (top-level)
# sysconfig module.
if (python_version < '2.7' or
    (python_version >= '3.0' and python_version < '3.2')):
    # Older Python versions (<=2.6 and <=3.1):
    from distutils.sysconfig import \
         get_config_h_filename, parse_config_h, customize_compiler, \
         get_config_var, get_config_vars, get_python_version
    from distutils.util import get_platform

else:
    # More recent Python versions (2.7 and 3.2+):
    from sysconfig import \
         get_config_h_filename, parse_config_h, get_path, \
         get_config_var, get_config_vars, get_python_version, get_platform

    # This API was moved from distutils.sysconfig to
    # distutils.ccompiler in Python 2.7... and then back again in
    # 2.7.3 (see #861); since the sysconfig version is deemed the
    # correct one and 3.x only has it there, we first try sysconfig
    # now and then revert to ccompiler in case it's not found
    try:
        from distutils.sysconfig import customize_compiler
    except ImportError:
        from distutils.ccompiler import customize_compiler


def get_python_include_dir():

    """ Return the path to the Python include dir.

        This is the location of the Python.h and other files.

    """
    # Can't use get_path('include') here, because Debian/Ubuntu
    # hack those paths to point to different installation paths
    # rather than the Python's own paths. See #762.
    try:
        config_h = get_config_h_filename()
        include_dir = os.path.split(config_h)[0]
        if not os.path.exists(os.path.join(include_dir, 'Python.h')):
            raise IOError('Python.h not found in include dir %s' %
                          include_dir)
        return include_dir

    except IOError, reason:
        if _debug:
            print 'get_config_h_filename: %s' % reason
        # Ok, we've hit a problem with the Python installation or
        # virtualenv setup, so let's try a common locations:
        pydir = 'python%i.%i' % sys.version_info[:2]
        for dir in (
            os.path.join(sys.prefix, 'include', pydir),
            os.path.join(sys.prefix, 'include', 'python'),
            os.path.join(sys.prefix, 'include'),
            os.path.join(sys.exec_prefix, 'include', pydir),
            os.path.join(sys.exec_prefix, 'include', 'python'),
            os.path.join(sys.exec_prefix, 'include'),
            ):
            if _debug > 1:
                print ('get_config_h_filename: trying include dir: %s' %
                       dir)
            if os.path.exists(os.path.join(dir, 'Python.h')):
                if _debug:
                    print ('get_config_h_filename: found include dir: %s' %
                           dir)
                return dir
        # Nothing much we can do...
        raise

### Optional distutils support

# Load the MSI bdist command (new in Python 2.5, only on Windows)
try:
    from distutils.command.bdist_msi import bdist_msi
    import msilib
except ImportError:
    bdist_msi = None

# The log object was added to distutils in Python 2.3; we provide a
# compatile emulation for earlier Python versions
try:
    from distutils import log
except ImportError:
    class Log:
        def log(self, level, msg, *args):
            print (msg % args)
            sys.stdout.flush()
        def debug(self, msg, *args):
            if _debug:
                self.log(1, msg, args)
        def info(self, msg, *args):
            if _debug:
                self.log(2, msg, args)
        def warn(self, msg, *args):
            self.log(3, msg, args)
        def error(self, msg, *args):
            self.log(4, msg, args)
        def fatal(self, msg, *args):
            self.log(5, msg, args)
    log = Log()

### Third-party distutils extensions

# This is bdist_ppm from ActiveState
if python_version >= '2.0':
    try:
        from distutils.command import bdist_ppm
    except ImportError:
        bdist_ppm = None
    try:
        from distutils.command import GenPPD
    except ImportError:
        GenPPD = None
else:
    bdist_ppm = None
    GenPPD = None

###

#
# Helpers
#

# prerelease parsers used for conversion to PEP 386 format
PRERELEASE_RX = re.compile(
    '(?P<alpha>alpha|a)'
    '(?P<beta>beta|b)'
    '(?P<rc>rc|rc|c)'
    '(?P<dev>dev)'
    '[-_]+'
    '(?P<version>[0-9]+)'
    )

def mx_version(major=1, minor=0, patch=0, prerelease='', snapshot=None,
               sub_version=None, pep_compatible=False, separator='_'):

    """ Format the package version number.

        If sub_version is given as tuple of integers, it is appended
        to the major.minor.patch version string, with dots as
        delimiters.

        If prerelease is given, this is appended to the version
        string. The string should use the format
        '(alpha/beta/rc/dev)_123', e.g. 'beta_1'.

        If snapshot is true, the current date is added to the version
        string. snapshot defaults to true, if prerelease is set to
        'dev'.

        separator is used to separate prerelease and snapshot from the
        version number. Note that RPM doesn't like '-' in the version
        string. For this reason we use '_' as default.

        When using pep_compatible=True, the functions returns a
        version string that conforms to PEP 386. It's turned off by
        default for now to keep the function backwards compatible.

    """
    s = '%i.%i.%i' % (major, minor, patch)
    if sub_version:
        # Add sub_version tuple
        s += ''.join(['.%i' % x for x in sub_version])
    if prerelease:
        pr = PRERELEASE_RX.match(prerelease)
        if pep_compatible:
            if pr is None:
                raise TypeError('unsupported prerelease string: %r' %
                                prerelease)
            if pr.group('version'):
                pr_version = int(pr.group('version'))
            else:
                pr_version = 0
            if pr.group('alpha'):
                s += 'a%i' % pr_version
            elif pr.group('beta'):
                s += 'b%i' % pr_version
            elif pr.group('rc'):
                s += 'rc%i' % pr_version
            elif pr.group('dev'):
                snapshot = 1
        else:
            # Old style format
            s += '_' + prerelease
            if prerelease == 'dev' and snapshot is None:
                snapshot = 1
    if snapshot:
        import time
        now = time.gmtime(time.time())
        date = time.strftime('%Y%m%d', now)
        if pep_compatible:
            s += '.dev' + date
        else:
            # Old style format
            s += '_' + date
    return s

MX_VERSION_RE = re.compile('(\d+)\.(\d+)\.(\d+)')

def parse_mx_version(version):

    """ Convert a version string created with mx_version() back to
        its components.

        Returns a tuple (major, minor, patch, prerelease, snapshot).

        prerelease and snapshot are currently not supported and always
        set to '' and None.

        Raises a ValueError in case the version string cannot be
        parsed.

    """
    m = MX_VERSION_RE.match(version)
    if m is None:
        raise ValueError('incompatible version string format: %r' %
                         version)
    major, minor, patch = m.groups()
    major = int(major)
    minor = int(minor)
    patch = int(patch)
    prerelease = ''
    snapshot = None
    return major, minor, patch, prerelease, snapshot

def get_env_var(name, default=None, integer_value=0, yesno_value=0):

    value = os.environ.get(name, None)
    if value is None:
        return default

    # Try to convert to an integer, if possible
    if integer_value:
        try:
            return int(value)
        except ValueError:
            return default

    # Try to convert a yes/no value, if possible
    if yesno_value:
        value = value.strip().lower()
        if not value:
            return default
        if value[0] in ('y', '1'):
            return 1
        elif value == 'on':
            return 1
        else:
            return 0
        
    # Return the string value
    return value

def convert_to_platform_path(distutils_path):

    """ Convert a distutils Unix path distutils_path to the platform
        format.

    """
    if os.sep == '/':
        return distutils_path
    return os.sep.join(distutils_path.split('/'))

def convert_to_distutils_path(platform_path):

    """ Convert a platform path to the distutils Unix format.

    """
    if os.sep == '/':
        return platform_path
    return '/'.join(platform_path.split(os.sep))

def remove_path_prefix(pathname, prefix):

    """ Return a relative path by removing prefix from pathname.

        If pathname doesn't begin with prefix, no change is applied
        and pathname returned as-is.

    """
    prefix_len = len(prefix)
    if pathname[:prefix_len] != prefix:
        # pathname doesn't begin with prefix
        return pathname
    if prefix[-1] != os.sep:
        # Remove leading separator as well
        prefix_len += 1
    return pathname[prefix_len:]

def find_file(filename, paths, pattern=None):

    """ Look for a file in the directories defined in the list
        paths.

        If pattern is given, the found files are additionally checked
        to include the given RE search pattern. Pattern matching is
        done case-insensitive per default.

        Returns the directory where the file can be found or None in
        case it was not found.

        filename may include path components, e.g. if a particular
        file in a subdirectory is used as token to match the
        subdirectory.

    """
    if _debug:
        print 'looking for %s ...' % filename
        if pattern:
            print ' applying pattern check using %r' % pattern
    for dir in paths:
        pathname = os.path.join(dir, filename)
        if os.path.exists(pathname):
            if pattern:
                data = open(pathname, 'rb').read()
                if re.search(pattern, data, re.I) is None:
                    data = None
                    if _debug:
                        print ' %s: found, but not matched' % dir
                    continue
                data = None
                if _debug:
                    print ' %s: found and matched' % dir
            else:
                if _debug:
                    print ' %s: found' % dir
            return dir
        elif _debug:
            print ' %s: not found' % dir
    if _debug:
        print 'not found'
    return None

def is_python_package(path):

    """ Return 1/0 depending on whether path points to a Python
        package directory or not.

    """
    marker = '__init__' + os.extsep
    for filename in os.listdir(path):
        if filename.startswith(marker):
            return True
    return False

def python_module_name(path):

    """ Return the Python module name for the Python module path.

        Returns None if path does not point to a (potential) Python
        module.

    """
    for suffix in PY_SUFFIXES:
        if path.endswith(suffix):
            return os.path.basename(path[:-len(suffix)])
    return None

def find_python_modules(path):

    """ Find Python modules/packages available in path.

        Returns a dictionary mapping the Python module/package name
        (without extension) to either 'package' or 'module'.

    """
    d = {}
    for filename in os.listdir(path):
        pathname = os.path.join(path, filename)
        if os.path.isdir(pathname) and is_python_package(pathname):
            d[os.path.basename(filename)] = 'package'
        else:
            module_name = python_module_name(pathname)
            if module_name:
                d[module_name] = 'module'
    return d

def add_dir(dir, pathlist, index=-1):

    if dir not in pathlist and \
       os.path.isdir(dir):
        if index < 0:
            index = index + len(pathlist) + 1
        pathlist.insert(index, dir)

def py_unicode_build():

    """ Return the Python Unicode version.

        Possible values:
         'ucs2' - UCS2 build (standard Python source build)
         'ucs4' - UCS4 build (used on most recent Linux distros)
         ''     - No Unicode support

    """
    if python_version >= '2.1':
        # UCS4 builds were introduced in Python 2.1; Note: RPM doesn't
        # like hyphens to be used in the Python version string which is
        # why we append the UCS information using an underscore.
        try:
            unichr(100000)
        except NameError:
            # No Unicode support
            return ''
        except ValueError:
            # UCS2 build (standard)
            return 'ucs2'
        else:
            # UCS4 build (most recent Linux distros)
            return 'ucs4'
    else:
        return ''


def py_version(unicode_aware=None, include_patchlevel=0):

    """ Return the Python version as short string.

        If unicode_aware is true (default on all platforms except
        win32, win16, os2 and dos), the function also tests whether a
        UCS2 or UCS4 built is running and modifies the version
        accordingly.

        If include_patchlevel is true (default is false), the patch
        level is also included in the version string.

    """
    if include_patchlevel:
        version = '%i.%i.%i' % sys.version_info[:3]
    else:
        version = '%i.%i' % sys.version_info[:2]
    if unicode_aware is None:
        # Chose default for unicode_aware based on platform
        if sys.platform in ('win32', 'dos', 'win16', 'os2'):
            # These platforms always use UCS2 builds (at least for all
            # versions up until Python 2.6)
            unicode_aware = 0
        else:
            unicode_aware = 1
    if unicode_aware and version >= '2.1':
        # UCS4 builds were introduced in Python 2.1; Note: RPM doesn't
        # like hyphens to be used in the Python version string which is
        # why we append the UCS information using an underscore.
        try:
            unichr(100000)
        except ValueError:
            # UCS2 build (standard)
            version = version + '_ucs2'
        else:
            # UCS4 build (most recent Linux distros)
            version = version + '_ucs4'
    return version

def check_zope_product_version(version, version_txt):

    """ Check whether the version string version matches the
        version data in the Zope product version.txt file
        version_txt.

    """
    data = open(version_txt, 'r').read().strip()
    return data[-len(version):] == version

def verify_package_version(package, version):

    """ Check whether the Python package's __version__ matches
        the given version string.

        The first 3 version components must match,
        i.e. major.minor.patch. parse_mx_version() is used to extract
        this information from both the version string and the packages
        __version__ attribute.

        Raises a ValueError in case the versions do not match.

    """
    package_path = package.replace('.', os.sep)
    try:
        m = __import__(package_path, None, None, ['*'])
    except ImportError, reason:
        raise ImportError('Cannot find %s package: %s' % (package, reason))
    dist_version = parse_mx_version(version)
    package_version = parse_mx_version(m.__version__)
    if dist_version[:3] != package_version[:3]:
        raise ValueError('%s.__version__ mismatch: expected %s, found %s' %
                         (package, version, m.__version__))

def find_code_version(filename):

    """ Tries to extract the code version information from the file
        filename and returns it.

        The following formats are recognized (first match wins):
        - __version__ = '1.2.3'
        - #define *MX*_VERSION* "1.2.3"
        - #define *VERSION* "1.2.3"

        Raises a ValueError in case the version cannot be parsed.

    """
    file = open(filename)
    try:
        code = file.read()
    finally:
        file.close()

    # Find version string
    for version_rx in (
        "__version__ *= *'([0-9]\.[0-9][^']*)'",
        '__version__ *= *"([0-9]\.[0-9][^"]*)"',
        '#define .*MX.*_VERSION.*"([0-9]\.[0-9]\.[0-9][^"]*)"',
        '#define .*VERSION.*"([0-9]\.[0-9]\.[0-9][^"]*)"',
        ):
        match = re.search(version_rx, code)
        if match is not None:
            return match.group(1)
    else:
        raise ValueError('%s code version could not be parsed' %
                         filename)

def verify_code_version(filename, version):

    """ Check whether the file filename's version string matches
        the given version string.

        The first 3 version components must match,
        i.e. major.minor.patch. parse_mx_version() is used to extract
        this information from both the version string and the file's
        version definition.

        The following find_code_version() formats are recognized.

        Raises a ValueError in case the versions do not match.

    """
    # Find code version
    code_version_string = find_code_version(filename)

    # Compare versions
    code_version = parse_mx_version(code_version_string)
    verify_version = parse_mx_version(version)
    if code_version[:3] != verify_version[:3]:
        raise ValueError('%s version mismatch: expected %s, found %s' %
                         (filename, version, code_version_string))

def quote_string_define_value(value):

    """ Quote the value used in a -D name=value compiler define in a
        command line safe way.

        This function makes sure that the string value is quoted in
        way that survives the command line interface used for talking
        to compilers.

    """
    # Remove any existing quotes
    if ((value[:1] == '"' and value[-1:] == '"') or
        (value[:1] == "'" and value[-1:] == "'")):
        value = value[1:-1]
    elif ((value[:2] == '\\"' and value[-2:] == '\\"') or
          (value[:2] == "\\'" and value[-2:] == "\\'")):
        value = value[2:-2]

    # Setup the value formatting template
    if sys.platform[:3] == 'win':
        # Use escaped quotes that survive the VC++ command line
        # interface
        value_template = '\\"%s\\"'
    else:
        # Regular quotes for all other platforms
        value_template = '"%s"'

    return value_template % value

# Keep the symbol around for backwards compatibility, but don't use it
# anymore.  See #943.
mx_customize_compiler = customize_compiler

compression_programs = {
    'gzip': ('.gz', '-f9'),
    'bzip2': ('.bz2', 'f9'),
    'compress': ('.Z', '-f'),
    }

def mx_make_tarball(base_name, base_dir, compression='gzip', verbose=0,
                    dry_run=0, owner=None, group=None,
                    tar_options='-h', **kws):

    # Much like archive_util.make_tarball, except that this version
    # dereferences symbolic links.
    tar_archive = base_name + '.tar'

    # Create the directory for the archive
    mkpath(os.path.dirname(tar_archive), dry_run=dry_run)

    # Create the archive
    if owner:
        tar_options += ' --owner="%s"' % owner
    if group:
        tar_options += ' --group="%s"' % group
    cmd = ['tar', '-c', tar_options, '-f', tar_archive, base_dir]
    spawn(cmd, verbose=verbose, dry_run=dry_run)

    # Compress if that's needed
    if compression:
        try:
            ext, options = compression_programs[compression]
        except KeyError:
            raise ValueError('unknown compression program: %s' % compression)
        cmd = [compression, options, tar_archive]
        spawn(cmd, verbose=verbose, dry_run=dry_run)
        tar_archive = tar_archive + ext
        
    return tar_archive

# Register our version of make_tarball()
register_archive_formats = {
    'gztar': (mx_make_tarball, [('compression', 'gzip')], 'gzipped tar-file'),
    'bztar': (mx_make_tarball, [('compression', 'bzip2')], 'bzip2ed tar-file'),
    'ztar':  (mx_make_tarball, [('compression', 'compress')], 'compressed tar file'),
    'tar':   (mx_make_tarball, [('compression', None)], 'tar file'),
    }
if not hasattr(shutil, 'register_format'):
    # In Python <2.7, 3.0 and 3.1, we have to register straight with
    # the distutils ARCHIVE_FORMATS dictionary
    distutils.archive_util.ARCHIVE_FORMATS.update(register_archive_formats)
else:
    # Python 2.7+ and 3.2+ use the new shutil archive functions instead
    # of the ones from distutils, so register our archives there
    for archive_name, archive_params in register_archive_formats.items():
        shutil.register_format(archive_name, *archive_params)

def build_path(dirs):

    """ Builds a path list from a list of directories/paths.

        The dirs list may contain shell variable references and user
        dir references. These will get expanded
        automatically. Non-existing shell variables are replaced with
        an empty string. Path entries will get expanded to single
        directory entries.  Empty string entries are removed from the
        list.

    """
    try:
        expandvars = os.path.expandvars
    except AttributeError:
        expandvars = None
    try:
        expanduser = os.path.expanduser
    except AttributeError:
        expanduser = None
    path = []
    for i in range(len(dirs)):
        dir = dirs[i]
        if expanduser is not None:
            dir = expanduser(dir)
        if expandvars is not None:
            dir = expandvars(dir)
            if '$' in dir:
                dir = ''.join(re.split(r'\$\w+|\{[^}]*\}', dir))
        dir = dir.strip()
        if os.pathsep in dir:
            path.extend(dir.split(os.pathsep))
        elif dir:
            path.append(dir)
        # empty entries are omitted
    return path

def verify_path(path):

    """ Verify the directories in path for existence and their
        directory nature.

        Also removes duplicates from the list.

    """
    d = {}
    l = []
    for dir in path:
        if os.path.exists(dir) and \
           os.path.isdir(dir):
            if not d.has_key(dir):
                d[dir] = 1
                l.append(dir)
    path[:] = l

def get_msvc_paths():

    """ Return a tuple (libpath, inclpath) defining the search
        paths for library files and include files that the MS VC++
        compiler uses per default.

        Both entries are lists of directories.

        Only available on Windows platforms with installed compiler.

    """
    if python_version >= '2.6':
        # Python 2.6 distutils

        # If possible, assume that the environment is already
        # properly setup and read the setting from there - this is
        # important since we could be cross-compiling
        if os.environ.has_key('lib') and os.environ.has_key('include'):
            return (os.environ['lib'].split(os.pathsep),
                    os.environ['include'].split(os.pathsep))
        else:
            # Use the old compiler code
            from distutils.msvccompiler import OldMSVCCompiler
            try:
                msvccompiler = OldMSVCCompiler()
                inclpath = msvccompiler.get_msvc_paths('include')
                libpath = msvccompiler.get_msvc_paths('library')
            except Exception, why:
                log.error('*** Problem: %s' % why)
                import traceback
                traceback.print_exc()
                libpath = []
                inclpath = []
            msvccompiler = None

    elif python_version >= '2.3':
        # Python 2.3 - 2.5 distutils
        try:
            msvccompiler = MSVCCompiler()
            inclpath = msvccompiler.get_msvc_paths('include')
            libpath = msvccompiler.get_msvc_paths('library')
        except Exception, why:
            log.error('*** Problem: %s' % why)
            import traceback
            traceback.print_exc()
            libpath = []
            inclpath = []
        msvccompiler = None

    else:
        # distutils versions prior to the one that came with Python 2.3
        from distutils.msvccompiler import get_devstudio_versions, get_msvc_paths
        msvc_versions = get_devstudio_versions()
        if msvc_versions:
            msvc_version = msvc_versions[0] # choose most recent one
            inclpath = get_msvc_paths('include', msvc_version)
            libpath = get_msvc_paths('lib', msvc_version)
        else:
            libpath = []
            inclpath = []

    return libpath, inclpath

#
# Search paths
#
        
# Standard system directories which are automatically scanned by the
# compiler and linker for include files and libraries. LIB and INCLUDE
# are environment variables used on Windows platforms, other platforms
# may have different names.
STDLIBPATH = build_path(['/usr/lib', '/opt/lib', '$LIB'])
STDINCLPATH = build_path(['/usr/include', '/opt/include', '$INCLUDE'])

# Add additional dirs from Windows registry if available
if sys.platform[:3] == 'win':
    libpath, inclpath = get_msvc_paths()
    STDLIBPATH.extend(libpath)
    STDINCLPATH.extend(inclpath)

# Default paths for additional library and include file search (in
# addition to the standard system directories above); these are always
# added to the compile and link commands by mxSetup per default.
LIBPATH = build_path(['/usr/local/lib',
                      '/opt/local/lib',
                      os.path.join(sys.prefix, 'lib')])
INCLPATH = build_path(['/usr/local/include',
                       '/opt/local/include',
                       os.path.join(sys.prefix, 'include')])

# Add 32- or 64-bit dirs if needed by the Python version
if sys.maxint > 2147483647L:
    # 64-bit build
    STDLIBPATH.extend(['/usr/lib64', '/opt/lib64'])
    LIBPATH.extend(['/usr/local/lib64', '/opt/local/lib64'])
else:
    # 32-bit build
    STDLIBPATH.extend(['/usr/lib32', '/opt/lib32'])
    LIBPATH.extend(['/usr/local/lib32', '/opt/local/lib32'])

# Additional paths to scan in order to find third party libs and
# headers; these are used by mx_autoconf.find_*_file() APIs.
FINDLIBPATH = build_path([])
FINDINCLPATH = build_path([])

verify_path(STDLIBPATH)
verify_path(STDINCLPATH)
verify_path(LIBPATH)
verify_path(INCLPATH)
verify_path(FINDLIBPATH)
verify_path(FINDINCLPATH)

if 0:
    # Work-around for quirk on Solaris which happens to be a common
    # problem when compiling things with GCC: there's a non-GCC stdarg.h
    # header file in /usr/include which gets picked up by GCC instead of
    # its own compiler specific one, so we remove /usr/include from
    # INCLPATH in that situation.
    if sys.platform == 'sunos5' and \
       sys.version.find('GCC') >= 0:
        if os.path.exists('/usr/include/stdarg.h'):
            INCLPATH.remove('/usr/include')

#
# File extensions
#

# Library types to search for (see distutils.ccompiler)
if sys.platform == 'darwin':
    # Mac OS X uses .dylibs in addition to .so files, so we need to
    # search for those as well
    #
    # Note that only the distutils unixcompiler supports searching for
    # dylib, other compiler classes will simply exist with an
    # AttributeError
    LIB_TYPES = ('dylib', 'shared', 'static')
    
else:
    # These types are supported by all compiler classes
    LIB_TYPES = ('shared', 'static')


if _debug > 1:
    # Note that printing these lines to stdout can cause scripts that
    # use mxSetup for extracting information from the setup module to
    # fail, since they don't expect the extra output. This is why we
    # only show this information for higher _debug levels.
    print 'mxSetup will be using these search paths:'
    print ' std lib path:', STDLIBPATH
    print ' std include path:', STDINCLPATH
    print ' additional lib path:', LIBPATH
    print ' additional include path:', INCLPATH
    print ' additional autoconf lib path:', FINDLIBPATH
    print ' additional autoconf include path:', FINDINCLPATH
    print ' library types:', LIB_TYPES
    print ' Python include path: %r' % get_python_include_dir()

### Download tools

def fetch_url(url, timeout=30):

    """ Fetch the given url and return the data as file-like object.

        The function can raise urllib2.URLError exceptions in case of
        an error.

    """
    import urllib2
    if python_version >= '2.6':
        data_file = urllib2.urlopen(url, None, timeout)
    elif python_version >= '2.3':
        import socket
        socket.setdefaulttimeout(timeout)
        data_file = urllib2.urlopen(url)
    else:
        # Ignore timeout
        data_file = urllib2.urlopen(url)
    return data_file

def read_url(url, timeout=30):

    """ Fetch the given url and return the data as string.

        The function can raise urllib2.URLError exceptions in case of
        an error.

    """
    data_file = fetch_url(url, timeout)
    data = data_file.read()
    data_file.close()
    return data

def copy_file(source, target, chunksize=65000, close_files=True):

    """ Copy data from source to target.

        chunksize may be given to determine the chunk size used for
        the copy operation.

        If close_files is true (default), close the files after the
        copying operation.

    """
    try:
        while True:
            data = source.read(chunksize)
            if not data:
                break
            target.write(data)
    finally:
        if close_files:
            source.close()
            target.close()

def download_url(url, timeout=30, target=None, path=None,
                 chunksize=65000, index_name='index.html'):

    """ Fetch the given url and write the data to target.

        Returns the target pathname.

        target defaults to the filename given in the url, if not
        provided.

        If path is given, the path is joined with the target filename
        to build the output filename.

        chunksize may be given to determine the chunk size used for
        downloading the file.

        index_name is used as target filename for directory downloads
        (defaults to 'index.html').

    """
    # Determine target filename, if not given
    if not target:
        import urlparse, posixpath
        parsed_url = urlparse.urlparse(url)
        dirs, target = posixpath.split(parsed_url.path)
    if path is not None:
        target = os.path.join(path, target)

    # Make sure that the parent dirs exist
    target_dir = os.path.dirname(target)
    if target_dir and not os.path.exists(target_dir):
        os.makedirs(target_dir)

    # If target is a directory, use the index_name as target name
    if os.path.isdir(target):
        target = os.path.join(target, index_name)

    # Fetch the data in chunks and write it to target
    data_file = fetch_url(url, timeout)
    target_file = open(target, 'wb')
    copy_file(data_file, target_file, chunksize)
    data_file.close()
    target_file.close()
    return target

def hashed_download_url(download_url, mode='simple', data=None, download=None):

    """ Add a hash marker to the download_url.

        This is done by fetching the URL, creating
        one or more hash/size entries and appending these
        as fragment to the download_url.

        mode determines how the fragment is formatted. Possible
        values:
        
        * 'simple': add #md5=...
        
          This is compatible with setuptools/easy_install/pip.

        * 'pip': add #sha256=...

          This is compatible with pip only. setuptools/easy_install
          don't support sha256 hashes.

        * 'extended': add #md5=...&sha1=...&sha256=...&size=...

          This is not yet supported by the installers tools, but
          provides the most advanced hash tag format. sha256 is only
          included if supported by the Python version.

        data or download may be given to avoid having the function
        fetch the URL. data must be the raw data chunk, download a
        filename to read the data from, if given. If both are given,
        download takes precedence.

    """
    # Determine hash functions to use
    if python_version >= '2.5':
        import hashlib
        md5 = hashlib.md5
        sha1 = hashlib.sha1
        sha256 = hashlib.sha256
    else:
        import md5, sha
        md5 = md5.md5
        sha1 = sha.sha
        sha256 = None

    # Strip existing hash tags
    if '#' in download_url:
        download_url = download_url.split('#')[0]

    # Get data to hash
    if download is not None:
        file = open(download, 'rb')
        data = file.read()
        file.close()
    elif data is None:
        data = read_url(download_url)

    # Format hashed URL
    if mode == 'simple':
        return '%s#md5=%s' % (
            download_url,
            md5(data).hexdigest())
    elif mode == 'pip':
        if sha256 is None:
            raise TypeError('Python version does not support SHA-256')
        return '%s#sha256=%s' % (
            download_url,
            sha256(data).hexdigest())
    elif mode == 'extended':
        if sha256 is not None:
            return '%s#md5=%s&sha1=%s&sha256=%s&size=%i' % (
                download_url,
                md5(data).hexdigest(),
                sha1(data).hexdigest(),
                sha256(data).hexdigest(),
                len(data))
        else:
            return '%s#md5=%s&sha1=%s&size=%i' % (
                download_url,
                md5(data).hexdigest(),
                sha1(data).hexdigest(),
                len(data))

def verify_hashed_url_download(url, download=None, data=None):

    """ Verify a hashed URL url based on the downloaded content.

        Either download or data must be given. download must point to
        a downloaded file, data to the raw data block to verify.

        Raises a ValueError in case the content does not verify, no
        hash tag can be found or the hash tag uses an unknown format.

    """
    if download is not None:
        file = open(download, 'rb')
        data = file.read()
        file.close()
    elif data is None:
        raise TypeError('Either download or data must be given')
    if python_version >= '2.5':
        import hashlib
        md5 = hashlib.md5
        sha1 = hashlib.sha1
        sha256 = hashlib.sha256
    else:
        import md5, sha
        md5 = md5.md5
        sha1 = sha.sha
        sha256 = None
    l = url.rsplit('#')
    if len(l) < 2:
        raise ValueError('No URL hashtag found')
    hashtag = l[1]
    components = hashtag.split('&')
    for component in hashtag.split('&'):
        try:
            function, value = component.split('=')
        except ValueError:
            raise ValueError('Hash tag component %r has illegal format' %
                             component)
        if function == 'md5':
            if md5(data).hexdigest() != value:
                raise ValueError('MD5 hash tag does not match')
        elif function == 'sha1':
            if sha1(data).hexdigest() != value:
                raise ValueError('SHA1 hash tag does not match')
        elif function == 'sha256' and sha256 is not None:
            if sha256(data).hexdigest() != value:
                raise ValueError('SHA256 hash tag does not match')
        elif function == 'size':
            if len(data) != int(value):
                raise ValueError('Size does not match')
        else:
            raise ValueError('Unknown function in component %r' %
                             component)

def build_archive_name_map(names, target_path, strip_path=0):

    """ Return a dictionary mapping archive name entries from names to
        local file names based at target_path.

        The function applies security checks to make sure that the
        local names do not map to directories outside the target_path.

        strip_path may be given as integer to indicate how many
        leading path entries from the archive names to strip when
        building the local names.

    """
    name_map = {}
    pathsep = os.path.sep
    for name in names:
        if strip_path:
            local_name = pathsep.join(name.split('/')[strip_path:])
        else:
            local_name = name.replace('/', pathsep)
        if not local_name:
            continue
        target_name = os.path.normpath(os.path.join(target_path, local_name))
        if not target_name.startswith(target_path):
            raise ValueError(
                'Illegal archive entry %r - would map to local %r' %
                (name, target_name))
        name_map[name] = target_name
    return name_map

def unzip_file(file, path=None, strip_path=0):

    """ Unzip the given file to the path directory.

        If path is not given, the current work directory is used.

    """
    import zipfile

    # Determine target dir
    if path is None:
        path = os.getcwd()
    target_path = os.path.abspath(path)

    # Open ZIP file and extract names
    zf = zipfile.ZipFile(file, 'r')
    names = zf.namelist()

    # Build name map
    name_map = build_archive_name_map(names,
                                      target_path,
                                      strip_path=strip_path)

    # Extract files (Python 2.4 compatible, 2.6+ could use .open()
    # and .extractall(), but they don't exist in Python 2.4 and 2.5)
    for name in names:
        target_name = name_map.get(name, None)
        if target_name is None:
            continue
        # Make sure that the parent dirs exist
        target_dir = os.path.dirname(target_name)
        if target_dir and not os.path.exists(target_dir):
            os.makedirs(target_dir)
        target_file = open(target_name, 'wb')
        target_file.write(zf.read(name))
        target_file.close()
    zf.close()

def untar_file(file, path=None, strip_path=0):

    """ Untar the given file to the path directory.

        If path is not given, the current work directory is used.

    """
    import tarfile

    # Determine target dir
    if path is None:
        path = os.getcwd()
    target_path = os.path.abspath(path)

    # Open tar file and extract names
    tf = tarfile.open(file, 'r')
    names = tf.getnames()

    # Build name map
    name_map = build_archive_name_map(names,
                                      target_path,
                                      strip_path=strip_path)

    # Extract files (Python 2.4 compatible, 2.6+ could use .open()
    # and .extractall(), but they don't exist in Python 2.4 and 2.5)
    for name in names:
        target_name = name_map.get(name, None)
        if target_name is None:
            continue
        # Make sure that the parent dirs exist
        target_dir = os.path.dirname(target_name)
        if target_dir and not os.path.exists(target_dir):
            os.makedirs(target_dir)
        data_file = tf.extractfile(name)
        if data_file is None:
            # Probably a directory entry
            continue
        data = data_file.read()
        data_file.close()
        target_file = open(target_name, 'wb')
        target_file.write(data)
        target_file.close()
    tf.close()

def install_web_package(url, destdir):

    # Strip hash tags from url to not confuse Python 2.6
    if '#' in url:
        verify = True
        file_url = url.split('#', 1)[0]
    else:
        verify = False
        file_url = url

    # Download the package into the destdir
    log.info('downloading web package %s' % file_url)
    pkg_file = download_url(file_url, path=destdir)

    # Verify the download
    if verify:
        log.info('verifying web package %s' % pkg_file)
        try:
            verify_hashed_url_download(url, download=pkg_file)
        except ValueError:
            raise DistutilsError('downloaded package file does not verify: '
                                 '%s' % pkg_file)
        if _debug:
            print ('downloaded package %s verifies ok' % file_url)

    # Unzip the package into the destdir (in place)
    log.info('extracting web package %s into %s' % (pkg_file, destdir))
    import zipfile, tarfile
    if zipfile.is_zipfile(pkg_file):
        if _debug:
            print ('unzipping downloaded package %s' % pkg_file)
        unzip_file(pkg_file, destdir, strip_path=1)
    elif tarfile.is_tarfile(pkg_file):
        if _debug:
            print ('untaring downloaded package %s' % pkg_file)
        untar_file(pkg_file, destdir, strip_path=1)
    else:
        raise DistutilsError('unsupported package file format: %r' %
                             pkg_file)
    if _debug:
        print ('extracted package into %s' % destdir)

    # Remove the downloaded ZIP file
    os.remove(pkg_file)

# Tests

def _test_hashed_download_url():
    for url in (
        'file:///home/lemburg/projects/distribution/index/ucs2/egenix-pyopenssl/0.13.0_1.0.1c_1/index.html',
        'http://downloads.egenix.com/python/index/ucs2/egenix-pyopenssl/0.13.0_1.0.1c_1/',
        'http://downloads.egenix.com/python/index/ucs2/egenix-pyopenssl/0.13.0_1.0.1c_1/index.html',
        'https://downloads.egenix.com/python/index/ucs2/egenix-pyopenssl/0.13.0_1.0.1c_1/',
        'https://downloads.egenix.com/python/index/ucs2/egenix-pyopenssl/0.13.0_1.0.1c_1/index.html',
        ):
        for mode in (
            'simple',
            'pip',
            'extended'):
            hurl = hashed_download_url(url, mode)
            print (hurl)
            target = download_url(url, path='test-hashed-download')
            verify_hashed_url_download(hurl, download=target)
            assert hurl == hashed_download_url(url, mode, download=target)

def _test_download_url():

    for url in (
        'https://downloads.egenix.com/python/egenix-mx-base-3.2.7-py2.7_ucs2-linux-x86_64-prebuilt.zip',
        ):
        target = download_url(url)
        unzip_file(target, 'test-zip-file/')
        unzip_file(target, 'test-stripped-zip-file/', strip_path=1)

if 0:    
    # Tests
    _test_hashed_download_url()
    #_test_download_url()

### Package platform matching

def normalize_tags(tags):

    """ Normalize tags and return the normalized set.

        Removes empty and duplicate entries, makes sure that all tags
        are lower case.

    """
    # Create tag set, strip and convert to lower case
    tag_set = set(tag.strip().lower()
                  for tag in tags)

    # Remove empty tag
    tag_set.discard('')

    # Return set as list
    return tags

def write_package_data(datafile, data, base_url=None, replace=False,
                       add_hash_tag=False, hash_tag_mode='extended'):

    """ Write the entries url: tags from the data dictionary to the
        datafile, possibly replacing already existing entries.

        For each data entry url: tags, url has to be a URL or filename
        which refers to the package file and tags has to be a set or
        list of string tags associated with the package.
        
        base_url may be given to set a base URL which is then joined
        to url before adding the data tuple. If the URLs in the data
        dictionary already are absolute URLs, the parameter has no
        effect.

        replace may be set to True to have the function
        unconditionally overwrite a possibly existing datafile with
        the given new data, removing any old data entries.

        If add_hash_tag is true (default is false), the function will
        add hash tags to the urls. hash_tag_mode defines the hash tag
        mode to use for this. Note that failing downloads will result
        in DistutilsError to be raised.
        
    """
    if os.path.exists(datafile) and not replace:
        package_data = read_package_data(datafile)
    else:
        package_data = {}
    for url, tags in data.iteritems():
        if base_url is not None:
            url = urlparse.urljoin(base_url, url)
        if add_hash_tag:
            try:
                url = hashed_download_url(url, hash_tag_mode)
            except urllib2.URLError, reason:
                raise DistutilsError(
                    'could not generate hash for download URL %s' %
                    url)
        package_data[url] = normalize_tags(tags)

    # Write data to datafile
    f = open(datafile, 'w')
    items = package_data.items()
    items.sort()
    for urlkey, urltags in items:
        # When making changes to the format, also update
        # read_package_data()
        f.write('%s -> %s\n' % (urlkey, ', '.join(urltags)))
    f.close()

def write_package_tags(package_file, tags):

    """ Write the tag file for package package_file.

        Returns the tag file name.

    """
    tag_file = package_file + '.tags'
    url = os.path.split(package_file)[1]
    log.info('writing package tag file %s' % tag_file)
    write_package_data(tag_file, {url: tags})
    return tag_file

def read_package_data(datafile, remove_base_url=''):

    """ Open datafile, read the data dictionary and return it.

        The dictionary entries will have the format url: tags, mapping
        URLs to sets of tag strings. See write_package_data() for
        details.

        If remove_base_url is given, prefixes of this form ar removed
        from the urls in the data dictionary.

    """
    f = open(datafile, 'r')
    chunk = f.read()
    f.close()
    data = {}
    len_remove_base_url = len(remove_base_url)
    for line in chunk.splitlines():
        line = line.strip()
        if not line or line[0] == '#':
            continue
        url, tagstr = line.split(' -> ')
        tags = normalize_tags(tagstr.split(', '))
        if len_remove_base_url and url.startswith(remove_base_url):
            url = url[len_remove_base_url:]
        data[url] = tags
    return data

def read_package_tags(package_file, remove_base_url=''):

    """ Read the tag file for package package_file.

        See read_package_data() for details on the returned format and
        parameters.

    """
    tag_file = package_file + '.tags'
    return read_package_data(tag_file, remove_base_url=remove_base_url)

def _filter_packages(package_data, tag):

    """ Return a package_data dictionary filtered by tag.

        Only packages which have the tag set are included in the
        returned dictionary.

    """
    return dict(
        (url, tags)
        for (url, tags) in package_data.iteritems()
        if tag in tags)

def _subset_packages(package_data, tags):

    """ Return a package_data dictionary subset with packages that
        have at least one of the given tags.

    """
    if not isinstance(tags, set):
        tags_set = set(tags)
    return dict(
        (url, tags)
        for (url, tags) in package_data.iteritems()
        if tags_set.intersection(tags))

def _setminus_packages(package_data, tags):

    """ Return a package_data dictionary subset with packages that
        have none of the given tags.

    """
    if not isinstance(tags, set):
        tags_set = set(tags)
    return dict(
        (url, tags)
        for (url, tags) in package_data.iteritems()
        if not tags_set.intersection(tags))

def _find_source_package(package_data):

    """ Find a suitable source package and return its URL.

        Raises a KeyError if no package can be found.
    
    """
    source_packages = _filter_packages(package_data, 'source')
    if _debug:
        print ('filtered to source packages: %r' % source_packages)
    if not source_packages:
        raise KeyError(
            'No suitable package found for the current platform')

    if len(source_packages) > 1:
        # Prefer compiled faster code, if available
        compiler_packages = _filter_packages(source_packages,
                                             'compiler')
        if compiler_packages:
            source_packages = compiler_packages
            if _debug:
                print ('filtered to compiler packages: %r' %
                       compiler_packages)

    # Choose one of the available source packages (by sorting and
    # picking the last one to make the choice deterministic; this will
    # prefer zip files over tar files)
    package_list = sorted(source_packages.iteritems())
    package = package_list[-1]
    if _debug:
        print ('found source package: %r' % (package,))
    return package[0]

def platform_tags():

    """ Determine the platform parameters of the currently running
        Python script.

        Returns a tuple (os_name, code_type, alt_code_types,
        python_version_tag, ucs_version_tag) with the found information.

        os_name and code_types are set to 'unknown-os' and
        'unknown-code-type' in case the details cannot be determined.
        ucs_version_tag is only set for Python versions where this
        matters and left set to '' otherwise.  alt_code_types is a
        tuple of additional code types supported by the package,
        e.g. for fat builds on some platforms.
        
    """
    # Determine bitsize used by Python (not necessarily the same as
    # the one used by the platform)
    import struct
    bits = struct.calcsize('P') * 8

    # Determine OS name and code type
    import platform
    machine = platform.machine()
    os_name = 'unknown-os'
    code_type = 'unknown-code-type'
    alt_code_types = []
    if sys.platform.startswith('linux'):
        os_name = 'linux'
        if machine in ('x86_64', 'i686', 'i586', 'i386'):
            if bits == 32:
                code_type = 'x86'
            elif bits == 64:
                code_type = 'x64'
        elif machine == 'armv6l':
            code_type = 'raspi'

    elif sys.platform.startswith('freebsd'):
        os_name = 'freebsd'
        if machine == 'arm':
            # Most likely running on Raspi
            code_type = 'raspi'
        elif machine in ('amd64', 'i386'):
            # Intel processor
            if bits == 32:
                code_type = 'x86'
            elif bits == 64:
                code_type = 'x64'

    elif sys.platform.startswith('win32'):
        os_name = 'windows'
        # XXX I guess we should also include ARM platforms here
        if bits == 32:
            code_type = 'x86'
        elif bits == 64:
            code_type = 'x64'
           
    elif sys.platform.startswith('darwin'):
        os_name = 'macosx'
        if machine.startswith('Power'):
            if bits == 32:
                code_type = 'ppc'
            elif bits == 64:
                code_type = 'ppc64'
        elif machine == 'i386':
            if bits == 32:
                code_type = 'x86'
            elif bits == 64:
                code_type = 'x64'
        # For fat Python builds on Mac OS X, add additional code_types
        # to alt_code_types
        cflags = get_config_var('CFLAGS')
        if cflags is not None:
            # Should always be defined
            if '-arch ppc' in cflags and code_type != 'ppc':
                alt_code_types.append('ppc')
            if '-arch ppc64' in cflags and code_type != 'ppc64':
                alt_code_types.append('ppc64')
            if '-arch i386' in cflags and code_type != 'i386':
                alt_code_types.append('x86')
            # XXX What about x64 builds ?
    alt_code_types = tuple(alt_code_types)

    # Determine Python version
    python_version_tag = 'py%i.%i' % sys.version_info[:2]
    if sys.version_info < (3, 3):
        if sys.maxunicode > 66000:
            ucs_version_tag = 'ucs4'
        else:
            ucs_version_tag = 'ucs2'
    else:
        ucs_version_tag = ''

    return (os_name, code_type, alt_code_types, python_version_tag, ucs_version_tag)

def find_matching_package(package_data, required_tags=()):

    """ Find a package matching the current platform and return the
        URL for this.

        package_data has to be dictionary mapping URLs to sets of
        tags identifying the platform for which the URLs are best
        suited.

        required_tags may be given to define a list/set of supported
        tags, one of which packages must include. The available
        packages are subset filtered using this list/set.

        Valid tags:

         * 'prebuilt' - package is a prebuilt archive
         * 'sdist' - package is a source distribution
         * 'msi' - package is an MSI installer
         * 'egg' - package is an egg package

         * 'linux' - package works on Linux
         * 'windows' - package works on Windows
         * 'freebsd' - package works on FreeBSD
         * 'macosx' - package works on Mac OS X
         * 'anyos' - package works on any OS

         * 'x86' - Intel 32-bit code
         * 'x64' - Intel 64-bit code
         * 'ppc' - Power-PC 32-bit code
         * 'ppc64' - Power-PC 64-bit code
         * 'raspi' - Raspberry Pi (ARMv6) 32-bit code
         * 'purepython' - Pure Python code

         * 'pyX.X' - Python X.X compatible
         * 'ucs2' - Python UCS2 build compatible
         * 'ucs4' - Python UCS4 build compatible

         * 'source' - Source code
         * 'compiler' - C compiler needed

         * 'tar' - package is a tar file (may be compressed)
         * 'zip' - package is a zip file

        Raises a KeyError if no suitable package can be found.

    """
    # Determine platform parameters of the Python version that's
    # currently running
    (os_name,
     code_type,
     alt_code_types,
     python_version_tag,
     ucs_version_tag) = platform_tags()
    if _debug:
        print ('Running package matching algorithm on %r' % package_data)

    # Init: filter by required tags
    if required_tags:
        package_data = _subset_packages(package_data,
                                        required_tags)
        if _debug > 1:
            print ('filtered by required tags (%s): %r' %
                   (', '.join(required_tags), package_data))
        
    # First step: filter by OS
    os_packages = _filter_packages(package_data, os_name)
    if _debug > 1:
        print ('filtered to OS (%s): %r' %
               (os_name, os_packages))
    if not os_packages:
        os_packages = _filter_packages(package_data, 'anyos')
        if _debug > 1:
            print ('filtered to OS (anyos): %r' %
                   (os_packages,))
    if not os_packages:
        if _debug > 1:
            print ('no suitable OS package found, '
                   'trying source code')
        return _find_source_package(package_data)

    # Next: filter by code type
    code_packages = _filter_packages(os_packages, code_type)
    if _debug > 1:
        print ('filtered to code type (%s): %r' %
               (code_type, code_packages))
    if not code_packages:
        code_packages = _filter_packages(os_packages, 'purepython')
        if _debug > 1:
            print ('filtered to code type (purepython): %r' %
                   (code_packages,))
    if not code_packages:
        if _debug > 1:
            print ('no suitable code type package found, '
                   'trying source code')
        return _find_source_package(package_data)

    # Next: filter by Python version
    python_packages = _filter_packages(code_packages, python_version_tag)
    if _debug > 1:
        print ('filtered to Python version (%s): %r' %
               (python_version_tag, python_packages))
    if not python_packages:
        if _debug > 1:
            print ('no suitable Python version package found, '
                   'trying source code')
        return _find_source_package(package_data)

    # Next: filter by UCS version, if needed
    if ucs_version_tag:
        ucs2_python_packages = _filter_packages(python_packages, 'ucs2')
        ucs4_python_packages = _filter_packages(python_packages, 'ucs4')
        if not ucs2_python_packages and not ucs4_python_packages:
            # No restriction
            pass
        else:
            if ucs_version_tag == 'ucs4':
                python_packages = ucs4_python_packages
            else:
                python_packages = ucs2_python_packages
            if _debug > 1:
                print ('filtered to UCS version (%s): %r' %
                       (ucs_version_tag, python_packages))
            if not python_packages:
                if _debug > 1:
                    print ('no suitable Python version package found, '
                           'trying source code')
                return _find_source_package(package_data)

    if not python_packages:
        if _debug:
            print ('no matching package found')
        raise KeyError(
            'No suitable package found for the current platform')

    # Choose one of the available packages (by sorting and picking the
    # last one to make the choice deterministic)
    package_list = sorted(python_packages.iteritems())
    package = package_list[-1]
    if _debug:
        print ('found binary package: %r' % (package,))
    return package[0]

def _test_package_data_files():
    
    example_package_data = {
        'egenix-mx-base-3.2.7-py2.7_ucs2-linux-x86_64-prebuilt.zip': [
            'linux', 'x64', 'py2.7', 'ucs2'],
        'egenix-mx-base-3.2.7-py2.7_ucs4-linux-x86_64-prebuilt.zip': [
            'linux', 'x64', 'py2.7', 'ucs4'],
        'egenix-mx-base-3.2.7-py2.6_ucs2-linux-x86_64-prebuilt.zip': [
            'linux', 'x64', 'py2.6', 'ucs2'],
        'egenix-mx-base-3.2.7-py2.6_ucs4-linux-x86_64-prebuilt.zip': [
            'linux', 'x64', 'py2.6', 'ucs4'],
        'egenix-mx-base-3.2.7-py2.7_ucs2-linux-i686-prebuilt.zip': [
            'linux', 'x86', 'py2.7', 'ucs2'],
        'egenix-mx-base-3.2.7-py2.7_ucs4-linux-i686-prebuilt.zip': [
            'linux', 'x86', 'py2.7', 'ucs4'],
        'egenix-mx-base-3.2.7-py2.6_ucs2-linux-i686-prebuilt.zip': [
            'linux', 'x86', 'py2.6', 'ucs2'],
        'egenix-mx-base-3.2.7-py2.6_ucs4-linux-i686-prebuilt.zip': [
            'linux', 'x86', 'py2.6', 'ucs4'],
        'egenix-mx-base-3.2.7.zip': [
            'source', 'compiler'],
        }

    test_file = 'test-package-data.tags'
    if os.path.exists(test_file):
        os.remove(test_file)
    
    write_package_data(test_file, example_package_data)
    data = read_package_data(test_file)
    assert data == example_package_data, (data, example_package_data)
    os.remove(test_file)

    data['addthis'] = ['linux', 'x86']
    write_package_data(test_file, data)
    read_data = read_package_data(test_file)
    assert read_data == data, (read_data, data)
    os.remove(test_file)

    data = {}
    data['addmore'] = ['linux', 'x86']
    write_package_data(test_file, data)
    read_data = read_package_data(test_file)
    assert read_data == data, (read_data, data)
    os.remove(test_file)

    write_package_data(
        test_file, example_package_data,
        base_url='https://downloads.egenix.com/python/')
    data = read_package_data(
        test_file,
        remove_base_url='https://downloads.egenix.com/python/')
    assert data == example_package_data, (data, example_package_data)
    os.remove(test_file)

# Tests
if 0:
    _test_package_data_files()
    sys.exit(0)

#
# Mixin helpers
#

class CompilerSupportMixin:

    """ Compiler support mixin which makes sure that the .compiler
        attribute is properly setup.
    
    """
    # Internal flag
    prepared_compiler = 0

    def prepare_compiler(self):

        # Setup .compiler instance
        compiler = self._get_compiler_object()
        if compiler is None:
            if hasattr(self, '_check_compiler'):
                # The config command has this method to setup the
                # compiler
                self._check_compiler()
            else:
                raise CCompilerError('no C compiler setup; cannot continue')
            compiler = self._get_compiler_object()
            
        elif self.prepared_compiler:
            # Return the already prepared compiler
            return compiler
        
        # Work around a bug in distutils <= 1.0.3
        if compiler.exe_extension is None:
            compiler.exe_extension = ''

        # Make sure we have a typical setup for directory
        # searches
        for dir in LIBPATH:
            add_dir(dir, compiler.library_dirs)
        for dir in INCLPATH:
            add_dir(dir, compiler.include_dirs)

        # Add the Python include dir
        add_dir(get_python_include_dir(), compiler.include_dirs)

        # Customize the compiler according to system settings
        customize_compiler(compiler)

        log.info('configured compiler')
        self.prepared_compiler = 1

        return compiler

    def reset_compiler(self):

        """ Reset a possibly already prepared compiler.

            The compiler will have to be prepared again using
            .prepared_compiler().

        """
        self._set_compiler_object(None)
        self.prepared_compiler = 0

    def _get_compiler_object(self):

        """ Get the command's compiler object.

        """
        # See the change and discussion for
        # http://bugs.python.org/issue6377, new in Python 2.7
        # The change was later reverted, so this probably never
        # triggers. See http://bugs.python.org/issue13994
        if hasattr(self, 'compiler_obj'):
            return self.compiler_obj
        else:
            return self.compiler

    def _set_compiler_object(self, compiler):

        """ Set the command's compiler object.

        """
        # See the change and discussion for
        # http://bugs.python.org/issue6377, new in Python 2.7
        # The change was later reverted, so this probably never
        # triggers. See http://bugs.python.org/issue13994
        if hasattr(self, 'compiler_obj'):
            self.compiler_obj = compiler
        else:
            self.compiler = compiler

#
# mx Auto-Configuration command
#

class mx_autoconf(CompilerSupportMixin,
                  config):

    """ Auto-configuration class which adds some extra configuration
        settings to the packages.

    """
    # Command description
    description = "auto-configuration build step (for internal use only)"

    # Command line options
    user_options = config.user_options + [
        ('enable-debugging', None,
         'compile with debugging support (env var: MX_ENABLE_DEBUGGING)'),
        ('platform-id', None,
         'set MX_PLATFORM_ID (env var: MX_PLATFORM_ID)'),
        ('defines=', None,
         'define C macros (example: A=1,B=2,C,D)'),
        ('undefs=', None,
         'define C macros (example: A,B,C)'),
        ]

    # User option defaults
    enable_debugging = 0
    platform_id = None
    defines = None
    undefs = None

    # C APIs to check: (C API name, list of header files to include)
    api_checks = (
        ('strftime', ['time.h']),
        ('strptime', ['time.h']),
        ('timegm', ['time.h']),
        ('clock_gettime', ['time.h']),
        ('clock_getres', ['time.h']),
        #('this_always_fails', []), # For testing the detection mechanism
        )

    def initialize_options(self):

        config.initialize_options(self)
        self.noisy = 0
        self.dump_source = 0

        if not self.enable_debugging:
            # Enable debugging for dev snapshots
            version = self.distribution.metadata.get_version()
            if has_substring('dev', version):
                self.enable_debugging = 1
                log.warn('debugging enabled for development snapshots')

        if not self.enable_debugging:
            # Enable debugging via env variable MX_ENABLE_DEBUGGING
            if get_env_var('MX_ENABLE_DEBUGGING', default=0, yesno_value=1):
                self.enable_debugging = 1
                log.warn('debugging enabled via '
                         'MX_ENABLE_DEBUGGING environment variable')

        if not self.platform_id:
            # Set platform id via env variable MX_PLATFORM_ID
            platform_id = get_env_var('MX_PLATFORM_ID', default=None)
            if platform_id is not None:
                self.platform_id = platform_id
                log.warn('MX_PLATFORM_ID set to %r (via env var)' %
                         platform_id)

    def finalize_options(self):

        config.finalize_options(self)
        
        if self.defines:
            defines = []
            for defstr in self.defines.split(','):
                defstr = defstr.strip()
                if '=' in defstr:
                    defname, defvalue = defstr.split('=')
                    defname = defname.strip()
                    defvalue = defvalue.strip()
                else:
                    defname = defstr
                    defvalue = '1'
                defines.append((defname, defvalue))
            self.defines = defines
        else:
            self.defines = []

        if self.undefs:
            undefs = []
            for undefstr in self.undefs.split(','):
                undefname = undefstr.strip()
                undefs.append(undefname)
            self.undefs = undefs
        else:
            self.undefs = []

    def get_outputs(self):

        """ We don't generate any output.

        """
        return []

    def run(self):

        # Setup compiler
        compiler = self.prepare_compiler()        

        # Check compiler setup
        log.info('checking compiler setup')
        if not self.compiler_available():
            if sys.platform == 'darwin':
                # On Mac OS X, Apple removed the GCC compiler from
                # Xcode 4.1, but the Python installers are still
                # compiled with GCC, so distutils will default to
                # looking for GCC (see Python issue #13590). We'll try
                # clang as fallback solution.
                cc, cxx, ldshared = get_config_vars('CC', 'CXX', 'LDSHARED')
                log.info('compiler problem: "%s" not available, trying '
                         '"clang" instead' % cc)
                if 'CC' not in os.environ:
                    os.environ['CC'] = 'clang'
                if 'CXX' not in os.environ:
                    os.environ['CXX'] = 'clang'
                if 'LDSHARED' not in os.environ:
                    if ldshared.startswith(cc):
                        ldshared = 'clang ' + ldshared[len(cc):]
                    os.environ['LDSHARED'] = ldshared
                self.reset_compiler()
                compiler = self.prepare_compiler()
                if self.compiler_available():
                    log.info('using "clang" instead of "%s"' % cc)
                else:
                    log.info('no working compiler found; '
                             'please install Xcode first')
                    raise CCompilerError('no compiler available')
            else:
                log.error('compiler setup does not work or no compiler found; '
                          'try adjusting the CC/LDSHARED environemnt variables '
                          'to point to the installed compiler')
                raise CCompilerError('no compiler available')
        log.info('compiler setup works')

        # Add some static #defines which should not hurt
        compiler.define_macro('_GNU_SOURCE', '1')

        # Parse [py]config.h
        config_h = get_config_h_filename()
        try:
            configfile = open(config_h)
        except IOError,why:
            log.warn('could not open %s file' % config_h)
            configuration = {}
        else:
            configuration = parse_config_h(configfile)
            configfile.close()

        # Tweak configuration a little for some platforms
        if sys.platform[:5] == 'win32':
            configuration['HAVE_STRFTIME'] = 1

        # Build lists of #defines and #undefs
        define = []
        undef = []

        # Check APIs
        for apiname, includefiles in self.api_checks:
            macro_name = 'HAVE_' + apiname.upper()
            if not configuration.has_key(macro_name):
                log.info('checking for availability of %s() '
                         '(errors from this can safely be ignored)' % apiname)
                if self.check_function(apiname, includefiles):
                    log.info('%s() is available on this platform' %
                             apiname)
                    define.append((macro_name, '1'))
                else:
                    log.info('%s() is not available on this platform' %
                             apiname)
                    undef.append(macro_name)

        # Compiler tests
        if not configuration.has_key('BAD_STATIC_FORWARD'):
            log.info('checking compiler for bad static forward handling '
                     '(errors from this can safely be ignored)')
            if self.check_bad_staticforward():
                log.info('compiler has problems with static forwards '
                         '- enabling work-around')
                define.append(('BAD_STATIC_FORWARD', '1'))

        # Enable debugging support
        if self.enable_debugging:
            log.info('enabling mx debug support')
            define.append(('MAL_DEBUG', None))

        # Set platform id
        if self.platform_id:
            log.info('setting mx platform id to %r' % self.platform_id)
            define.append(('MX_PLATFORM_ID',
                           quote_string_define_value(self.platform_id)))

        # Add extra #defines and #undefs
        if self.defines:
            define.extend(self.defines)
        if self.undefs:
            undef.extend(self.undefs)

        log.info('macros to define: %s' % define)
        log.info('macros to undefine: %s' % undef)

        # Reinitialize build_ext command with extra defines
        build_ext = self.distribution.reinitialize_command('build_ext')
        build_ext.ensure_finalized()
        # We set these here, since distutils 1.0.2 introduced a
        # new incompatible way to process .define and .undef
        if build_ext.define:
            #print repr(build_ext.define)
            if type(build_ext.define) is types.StringType:
                # distutils < 1.0.2 needs this:
                l = build_ext.define.split(',')
                build_ext.define = map(lambda symbol: (symbol, '1'), l)
            build_ext.define = build_ext.define + define
        else:
            build_ext.define = define
        if build_ext.undef:
            #print repr(build_ext.undef)
            if type(build_ext.undef) is types.StringType:
                # distutils < 1.0.2 needs this:
                build_ext.undef = build_ext.undef.split(',')
            build_ext.undef = build_ext.undef + undef
        else:
            build_ext.undef = undef
        log.debug('updated build_ext with autoconf setup')

    def compiler_available(self):

        """ Check whether the compiler works.

            Return 1/0 depending on whether the compiler can produce
            code or not.
        
        """
        body = "int main (void) { return 0; }"
        return self.check_compiler(body)

    def check_compiler(self, sourcecode, headers=None, include_dirs=None,
                       libraries=None, library_dirs=None):

        """ Check whether sourcecode compiles and links with the current
            compiler and link environment.

            For documentation of the other arguments see the base
            class' .try_link().
        
        """
        self.prepare_compiler()
        return self.try_link(sourcecode, headers, include_dirs,
                             libraries, library_dirs)

    def check_bad_staticforward(self):

        """ Check whether the compiler does not supports forward declaring
            static arrays.

            For documentation of the other arguments see the base
            class' .try_link().
        
        """
        body = """
        typedef struct _mxstruct {int a; int b;} mxstruct;
        staticforward mxstruct mxarray[];
        statichere mxstruct mxarray[] = {{0,2},{3,4},};
        int main(void) {return mxarray[0].a;}
        """
        self.prepare_compiler()
        return not self.try_compile(body,
                                    headers=('Python.h',),
                                    include_dirs=[get_python_include_dir()])

    def check_function(self, function, headers=None, include_dirs=None,
                       libraries=None, library_dirs=None,
                       prototype=0, call=0):

        """ Check whether function is available in the given
            compile and link environment.

            If prototype is true, a function prototype is included in
            the test. If call is true, a function call is generated
            (rather than just a reference of the function symbol).

            For documentation of the other arguments see the base
            class' .try_link().
        
        """
        body = []
        if prototype:
            body.append("int %s (void);" % function)
        body.append("int main (void) {\n"
                    "  void *ptr = 0; ")
        if call:
            body.append("  %s();" % function)
        else:
            body.append("  ptr = &%s;" % function)
        body.append("return 0; }")
        body = "\n".join(body) + "\n"
        return self.check_compiler(body, headers, include_dirs,
                                   libraries, library_dirs)

    def check_library(self, library, library_dirs=None,
                      headers=None, include_dirs=None, other_libraries=[]):

        """ Check whether we can link against the given library.

            For documentation of the other arguments see the base
            class' .try_link().
        
        """
        body = "int main (void) { return 0; }"
        return self.check_compiler(body, headers, include_dirs,
                                   [library]+other_libraries, library_dirs)

    def find_include_file(self, filename, paths, pattern=None):

        """ Find an include file of the given name.

            Returns a tuple (directory, filename) or (None, None) in
            case no library can be found.

            The search path is determined by the paths parameter, the
            compiler's .include_dirs attribute and the STDINCLPATH and
            FINDINCLPATH globals. The search is done in this order.

        """
        compiler = self.prepare_compiler()
        paths = (paths
                 + compiler.include_dirs
                 + STDINCLPATH
                 + FINDINCLPATH)
        verify_path(paths)
        if _debug:
            print 'INCLPATH', paths
        dir = find_file(filename, paths, pattern)
        if dir is None:
            return (None, None)
        return (dir, os.path.join(dir, filename))

    def find_library_file(self, libname, paths, pattern=None,
                          lib_types=LIB_TYPES):

        """ Find a library of the given name.

            Returns a tuple (directory, filename) or (None, None) in
            case no library can be found.

            The search path is determined by the paths parameter, the
            compiler's .library_dirs attribute and the STDLIBPATH and
            FINDLIBPATH globals. The search is done in this order.

            Shared libraries are prefered over static ones if both
            types are given in lib_types.

        """
        compiler = self.prepare_compiler()
        paths = (paths
                 + compiler.library_dirs
                 + STDLIBPATH
                 + FINDLIBPATH)
        verify_path(paths)
        if _debug:
            print 'LIBPATH: %r' % paths
            print 'Library types: %r' % lib_types

        # Try to first find a shared library to use and revert
        # to a static one, if no shared lib can be found
        for lib_type in lib_types:
            filename = compiler.library_filename(libname,
                                                 lib_type=lib_type)
            if _debug:
                print 'looking for library file %s' % filename
            dir = find_file(filename, paths, pattern)
            if dir is not None:
                return (dir, os.path.join(dir, filename))
            
        return (None, None)

#
# mx MSVC Compiler extension
#
# We want some extra options for the MSVCCompiler, so we add them
# here. This is an awful hack, but there seems to be no other way to
# subclass a standard distutils C compiler class...

if python_version < '2.4':
    # VC6
    MSVC_COMPILER_FLAGS = ['/O2', '/Gf', '/GB', '/GD', '/Ob2']
elif python_version < '2.6':
    # VC7.1
    MSVC_COMPILER_FLAGS = ['/O2', '/GF', '/GB', '/Ob2']
else:
    # VC9
    MSVC_COMPILER_FLAGS = ['/O2', '/GF', '/Ob2']

if hasattr(MSVCCompiler, 'initialize'):
    # distutils 2.5.0 separates the initialization of the
    # .compile_options out into a new method .initialize()

    # remember old _initialize
    old_MSVCCompiler_initialize = MSVCCompiler.initialize

    def mx_msvccompiler_initialize(self, *args, **kws):

        apply(old_MSVCCompiler_initialize, (self,) + args, kws)

        # Add our extra options
        self.compile_options.extend(MSVC_COMPILER_FLAGS)

    # "Install" new initialize
    MSVCCompiler.initialize = mx_msvccompiler_initialize

else:
    # distutils prior to 2.5.0 only allow to hook into the .__init__()
    # method

    # remember old __init__
    old_MSVCCompiler__init__ = MSVCCompiler.__init__

    def mx_msvccompiler__init__(self, *args, **kws):

        apply(old_MSVCCompiler__init__, (self,) + args, kws)

        # distutils 2.5.0 separates the initialization of the
        # .compile_options out into a new method
        if hasattr(self, 'initialized') and not self.initialized:
            self.initialize()

        # Add out extra options
        self.compile_options.extend(MSVC_COMPILER_FLAGS)

    # "Install" new __init__
    MSVCCompiler.__init__ = mx_msvccompiler__init__

#
# mx Distribution class
#

class mx_Distribution(Distribution):

    """ Distribution class which knows about our distutils extensions.
        
    """
    # List of UnixLibrary instances
    unixlibs = None

    # Option to override the package version number
    set_version = None

    # List of setuptools namespace package names
    namespace_packages = None

    # List of setuptools dependency links
    dependency_links = None

    # Add classifiers dummy option if needed
    display_options = Distribution.display_options[:]
    display_option_names = Distribution.display_option_names[:]
    if 'classifiers' not in display_options:
        display_options = display_options + [
            ('classifiers', None,
             "print the list of classifiers (not yet supported)"),
        ]
        display_option_names = display_option_names + [
            'classifiers'
            ]

    # Add set-version option
    global_options = Distribution.global_options[:]
    global_options = global_options + [
        ('set-version=', None, "override the package version number"),
        ]

    def finalize_options(self):

        if self.namespace_packages is None:
            self.namespace_packages = []
        if self.dependency_links is None:
            self.dependency_links = []

        # Call base method
        Distribution.finalize_options(self)

    def parse_command_line(self):

        if not Distribution.parse_command_line(self):
            return

        # Override the version information from the package with a
        # command-line given version string
        if self.set_version is not None:
            self.metadata.version = self.set_version
            self.version = self.set_version

        return 1

    def has_unixlibs(self):
        return self.unixlibs and len(self.unixlibs) > 0

    def get_namespace_packages(self):
        return self.namespace_packages or []

    def get_dependency_links(self):
        return self.dependency_links or []

#
# mx Extension class
#

class mx_Extension(Extension):

    """ Extension class which allows specifying whether the extension
        is required to build or optional.
        
    """
    # Is this Extension required to build or can we issue a Warning in
    # case it fails to build and continue with the remaining build
    # process ?
    required = 1

    # Warn, if an optional extension cannot be built ?
    warn = 1

    # List of optional libaries (libname, list of header files to
    # check) to include in the link step; the availability of these is
    # tested prior to compiling the extension. If found, the symbol
    # HAVE_<upper(libname)>_LIB is defined and the library included in
    # the list of libraries to link against.
    optional_libraries = ()

    # List of needed include files in form of tuples (filename,
    # [dir1, dir2,...], pattern); see mx_autoconf.find_file()
    # for details
    needed_includes = ()

    # List of needed include files in form of tuples (libname,
    # [dir1, dir2,...], pattern); see mx_autoconf.find_library_file()
    # for details
    needed_libraries = ()

    # Include the found library files in the extension output ?  This
    # causes the files to be copied into the same location as the
    # extension itself.
    include_needed_libraries = 0

    # Library types to check (for needed libraries)
    lib_types = LIB_TYPES

    # Data files needed by this extension (these are only
    # installed if building the extension succeeded)
    data_files = ()

    # Python packages needed by this extension (these are only
    # installed if building the extension succeeded)
    packages = ()

    # Building success marker
    successfully_built = 0

    # NOTE: If you add new features to this class, also adjust
    # rebase_extensions()

    def __init__(self, *args, **kws):
        for attr in ('required',
                     'warn',
                     'lib_types',
                     'optional_libraries',
                     'needed_includes',
                     'needed_libraries',
                     'include_needed_libraries',
                     'data_files',
                     'packages'):
            if kws.has_key(attr):
                setattr(self, attr, kws[attr])
                del kws[attr]
            else:
                value = getattr(self, attr)
                if type(value) == type(()):
                    setattr(self, attr, list(value))
        apply(Extension.__init__, (self,) + args, kws)

#
# mx Build command
#

class mx_build(build):

    """ build command which knows about our distutils extensions.

        This build command builds extensions in properly separated
        directories (which includes building different Unicode
        variants in different directories).
        
    """
    # Skip the build process ?
    skip = None

    # Assume prebuilt archive ?
    prebuilt = None
   
    user_options = build.user_options + [
            ('skip', None,
             'skip build and reuse the existing build files'),
            ('prebuilt', None,
             'assume we have a prebuilt archive (even without %s file)' %
             PREBUILT_MARKER),
            ]

    def finalize_options(self):

        # Make sure different Python versions are built in separate
        # directories
        python_platform = '.%s-%s' % (mx_get_platform(), py_version())
        if self.build_platlib is None:
            self.build_platlib = os.path.join(
                self.build_base,
                'lib' + python_platform)
        if self.build_purelib is None:
            self.build_purelib = os.path.join(
                self.build_base,
                'lib.' + py_version(unicode_aware=0))
        if self.build_temp is None:
            self.build_temp = os.path.join(
                self.build_base,
                'temp' + python_platform)

        # Call the base method
        build.finalize_options(self)

        if self.skip is None:
            self.skip = 0
            
        # Handle prebuilt archives
        if os.path.exists(PREBUILT_MARKER):
            self.prebuilt = 1
        elif self.prebuilt is None:
            self.prebuilt = 0

        if self.prebuilt:
            if not self.have_build_pickle():
                # Either the build pickle is missing or not compatible
                # with the currently running Python interpreter.  Read
                # setup information from the build pickle; the path to
                # this file is stored in the PREBUILT_MARKER file.
                if os.path.exists(PREBUILT_MARKER):
                    build_pickle_pathname = (
                        open(PREBUILT_MARKER, 'rb').read().strip())
                    build_pickle = self.read_build_pickle(
                        build_pickle_pathname)
                    mxSetup = build_pickle.get('mxSetup', {})
                else:
                    mxSetup = {}
                    
                print """
--------------------------------------------------------------------

ERROR: Cannot find the build information needed for your platform.
                
Please check that you have downloaded the right prebuilt archive for
your platform and Python version.

Product name:        %s
Product version:     %s
    
Your Python installation uses these settings:

    Python version:  %s
    Platform:        %s

The prebuilt archive was built for:

    Python version:  %s
    Platform:        %s

--------------------------------------------------------------------
                """.strip() % (
                self.distribution.metadata.get_name(),
                self.distribution.metadata.get_version(),
                py_version(unicode_aware=1),
                mx_get_platform(),
                mxSetup.get('py_version', 'unknown'),
                mxSetup.get('get_platform', 'unknown'))
                sys.exit(1)
            if self.force:
                log.info('prebuilt archive found: ignoring the --force option')
                self.force = 0
                
            # Override settings with data from a possibly existing
            # build pickle
            log.info('prebuilt archive found: skipping the build process and '
                     'reusing the existing build files and data')
            self.load_build_pickle()

            # Make sure loading the pickle does not overwrite the
            # .prebuilt flag
            self.prebuilt = 1
            self.skip = 1

        # Use the build pickle, in case we are supposed to skip the
        # build
        elif self.skip and self.have_build_pickle():
            log.info('skipping the build process and '
                     'reusing the existing build files and data')
            self.load_build_pickle()

            # Make sure loading the pickle did not overwrite the .skip
            # flag
            self.skip = 1

    def get_outputs(self):

        """ Collect the outputs of all sub-commands (this knows about
            the extra commands we added).

            This is needed by mx_bdist_prebuilt and mx_uninstall.

        """
        outputs = {}
        for sub_command in self.get_sub_commands():
            cmd = self.get_finalized_command(sub_command)
            if not hasattr(cmd, 'get_outputs'):
                log.error('problem: missing .get_outputs() ... %r' % cmd)
                raise ValueError('missing .get_outputs() implementation '
                                 'for command %r' % cmd)
            for filename in cmd.get_outputs():
                outputs[filename] = 1

        # Add pickle, if available
        pickle_filename = self.get_build_pickle_pathname()
        if os.path.exists(pickle_filename):
            if _debug:
                print 'added pickle:', pickle_filename
            outputs[pickle_filename] = 1

        # Return a sorted list
        outputs = outputs.keys()
        outputs.sort()
        return outputs

    def pure_python_build(self):

        """ Return 1/0 depending on whether this is a pure Python
            build or not.

            Pure Python builds do not include extensions.

        """
        return not self.distribution.ext_modules

    def get_build_pickle_pathname(self):

        """ Return the path name for the build pickle file.

            Note that the target system loading these pickles may well
            return different values for get_platform() than the system
            used to compile the build.

            We therefore do not include the platform in the pathname,
            only the Python version (to prevent obvious user errors
            related to downloading the wrong prebuilt archive for
            their Python version).

            For pure Python distributions (ones without extensions),
            we also don't include the Unicode tag in the pickle name.

        """
        unicode_aware = not self.pure_python_build()
        return os.path.join(
            self.build_base,
            'build-py%s.pck' % (py_version(unicode_aware=unicode_aware)))

    def write_build_pickle(self, pathname=None):

        """ Write the current state of the distribution, the
            build command and all sub-commands to a Python
            pickle in the build directory.

            If pathname is not given, .get_build_pickle_pathname() is
            used as default.

        """
        if pathname is None:
            pathname = self.get_build_pickle_pathname()

        # Prepare the sub commands for pickle'ing
        self.get_outputs()

        # Remove data that would cause conflicts when restoring the
        # build pickle
        data = self.__dict__.copy()
        if data.has_key('distribution'):
            del data['distribution']
        if data.has_key('compiler'):
            del data['compiler']
        state = {'build': data}
        for sub_command, needed in self.sub_commands:
            cmd = self.distribution.get_command_obj(sub_command)
            data = cmd.__dict__.copy()
            if data.has_key('distribution'):
                del data['distribution']
            if data.has_key('compiler'):
                del data['compiler']
            if data.has_key('extensions'):
                # Convert mx_Extension instances into instance dicts;
                # this is needed to make easy_install happy
                extensions = data['extensions']
                data['extensions'] = [
                    ext.__dict__.copy()
                    for ext in extensions]
            if data.has_key('autoconf'):
                del data['autoconf']
            state[sub_command] = data
        data = {'have_run': self.distribution.have_run,
                'data_files': self.distribution.data_files,
                }
        state['distribution'] = data
        if 0:
            data = self.distribution.__dict__.copy()
            if data.has_key('distribution'):
                del data['distribution']
            if data.has_key('metadata'):
                del data['metadata']
            if data.has_key('ext_modules'):
                del data['ext_modules']
            if data.has_key('command_obj'):
                del data['command_obj']
            if data.has_key('cmdclass'):
                del data['cmdclass']
            for key, value in data.items():
                if type(value) is type(self.distribution.get_url):
                    # Bound method
                    del data[key]
                elif type(value) is type(self.distribution):
                    # Instance
                    del data[key]
            state['distribution'] = data


        # Save additional meta-data
        pure_python_build = self.pure_python_build()
        unicode_aware = not pure_python_build
        state['mxSetup'] = {
            '__version__': __version__,
            'unicode_aware': unicode_aware, 
            'py_version': py_version(unicode_aware=unicode_aware),
            'sys_platform': sys.platform,
            'get_name': self.distribution.metadata.get_name(),
            'get_version': self.distribution.metadata.get_version(),
            'get_platform': mx_get_platform(),
            'pure_python_build': pure_python_build,
            }

        # Save pickle
        if _debug:
            print ('saving build pickle: %s' % repr(state))
        pickle_file = open(self.get_build_pickle_pathname(),
                           'wb')
        cPickle.dump(state, pickle_file)
        pickle_file.close()

    def have_build_pickle(self, pathname=None,
                          ignore_distutils_platform=True,
                          ignore_distribution_version=True,
                          ignore_sys_platform=None):

        """ Return 1/0 depending on whether there is a build pickle
            that could be used or not.

            If pathname is not given, .get_build_pickle_pathname() is
            used as default.

            If ignore_platform is set (default), the platform
            information stored in the pickle is ignored in the check.
            This is useful, since the value of the build system may
            differ from the target system (e.g. for fat builds on Mac
            OS X that get installed on Intel Macs).

            If ignore_distribution_version is set (default), the
            distribution version information in the pickle is ignored.
            This is useful for cases where the version determined at
            build time can be different than at installation time,
            e.g. due to a timestamp being created dynamically and
            added to the version.

            If ignore_sys_platform is set (default is false for builds
            with C extensions and true for pure Python builds), the
            sys.platform version information in the pickle is ignored.

        """
        if pathname is None:
            pathname = self.get_build_pickle_pathname()
        try:
            pickle_file = open(pathname, 'rb')
        except IOError:
            log.info('no build data file %r found' % pathname)
            return 0
        state = cPickle.load(pickle_file)
        pickle_file.close()

        # Check whether this is a valid build file for this Python
        # version
        mxSetup = state.get('mxSetup', None)
        if mxSetup is None:
            return 0
        unicode_aware = mxSetup.get('unicode_aware', 1)
        pure_python_build = mxSetup.get('pure_python_build', 0)
        if ignore_sys_platform is None:
            if pure_python_build:
                if mxSetup['sys_platform'].startswith('win'):
                    # Prebuilt archives built on win32 can only be
                    # installed there, since the os.sep does not
                    # correspond to the distutils standard of '/'.
                    ignore_sys_platform = False
                else:
                    ignore_sys_platform = True
        if mxSetup['__version__'] != __version__:
            log.info('build data file %r found, '
                     'but mxSetup version does not match; not using it' %
                     pathname)
            return 0
        if mxSetup['py_version'] != py_version(unicode_aware=unicode_aware):
            log.info('build data file %r found, '
                     'but Python version does not match; not using it' %
                     pathname)
            return 0
        if ((not ignore_sys_platform) and
            mxSetup['sys_platform'] != sys.platform):
            log.info('build data file %r found, '
                     'but sys.platform does not match; not using it' %
                     pathname)
            return 0
        if mxSetup['get_name'] != self.distribution.metadata.get_name():
            log.info('build data file %r found, '
                     'but distribution name does not match; not using it' %
                     pathname)
            return 0
        if ((not ignore_distribution_version) and
            mxSetup['get_version'] != self.distribution.metadata.get_version()):
            log.info('build data file %r found, '
                     'but distribution version does not match; not using it' %
                     pathname)
            return 0
        if ((not ignore_distutils_platform) and
            mxSetup['get_platform'] != mx_get_platform()):
            log.info('build data file %r found, '
                     'but distutils platform does not match; not using it' %
                     pathname)
            return 0

        log.info('found usable build data file %r' % pathname)
        return 1

    def read_build_pickle(self, pathname=None):

        """ Read the pickle written by the .write_build_pickle() method.

            If pathname is not given, .get_build_pickle_pathname() is
            used as default.

        """
        if pathname is None:
            pathname = self.get_build_pickle_pathname()
        pickle_file = open(pathname, 'rb')
        state = cPickle.load(pickle_file)
        pickle_file.close()
        if _debug:
            print 'read build pickle:'
            import pprint
            pprint.pprint(state)
        return state

    def load_build_pickle(self):

        """ Restore the state written by the .write_build_pickle() method.

        """
        # Read pickle file
        state = self.read_build_pickle()

        # Adjust distutils platform string, if needed
        platform = state['mxSetup'].get('get_platform', None)
        if platform is not None:
            log.info('setting platform to %r' % platform)
            mx_set_platform(platform)
        
        log.info('restoring build data from a previous build run')
        for sub_command, data in state.items():
            if _debug:
                print 'restoring build data for command %r' % sub_command
            if sub_command == 'mxSetup':
                self.distribution.metadata.version = data['get_version']
                self.distribution.metadata.name = data['get_name']
            elif sub_command == 'build':
                for key, value in data.items():
                    self.__dict__[key] = value
            elif sub_command == 'distribution':
                for key, value in data.items():
                    self.distribution.__dict__[key] = value
            else:
                cmd = self.distribution.get_command_obj(sub_command)
                for key, value in data.items():
                    cmd.__dict__[key] = value
                if sub_command == 'build_ext':
                    if cmd.extensions:
                        # Convert Extension dicts back into mx_Extension
                        # instances
                        extensions_dicts = cmd.extensions
                        cmd.extensions = []
                        class Ext: pass
                        for ext_dict in extensions_dicts:
                            ext = Ext()
                            ext.__dict__.update(ext_dict)
                            ext.__class__ = mx_Extension
                            cmd.extensions.append(ext)

    def run(self):

        # Check whether we need to skip the build process
        if self.skip:
            log.info('skipping the build process and '
                     'reusing existing build files')
            return

        # Run the build command
        build.run(self)

        # Save the build data in a build pickle for later reuse,
        # unless this is a prebuilt initiated run
        if not self.prebuilt:
            self.write_build_pickle()

    def has_unixlibs(self):
        return self.distribution.has_unixlibs()

    def has_data_files(self):
        return self.distribution.has_data_files()

    # Add new sub-commands:
    if len(build.sub_commands) > 4:
        raise DistutilsError, 'incompatible distutils version !'
    sub_commands = [('build_clib',    build.has_c_libraries),
                    ('build_unixlib', has_unixlibs),
                    ('mx_autoconf',   build.has_ext_modules),
                    ('build_ext',     build.has_ext_modules),
                    ('build_py',      build.has_pure_modules),
                    ('build_scripts', build.has_scripts),
                    ('build_data',    has_data_files),
                   ]

#
# mx Build C Lib command
#

class mx_build_clib(CompilerSupportMixin,
                    build_clib):

    """ build_clib command which builds the libs using
        separate temp dirs
        
    """
    # Lib of output files
    outfiles = None

    def finalize_options(self):
        
        build_clib.finalize_options(self)
        self.outfiles = []

    def build_library(self, lib_name, build_info):

        # Build each extension in its own subdir of build_temp (to
        # avoid accidental sharing of object files between extensions
        # having the same name, e.g. mxODBC.o).
        build_temp_base = self.build_temp
        self.build_temp = os.path.join(build_temp_base, lib_name)
        compiler = self.prepare_compiler()

        try:

            #
            # This is mostly identical to the original build_clib command.
            #
            sources = build_info.get('sources')
            if sources is None or \
               type(sources) not in (types.ListType, types.TupleType):
                raise DistutilsSetupError, \
                      ("in 'libraries' option (library '%s'), " +
                       "'sources' must be present and must be " +
                       "a list of source filenames") % lib_name
            sources = list(sources)

            log.info("building '%s' library" % lib_name)

            # First, compile the source code to object files in the
            # library directory.  (This should probably change to
            # putting object files in a temporary build directory.)
            macros = build_info.get('macros')
            include_dirs = build_info.get('include_dirs')
            objects = compiler.compile(sources,
                                       output_dir=self.build_temp,
                                       macros=macros,
                                       include_dirs=include_dirs,
                                       debug=self.debug)

            # Now "link" the object files together into a static library.
            # (On Unix at least, this isn't really linking -- it just
            # builds an archive.  Whatever.)
            compiler.create_static_lib(objects, lib_name,
                                       output_dir=self.build_clib,
                                       debug=self.debug)
            
            # Record the name of the library we just created
            self.outfiles.append(
                compiler.library_filename(lib_name,
                                          output_dir=self.build_clib))

        finally:
            # Cleanup local changes to the configuration
            self.build_temp = build_temp_base
        
    def build_libraries(self, libraries):

        for (lib_name, build_info) in libraries:
            self.build_library(lib_name, build_info)

    def get_outputs(self):

        """ Return a list of the created library files.

            This is needed by mx_bdist_prebuilt on all build commands
            and build_clib doesn't provide it.

        """
        return self.outfiles

#
# mx Build Extensions command
#

class mx_build_ext(CompilerSupportMixin,
                   build_ext):

    """ build_ext command which runs mx_autoconf command before
        trying to build anything.
        
    """
    user_options = build_ext.user_options + [
        ('disable-build=', None,
         'disable building an optional extensions (comma separated list of '
         'dotted package names); default is to try building all'),
        ('enable-build=', None,
         'if given, only these optional extensions are built (comma separated '
         'list of dotted package names)'),
        ]

    # mx_autoconf command object (finalized and run)
    autoconf = None

    # Default values for command line options
    disable_build = None
    enable_build = None

    # Extra output files
    extra_output = ()

    # Output files
    output_files = None
    
    def finalize_options(self):

        build_ext.finalize_options(self)
        if self.disable_build is None:
            self.disable_build = ()
        elif type(self.disable_build) is types.StringType:
            self.disable_build = [x.strip()
                                  for x in self.disable_build.split(',')]
        if type(self.enable_build) is types.StringType:
            self.enable_build = [x.strip()
                                 for x in self.enable_build.split(',')]
        self.extra_output = []

    def run(self):

        # Add unixlibs install-dirs to library_dirs, so that linking
        # against them becomes easy
        if self.distribution.has_unixlibs():
            build_unixlib = self.get_finalized_command('build_unixlib')
            paths, libs = build_unixlib.get_unixlib_lib_options()
            # Libraries have to be linked against by defining them
            # in the mx_Extension() setup, otherwise, all extensions
            # get linked against all Unix libs that were built...
            #self.libraries[:0] = libs
            self.library_dirs[:0] = paths
            
        # Assure that mx_autoconf has been run and store a reference
        # in .autoconf
        self.run_command('mx_autoconf')
        self.autoconf = self.get_finalized_command('mx_autoconf')

        # Now, continue with the standard build process
        build_ext.run(self)

    def build_extensions(self):

        # Make sure the compiler is setup correctly
        compiler = self.prepare_compiler()
        
        # Make sure that any autoconf actions use the same compiler
        # settings as we do (the .compiler is set up in build_ext.run()
        # just before calling .build_extensions())
        self.autoconf._set_compiler_object(compiler)

        # Build the extensions
        self.check_extensions_list(self.extensions)
        for ext in self.extensions:
            self.build_extension(ext)

        # Cleanup .extensions list (remove entries which did not build correctly)
        l = []
        for ext in self.extensions:
            if not isinstance(ext, mx_Extension):
                l.append(ext)
            else:
                if ext.successfully_built:
                    l.append(ext)
        self.extensions = l
        if _debug:
            print ('built these extensions: %s' % (
                ', '.join(ext.name for ext in self.extensions)))
        log.info('')
         
    def build_extension(self, ext):

        # Prevent duplicate builds of the same extension
        if ext.successfully_built and not self.force:
            log.info('reusing existing build of extension "%s"' % ext.name)
            return

        # Prepare compilation
        required = not hasattr(ext, 'required') or ext.required
        warn = not hasattr(ext, 'warn') or ext.warn
        log.info('')
        log.info('building extension "%s" %s' %
                 (ext.name,
                  required * '(required)' or '(optional)'))
        compiler = self.prepare_compiler()

        # Optional extension building can be adjusted via command line options
        if not required:
            if self.enable_build is not None and \
               ext.name not in self.enable_build:
                log.info('skipped -- build not enabled by command line option')
                return
            elif ext.name in self.disable_build:
                log.info('skipped -- build disabled by command line option')
                return

        # Search for include files
        if (isinstance(ext, mx_Extension) and \
            ext.needed_includes):
            log.info('looking for header files needed by extension '
                     '"%s"' % ext.name)
            for filename, dirs, pattern in ext.needed_includes:
                (dir, pathname) = self.autoconf.find_include_file(
                    filename,
                    dirs,
                    pattern)
                if dir is not None:
                    log.info('found needed include file "%s" '
                             'in directory %s' % (filename, dir))
                    if dir not in ext.include_dirs and \
                       dir not in STDINCLPATH and \
                       dir not in INCLPATH:
                        ext.include_dirs.append(dir)
                else:
                    if required:
                        raise CompileError(
                              'could not find needed header file "%s"' %
                              filename)
                    else:
                        if warn:
                            log.warn(
                                '*** WARNING: Building of extension '
                                '"%s" failed: needed include file "%s" '
                                'not found' %
                                (ext.name, filename))
                        else:
                            log.warn(
                                'optional extension "%s" could not be built: '
                                'needed include file "%s" not found' %
                                (ext.name, filename))
                        return
                    
        # Search for libraries
        if hasattr(ext, 'needed_libraries') and \
           ext.needed_libraries:
            log.info('looking for libraries needed by extension '
                     '"%s"' % ext.name)
            for libname, dirs, pattern in ext.needed_libraries:
                dir, pathname = self.autoconf.find_library_file(
                    libname,
                    dirs,
                    pattern=pattern,
                    lib_types=ext.lib_types)
                if dir is not None:
                    log.info('found needed library "%s" '
                             'in directory %s (%s)' % (libname,
                                                       dir,
                                                       pathname))
                    if 'shared' not in ext.lib_types:
                        # Force static linking and append the library
                        # pathname to the linker arguments
                        if libname in ext.libraries:
                            ext.libraries.remove(libname)
                        if not ext.extra_link_args:
                            ext.extra_link_args = []
                        ext.extra_link_args.append(
                            pathname)
                    else:
                        # Prefer dynamic linking
                        if dir not in ext.library_dirs and \
                           dir not in STDLIBPATH and \
                           dir not in LIBPATH:
                            ext.library_dirs.append(dir)
                        if libname not in ext.libraries:
                            ext.libraries.append(libname)
                    if ext.include_needed_libraries:
                        ext_package_dir = os.path.split(
                            self.get_ext_filename(
                            self.get_ext_fullname(ext.name)))[0]
                        # The linker will always link against the
                        # real name, not a symbolic name (XXX Hope this
                        # is true for all platforms)
                        realpathname = os.path.realpath(pathname)
                        realfilename = os.path.split(realpathname)[1]
                        # Copy the share lib to the package dir, using
                        # the real filename
                        data_entry = (realpathname,
                                      ext_package_dir + os.sep)
                        if data_entry not in ext.data_files:
                            if _debug:
                                print ('adding library data entry %r' %
                                       (data_entry,))
                            ext.data_files.append(data_entry)
                else:
                    if required:
                        raise CompileError(
                              'could not find needed library "%s"' %
                              libname)
                    else:
                        if warn:
                            log.warn(
                                '*** WARNING: Building of extension '
                                '"%s" failed: needed library "%s" '
                                'not found' %
                                (ext.name, libname))
                        else:
                            log.warn(
                                'optional extension "%s" could not be built: '
                                'needed library "%s" not found' %
                                (ext.name, libname))
                        return
                    
        # Build each extension in its own subdir of build_temp (to
        # avoid accidental sharing of object files between extensions
        # having the same name, e.g. mxODBC.o).
        build_temp_base = self.build_temp
        extpath = ext.name.replace('.', '-')
        self.build_temp = os.path.join(build_temp_base, extpath)

        # Check for availability of optional libs which can be used
        # by the extension; note: this step includes building small
        # object files to test for the availability of the libraries
        if isinstance(ext, mx_Extension) and \
           ext.optional_libraries:
            log.info("checking for optional libraries")
            for libname, headerfiles in ext.optional_libraries:
                if self.autoconf.check_library(libname, headers=headerfiles):
                    symbol = 'HAVE_%s_LIB' % libname.upper()
                    log.info("found optional library '%s'"
                             " -- defining %s" % (libname, symbol))
                    ext.libraries.append(libname)
                    ext.define_macros.append((symbol, '1'))
                else:
                    log.warn("could not find optional library '%s'"
                             " -- omitting it" % libname)

        if _debug:
            print 'Include dirs:', repr(ext.include_dirs +
                                        compiler.include_dirs)
            print 'Libary dirs:', repr(ext.library_dirs +
                                       compiler.library_dirs)
            print 'Libaries:', repr(ext.libraries)
            print 'Macros:', repr(ext.define_macros)

        # Build the extension
        successfully_built = 0
        try:
            
            # Skip extensions which cannot be built if the required
            # option is given and set to false.
            required = not hasattr(ext, 'required') or ext.required
            if required:
                build_ext.build_extension(self, ext)
                successfully_built = 1
            else:
                try:
                    build_ext.build_extension(self, ext)
                except (CCompilerError, DistutilsError), why:
                    if warn:
                        log.warn(
                            '*** WARNING: Building of extension "%s" '
                            'failed: %s' %
                            (ext.name, why))
                    else:
                        log.warn(
                            'optional extension "%s" '
                            'did not build correctly -- skipped' %
                            (ext.name,))
                else:
                    successfully_built = 1

        finally:
            # Cleanup local changes to the configuration
            self.build_temp = build_temp_base

        # Processing for successfully built extensions
        if successfully_built:

            if isinstance(ext, mx_Extension):
                # Add Python packages needed by this extension
                self.distribution.packages.extend(ext.packages)

                # Add data files needed by this extension
                self.distribution.data_files.extend(ext.data_files)

            # Mark as having been built successfully
            ext.successfully_built = 1

    def get_outputs(self):

        # Note: The cache is needed for mx_uninstall when used
        # together with mx_bdist_prebuilt. mx_build will run a
        # recursive .get_outputs() on all sub-commands and then store
        # the sub-command objects in the build pickle.  By using a
        # cache, we can avoid to have the command object to have to
        # rebuild the outputs (this may not be possible due to the
        # missing source files).
        if self.output_files is not None:
            return self.output_files
        else:
            files = build_ext.get_outputs(self)
            self.output_files = files
            return files

#
# mx Build Python command
#

class mx_build_py(build_py):

    """ build_py command which also allows removing Python source code
        after the byte-code compile process.
        
    """
    user_options = build_py.user_options + [
        ('without-source', None, "only include Python byte-code"),
        ]

    boolean_options = build_py.boolean_options + ['without-source']

    # Omit source files ?
    without_source = 0

    # Output cache
    output_files = None
    
    def run(self):

        if self.without_source:
            # --without-source implies byte-code --compile
            if (not self.compile and
                not self.optimize):
                self.compile = 1

        # Build the Python code
        build_py.run(self)

        # Optionally remove source code
        if self.without_source:
            log.info('removing Python source files (--without-source)')
            verbose = self.verbose
            dry_run = self.dry_run
            for file in build_py.get_outputs(self, include_bytecode=0):
                # Only process .py files
                if file[-3:] != '.py':
                    continue
                # Remove source code
                execute(os.remove, (file,), "removing %s" % file,
                        verbose=verbose, dry_run=dry_run)
                # Remove .pyc files (if not requested)
                if not self.compile:
                    filename = file + "c"
                    if os.path.isfile(filename):
                        execute(os.remove, (filename,),
                                "removing %s" % filename,
                                verbose=verbose, dry_run=dry_run)
                # Remove .pyo files (if not requested)
                if self.optimize == 0:
                    filename = file + "o"
                    if os.path.isfile(filename):
                        execute(os.remove, (filename,),
                                "removing %s" % filename,
                                verbose=verbose, dry_run=dry_run)

    def get_outputs(self, include_bytecode=1):

        # Note: The cache is needed for mx_uninstall when used
        # together with mx_bdist_prebuilt. See
        # mx_build_ext.get_outputs() for details.

        # Try cache first
        cache_key = include_bytecode
        if self.output_files is not None:
            files = self.output_files.get(cache_key, None)
            if files is not None:
                return files
        else:
            self.output_files = {}

        # Regular processing
        if (not self.without_source or
            not include_bytecode):
            files = build_py.get_outputs(self, include_bytecode)
            self.output_files[cache_key] = files
            return files

        # Remove source code files from outputs
        files = []
        for file in build_py.get_outputs(self, include_bytecode=1):
            if ((self.without_source and file[-3:] == '.py') or
                (not self.compile and file[-4:] == '.pyc') or
                (not self.optimize and file[-4:] == '.pyo')):
                continue
            files.append(file)
        self.output_files[cache_key] = files
        return files
        
#
# mx Build Data command
#

class mx_build_data(Command):

    """ build_data command which allows copying (external) data files
        into the build tree.
        
    """
    description = "build data files (copy them to the build directory)"

    user_options = [
        ('build-lib=', 'b',
         "directory to store built Unix libraries in"),
        ]
    
    boolean_options = []

    # Location of the build directory
    build_lib = None

    def initialize_options(self):

        self.build_lib = None
        self.outfiles = []

    def finalize_options(self):

        self.set_undefined_options('build',
                                   ('verbose', 'verbose'),
                                   ('build_lib', 'build_lib'),
                                   )
        if _debug:
            # For debugging we are always in very verbose mode...
            self.verbose = 2

    def get_outputs(self):

        return self.outfiles

    def build_data_files(self, data_files):

        """ Copy the data_files to the build_lib directory.

            For tuple entries, this updates the data_files list in
            place and adjusts it, so that the data files are picked
            up from the build directory rather than their original
            location.

        """
        build_lib = self.build_lib
        for i in range(len(data_files)):

            entry = data_files[i]
            copied_data_files = []
            
            if type(entry) == types.StringType:
                # Unix- to platform-convention conversion
                entry = convert_to_platform_path(entry)
                filenames = glob.glob(entry)
                for filename in filenames:
                    dst = os.path.join(build_lib, filename)
                    dstdir = os.path.split(dst)[0]
                    if not self.dry_run:
                        self.mkpath(dstdir)
                        outfile = self.copy_file(filename, dst)[0]
                    else:
                        outfile = dst
                    self.outfiles.append(outfile)
                    # Add to the copied_data_files list (using the
                    # distutils internal Unix path format)
                    copied_data_files.append(
                        convert_to_distutils_path(filename))
                    
            elif type(entry) == types.TupleType:
                origin, target = entry
                # Unix- to platform-convention conversion
                origin = convert_to_platform_path(origin)
                target = convert_to_platform_path(target)
                targetdir, targetname = os.path.split(target)
                origin_pathnames = glob.glob(origin)
                if targetname:
                    # Make sure that we don't copy multiple files to
                    # the same target filename
                    if len(origin_pathnames) > 1:
                        raise ValueError(
                            'cannot copy multiple files to %s' %
                            target)
                for pathname in origin_pathnames:
                    if targetname:
                        # Use the new targetname filename
                        filename = targetname
                    else:
                        # Use the original filename
                        filename = os.path.split(pathname)[1]
                    dst = os.path.join(build_lib,
                                       targetdir,
                                       filename)
                    dstdir = os.path.split(dst)[0]
                    if not self.dry_run:
                        self.mkpath(dstdir)
                        outfile = self.copy_file(pathname, dst)[0]
                    else:
                        outfile = dst
                    self.outfiles.append(outfile)
                    # Add to the copied_data_files list (using the
                    # distutils internal Unix path format)
                    copied_data_files.append(
                        convert_to_distutils_path(
                            os.path.join(targetdir,
                                         filename)))

            else:
                raise ValueError('unsupported data_files item format: %r' %
                                 entry)

            # Clear the data_files entry (we'll clean up the list
            # later on)
            data_files[i] = None

            # Add the new copied_data_files to the data_files, so
            # that install_data can pick up the build version of
            # the data file
            data_files.extend(copied_data_files)

        # Cleanup data_files (remove all None, empty and duplicate
        # entries)
        d = {}
        for filename in data_files:
            if not filename:
                continue
            d[filename] = 1
        data_files[:] = d.keys()

        if _debug:
            print 'After build_data: data_files=%r' % data_files
            print 'build_data output=%r' % self.outfiles
        
    def run(self):

        if not self.distribution.data_files:
            return
        self.build_data_files(self.distribution.data_files)

#
# mx Build Unix Libs command
#
class UnixLibrary:

    """ Container for library configuration data.
    """
    # Name of the library
    libname = ''

    # Source tree where the library lives
    sourcetree = ''

    # List of library files and where to install them in the
    # build tree
    libfiles = None

    # Name of the configure script
    configure = 'configure'

    # Configure options
    configure_options = None

    # Make options
    make_options = None
    
    def __init__(self, libname, sourcetree, libfiles,
                 configure=None, configure_options=None,
                 make_options=None):

        self.libname = libname
        self.sourcetree = sourcetree
        # Check for 2-tuples...
        for libfile, targetdir in libfiles:
            pass
        self.libfiles = libfiles

        # Optional settings
        if configure:
            self.configure = configure
        if configure_options:
            self.configure_options = configure_options
        else:
            self.configure_options = []
        if make_options:
            self.make_options = make_options
        else:
            self.make_options = []
            
    def get(self, option, alternative=None):

        return getattr(self, option, alternative)

class mx_build_unixlib(Command):

    """ This command compiles external libs using the standard Unix
        procedure for this:
        
        ./configure
        make

    """
    description = "build Unix libraries used by Python extensions"

    # make program to use
    make = None
    
    user_options = [
        ('build-lib=', 'b',
         "directory to store built Unix libraries in"),
        ('build-temp=', 't',
         "directory to build Unix libraries to"),
        ('make=', None,
         "make program to use"),
        ('makefile=', None,
         "makefile to use"),
        ('force', 'f',
         "forcibly reconfigure"),
        ]
    
    boolean_options = ['force']

    def initialize_options(self):

        self.build_lib = None
        self.build_temp = None
        self.make = None
        self.makefile = None
        self.force = 0

    def finalize_options(self):

        self.set_undefined_options('build',
                                   ('verbose', 'verbose'),
                                   ('build_lib', 'build_lib'),
                                   ('build_temp', 'build_temp')
                                   )
        if self.make is None:
            self.make = 'make'
        if self.makefile is None:
            self.makefile = 'Makefile'

        if _debug:
            # For debugging we are always in very verbose mode...
            self.verbose = 2

    def run_script(self, script, options=[]):

        if options:
            l = []
            for k, v in options:
                if v is not None:
                    l.append('%s=%s' % (k, v))
                else:
                    l.append(k)
            script = script + ' ' + ' '.join(l)
        log.info('executing script %s' % repr(script))
        if self.dry_run:
            return 0
        try:
            rc = os.system(script)
        except DistutilsExecError, msg:
            raise CompileError, msg
        return rc
    
    def run_configure(self, options=[], dir=None, configure='configure'):

        """ Run the configure script using options is given.

            Options must be a list of tuples (optionname,
            optionvalue).  If an option should not have a value,
            passing None as optionvalue will have the effect of using
            the option without value.

            dir can be given to have the configure script execute in
            that directory instead of the current one.

        """
        cmd = './%s' % configure
        if dir:
            cmd = 'cd %s; ' % dir + cmd
        log.info('running %s in %s' % (configure, dir or '.'))
        rc = self.run_script(cmd, options)
        if rc != 0:
            raise CompileError, 'configure script failed'

    def run_make(self, targets=[], dir=None, make='make', options=[]):

        """ Run the make command for the given targets.

            Targets must be a list of valid Makefile targets.

            dir can be given to have the make program execute in that
            directory instead of the current one.

        """
        cmd = '%s' % make
        if targets:
            cmd = cmd + ' ' + ' '.join(targets)
        if dir:
            cmd = 'cd %s; ' % dir + cmd
        log.info('running %s in %s' % (make, dir or '.'))
        rc = self.run_script(cmd, options)
        if rc != 0:
            raise CompileError, 'make failed'

    def build_unixlib(self, unixlib):

        # Build each lib in its own subdir of build_temp (to
        # avoid accidental sharing of object files)
        build_temp_base = self.build_temp
        libpath = unixlib.libname
        self.build_temp = os.path.join(build_temp_base, libpath)

        try:

            # Get options
            configure = unixlib.configure
            configure_options = unixlib.configure_options
            make_options = unixlib.make_options
            sourcetree = unixlib.sourcetree
            buildtree = os.path.join(self.build_temp, sourcetree)
            libfiles = unixlib.libfiles
            if not libfiles:
                raise DistutilsError, \
                      'no libfiles defined for unixlib "%s"' % \
                      unixlib.name
            log.info('building C lib %s in %s' % (unixlib.libname,
                                                  buildtree))
            # Prepare build
            log.info('preparing build of %s' % unixlib.libname)
            self.mkpath(buildtree)
            self.copy_tree(sourcetree, buildtree)

            # Run configure to build the Makefile
            if not os.path.exists(os.path.join(buildtree, self.makefile)) or \
               self.force:
                self.run_configure(configure_options,
                                   dir=buildtree,
                                   configure=configure)
            else:
                log.info("skipping configure: "
                         "%s is already configured" %
                         unixlib.libname)

            # Run make
            self.run_make(dir=buildtree,
                          make=self.make,
                          options=make_options)

            # Copy libs to destinations
            for sourcefile, destination in libfiles:
                if not destination:
                    continue
                sourcefile = os.path.join(self.build_temp, sourcefile)
                destination = os.path.join(self.build_lib, destination)
                if not os.path.exists(sourcefile):
                    raise CompileError, \
                          'library "%s" failed to build' % sourcefile
                self.mkpath(destination)
                self.copy_file(sourcefile, destination)

        finally:
            # Cleanup local changes to the configuration
            self.build_temp = build_temp_base

    def build_unixlibs(self, unixlibs):

        for unixlib in unixlibs:
            self.build_unixlib(unixlib)

    def get_unixlib_lib_options(self):

        libs = []
        paths = []
        for unixlib in self.distribution.unixlibs:
            for sourcefile, destination in unixlib.libfiles:
                if not destination:
                    # direct linking
                    sourcefile = os.path.join(self.build_temp,
                                              sourcefile)
                    libs.append(sourcefile)
                else:
                    # linking via link path and lib name
                    sourcefile = os.path.basename(sourcefile)
                    libname = os.path.splitext(sourcefile)[0]
                    if libname[:3] == 'lib':
                        libname = libname[3:]
                    libs.append(libname)
                    destination = os.path.join(self.build_lib,
                                               destination)
                    paths.append(destination)
        #print paths, libs
        return paths, libs

    def run(self):

        if not self.distribution.unixlibs:
            return
        self.build_unixlibs(self.distribution.unixlibs)

#
# mx Install command
#

class mx_install(install):

    """ We want install_data to default to install_purelib, if it is
        not given.

    """
    # build_lib attribute copied to the install command from the
    # build command during .finalize_options()
    build_lib = None

    # Force installation into the platlib, even if the package is a
    # pure Python library
    force_non_pure = 0
    
    user_options = install.user_options + [
        ('force-non-pure', None,
         "force installation into the platform dependent directory"),
        ]

    def initialize_options(self):

        install.initialize_options(self)
        self.force_non_pure = 0
    
    def finalize_options(self):

        fix_install_data = (self.install_data is None)
            
        install.finalize_options(self)

        # Force installation into the platform dependent directories,
        # even if this package is a pure Python package
        if self.force_non_pure:
            self.install_purelib = self.install_platlib
            self.install_libbase = self.install_platlib
            self.install_lib = os.path.join(self.install_platlib,
                                            self.extra_dirs)

        # We want install_data to default to install_purelib, if it is
        # not given.
        if fix_install_data:
            # We want data to be installed alongside the Python
            # modules
            self.install_data = self.install_purelib

        # Undo the change introduced in Python 2.4 to bdist_wininst
        # which manipulates the build.build_lib path and adds
        # a target_version specific ending; we simply override
        # the value here (rather than in build), since all install_*
        # commands pick up the .build_lib value from this command
        # rather than build.
        if self.distribution.has_ext_modules():
            build = self.get_finalized_command('build')
            if _debug:
                print ('resetting build_lib from "%s" to "%s"' %
                       (self.build_lib,
                        build.build_platlib))
            self.build_lib = build.build_platlib

        self.dump_dirs('after applying mx_install fixes')

    def ensure_finalized(self):

        install.ensure_finalized(self)

        # Hack needed for bdist_wininst
        if self.install_data[-5:] == '\\DATA':
            # Install data into the Python module part
            self.install_data = self.install_data[:-5] + '\\PURELIB'

#
# mx Install Data command
#

class mx_install_data(install_data):

    """ Rework the install_data command to something more useful.

        Two data_files formats are supported:

        * string entries
        
            The files (which may include wildcards) are copied to the
            installation directory using the same relative path.

        * tuple entries of the form (orig, dest)

            The files given in orig (which may include wildcards) are
            copied to the dest directory relative to the installation
            directory.

            If dest includes a filename, the file orig is copied to
            the file dest. Otherwise, the original filename is used
            and the file copied to the dest directory.
    
    """

    user_options = install_data.user_options + [
        ('build-lib=', 'b',
         "directory to store built Unix libraries in"),
        ]

    def initialize_options(self):

        self.build_lib = None
        install_data.initialize_options(self)

    def finalize_options(self):

        if self.install_dir is None:
            installobj = self.distribution.get_command_obj('install')
            self.install_dir = installobj.install_data
        if _debug:
            print 'Installing data files to %s' % self.install_dir
        self.set_undefined_options('install',
                                   ('build_lib', 'build_lib'),
                                   )
        install_data.finalize_options(self)

    def run(self):

        if not self.dry_run:
            self.mkpath(self.install_dir)
        data_files = self.get_inputs()
        if _debug:
            print 'install_data: data_files=%r' % self.data_files
        for entry in data_files:

            if type(entry) == types.StringType:
                # Unix- to platform-convention conversion
                entry = convert_to_platform_path(entry)
                # Names in data_files are now relative to the build
                # directory, since mx_build_data has copied them there
                entry = os.path.join(self.build_lib, entry)
                filenames = glob.glob(entry)
                for filename in filenames:
                    relative_filename = remove_path_prefix(
                        filename, self.build_lib)
                    dst = os.path.join(self.install_dir, relative_filename)
                    dstdir = os.path.split(dst)[0]
                    if not self.dry_run:
                        self.mkpath(dstdir)
                        outfile = self.copy_file(filename, dst)[0]
                    else:
                        outfile = dst
                    self.outfiles.append(outfile)

            else:
                raise ValueError('unsupported data_files item format: %r' %
                                 (entry,))

        if _debug:
            print 'install_data: outfiles=%r' % self.outfiles

#
# mx Install Lib command
#

class mx_install_lib(install_lib):

    """ Patch the install_lib to work around a problem where the
        .get_outputs() method would return filenames like '.pyoo',
        '.pyco', etc.
        
    """
    def _bytecode_filenames (self, filenames):

        """ Create a list of byte-code filenames from the list of
            filenames.

            Files in filenames that are not Python source code are
            ignored.

        """
        bytecode_filenames = []
        for py_file in filenames:
            if py_file[-3] != '.py':
                continue
            if self.compile:
                bytecode_filenames.append(py_file + "c")
            if self.optimize > 0:
                bytecode_filenames.append(py_file + "o")
        return bytecode_filenames

#
# mx Uninstall command
#
# Credits are due to Thomas Heller for the idea and the initial code
# base for this command (he posted a different version to
# distutils-sig@python.org) in 02/2001.
#

class mx_uninstall(Command):

    description = "uninstall the package files and directories"

    user_options = []

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass
        
    def run(self):

        # Execute build
        log.info('determining installation files')
        log.info('(re)building package')
        savevalue = self.distribution.dry_run
        self.distribution.dry_run = 0
        self.run_command('build')

        # Execute install in dry-run mode
        log.debug('dry-run package install (to determine the installed files)')
        self.distribution.dry_run = 1
        self.run_command('install')
        self.distribution.dry_run = savevalue
        build = self.get_finalized_command('build')
        install = self.get_finalized_command('install')

        # Remove all installed files
        log.info('removing installed files')
        dirs = {}
        filenames = install.get_outputs()
        for filename in filenames:
            if not os.path.isabs(filename):
                raise DistutilsError,\
                      'filename %s from .get_output() not absolute' % \
                      filename

            if os.path.isfile(filename):
                log.info('removing %s' % filename)
                if not self.dry_run:
                    try:
                        os.remove(filename)
                    except OSError, details:
                        log.warn('could not remove file: %s' % details)
                    dir = os.path.split(filename)[0]
                    if not dirs.has_key(dir):
                        dirs[dir] = 1
                    if os.path.splitext(filename)[1] == '.py':
                        # Remove byte-code files as well
                        try:
                            os.remove(filename + 'c')
                        except OSError:
                            pass
                        try:
                            os.remove(filename + 'o')
                        except OSError:
                            pass

            elif os.path.isdir(filename):
                if not dirs.has_key(dir):
                    dirs[filename] = 1

            elif not os.path.splitext(filename)[1] in ('.pyo', '.pyc'):
                log.warn('skipping removal of %s (not found)' %
                         filename)

        # Remove the installation directories
        log.info('removing directories')
        dirs = dirs.keys()
        dirs.sort(); dirs.reverse() # sort descending
        for dir in dirs:
            # Don't remove directories which are listed on sys.path
            if dir in sys.path:
                continue
            # Check the the directory is empty
            dir_content = os.listdir(dir)
            if dir_content:
                log.info('directory %s is not empty - not removing it' %
                         dir)
                continue
            # Remove the directory and warn if this fails
            log.info('removing directory %s' % dir)
            if not self.dry_run:
                try:
                    os.rmdir(dir)
                except OSError, details:
                    log.warn('could not remove directory: %s' % details)

#
# mx register command
#

class mx_register(register):

    """ Register a package with PyPI.

        This version enhances the download_url by optionally adding a
        hash tag to it. The command fetches the current contents of
        the referenced URL and calculates the hash value from it.

    """
    # Add new option --add-hash-tag
    user_options = register.user_options + [
        ('add-hash-tag', None,
         'add a hash tag to the download_url'),
        ]
    boolean_options = register.boolean_options + [
        'add-hash-tag',
        ]

    def initialize_options(self):

        self.add_hash_tag = None
        register.initialize_options(self)

    def finalize_options(self):

        if self.add_hash_tag is None:
            self.add_hash_tag = False
        register.finalize_options(self)

    def run(self):

        if self.add_hash_tag:
            download_url = self.distribution.metadata.download_url
            if download_url:
                download_url = hashed_download_url(
                    download_url,
                    'simple')
                log.info('updating download_url to %r' % download_url)
                self.distribution.metadata.download_url = download_url
        register.run(self)

#
# mx clean command
#

class mx_clean(clean):

    """ Clean up the build directories.

        This version knows about the build pickle.

    """
    def run(self):

        if self.all:
            # Only remove the build pickle, if --all is requested
            build = self.get_finalized_command('build')
            build_pickle = build.get_build_pickle_pathname()
            if os.path.exists(build_pickle):
                log.info('removing %r' % build_pickle)
                try:
                    os.remove(build_pickle)
                except OSError, details:
                    log.warn('could not remove build pickle %s: %s' %
                             (build_pickle, details))

        clean.run(self)

#
# mx sdist command
#

class mx_sdist(sdist):

    """ Build a source distribution.

        This version does not use hard links when preparing the source
        archive - hard links don't match well with symlinks which we
        use in our source repositories.

    """
    # Tags for the build
    tags = None

    def finalize_options(self):

        # Determine tags
        if self.tags is None:
            self.tags = []
        if 'prebuilt' not in self.tags:
            self.tags.append('sdist')
            self.tags.append('source')
            build = self.get_finalized_command('build')
            if (build.has_ext_modules() or
                build.has_c_libraries() or
                build.has_unixlibs()):
                self.tags.append('compiler')

        # Call base method
        sdist.finalize_options(self)

    def make_release_tree(self, base_dir, files):

        # Prepare release dir
        self.mkpath(base_dir)
        self.distribution.metadata.write_pkg_info(base_dir)
        if not files:
            log.warn('no files found in release !')
            return

        # Create dir structure
        log.info('preparing release tree in %s...' % base_dir)
        create_tree(base_dir, files, dry_run=self.dry_run)
        for file in files:
            if not os.path.isfile(file):
                log.warn('%s is not a regular file -- skipping' % file)
            else:
                dest = os.path.join(base_dir, file)
                self.copy_file(file, dest, link=None)

    def make_distribution(self):

        # Call base method to have the distribution(s) built
        sdist.make_distribution(self)

        # Write tag files
        import tarfile, zipfile
        for archive in self.archive_files:
            if zipfile.is_zipfile(archive):
                format_tag = 'zip'
            elif tarfile.is_tarfile(archive):
                format_tag = 'tar'
            write_package_tags(archive, self.tags + [format_tag])

#
# mx sdist_web command
#

# Name of the tags file which hold the tagged download URLs of the web
# installer packages
SETUP_TAGS = 'setup.tags'

# Web installer supported distribution format tags. The web installer
# can only handle packages of these distribution formats.
WEB_INSTALLER_DIST_FORMS = ('prebuilt', 'sdist')

class mx_sdist_web(mx_sdist):

    """ Build a source distribution which works as web installer.

        The distribution only contains a minimal set of files and then
        downloads the remaining files when setup.py is used for the
        first time.

        Note that the corresponding setup.py will have to include the
        needed code to actually implement the web installation of the
        missing files.

    """
    # Files allowed in the web installer version of the sdist
    # distribution; case does not matter
    allowed_files = [
        'README.*',
        'LICENSE.*',
        'COPYRIGHT.*',
        'MANIFEST',
        'PKG-INFO',
        '[\w-]+\.py', # top-level .py files
        'setup\.cfg',
        # SETUP_TAGS is always included
        ]

    user_options = mx_sdist.user_options + [
        ('package-tags=', None,
         'tags file with the available package files'),
        ('base-url=', None,
         'base URL to add to the tag file entries'),
        ('exclude-tags=', None,
         'exclude package files which have these tags set '
         '(comma separated list of tags)'),
        ]

    # Defaults
    package_tags = None
    base_url = None
    exclude_tags = None

    def initialize_options(self):

        # Call base method first
        mx_sdist.initialize_options(self)

        # Create .zip archives per default, instead of using a
        # platform-dependent default
        self.formats = ['zip']

    def finalize_options(self):

        mx_sdist.finalize_options(self)

        # Set package tags file
        if self.package_tags is None:
            self.package_tags = SETUP_TAGS
        if _debug:
            print ('using package tags file: %s' % self.package_tags)

        # Set base URL
        if self.base_url is None:
            self.base_url = ''
        if _debug:
            print ('using base URL: %r' % self.base_url)

        # Set exclude tags
        if self.exclude_tags is None:
            self.exclude_tags = set()
        else:
            self.exclude_tags = normalize_tags(self.exclude_tags.split(','))
        if _debug:
            print ('excluding files with tags: %s' % self.exclude_tags)

        # Check setup.py for web installer
        self.check_setup_py()

        # Create setup.tags file
        self.create_setup_tags()

    def check_setup_py(self):

        """ Check setup.py for web installer code.

        """
        try:
            f = open('setup.py', 'r')
        except IOError:
            log.warn('WARNING: Could not open setup.py')
            return
        else:
            data = f.read()
            f.close()
        if 'run_web_installer(' not in data:
            log.warn('WARNING: No call to mxSetup.run_web_installer() found '
                     'in setup.py - missing web installer ?')
        log.info('found call to web installer in setup.py')

    def create_setup_tags(self):

        """ Create a new setup.tags file

        """
        log.info('reading package tags from %s' % self.package_tags)
        package_data = read_package_data(self.package_tags)

        # Filter out the packages that the web installer can actually
        # handle
        log.info('filtering packages to include only '
                 'compatible distribution formats')
        filtered_packages = _subset_packages(package_data,
                                             WEB_INSTALLER_DIST_FORMS)

        # Filter out the packages that have tags in .exclude_tags
        if self.exclude_tags:
            log.info('excluding packages to which have tags %s' %
                     self.exclude_tags)
            filtered_packages = _setminus_packages(filtered_packages,
                                                   self.exclude_tags)

        # Write package tags
        if not filtered_packages:
            raise DistutilsError(
                'no web installable packages found in %s' %
                self.package_tags)
        log.info('writing package tags (with hashes) to %s' % SETUP_TAGS)
        write_package_data(SETUP_TAGS,
                           filtered_packages,
                           base_url=self.base_url,
                           replace=True,
                           add_hash_tag=True)

    def get_file_list(self):

        # Call base method to build the file list
        mx_sdist.get_file_list(self)

        files = self.filelist.files
        if _debug:
            print ('Original file list: %r' % files)

        # Strip all files which do not match the files in
        # .allowed_files
        pattern_re = re.compile('|'.join(self.allowed_files), re.I)
        for i in range(len(files)-1, -1, -1):
            if pattern_re.match(files[i]) is None:
                if _debug:
                    print (' removing %r' % files[i])
                del files[i]

        # Add setup.tags
        files.append(SETUP_TAGS)
        
        if _debug:
            print ('Updated file list: %r' % files)

        # Note: We do not write this as MANIFEST, since this should
        # contain the files of the web installed package, not those of
        # the web installer package.

# Support functions for web installer compatible setup.py's

def _check_landmarks(dir, landmarks):

    for landmark in landmarks:
        landmark_path = os.path.join(dir, landmark)
        if os.path.exists(landmark_path):
            return landmark
    return None

def run_web_installer(dir, landmarks):

    """ Run the web installer for the package unpacked in dir.

        The function will only run the download in case none of the
        landmarks file/dirs are not found in dir. The landmarks should
        not be present in the web installer itself.

        Returns True/False depending on whether the web installer
        downloaded a package or not.

    """
    # Check whether the web installer already ran
    landmark = _check_landmarks(dir, landmarks)
    if landmark:
        # Web installer already ran
        if _debug:
            print ('web installer found landmark %r: skipping download' %
                   landmark)
        return False
    log.info('web installer running')

    # Load package data from setup.tags file
    package_data = read_package_data(SETUP_TAGS)
    log.info('trying to find suitable download package')
    try:
        url = find_matching_package(package_data,
                                    required_tags=WEB_INSTALLER_DIST_FORMS)
    except KeyError, reason:
        log.error('web installer could not find a suitable '
                  'download package')
        raise DistutilsError(
            'web installer could not find a suitable '
            'download package: %s' % reason)

    # Download the installer package
    log.info('found package URL %s' % url)
    try:
        install_web_package(url, dir)
    except urllib2.URLError, reason:
        log.error('web installer could not download the package %s: %s' %
                  (url, reason))
        raise DistutilsError(
            'web installer could not download the package %s: %s' %
            (url, reason))

    # Make sure that the landmark exists now
    landmark = _check_landmarks(dir, landmarks)
    if not landmark:
        log.error('downloaded package is missing landmark files/dirs')
        raise DistutilsError(
            'downloaded package is missing landmark files/dirs')

    return True

#
# mx generic binary distribution command
#

class mx_bdist(bdist):

    """ Generic binary distribution command.
    
    """
    
    def finalize_options(self):

        # Default to <platform>-<pyversion> on all platforms
        if self.plat_name is None:
            self.plat_name = '%s-py%s' % (mx_get_platform(), py_version())
        bdist.finalize_options(self)


#
# mx RPM distribution command
#

class mx_bdist_rpm(bdist_rpm):

    """ bdist_rpm command which allows passing in distutils
        options.

        XXX Note: bdist_rpm no longer works for some reason, probably
            related to distutils rather than our small modification.

    """
    user_options = bdist_rpm.user_options + [
        ('distutils-build-options=', None,
         'extra distutils build options to use before the "build" command'),
        ('distutils-install-options=', None,
         'extra distutils install options to use after the "install" command'),
        ]

    # Defaults
    distutils_build_options = None
    distutils_install_options = None

    def finalize_options(self):

        bdist_rpm.finalize_options(self)
        if self.distutils_build_options is None:
            self.distutils_build_options = ''
        if self.distutils_install_options is None:
            self.distutils_install_options = ''

    def _make_spec_file(self):

        # Generate .spec file
        l = bdist_rpm._make_spec_file(self)

        # Insert into build command
        i = l.index('%build')
        buildcmd = l[i + 1]
        inspos = l[i + 1].find(' build')
        if inspos >= 0:
            l[i + 1] = '%s %s %s' % (buildcmd[:inspos],
                                     self.distutils_build_options,
                                     buildcmd[inspos:])
        else:
            raise DistutilsError, \
                  'could not insert distutils command in RPM build command'
        
        # Insert into install command
        i = l.index('%install')
        installcmd = l[i + 1]
        inspos = l[i + 1].find(' install')
        if inspos >= 0:
            l[i + 1] = '%s %s %s %s' % (installcmd[:inspos],
                                        self.distutils_build_options,
                                        installcmd[inspos:],
                                        self.distutils_install_options)
        else:
            raise DistutilsError, \
                  'could not insert distutils command in RPM install command'

        return l
    
#
# mx bdist_wininst command
#

class mx_bdist_wininst(bdist_wininst):

    """ We want bdist_wininst to include the Python version number
        even for pure Python distribution - in case we don't include
        the Python source code.

    """
    
    def finalize_options(self):

        bdist_wininst.finalize_options(self)

        # Force a target version if without_source was used for
        # build_py
        if not self.target_version:
            build_py = self.get_finalized_command('build_py')
            if build_py.without_source:
                self.target_version = py_version(unicode_aware=0)

#
# mx in-place binary distribution command
#

class mx_bdist_inplace(bdist_dumb):

    """ Build an in-place binary distribution.
    
    """

    # Path prefix to use in the distribution (all files will be placed
    # under this prefix)
    dist_prefix = ''

    user_options = bdist_dumb.user_options + [
        ('dist-prefix=', None,
         'path prefix the binary distribution with'),
        ]

    def finalize_options(self):

        # Default to ZIP as format on all platforms
        if self.format is None:
            self.format = 'zip'
        bdist_dumb.finalize_options(self)

    # Hack to reuse bdist_dumb for our purposes; .run() calls
    # reinitialize_command() with 'install' as command.
    def reinitialize_command(self, command, reinit_subcommands=0):

        cmdobj = bdist_dumb.reinitialize_command(self, command,
                                                 reinit_subcommands)
        if command == 'install':
            cmdobj.install_lib = self.dist_prefix
            cmdobj.install_data = self.dist_prefix
        return cmdobj

#
# mx Zope binary distribution command
#

class mx_bdist_zope(mx_bdist_inplace):

    """ Build binary Zope product distribution.
    
    """

    # Path prefix to use in the distribution (all files will be placed
    # under this prefix); for Zope instances, all code can be placed
    # into the Zope instance directory since this is on sys.path when
    # Zope starts
    dist_prefix = ''

#
# mx bdist_prebuilt command
#

class mx_bdist_prebuilt(mx_sdist):

    """ Build a pre-built distribution.

        The idea is to ship a version of the package that is already
        built, but not yet packaged or installed.

        This makes it possible to do that last step on the client's
        target machine, giving much more flexibility in the way
        software is installed.

    """
    # Skip the build process ?
    skip_build = None

    # Platform name to use
    plat_name = None

    # Keep data source files
    include_data_source_files = None

    # Files to exclude from the prebuilt archives
    exclude_files = None

    # File name version
    #
    # Version 1 format (not easy_install compatible):
    # product_name + '-' + version + '.' + platform + '-' + python_ucs + '.prebuilt.zip'
    #
    # Version 2 format (easy_install compatible):
    # product_name + '-' + version + '-' + python_ucs + '-' + platform + '-prebuilt.zip'
    #
    filename_version = 2

    user_options = mx_sdist.user_options + [
        ('plat-name=', None,
         'platform name to use'),
        ('skip-build', None,
         'skip build and reuse the existing build files'),
        ('include-data-source-files', None,
         'include the data source files '
         'in addition to the build versions'),
        ('exclude-files', None,
         'exclude files matching the given RE patterns '
         '(separated by whitespace)'),
        ]

    def finalize_options(self):

        # Determine tags
        if self.tags is None:
            self.tags = []
        self.tags.append('prebuilt')
        (os_tag,
         code_tag,
         alt_code_tags,
         python_tag,
         ucs_tag) = platform_tags()

        build = self.get_finalized_command('build')
        if (not build.has_ext_modules() and
            not build.has_c_libraries() and
            not build.has_unixlibs() and
            not sys.platform.startswith('win')):
            # We can build a platform independent distribution;
            # note that we cannot build platform independent
            # distributions on Windows, since all path names will
            # use the Windows os.sep which doesn't work on Unix
            # platforms.
            if self.plat_name is None:
                self.plat_name = 'py%s' % py_version(unicode_aware=0)
            self.tags.extend(['purepython', python_tag])
        else:
            # Include the platform name
            if self.plat_name is None:
                if self.filename_version == 1:
                    self.plat_name = '%s-py%s' % (
                        mx_get_platform(),
                        py_version())
                elif self.filename_version == 2:
                    self.plat_name = 'py%s-%s' % (
                        py_version(),
                        mx_get_platform())
                else:
                    raise TypeError('unsupported .filename_version')
            # Add tags
            if os_tag == 'unknown-os':
                raise DistutilsError('unknown OS found by platform_tags()')
            if code_tag == 'unknown-code-type':
                raise DistutilsError('unknown code type found by platform_tags()')
            self.tags.extend([os_tag, code_tag, python_tag, ucs_tag])
            if alt_code_tags:
                # Add fat build tags as well
                self.tags.extend(alt_code_tags)

        # Skip the build step ?
        if self.skip_build is None:
            self.skip_build = 0

        # Include data source files ?
        if self.include_data_source_files is None:
            self.include_data_source_files = 0
        else:
            # If source files are included, the package can also be
            # used to rebuild the package for other platforms, so
            # include the corresponding tags
            self.tags.append('source')
            if (build.has_ext_modules() or
                build.has_c_libraries() or
                build.has_unixlibs()):
                self.tags.append('compiler')

        # Exclude files ?
        if self.exclude_files is None:
            self.exclude_files = []
        else:
            self.exclude_files = [re.compile(pattern.strip())
                                  for pattern in self.exclude_files.split()]

        # Default to ZIP files for all platforms
        if self.formats is None:
            self.formats = ['zip']

        # Call the base method
        mx_sdist.finalize_options(self)

    def get_file_list(self):

        log.info('building list of files to include in the pre-built distribution')
        if not os.path.isfile(self.manifest):
            log.error('manifest missing: '
                      'cannot create pre-built distribution')
            return
        self.read_manifest()

        # Prepare a list of source files needed for install_data
        data_source_files = []
        for entry in self.distribution.data_files:
            if type(entry) is types.TupleType:
                (source_file, dest_file) = entry
            else:
                source_file = entry
            source_file = convert_to_platform_path(source_file)
            data_source_files.append(source_file)
        if _debug:
            print 'found these data source files: %r' % data_source_files

        # Remove most subdirs from the MANIFEST file list
        files = []
        for path in self.filelist.files:
            
            # Note: the MANIFEST file list can use both os.sep and the
            # distutils dir separator as basis

            # Filter files which are not to be included in the archive;
            # we use the distutils path for the filtering
            distutils_path = convert_to_distutils_path(path)
            skip_path = False
            for pattern_re in self.exclude_files:
                if pattern_re.match(distutils_path) is not None:
                    skip_path = True
                    break
            if skip_path:
                if _debug:
                    print '  skipping excluded file: %s' % path
                continue

            # Now filter the remaining files; we'll convert the path
            # the platform version for this
            path = convert_to_platform_path(path)
            if os.sep in path:
                # Subdir entry
                path_components = path.split(os.sep)
                if path_components[0].lower().startswith('doc'):
                    # Top-level documentation directories are always included
                    if _debug:
                        print '  found documentation file: %s' % path
                elif (self.include_data_source_files and
                      path in data_source_files):
                    # Data source files can optionally be included as
                    # well; these will already be in the build area
                    # due to mx_build_data
                    if _debug:
                        print '  found data source file: %s' % path
                else:
                    # Skip all other files in subdirectories
                    if _debug:
                        print '  skipping file: %s' % path
                    continue
            elif _debug:
                print '  found top-level file: %s' % path
            log.info('adding %s' % path)
            files.append(path)

        self.filelist.files = files

        # Add build files
        build = self.get_finalized_command('build')
        self.filelist.files.extend(build.get_outputs())

        if _debug:
            print 'pre-built files:', repr(self.filelist.files)
                
    def run(self):

        if not self.skip_build:
            self.run_command('build')

        mx_sdist.run(self)

    def make_distribution(self):

        if self.filename_version == 1:
            # Version 1 format (not easy_install compatible)
            archive_name = '%s.%s.prebuilt' % (
                self.distribution.get_fullname(),
                self.plat_name)
        elif self.filename_version == 2:
            # Version 2 format (easy_install compatible)
            archive_name = '%s-%s-prebuilt' % (
                self.distribution.get_fullname(),
                self.plat_name)
        else:
            raise TypeError('unsupported .filename_version')
        archive_path = os.path.join(self.dist_dir, archive_name)

        # Create the release tree
        self.make_release_tree(archive_name, self.filelist.files)

        # Add pre-built marker file containing the path to the build
        # pickle with the build information
        prebuilt_pathname = os.path.join(archive_name, PREBUILT_MARKER)
        prebuilt_file = open(prebuilt_pathname, 'w')
        build = self.get_finalized_command('build')
        prebuilt_file.write(build.get_build_pickle_pathname())
        prebuilt_file.close()

        # Create the requested archives
        archive_files = []
        for fmt in self.formats:
            archive = self.make_archive(
                archive_path, fmt, base_dir=archive_name)
            archive_files.append(archive)
            # XXX Not sure what .dist_files is good for...
            #self.distribution.dist_files.append(('sdist', '', archive))
            # Create tag files
            write_package_tags(archive, self.tags)
        self.archive_files = archive_files

        # Remove the release tree
        if not self.keep_temp:
            remove_tree(archive_name, dry_run=self.dry_run)

#
# mx egg binary distribution command
#

class mx_bdist_egg(bdist_dumb):

    """ Build an egg binary distribution.

        This is essentially a bdist_dumb ZIP archive with a special
        name and an .egg extension.

        In addition to the distribution files, it also contains an
        EGG-INFO directory with some additional meta-information about
        the package.
    
    """
    # Build a Unicode-aware egg ? easy_install does not support having
    # UCS2/UCS4 markers in the filename, so we place the egg files
    # into ucs2/ucs4 subdirectories of the --dist-dir if this option
    # is set. Default is not to use these subdirs.
    unicode_aware = None

    user_options = [
        ('plat-name=', 'p',
         'platform name to embed in generated filenames '
         '(default: %s)' % mx_get_platform()),
        ('skip-build', None,
         'skip rebuilding everything (for testing/debugging)'),
        ('dist-dir=', 'd',
         'directory where to put the .egg file'),
        ('unicode-aware', None,
         'put eggs into ucs2/ucs4 subdirectories of --dist-dir'),
        ]

    def finalize_options(self):

        if self.plat_name is None:
            build = self.get_finalized_command('build')
            if (not build.has_ext_modules() and
                not build.has_c_libraries() and
                not build.has_unixlibs()):
                # We can build a platform independent distribution
                self.plat_name = ''
            else:
                # Include the platform name
                self.plat_name = mx_get_platform()

        if self.unicode_aware is None:
            self.unicode_aware = 0

        bdist_dumb.finalize_options(self)

        if self.unicode_aware:
            # Put the eggs into separate dist_dir subdirectories in
            # case unicode aware eggs are to be built
            unicode_subdir = py_unicode_build()
            if unicode_subdir:
                self.dist_dir = os.path.join(self.dist_dir,
                                             unicode_subdir)

    def write_egg_info_file(self, egg_info_dir, filename, lines=()):

        f = open(os.path.join(egg_info_dir, filename), 'wb')
        f.write('\n'.join(lines))
        if lines:
            f.write('\n')
        f.close()

    def run(self):

        if not self.skip_build:
            self.run_command('build')

        # Install the package in the .bdist_dir
        install = self.reinitialize_command('install', reinit_subcommands=1)
        install.root = self.bdist_dir
        install.skip_build = self.skip_build
        install.warn_dir = 0
        # Use an in-place install without prefix
        install.install_lib = ''
        install.install_data = ''
        log.info("installing to %s" % self.bdist_dir)
        self.run_command('install')

        # Remove .egg-info file
        if python_version >= '2.5':
            # install_egg_info was added in Python 2.5 distutils
            install_egg_info = self.get_finalized_command('install_egg_info')
            for filename in install_egg_info.outputs:
                execute(os.remove, (filename,),
                        "removing %s" % filename,
                        verbose=self.verbose, dry_run=self.dry_run)
            install_egg_info.output = []

        # Create EGG-INFO dir in .bdist_dir
        egg_info_dir = os.path.join(self.bdist_dir, 'EGG-INFO')
        self.mkpath(egg_info_dir)
        if not self.dry_run:
            
            # add PKG-INFO file
            self.distribution.metadata.write_pkg_info(egg_info_dir)

            # add not-zip-safe marker to force unzipping the .egg file
            self.write_egg_info_file(egg_info_dir,
                                     'not-zip-safe')

            # add requires.txt
            if python_version >= '2.5':
                self.write_egg_info_file(egg_info_dir,
                                         'requires.txt',
                                         self.distribution.metadata.get_requires())

            # add namespace_packages.txt
            self.write_egg_info_file(egg_info_dir,
                                     'namespace_packages.txt',
                                     self.distribution.namespace_packages)

            # add top_level.txt
            top_level_modules = find_python_modules(self.bdist_dir)
            for namespace_package in self.distribution.namespace_packages:
                if '.' in namespace_package:
                    namespace_package = namespace_package.split('.')[0]
                if namespace_package not in top_level_modules:
                    top_level_modules[namespace_package] = 'namespace'
            self.write_egg_info_file(egg_info_dir,
                                     'top_level.txt',
                                     top_level_modules.keys())
                
            # add dependency_links.txt
            self.write_egg_info_file(egg_info_dir,
                                     'dependency_links.txt',
                                     self.distribution.dependency_links)

            # Add namespace module __init__.py files
            for namespace_package in self.distribution.namespace_packages:
                package_dir = os.path.join(self.bdist_dir,
                                           namespace_package.replace('.', os.sep))
                # We overwrite any existing files, if necessary
                init_file = os.path.join(package_dir, '__init__.py')
                if os.path.exists(init_file):
                    log.info('overwriting %s with setuptools namespace version' %
                             init_file)
                else:
                    log.info('adding %s with setuptools namespace marker' %
                             init_file)
                open(init_file, 'w').write(SETUPTOOLS_NAMESPACE_INIT)
                # Remove any existing byte code files
                for bytecode_suffix in ('c', 'o'):
                    filename = init_file + bytecode_suffix
                    if os.path.exists(filename):
                        execute(os.remove, (filename,),
                                "removing %s" % filename,
                                verbose=self.verbose)

        # Build egg file from .bdist_dir
        egg_name = self.distribution.metadata.get_name().replace('-', '_')
        if self.plat_name:
            unicode_aware = False # self.unicode_aware
        else:
            unicode_aware = False
        archive_basename = "%s-%s-py%s" % (
            egg_name,
            self.distribution.metadata.get_version(),
            py_version(unicode_aware=unicode_aware))
        if self.plat_name:
            archive_basename += '-' + self.plat_name
        archive_basepath = os.path.join(self.dist_dir, archive_basename)
        archive_root = self.bdist_dir
        zip_filename = self.make_archive(archive_basepath, format='zip',
                                         root_dir=archive_root)
        assert zip_filename.endswith('.zip')
        egg_filename = zip_filename[:-4] + '.egg'
        if os.path.exists(egg_filename):
            execute(os.remove, (egg_filename,),
                    "removing %s" % egg_filename,
                    verbose=self.verbose, dry_run=self.dry_run)
        self.move_file(zip_filename, egg_filename)

        # Add to distribution files
        if python_version >= '2.5':
            # This features was added in Python 2.5 distutils
            if self.distribution.has_ext_modules():
                pyversion = get_python_version()
            else:
                pyversion = 'any'
            self.distribution.dist_files.append(('bdist_egg',
                                                 pyversion,
                                                 egg_filename))

        # Cleanup
        if not self.keep_temp:
            remove_tree(self.bdist_dir, dry_run=self.dry_run)

#
# mx MSI distribution command
#

if bdist_msi is not None:

    class mx_bdist_msi(bdist_msi):

        """ Build an MSI installer.

            This version allows to customize the product name used for
            the installer.

        """
        # Product name to use for the installer (this is the name that
        # gets displayed in the dialogs and on the installed software
        # list)
        product_name = None

        # Product title to use for the installer
        title = None

        # Platform name to use in the installer filename (new in Python 2.6)
        plat_name = None

        user_options = bdist_msi.user_options + [
            ('product-name=', None,
             'product name to use for the installer'),
            ('title=', None,
             'product title to use for the installer'),
            ]

        def finalize_options(self):

            # Force a target version if without_source was used for
            # build_py; this is needed since bdist_msi start to
            # default to installing to all available Python versions,
            # if no .target_version is given for Python 2.7+
            build_py = self.get_finalized_command('build_py')
            if build_py.without_source:
                self.target_version = py_version(unicode_aware=0)

            bdist_msi.finalize_options(self)
            
            if self.title is None:
                self.title = self.distribution.get_fullname()

            # Inherit the .plat_name from the bdist command
            self.set_undefined_options('bdist',
                                       ('plat_name', 'plat_name'),
                                       )


        # XXX This is basically a copy of bdist_msi.run(), restructured
        #     a bit.

        def run_install(self):

            if not self.skip_build:
                self.run_command('build')

            install = self.reinitialize_command('install', reinit_subcommands=1)
            install.prefix = self.bdist_dir
            install.skip_build = self.skip_build
            install.warn_dir = 0

            install_lib = self.reinitialize_command('install_lib')
            # we do not want to include pyc or pyo files
            install_lib.compile = 0
            install_lib.optimize = 0

            if self.distribution.has_ext_modules():
                # If we are building an installer for a Python version other
                # than the one we are currently running, then we need to ensure
                # our build_lib reflects the other Python version rather than ours.
                # Note that for target_version!=sys.version, we must have skipped the
                # build step, so there is no issue with enforcing the build of this
                # version.
                target_version = self.target_version
                if not target_version:
                    assert self.skip_build, "Should have already checked this"
                    target_version = python_version
                plat_specifier = ".%s-%s" % (self.plat_name, target_version)
                build = self.get_finalized_command('build')
                build.build_lib = os.path.join(build.build_base,
                                               'lib' + plat_specifier)

            log.info("installing to %s", self.bdist_dir)
            install.ensure_finalized()

            # avoid warning of 'install_lib' about installing
            # into a directory not in sys.path
            sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))

            install.run()

            del sys.path[0]

        def get_product_version(self):

            # ProductVersion must be strictly numeric
            version = self.distribution.metadata.get_version()
            try:
                return '%d.%d.%d' % StrictVersion(version).version
            except ValueError:
                # Remove any pre-release or snapshot parts
                try:
                    verstuple = parse_mx_version(version)
                except ValueError:
                    raise DistutilsError(
                        'package version must be formatted with mx_version()')
                major, minor, patch = verstuple[:3]
                new_version = mx_version(major, minor, patch)
                log.warn(
                    "bdist_msi requires strictly numeric "
                    "version numbers: "
                    "using %r for MSI installer, instead of %r" %
                    (new_version, version))
                return new_version

        def get_product_name(self):

            # User defined product name
            if self.product_name is not None:
                product_name = self.product_name

            else:
                # Emulate bdist_msi default behavior, but make the
                # product title changeable.
                #
                # Prefix ProductName with Python x.y, so that
                # it sorts together with the other Python packages
                # in Add-Remove-Programs (APR)
                if self.target_version:
                    product_name = (
                        'Python %s %s %s' % (
                            self.target_version,
                            self.title,
                            self.distribution.metadata.get_version()))
                else:
                    # Group packages under "Python 2.x" if no
                    # .target_version is given
                    product_name = (
                        'Python 2.x %s %s' % (
                            self.title,
                            self.distribution.metadata.get_version()))
            log.info('using %r as product name.' % product_name)
            return product_name

        def run (self):

            self.run_install()

            # Create the installer
            self.mkpath(self.dist_dir)
            fullname = self.distribution.get_fullname()
            installer_name = self.get_installer_filename(fullname)
            log.info('creating MSI installer %s' % installer_name)
            installer_name = os.path.abspath(installer_name)
            if os.path.exists(installer_name):
                os.unlink(installer_name)

            metadata = self.distribution.metadata
            author = metadata.author
            if not author:
                author = metadata.maintainer
            if not author:
                author = "UNKNOWN"
            version = metadata.get_version()
            product_version = self.get_product_version()
            product_name = self.get_product_name()
            self.db = msilib.init_database(
                installer_name,
                msilib.schema,
                product_name,
                msilib.gen_uuid(),
                product_version,
                author)

            # Add tables
            msilib.add_tables(self.db, msilib.sequence)

            # Add meta-data
            props = [('DistVersion', version)]
            email = metadata.author_email or metadata.maintainer_email
            if email:
                props.append(("ARPCONTACT", email))
            if metadata.url:
                props.append(("ARPURLINFOABOUT", metadata.url))
            if props:
                msilib.add_data(self.db, 'Property', props)

            # Add sections
            self.add_find_python()
            self.add_files()
            self.add_scripts()
            self.add_ui()

            # Write the file and append to distribution's .dist_files
            self.db.Commit()
            if hasattr(self.distribution, 'dist_files'):
                self.distribution.dist_files.append(
                    ('bdist_msi', self.target_version, fullname))

            # Cleanup
            if not self.keep_temp:
                remove_tree(self.bdist_dir, dry_run=self.dry_run)

        def get_installer_filename(self, fullname):

            return os.path.join(self.dist_dir,
                                "%s.%s.msi" %
                                (fullname,
                                 self.plat_name))

else:
    
    class mx_bdist_msi:
        pass

if 0:
    # Hack to allow quick debugging of the mx_bdist_msi command
    if os.name == 'nt' and bdist_msi is None:
        raise TypeError('just testing...')

### Helper to make prebuilt packages compatible with setuptools' bdist_egg

if setuptools is not None:

    from setuptools.command import egg_info
    from distutils.filelist import FileList

    class mx_egg_info(egg_info.egg_info):

        def find_sources(self):
            
            """ This method is used to generate the SOURCES.txt
                manifest file in the EGG-INFO directory.

                Since there's no clear use of that file and it
                prevents building eggs from prebuilt binaries, we'll
                just return a list with the EGG-INFO files.

            """
            self.filelist = FileList()
            egg_info = self.get_finalized_command('egg_info')
            self.filelist.include_pattern('*',
                                          prefix=egg_info.egg_info)

else:
    mx_egg_info = None

#
# mx Upload command
#
if upload is not None:

    class mx_upload(upload):

        """ Upload command which allows to separate uploads from distribution
            builds.

            The distribution files can be given using the --dist-file command. 
            The argument has to be of the form "command,pyversion,filename"
            with:
            
            * command == 'bdist_egg', 'sdist' or 'bdist_msi'
            * pyversion == '', '2.6', '2.7', etc.
            * filename == filename of the distribution file

            command has to be the distribution command which results in
            the given distribution file.
            
        """

        # Distribution file entry
        dist_file = None

        user_options = upload.user_options + [
            ('dist-file=', None,
             'distribution file given as "command,pyversion,filename"'),
            ]

        def finalize_options(self):

            if self.dist_file is not None:
                dist_file = self.dist_file.split(',')
                if not len(dist_file) == 3:
                    raise DistutilsError(
                        '--dist-file format is "command,pyversion,filename"'
                        ' %r given' %
                        self.dist_file)
                self.dist_file = dist_file

            # Call base method
            upload.finalize_options(self)

        def run(self):
        
            dist_files = self.distribution.dist_files
            if not dist_files and self.dist_file is not None:
                dist_files = [self.dist_file]
            else:
                raise DistutilsOptionError("No dist file created in earlier command")
            for command, pyversion, filename in dist_files:
                self.upload_file(command, pyversion, filename)

else:
    mx_upload = None
    
### Helpers to allow rebasing packages within setup.py files

def rebase_packages(packages, new_base_package, filter=None):

    rebased_packages = []
    for package in packages:
        # Apply filter (only rebase packages for which the filter
        # returns true)
        if (filter is not None and
            not filter(package)):
            rebased_packages.append(package)
        else:
            # Rebase the package
            rebased_packages.append(new_base_package + '.' + package)
    return rebased_packages

def rebase_files(files, new_base_dir, filter=None):

    rebased_files = []
    for file in files:
        # Apply filter (only rebase packages for which the filter
        # returns true)
        if (filter is not None and
            not filter(file)):
            rebased_files.append(file)
        else:
            # Rebase the file
            rebased_files.append(os.path.join(new_base_dir, file))
    return rebased_files

def rebase_extensions(extensions,
                      new_base_package, new_base_dir,
                      filter_packages=None, filter_files=None):

    rebased_extensions = []
    for ext in extensions:

        # Apply package filter to the extension name
        if (filter_packages is not None and
            not filter_packages(ext)):
            rebased_extensions.append(ext)
            continue

        # Create a shallow copy
        new_ext = copy.copy(ext)
        rebased_extensions.append(new_ext)

        # Standard distutils Extension
        new_ext.name = new_base_package + '.' + ext.name
        new_ext.sources = rebase_files(
            ext.sources,
            new_base_dir,
            filter_files)
        new_ext.include_dirs = rebase_files(
            ext.include_dirs,
            new_base_dir,
            filter_files)
        new_ext.library_dirs = rebase_files(
            ext.library_dirs,
            new_base_dir,
            filter_files)
        new_ext.runtime_library_dirs = rebase_files(
            ext.runtime_library_dirs,
            new_base_dir,
            filter_files)
        if not isinstance(ext, mx_Extension):
            continue

        # mx_Extension
        new_ext.data_files = rebase_files(
            ext.data_files,
            new_base_dir,
            filter_files)
        new_ext.packages = rebase_packages(
            ext.packages,
            new_base_package,
            filter_packages)

        if 0:
            # optional_libraries will not need any rebasing, since
            # the header files rely on the standard search path
            new_optional_libraries = []
            for (libname, header_files) in ext.optional_libraries:
                new_optional_libraries.append(
                    (libname,
                     rebase_files(header_files,
                                  new_base_dir,
                                  filter_files)))
            new_ext.optional_libraries = new_optional_libraries

        new_needed_includes = []
        for (filename, dirs, pattern) in ext.needed_includes:
            new_needed_includes.append(
                (filename,
                 rebase_files(dirs,
                              new_base_dir,
                              filter_files),
                 pattern))
        new_ext.needed_includes = new_needed_includes

        new_needed_libraries = []
        for (filename, dirs, pattern) in ext.needed_libraries:
            new_needed_libraries.append(
                (filename,
                 rebase_files(dirs,
                              new_base_dir,
                              filter_files),
                 pattern))
        new_ext.needed_libraries = new_needed_libraries

    return rebased_extensions
    
###

def run_setup(configurations):

    """ Run distutils setup.

        The parameters passed to setup() are extracted from the list
        of modules, classes or instances given in configurations.

        Names with leading underscore are removed from the parameters.
        Parameters which are not strings, lists or tuples are removed
        as well.  Configurations which occur later in the
        configurations list override settings of configurations
        earlier in the list.

    """
    # Build parameter dictionary
    kws = {
        # Defaults for distutils and our add-ons
        #'version': '0.0.0',
        #'name': '',
        #'description': '',
        'license': ('(c) eGenix.com Sofware, Skills and Services GmbH, '
                    'All Rights Reserved.'),
        'author': 'eGenix.com Software, Skills and Services GmbH',
        'author_email': 'info@egenix.com',
        'maintainer': 'eGenix.com Software, Skills and Services GmbH',
        'maintainer_email': 'info@egenix.com',
        'url': 'http://www.egenix.com/',
        'download_url': 'http://www.egenix.com/',
        'platforms': [],
        'classifiers': [],
        'packages': [],
        'ext_modules': [],
        'data_files': [],
        'libraries': [],
    }
    if setuptools is not None:
        # Add defaults for setuptools
        kws.update({
            # Default to not install eggs as ZIP archives
            'zip_safe': 0,
            })
    for configuration in configurations:
        kws.update(vars(configuration))

    # Type and name checking
    for name, value in kws.items():
        if (name[:1] == '_' or
            name in UNSUPPORTED_SETUP_KEYWORDS):
            del kws[name]
            continue
        if not isinstance(value, ALLOWED_SETUP_TYPES):
            if isinstance(value, types.UnicodeType):
                # Convert Unicode values to UTF-8 encoded strings
                kws[name] = value.encode('utf-8')
            else:
                del kws[name]
            continue
        #if type(value) is types.FunctionType:
        #    kws[name] = value()

    if setuptools is not None:
        # Map requires to install_requires for setuptools
        if ('requires' in kws and
            'install_requires' not in kws):
            # Setuptools doesn't like hypens in package names, so we
            # convert them to underscores
            deps = [dep.replace('-', '_') for dep in kws['requires']]
            kws['install_requires'] = deps

    # Add setup extensions
    kws['distclass'] = mx_Distribution
    extensions = {'build': mx_build,
                  'build_unixlib': mx_build_unixlib,
                  'mx_autoconf': mx_autoconf,
                  'build_ext': mx_build_ext,
                  'build_clib': mx_build_clib,
                  'build_py': mx_build_py,
                  'build_data': mx_build_data,
                  'install': mx_install,
                  'install_data': mx_install_data,
                  'install_lib': mx_install_lib,
                  'uninstall': mx_uninstall,
                  'register': mx_register,
                  'bdist': mx_bdist,
                  'bdist_rpm': mx_bdist_rpm,
                  'bdist_zope': mx_bdist_zope,
                  'bdist_inplace': mx_bdist_inplace,
                  'bdist_wininst': mx_bdist_wininst,
                  'bdist_msi': mx_bdist_msi,
                  'bdist_prebuilt': mx_bdist_prebuilt,
                  'mx_bdist_egg': mx_bdist_egg,
                  'sdist': mx_sdist,
                  'sdist_web': mx_sdist_web,
                  'clean': mx_clean,
                  'upload': mx_upload,
                  }
    if bdist_ppm is not None:
        extensions['bdist_ppm'] = bdist_ppm.bdist_ppm
    if GenPPD is not None:
        extensions['GenPPD'] = GenPPD.GenPPD
    if mx_egg_info is not None:
        extensions['egg_info'] = mx_egg_info
    if setuptools is None:
        extensions['bdist_egg'] = mx_bdist_egg
        
    kws['cmdclass'] = extensions

    # Invoke distutils setup
    apply(setup, (), kws)