1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
|
""" Tools for compiling C/C++ code to extension modules
The main function, build_extension(), takes the C/C++ file
along with some other options and builds a Python extension.
It uses distutils for most of the heavy lifting.
choose_compiler() is also useful (mainly on windows anyway)
for trying to determine whether MSVC++ or gcc is available.
MSVC doesn't handle templates as well, so some of the code emitted
by the python->C conversions need this info to choose what kind
of code to create.
The other main thing here is an alternative version of the MingW32
compiler class. The class makes it possible to build libraries with
gcc even if the original version of python was built using MSVC. It
does this by converting a pythonxx.lib file to a libpythonxx.a file.
Note that you need write access to the pythonxx/lib directory to do this.
"""
from __future__ import absolute_import, print_function
import sys
import os
import time
import tempfile
import commands
import subprocess
import warnings
from . import platform_info
# If linker is 'gcc', this will convert it to 'g++'
# necessary to make sure stdc++ is linked in cross-platform way.
import distutils.sysconfig
import distutils.dir_util
# some (most?) platforms will fail to link C++ correctly
# unless numpy.distutils is used.
from numpy.distutils.core import Extension
old_init_posix = distutils.sysconfig._init_posix
def _init_posix():
old_init_posix()
ld = distutils.sysconfig._config_vars['LDSHARED']
# distutils.sysconfig._config_vars['LDSHARED'] = ld.replace('gcc','g++')
# FreeBSD names gcc as cc, so the above find and replace doesn't work.
# So, assume first entry in ld is the name of the linker -- gcc or cc or
# whatever. This is a sane assumption, correct?
# If the linker is gcc, set it to g++
link_cmds = ld.split()
if gcc_exists(link_cmds[0]):
link_cmds[0] = 'g++'
ld = ' '.join(link_cmds)
if (sys.platform == 'darwin'):
# The Jaguar distributed python 2.2 has -arch i386 in the link line
# which doesn't seem right. It omits all kinds of warnings, so
# remove it.
ld = ld.replace('-arch i386','')
# The following line is a HACK to fix a problem with building the
# freetype shared library under Mac OS X:
ld += ' -framework AppKit'
# 2.3a1 on OS X emits a ton of warnings about long double. OPT
# appears to not have all the needed flags set while CFLAGS does.
cfg_vars = distutils.sysconfig._config_vars
cfg_vars['OPT'] = cfg_vars['CFLAGS']
distutils.sysconfig._config_vars['LDSHARED'] = ld
distutils.sysconfig._init_posix = _init_posix
# end force g++
class CompileError(Exception):
pass
def create_extension(module_path, **kw):
""" Create an Extension that can be buil by setup.py
See build_extension for information on keyword arguments.
"""
# this is a screwy trick to get rid of a ton of warnings on Unix
distutils.sysconfig.get_config_vars()
if 'OPT' in distutils.sysconfig._config_vars:
flags = distutils.sysconfig._config_vars['OPT']
flags = flags.replace('-Wall','')
distutils.sysconfig._config_vars['OPT'] = flags
# get the name of the module and the extension directory it lives in.
module_dir,cpp_name = os.path.split(os.path.abspath(module_path))
module_name,ext = os.path.splitext(cpp_name)
# the business end of the function
sources = kw.get('sources',[])
kw['sources'] = [module_path] + sources
#--------------------------------------------------------------------
# added access to environment variable that user can set to specify
# where python (and other) include files are located. This is
# very useful on systems where python is installed by the root, but
# the user has also installed numerous packages in their own
# location.
#--------------------------------------------------------------------
if 'PYTHONINCLUDE' in os.environ:
path_string = os.environ['PYTHONINCLUDE']
if sys.platform == "win32":
extra_include_dirs = path_string.split(';')
else:
extra_include_dirs = path_string.split(':')
include_dirs = kw.get('include_dirs',[])
kw['include_dirs'] = include_dirs + extra_include_dirs
# SunOS specific
# fix for issue with linking to libstdc++.a. see:
# http://mail.python.org/pipermail/python-dev/2001-March/013510.html
platform = sys.platform
version = sys.version.lower()
if platform[:5] == 'sunos' and 'gcc' in version:
extra_link_args = kw.get('extra_link_args',[])
kw['extra_link_args'] = ['-mimpure-text'] + extra_link_args
ext = Extension(module_name, **kw)
return ext
def build_extension(module_path,compiler_name='',build_dir=None,
temp_dir=None, verbose=0, **kw):
""" Build the file given by module_path into a Python extension module.
build_extensions uses distutils to build Python extension modules.
kw arguments not used are passed on to the distutils extension
module. Directory settings can handle absoulte settings, but don't
currently expand '~' or environment variables.
module_path -- the full path name to the c file to compile.
Something like: /full/path/name/module_name.c
The name of the c/c++ file should be the same as the
name of the module (i.e. the initmodule() routine)
compiler_name -- The name of the compiler to use. On Windows if it
isn't given, MSVC is used if it exists (is found).
gcc is used as a second choice. If neither are found,
the default distutils compiler is used. Acceptable
names are 'gcc', 'msvc' or any of the compiler names
shown by distutils.ccompiler.show_compilers()
build_dir -- The location where the resulting extension module
should be placed. This location must be writable. If
it isn't, several default locations are tried. If the
build_dir is not in the current python path, a warning
is emitted, and it is added to the end of the path.
build_dir defaults to the current directory.
temp_dir -- The location where temporary files (*.o or *.obj)
from the build are placed. This location must be
writable. If it isn't, several default locations are
tried. It defaults to tempfile.gettempdir()
verbose -- 0, 1, or 2. 0 is as quiet as possible. 1 prints
minimal information. 2 is noisy.
**kw -- keyword arguments. These are passed on to the
distutils extension module. Most of the keywords
are listed below.
Distutils keywords. These are cut and pasted from Greg Ward's
distutils.extension.Extension class for convenience:
sources : [string]
list of source filenames, relative to the distribution root
(where the setup script lives), in Unix form (slash-separated)
for portability. Source files may be C, C++, SWIG (.i),
platform-specific resource files, or whatever else is recognized
by the "build_ext" command as source for a Python extension.
Note: The module_path file is always appended to the front of this
list
include_dirs : [string]
list of directories to search for C/C++ header files (in Unix
form for portability)
define_macros : [(name : string, value : string|None)]
list of macros to define; each macro is defined using a 2-tuple,
where 'value' is either the string to define it to or None to
define it without a particular value (equivalent of "#define
FOO" in source or -DFOO on Unix C compiler command line)
undef_macros : [string]
list of macros to undefine explicitly
library_dirs : [string]
list of directories to search for C/C++ libraries at link time
libraries : [string]
list of library names (not filenames or paths) to link against
runtime_library_dirs : [string]
list of directories to search for C/C++ libraries at run time
(for shared extensions, this is when the extension is loaded)
extra_objects : [string]
list of extra files to link with (eg. object files not implied
by 'sources', static library that must be explicitly specified,
binary resource files, etc.)
extra_compile_args : [string]
any extra platform- and compiler-specific information to use
when compiling the source files in 'sources'. For platforms and
compilers where "command line" makes sense, this is typically a
list of command-line arguments, but for other platforms it could
be anything.
extra_link_args : [string]
any extra platform- and compiler-specific information to use
when linking object files together to create the extension (or
to create a new static Python interpreter). Similar
interpretation as for 'extra_compile_args'.
export_symbols : [string]
list of symbols to be exported from a shared extension. Not
used on all platforms, and not generally necessary for Python
extensions, which typically export exactly one symbol: "init" +
extension_name.
"""
success = 0
from numpy.distutils.core import setup
from numpy.distutils.log import set_verbosity
set_verbosity(-1)
# this is a screwy trick to get rid of a ton of warnings on Unix
import distutils.sysconfig
distutils.sysconfig.get_config_vars()
if 'OPT' in distutils.sysconfig._config_vars:
flags = distutils.sysconfig._config_vars['OPT']
flags = flags.replace('-Wall','')
distutils.sysconfig._config_vars['OPT'] = flags
# get the name of the module and the extension directory it lives in.
module_dir,cpp_name = os.path.split(os.path.abspath(module_path))
module_name,ext = os.path.splitext(cpp_name)
# configure temp and build directories
temp_dir = configure_temp_dir(temp_dir)
build_dir = configure_build_dir(build_dir or module_dir)
# dag. We keep having to add directories to the path to keep
# object files separated from each other. gcc2.x and gcc3.x C++
# object files are not compatible, so we'll stick them in a sub
# dir based on their version. This will add a SHA-256 check sum
# (truncated to 32 characters) of the compiler binary to the directory
# name to keep objects from different compilers in different locations.
compiler_dir = platform_info.get_compiler_dir(compiler_name)
temp_dir = os.path.join(temp_dir,compiler_dir)
distutils.dir_util.mkpath(temp_dir)
compiler_name = choose_compiler(compiler_name)
configure_sys_argv(compiler_name,temp_dir,build_dir)
# the business end of the function
try:
if verbose == 1:
print('Compiling code...')
# set compiler verboseness 2 or more makes it output results
if verbose > 1:
verb = 1
else:
verb = 0
t1 = time.time()
ext = create_extension(module_path,**kw)
# the switcheroo on SystemExit here is meant to keep command line
# sessions from exiting when compiles fail.
builtin = sys.modules['__builtin__']
old_SysExit = builtin.__dict__['SystemExit']
builtin.__dict__['SystemExit'] = CompileError
# change current working directory to 'build_dir' so compiler won't
# pick up anything by mistake
oldcwd = os.path.abspath(os.getcwd())
os.chdir(build_dir)
# distutils for MSVC messes with the environment, so we save the
# current state and restore them afterward.
import copy
environ = copy.deepcopy(os.environ)
try:
setup(name=module_name, ext_modules=[ext],verbose=verb)
finally:
# restore state
os.environ = environ
# restore SystemExit
builtin.__dict__['SystemExit'] = old_SysExit
# restore working directory to one before setup
os.chdir(oldcwd)
t2 = time.time()
if verbose == 1:
print('finished compiling (sec): ', t2 - t1)
success = 1
configure_python_path(build_dir)
except SyntaxError: # TypeError:
success = 0
except Exception, e:
restore_sys_argv()
raise e
# restore argv after our trick...
restore_sys_argv()
return success
old_argv = []
def configure_sys_argv(compiler_name,temp_dir,build_dir):
# We're gonna play some tricks with argv here to pass info to distutils
# which is really built for command line use. better way??
global old_argv
old_argv = sys.argv[:]
sys.argv = ['','build_ext','--build-lib', build_dir,
'--build-temp',temp_dir]
if compiler_name == 'gcc':
sys.argv.insert(2,'--compiler='+compiler_name)
elif compiler_name:
sys.argv.insert(2,'--compiler='+compiler_name)
def restore_sys_argv():
sys.argv = old_argv
def configure_python_path(build_dir):
# make sure the module lives in a directory on the python path.
python_paths = [os.path.abspath(x) for x in sys.path]
if os.path.abspath(build_dir) not in python_paths:
# print "warning: build directory was not part of python path."\
# " It has been appended to the path."
sys.path.append(os.path.abspath(build_dir))
def choose_compiler(compiler_name=''):
""" Try and figure out which compiler is gonna be used on windows.
On other platforms, it just returns whatever value it is given.
converts 'gcc' to 'mingw32' on win32
"""
if sys.platform == 'win32':
if not compiler_name:
# On Windows, default to MSVC and use gcc if it wasn't found
# wasn't found. If neither are found, go with whatever
# the default is for distutils -- and probably fail...
if msvc_exists():
compiler_name = 'msvc'
elif gcc_exists():
compiler_name = 'mingw32'
elif compiler_name == 'gcc':
compiler_name = 'mingw32'
else:
# don't know how to force gcc -- look into this.
if compiler_name == 'gcc':
compiler_name = 'unix'
return compiler_name
def gcc_exists(name='gcc'):
""" Test to make sure gcc is found."""
result = 0
cmd = [str(name), '-v']
try:
if sys.platform == 'win32':
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
else:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
str_result = p.stdout.read()
if 'specs' in str_result:
result = 1
except:
# This was needed because the msvc compiler messes with
# the path variable. and will occasionlly mess things up
# so much that gcc is lost in the path. (Occurs in test
# scripts)
result = not os.system(" ".join(cmd))
return result
def msvc_exists():
""" Determine whether MSVC is available on the machine.
"""
result = 0
try:
p = subprocess.Popen(['cl'], shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
str_result = p.stdout.read()
# print str_result
if 'Microsoft' in str_result:
result = 1
except:
# assume we're ok if devstudio exists
import distutils.msvccompiler
try:
cc = distutils.msvccompiler.MSVCCompiler()
cc.initialize()
p = subprocess.Popen([cc.cc])
result = 1
except distutils.errors.DistutilsPlatformError:
pass
except WindowsError:
pass
return result
if os.name == 'nt':
def run_command(command):
""" not sure how to get exit status on nt. """
p = subprocess.Popen(['cl'], shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
text = p.stdout.read()
return 0, text
else:
run_command = commands.getstatusoutput
def configure_temp_dir(temp_dir=None):
if temp_dir is None:
temp_dir = tempfile.gettempdir()
elif not os.path.exists(temp_dir) or not os.access(temp_dir,os.W_OK):
print("warning: specified temp_dir '%s' does not exist "
"or is not writable. Using the default temp directory" %
temp_dir)
temp_dir = tempfile.gettempdir()
# Final check that directories are writable.
if not os.access(temp_dir,os.W_OK):
msg = "Either the temp or build directory wasn't writable. Check" \
" these locations: '%s'" % temp_dir
raise ValueError(msg)
return temp_dir
def configure_build_dir(build_dir=None):
# make sure build_dir exists and is writable
if build_dir and (not os.path.exists(build_dir) or
not os.access(build_dir,os.W_OK)):
msg = "specified build_dir '%s' does not exist " \
"or is not writable. Trying default locations" % build_dir
warnings.warn(msg)
build_dir = None
if build_dir is None:
# Default to building in the home directory of the given module.
build_dir = os.curdir
# If it doesn't work use the current directory. This should always
# be writable.
if not os.access(build_dir,os.W_OK):
print("warning:, neither the module's directory nor the "
"current directory are writable. Using the temporary"
"directory.")
build_dir = tempfile.gettempdir()
# Final check that directories are writable.
if not os.access(build_dir,os.W_OK):
msg = "The build directory wasn't writable. Check" \
" this location: '%s'" % build_dir
raise ValueError(msg)
return os.path.abspath(build_dir)
if sys.platform == 'win32':
import distutils.cygwinccompiler
from distutils.version import StrictVersion
from distutils.errors import CompileError
from distutils.unixccompiler import UnixCCompiler
# the same as cygwin plus some additional parameters
class Mingw32CCompiler(distutils.cygwinccompiler.CygwinCCompiler):
""" A modified MingW32 compiler compatible with an MSVC built Python.
"""
compiler_type = 'mingw32'
def __init__(self,
verbose=0,
dry_run=0,
force=0):
distutils.cygwinccompiler.CygwinCCompiler.__init__(self,
verbose,dry_run, force)
# we need to support 3.2 which doesn't match the standard
# get_versions methods regex
if self.gcc_version is None:
import re
p = subprocess.Popen(['gcc', ' -dumpversion'], shell=True,
stdout=subprocess.PIPE)
out_string = p.stdout.read()
result = re.search('(\d+\.\d+)',out_string)
if result:
self.gcc_version = StrictVersion(result.group(1))
# A real mingw32 doesn't need to specify a different entry point,
# but cygwin 2.91.57 in no-cygwin-mode needs it.
if self.gcc_version <= "2.91.57":
entry_point = '--entry _DllMain@12'
else:
entry_point = ''
if self.linker_dll == 'dllwrap':
self.linker = 'dllwrap' + ' --driver-name g++'
elif self.linker_dll == 'gcc':
self.linker = 'g++'
# **changes: eric jones 4/11/01
# 1. Check for import library on Windows. Build if it doesn't exist.
if not import_library_exists():
build_import_library()
# **changes: eric jones 4/11/01
# 2. increased optimization and turned off all warnings
# 3. also added --driver-name g++
# self.set_executables(compiler='gcc -mno-cygwin -O2 -w',
# compiler_so='gcc -mno-cygwin -mdll -O2 -w',
# linker_exe='gcc -mno-cygwin',
# linker_so='%s --driver-name g++ -mno-cygwin -mdll -static %s'
# % (self.linker, entry_point))
if self.gcc_version <= "3.0.0":
self.set_executables(compiler='gcc -mno-cygwin -O2 -w',
compiler_so='gcc -mno-cygwin -mdll -O2 -w -Wstrict-prototypes',
linker_exe='g++ -mno-cygwin',
linker_so='%s -mno-cygwin -mdll -static %s'
% (self.linker, entry_point))
else:
self.set_executables(compiler='gcc -mno-cygwin -O2 -w',
compiler_so='gcc -O2 -w -Wstrict-prototypes',
linker_exe='g++ ',
linker_so='g++ -shared')
# added for python2.3 support
# we can't pass it through set_executables because pre 2.2 would fail
self.compiler_cxx = ['g++']
# Maybe we should also append -mthreads, but then the finished
# dlls need another dll (mingwm10.dll see Mingw32 docs)
# (-mthreads: Support thread-safe exception handling on `Mingw32')
# no additional libraries needed
self.dll_libraries = []
# __init__ ()
def link(self,
target_desc,
objects,
output_filename,
output_dir,
libraries,
library_dirs,
runtime_library_dirs,
export_symbols=None, # export_symbols, we do this in our def-file
debug=0,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None):
if self.gcc_version < "3.0.0":
distutils.cygwinccompiler.CygwinCCompiler.link(self,
target_desc,
objects,
output_filename,
output_dir,
libraries,
library_dirs,
runtime_library_dirs,
None, # export_symbols, we do this in our def-file
debug,
extra_preargs,
extra_postargs,
build_temp,
target_lang)
else:
UnixCCompiler.link(self,
target_desc,
objects,
output_filename,
output_dir,
libraries,
library_dirs,
runtime_library_dirs,
None, # export_symbols, we do this in our def-file
debug,
extra_preargs,
extra_postargs,
build_temp,
target_lang)
# On windows platforms, we want to default to mingw32 (gcc)
# because msvc can't build blitz stuff.
# We should also check the version of gcc available...
# distutils.ccompiler._default_compilers['nt'] = 'mingw32'
# distutils.ccompiler._default_compilers = (('nt', 'mingw32'))
# reset the Mingw32 compiler in distutils to the one defined above
distutils.cygwinccompiler.Mingw32CCompiler = Mingw32CCompiler
def import_library_exists():
""" on windows platforms, make sure a gcc import library exists
"""
if os.name == 'nt':
lib_name = "libpython%d%d.a" % tuple(sys.version_info[:2])
full_path = os.path.join(sys.prefix,'libs',lib_name)
if not os.path.exists(full_path):
return 0
return 1
def build_import_library():
""" Build the import libraries for Mingw32-gcc on Windows
"""
from numpy.distutils import lib2def
# libfile, deffile = parse_cmd()
# if deffile is None:
# deffile = sys.stdout
# else:
# deffile = open(deffile, 'w')
lib_name = "python%d%d.lib" % tuple(sys.version_info[:2])
lib_file = os.path.join(sys.prefix,'libs',lib_name)
def_name = "python%d%d.def" % tuple(sys.version_info[:2])
def_file = os.path.join(sys.prefix,'libs',def_name)
nm_cmd = '%s %s' % (lib2def.DEFAULT_NM, lib_file)
nm_output = lib2def.getnm(nm_cmd)
dlist, flist = lib2def.parse_nm(nm_output)
lib2def.output_def(dlist, flist, lib2def.DEF_HEADER, open(def_file, 'w'))
out_name = "libpython%d%d.a" % tuple(sys.version_info[:2])
out_file = os.path.join(sys.prefix,'libs',out_name)
dll_name = "python%d%d.dll" % tuple(sys.version_info[:2])
args = (dll_name,def_file,out_file)
cmd = 'dlltool --dllname %s --def %s --output-lib %s' % args
success = not os.system(cmd)
# for now, fail silently
if not success:
print('WARNING: failed to build import library for gcc. Linking will fail.')
# if not success:
# msg = "Couldn't find import library, and failed to build it."
# raise DistutilsPlatformError, msg
|