from __future__ import absolute_import import sys, os, re, inspect import imp try: import hashlib except ImportError: import md5 as hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext import Cython from ..Compiler.Main import Context, CompilationOptions, default_options from ..Compiler.ParseTreeTransforms import (CythonTransform, SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform) from ..Compiler.TreeFragment import parse_from_strings from .Dependencies import strip_string_literals, cythonize, cached_function from ..Compiler import Pipeline, Nodes from ..Utils import get_cython_cache_dir import cython as cython_module # A utility function to convert user-supplied ASCII strings to unicode. if sys.version_info[0] < 3: def to_unicode(s): if not isinstance(s, unicode): return s.decode('ascii') else: return s else: to_unicode = lambda x: x class UnboundSymbols(EnvTransform, SkipDeclarations): def __init__(self): CythonTransform.__init__(self, None) self.unbound = set() def visit_NameNode(self, node): if not self.current_env().lookup(node.name): self.unbound.add(node.name) return node def __call__(self, node): super(UnboundSymbols, self).__call__(node) return self.unbound @cached_function def unbound_symbols(code, context=None): code = to_unicode(code) if context is None: context = Context([], default_options) from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform tree = parse_from_strings('(tree fragment)', code) for phase in Pipeline.create_pipeline(context, 'pyx'): if phase is None: continue tree = phase(tree) if isinstance(phase, AnalyseDeclarationsTransform): break try: import builtins except ImportError: import __builtin__ as builtins return UnboundSymbols()(tree) - set(dir(builtins)) def unsafe_type(arg, context=None): py_type = type(arg) if py_type is int: return 'long' else: return safe_type(arg, context) def safe_type(arg, context=None): py_type = type(arg) if py_type in [list, tuple, dict, str]: return py_type.__name__ elif py_type is complex: return 'double complex' elif py_type is float: return 'double' elif py_type is bool: return 'bint' elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray): return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim) else: for base_type in py_type.mro(): if base_type.__module__ in ('__builtin__', 'builtins'): return 'object' module = context.find_module(base_type.__module__, need_pxd=False) if module: entry = module.lookup(base_type.__name__) if entry.is_type: return '%s.%s' % (base_type.__module__, base_type.__name__) return 'object' def _get_build_extension(): dist = Distribution() # Ensure the build respects distutils configuration by parsing # the configuration files config_files = dist.find_config_files() dist.parse_config_files(config_files) build_extension = build_ext(dist) build_extension.finalize_options() return build_extension @cached_function def _create_context(cython_include_dirs): return Context(list(cython_include_dirs), default_options) def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=['.'], force=False, quiet=False, locals=None, globals=None, **kwds): if get_type is None: get_type = lambda x: 'object' code = to_unicode(code) orig_code = code code, literals = strip_string_literals(code) code = strip_common_indent(code) ctx = _create_context(tuple(cython_include_dirs)) if locals is None: locals = inspect.currentframe().f_back.f_back.f_locals if globals is None: globals = inspect.currentframe().f_back.f_back.f_globals try: for symbol in unbound_symbols(code): if symbol in kwds: continue elif symbol in locals: kwds[symbol] = locals[symbol] elif symbol in globals: kwds[symbol] = globals[symbol] else: print("Couldn't find ", symbol) except AssertionError: if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") cimports = [] for name, arg in kwds.items(): if arg is cython_module: cimports.append('\ncimport cython as %s' % name) del kwds[name] arg_names = kwds.keys() arg_names.sort() arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() if module_name in sys.modules: module = sys.modules[module_name] else: build_extension = None if cython_inline.so_ext is None: # Figure out and cache current extension suffix build_extension = _get_build_extension() cython_inline.so_ext = build_extension.get_ext_filename('') module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext) if not os.path.exists(lib_dir): os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: m = qualified.match(type) if m: cimports.append('\ncimport %s' % m.groups()[0]) # one special case if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) module_code = """ %(module_body)s %(cimports)s def __invoke(%(params)s): %(func_body)s return locals() """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body } for key, value in literals.items(): module_code = module_code.replace(key, value) pyx_file = os.path.join(lib_dir, module_name + '.pyx') fh = open(pyx_file, 'w') try: fh.write(module_code) finally: fh.close() extension = Extension( name = module_name, sources = [pyx_file], include_dirs = c_include_dirs, extra_compile_args = cflags) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() module = imp.load_dynamic(module_name, module_path) arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list) # Cached suffix used by cython_inline above. None should get # overridden with actual value upon the first cython_inline invocation cython_inline.so_ext = None non_space = re.compile('[^ ]') def strip_common_indent(code): min_indent = None lines = code.split('\n') for line in lines: match = non_space.search(line) if not match: continue # blank indent = match.start() if line[indent] == '#': continue # comment elif min_indent is None or min_indent > indent: min_indent = indent for ix, line in enumerate(lines): match = non_space.search(line) if not match or line[indent] == '#': continue else: lines[ix] = line[min_indent:] return '\n'.join(lines) module_statement = re.compile(r'^((cdef +(extern|class))|cimport|(from .+ cimport)|(from .+ import +[*]))') def extract_func_code(code): module = [] function = [] current = function code = code.replace('\t', ' ') lines = code.split('\n') for line in lines: if not line.startswith(' '): if module_statement.match(line): current = module else: current = function current.append(line) return '\n'.join(module), ' ' + '\n '.join(function) try: from inspect import getcallargs except ImportError: def getcallargs(func, *arg_values, **kwd_values): all = {} args, varargs, kwds, defaults = inspect.getargspec(func) if varargs is not None: all[varargs] = arg_values[len(args):] for name, value in zip(args, arg_values): all[name] = value for name, value in kwd_values.items(): if name in args: if name in all: raise TypeError("Duplicate argument %s" % name) all[name] = kwd_values.pop(name) if kwds is not None: all[kwds] = kwd_values elif kwd_values: raise TypeError("Unexpected keyword arguments: %s" % kwd_values.keys()) if defaults is None: defaults = () first_default = len(args) - len(defaults) for ix, name in enumerate(args): if name not in all: if ix >= first_default: all[name] = defaults[ix - first_default] else: raise TypeError("Missing argument: %s" % name) return all def get_body(source): ix = source.index(':') if source[:5] == 'lambda': return "return %s" % source[ix+1:] else: return source[ix+1:] # Lots to be done here... It would be especially cool if compiled functions # could invoke each other quickly. class RuntimeCompiledFunction(object): def __init__(self, f): self._f = f self._body = get_body(inspect.getsource(f)) def __call__(self, *args, **kwds): all = getcallargs(self._f, *args, **kwds) return cython_inline(self._body, locals=self._f.func_globals, globals=self._f.func_globals, **all)