import contextlib import py import sys, os from rpython.rlib import exports from rpython.rtyper.lltypesystem.lltype import getfunctionptr from rpython.rtyper.lltypesystem import lltype from rpython.tool import runsubprocess from rpython.tool.nullpath import NullPyPathLocal from rpython.tool.udir import udir from rpython.translator.c import gc from rpython.translator.c.database import LowLevelDatabase from rpython.translator.c.extfunc import pre_include_code_lines from rpython.translator.c.support import log from rpython.translator.gensupp import uniquemodulename, NameManager from rpython.translator.tool.cbuild import ExternalCompilationInfo _CYGWIN = sys.platform == 'cygwin' _CPYTHON_RE = py.std.re.compile('^Python 2.[567]') def get_recent_cpython_executable(): if sys.platform == 'win32': python = sys.executable.replace('\\', '/') else: python = sys.executable # Is there a command 'python' that runs python 2.5-2.7? # If there is, then we can use it instead of sys.executable returncode, stdout, stderr = runsubprocess.run_subprocess( "python", "-V") if _CPYTHON_RE.match(stdout) or _CPYTHON_RE.match(stderr): python = 'python' return python class CCompilerDriver(object): def __init__(self, platform, cfiles, eci, outputfilename=None, profbased=False): # XXX config might contain additional link and compile options. # We need to fish for it somehow. self.platform = platform self.cfiles = cfiles self.eci = eci self.outputfilename = outputfilename # self.profbased = profbased def _build(self, eci=ExternalCompilationInfo(), shared=False): outputfilename = self.outputfilename if shared: if outputfilename: basename = outputfilename else: basename = self.cfiles[0].purebasename outputfilename = 'lib' + basename return self.platform.compile(self.cfiles, self.eci.merge(eci), outputfilename=outputfilename, standalone=not shared) class CBuilder(object): c_source_filename = None _compiled = False modulename = None split = False def __init__(self, translator, entrypoint, config, gcpolicy=None, gchooks=None, secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) self.originalentrypoint = entrypoint self.config = config self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/ self.gchooks = gchooks self.eci = self.get_eci() self.secondary_entrypoints = secondary_entrypoints def get_eci(self): pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] if self.config.translation.reverse_debugger: include_dirs.append(pypy_include_dir.join('..', 'revdb')) return ExternalCompilationInfo(include_dirs=include_dirs) def build_database(self): translator = self.translator gcpolicyclass = self.get_gcpolicyclass() if self.config.translation.gcrootfinder == "asmgcc": if not self.standalone: raise NotImplementedError("--gcrootfinder=asmgcc requires standalone") exctransformer = translator.getexceptiontransformer() db = LowLevelDatabase(translator, standalone=self.standalone, gcpolicyclass=gcpolicyclass, gchooks=self.gchooks, exctransformer=exctransformer, thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox, split_gc_address_space= self.config.translation.split_gc_address_space, reverse_debugger= self.config.translation.reverse_debugger) self.db = db # give the gc a chance to register interest in the start-up functions it # need (we call this for its side-effects of db.get()) list(db.gcpolicy.gc_startup_code()) # build entrypoint and eventually other things to expose pf = self.getentrypointptr() if isinstance(pf, list): for one_pf in pf: db.get(one_pf) self.c_entrypoint_name = None else: pfname = db.get(pf) for func, _ in self.secondary_entrypoints: bk = translator.annotator.bookkeeper db.get(getfunctionptr(bk.getdesc(func).getuniquegraph())) self.c_entrypoint_name = pfname if self.config.translation.reverse_debugger: from rpython.translator.revdb import gencsupp gencsupp.prepare_database(db) for obj in exports.EXPORTS_obj2name.keys(): db.getcontainernode(obj) exports.clear() for ll_func in db.translator._call_at_startup: db.get(ll_func) db.complete() self.collect_compilation_info(db) return db have___thread = None def merge_eci(self, *ecis): self.eci = self.eci.merge(*ecis) def collect_compilation_info(self, db): # we need a concrete gcpolicy to do this self.merge_eci(db.gcpolicy.compilation_info()) all = [] for node in self.db.globalcontainers(): eci = node.compilation_info() if eci: all.append(eci) for node in self.db.getstructdeflist(): try: all.append(node.STRUCT._hints['eci']) except (AttributeError, KeyError): pass self.merge_eci(*all) def get_gcpolicyclass(self): if self.gcpolicy is None: name = self.config.translation.gctransformer if name == "framework": name = "%s+%s" % (name, self.config.translation.gcrootfinder) return gc.name_to_gcpolicy[name] return self.gcpolicy # use generate_source(defines=DEBUG_DEFINES) to force the #definition # of the macros that enable debugging assertions DEBUG_DEFINES = {'RPY_ASSERT': 1, 'RPY_LL_ASSERT': 1, 'RPY_REVDB_PRINT_ALL': 1} def generate_source(self, db=None, defines={}, exe_name=None): assert self.c_source_filename is None if db is None: db = self.build_database() pf = self.getentrypointptr() if self.modulename is None: self.modulename = uniquemodulename('testing') modulename = self.modulename targetdir = udir.ensure(modulename, dir=1) if self.config.translation.dont_write_c_files: targetdir = NullPyPathLocal(targetdir) self.targetdir = targetdir defines = defines.copy() if self.config.translation.countmallocs: defines['COUNT_OP_MALLOCS'] = 1 if self.config.translation.sandbox: defines['RPY_SANDBOXED'] = 1 if self.config.translation.reverse_debugger: defines['RPY_REVERSE_DEBUGGER'] = 1 if CBuilder.have___thread is None: CBuilder.have___thread = self.translator.platform.check___thread() if not self.standalone: assert not self.config.translation.instrument else: defines['PYPY_STANDALONE'] = db.get(pf) if self.config.translation.instrument: defines['PYPY_INSTRUMENT'] = 1 if CBuilder.have___thread: if not self.config.translation.no__thread: defines['USE___THREAD'] = 1 if self.config.translation.shared: defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup" self.eci, cfile, extra, headers_to_precompile = \ gen_source(db, modulename, targetdir, self.eci, defines=defines, split=self.split) self.c_source_filename = py.path.local(cfile) self.extrafiles = self.eventually_copy(extra) self.gen_makefile(targetdir, exe_name=exe_name, headers_to_precompile=headers_to_precompile) return cfile def eventually_copy(self, cfiles): extrafiles = [] for fn in cfiles: fn = py.path.local(fn) if not fn.relto(udir): newname = self.targetdir.join(fn.basename) fn.copy(newname) fn = newname extrafiles.append(fn) return extrafiles class CStandaloneBuilder(CBuilder): standalone = True split = True executable_name = None shared_library_name = None _entrypoint_wrapper = None make_entrypoint_wrapper = True # for tests def getentrypointptr(self): # XXX check that the entrypoint has the correct # signature: list-of-strings -> int if not self.make_entrypoint_wrapper: bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) if self._entrypoint_wrapper is not None: return self._entrypoint_wrapper # from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem import rffi from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.rtyper.llannotation import lltype_to_annotation entrypoint = self.entrypoint # def entrypoint_wrapper(argc, argv): """This is a wrapper that takes "Signed argc" and "char **argv" like the C main function, and puts them inside an RPython list of strings before invoking the real entrypoint() function. """ list = [""] * argc i = 0 while i < argc: list[i] = rffi.charp2str(argv[i]) i += 1 return entrypoint(list) # mix = MixLevelHelperAnnotator(self.translator.rtyper) args_s = [annmodel.SomeInteger(), lltype_to_annotation(rffi.CCHARPP)] s_result = annmodel.SomeInteger() graph = mix.getgraph(entrypoint_wrapper, args_s, s_result) mix.finish() res = getfunctionptr(graph) self._entrypoint_wrapper = res return res def cmdexec(self, args='', env=None, err=False, expect_crash=False, exe=None): assert self._compiled if sys.platform == 'win32': #Prevent opening a dialog box import ctypes winapi = ctypes.windll.kernel32 SetErrorMode = winapi.SetErrorMode SetErrorMode.argtypes=[ctypes.c_int] SEM_FAILCRITICALERRORS = 1 SEM_NOGPFAULTERRORBOX = 2 SEM_NOOPENFILEERRORBOX = 0x8000 flags = SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX \ | SEM_NOOPENFILEERRORBOX #Since there is no GetErrorMode, do a double Set old_mode = SetErrorMode(flags) SetErrorMode(old_mode | flags) if env is None: envrepr = '' else: envrepr = ' [env=%r]' % (env,) if exe is None: exe = self.executable_name log.cmdexec('%s %s%s' % (exe, args, envrepr)) res = self.translator.platform.execute(exe, args, env=env) if sys.platform == 'win32': SetErrorMode(old_mode) if res.returncode != 0: if expect_crash: if type(expect_crash) is int and expect_crash != res.returncode: raise Exception("Returned %d, but expected %d" % ( res.returncode, expect_crash)) return res.out, res.err print >> sys.stderr, res.err raise Exception("Returned %d" % (res.returncode,)) if expect_crash: raise Exception("Program did not crash!") if err: return res.out, res.err return res.out def build_main_for_shared(self, shared_library_name, entrypoint, exe_name): import time time.sleep(1) self.shared_library_name = shared_library_name # build main program eci = self.get_eci() kw = {} if self.translator.platform.cc == 'gcc': kw['libraries'] = [self.shared_library_name.purebasename[3:]] kw['library_dirs'] = [self.targetdir] else: kw['libraries'] = [self.shared_library_name.new(ext='')] eci = eci.merge(ExternalCompilationInfo( separate_module_sources=[''' int %s(int argc, char* argv[]); int main(int argc, char* argv[]) { return %s(argc, argv); } ''' % (entrypoint, entrypoint) ], **kw )) eci = eci.convert_sources_to_files( cache_dir=self.targetdir) return self.translator.platform.compile( [], eci, outputfilename=exe_name) def compile(self, exe_name=None): assert self.c_source_filename assert not self._compiled shared = self.config.translation.shared extra_opts = [] if self.config.translation.profopt: extra_opts += ["profopt"] if self.config.translation.make_jobs != 1: extra_opts += ['-j', str(self.config.translation.make_jobs)] if self.config.translation.lldebug: extra_opts += ["lldebug"] elif self.config.translation.lldebug0: extra_opts += ["lldebug0"] self.translator.platform.execute_makefile(self.targetdir, extra_opts) if shared: self.shared_library_name = self.executable_name.new( purebasename='lib' + self.executable_name.purebasename, ext=self.translator.platform.so_ext) self._compiled = True return self.executable_name def gen_makefile(self, targetdir, exe_name=None, headers_to_precompile=[]): module_files = self.eventually_copy(self.eci.separate_module_files) self.eci.separate_module_files = [] cfiles = [self.c_source_filename] + self.extrafiles + list(module_files) if exe_name is not None: exe_name = targetdir.join(exe_name) mk = self.translator.platform.gen_makefile( cfiles, self.eci, path=targetdir, exe_name=exe_name, headers_to_precompile=headers_to_precompile, no_precompile_cfiles = module_files, shared=self.config.translation.shared, profopt = self.config.translation.profopt, config=self.config) if exe_name is None: short = targetdir.basename exe_name = targetdir.join(short) rules = [ ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'), ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'), ('debug_mem', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DPYPY_USE_TRIVIAL_MALLOC" debug_target'), ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'), ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'), ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'), ] # added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument # Also made a difference between translating with shared or not, because this affects profopt's target if self.config.translation.profopt: if self.config.translation.profoptargs is None: raise Exception("No profoptargs specified, neither in the command line, nor in the target. If the target is not PyPy, please specify profoptargs") # Set the correct PGO params based on OS and CC profopt_gen_flag = "" profopt_use_flag = "" profopt_merger = "" profopt_file = "" llvm_profdata = "" cc = self.translator.platform.cc # Locate llvm-profdata if "clang" in cc: clang_bin = cc path = os.environ.get("PATH").split(":") profdata_found = False # Try to find it in $PATH (Darwin and Linux) for dir in path: bin = "%s/llvm-profdata" % dir if os.path.isfile(bin): llvm_profdata = bin profdata_found = True break # If not found, try to find it where clang is actually installed (Darwin and Linux) if not profdata_found: # If the full path is not given, find where clang is located if not os.path.isfile(clang_bin): for dir in path: bin = "%s/%s" % (dir, cc) if os.path.isfile(bin): clang_bin = bin break # Some systems install clang elsewhere as a symlink to the real path, # which is where the related llvm tools are located. if os.path.islink(clang_bin): clang_bin = os.path.realpath(clang_bin) # the real clang binary # llvm-profdata must be in the same directory as clang llvm_profdata = "%s/llvm-profdata" % os.path.dirname(clang_bin) profdata_found = os.path.isfile(llvm_profdata) # If not found, and Darwin is used, try to find it in the development environment # More: https://apple.stackexchange.com/questions/197053/ if not profdata_found and sys.platform == 'darwin': code = os.system("/usr/bin/xcrun -find llvm-profdata 2>/dev/null") if code == 0: llvm_profdata = "/usr/bin/xcrun llvm-profdata" profdata_found = True # If everything failed, throw Exception, sorry if not profdata_found: raise Exception( "Error: Cannot perform profopt build because llvm-profdata was not found in PATH. " "Please add it to PATH and run the translation again.") # Set the PGO flags if "clang" in cc: # Any changes made here should be reflected in the GCC+Darwin case below profopt_gen_flag = "-fprofile-instr-generate" profopt_use_flag = "-fprofile-instr-use=code.profclangd" profopt_merger = "%s merge -output=code.profclangd *.profclangr" % llvm_profdata profopt_file = 'LLVM_PROFILE_FILE="code-%p.profclangr"' elif "gcc" in cc: if sys.platform == 'darwin': profopt_gen_flag = "-fprofile-instr-generate" profopt_use_flag = "-fprofile-instr-use=code.profclangd" profopt_merger = "%s merge -output=code.profclangd *.profclangr" % llvm_profdata profopt_file = 'LLVM_PROFILE_FILE="code-%p.profclangr"' else: profopt_gen_flag = "-fprofile-generate" profopt_use_flag = "-fprofile-use -fprofile-correction" profopt_merger = "true" profopt_file = "" if self.config.translation.shared: mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o', ['$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS) -lgcov', '$(MAKE) postcompile BIN=$(PROFOPT_TARGET)']) else: mk.definition('PROFOPT_TARGET', '$(TARGET)') rules.append( ('profopt', '', [ '$(MAKE) CFLAGS="%s -fPIC $(CFLAGS)" LDFLAGS="%s $(LDFLAGS)" $(PROFOPT_TARGET)' % (profopt_gen_flag, profopt_gen_flag), '%s %s %s ' % (profopt_file, exe_name, self.config.translation.profoptargs), '%s' % (profopt_merger), '$(MAKE) clean_noprof', '$(MAKE) CFLAGS="%s -fPIC $(CFLAGS)" LDFLAGS="%s $(LDFLAGS)" $(PROFOPT_TARGET)' % (profopt_use_flag, profopt_use_flag), ])) for rule in rules: mk.rule(*rule) if self.translator.platform.name == 'msvc': mk.rule('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -Od -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'), wildcards = '..\*.obj ..\*.pdb ..\*.lib ..\*.dll ..\*.manifest ..\*.exp *.pch' cmd = r'del /s %s $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)' % wildcards mk.rule('clean', '', cmd + ' *.gc?? ..\module_cache\*.gc??') mk.rule('clean_noprof', '', cmd) else: mk.rule('lldebug0','', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -O0 -DMAX_STACK_SIZE=8192000 -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'), mk.rule('clean', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES) *.gc?? ../module_cache/*.gc??') mk.rule('clean_noprof', '', 'rm -f $(OBJECTS) $(DEFAULT_TARGET) $(TARGET) $(GCMAPFILES) $(ASMFILES)') #XXX: this conditional part is not tested at all if self.config.translation.gcrootfinder == 'asmgcc': if self.translator.platform.name == 'msvc': raise Exception("msvc no longer supports asmgcc") _extra = '' if self.config.translation.shared: _extra = ' -fPIC' _extra += ' -fdisable-tree-fnsplit' # seems to help mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g'+ _extra) if self.config.translation.shared: mk.definition('PYPY_MAIN_FUNCTION', "pypy_main_startup") else: mk.definition('PYPY_MAIN_FUNCTION', "main") mk.definition('PYTHON', get_recent_cpython_executable()) mk.definition('GCMAPFILES', '$(subst .vmprof.s,.gcmap,$(subst .c,.gcmap,$(SOURCES)))') mk.definition('OBJECTS1', '$(subst .vmprof.s,.o,$(subst .c,.o,$(SOURCES)))') mk.definition('OBJECTS', '$(OBJECTS1) gcmaptable.s') # the CFLAGS passed to gcc when invoked to assembler the .s file # must not contain -g. This confuses gcc 5.1. (Note that it # would seem that gcc 5.1 with "-g" does not produce debugging # info in a format that gdb 4.7.1 can read.) mk.definition('CFLAGS_AS', '$(patsubst -g,,$(CFLAGS))') # the rule that transforms %.c into %.o, by compiling it to # %.s, then applying trackgcroot to get %.lbl.s and %.gcmap, and # finally by using the assembler ($(CC) again for now) to get %.o mk.rule('%.o %.gcmap', '%.c', [ '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< ' '-o $*.s -S $< $(INCLUDEDIRS)', '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '-t $*.s > $*.gctmp', '$(CC) $(CFLAGS_AS) -o $*.o -c $*.lbl.s', 'mv $*.gctmp $*.gcmap', 'rm $*.s $*.lbl.s']) # this is for manually written assembly files which needs to be parsed by asmgcc mk.rule('%.o %.gcmap', '%.vmprof.s', [ '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '-t $*.vmprof.s > $*.gctmp', '$(CC) -o $*.o -c $*.vmprof.lbl.s', 'mv $*.gctmp $*.gcmap', 'rm $*.vmprof.lbl.s']) # the rule to compute gcmaptable.s mk.rule('gcmaptable.s', '$(GCMAPFILES)', [ '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '$(GCMAPFILES) > $@.tmp', 'mv $@.tmp $@']) else: if self.translator.platform.name == 'msvc': mk.definition('DEBUGFLAGS', '-MD -Zi') else: if self.config.translation.shared: mk.definition('DEBUGFLAGS', '-O1 -g -fPIC') else: mk.definition('DEBUGFLAGS', '-O1 -g') if self.translator.platform.name == 'msvc': mk.rule('debug_target', '$(DEFAULT_TARGET) $(WTARGET)', 'rem') else: mk.rule('debug_target', '$(DEFAULT_TARGET)', '#') mk.write() #self.translator.platform, # , # self.eci, profbased=self.getprofbased() self.executable_name = mk.exe_name # ____________________________________________________________ SPLIT_CRITERIA = 65535 # support VC++ 7.2 #SPLIT_CRITERIA = 32767 # enable to support VC++ 6.0 MARKER = '/*/*/' # provide an easy way to split after generating class SourceGenerator: one_source_file = True def __init__(self, database): self.database = database self.extrafiles = [] self.headers_to_precompile = [] self.path = None self.namespace = NameManager() def set_strategy(self, path, split=True): all_nodes = list(self.database.globalcontainers()) # split off non-function nodes. We don't try to optimize these, yet. funcnodes = [] othernodes = [] for node in all_nodes: if node.nodekind == 'func': funcnodes.append(node) else: othernodes.append(node) if split: self.one_source_file = False self.funcnodes = funcnodes self.othernodes = othernodes self.path = path def uniquecname(self, name): assert name.endswith('.c') return self.namespace.uniquename(name[:-2]) + '.c' def makefile(self, name): log.writing(name) filepath = self.path.join(name) if name.endswith('.c'): self.extrafiles.append(filepath) if name.endswith('.h'): self.headers_to_precompile.append(filepath) return filepath.open('w') def getextrafiles(self): return self.extrafiles def getothernodes(self): return self.othernodes[:] def getbasecfilefornode(self, node, basecname): # For FuncNode instances, use the python source filename (relative to # the top directory): def invent_nice_name(g): # Lookup the filename from the function. # However, not all FunctionGraph objs actually have a "func": if hasattr(g, 'func'): if g.filename.endswith('.py'): localpath = py.path.local(g.filename) pypkgpath = localpath.pypkgpath() if pypkgpath: relpypath = localpath.relto(pypkgpath.dirname) assert relpypath, ("%r should be relative to %r" % (localpath, pypkgpath.dirname)) if len(relpypath.split(os.path.sep)) > 2: # pypy detail to agregate the c files by directory, # since the enormous number of files was causing # memory issues linking on win32 return os.path.split(relpypath)[0] + '.c' return relpypath.replace('.py', '.c') return None if hasattr(node.obj, 'graph'): # Regular RPython functions name = invent_nice_name(node.obj.graph) if name is not None: return name elif node._funccodegen_owner is not None: # Data nodes that belong to a known function graph = getattr(node._funccodegen_owner, 'graph', None) name = invent_nice_name(graph) if name is not None: return "data_" + name return basecname def splitnodesimpl(self, basecname, nodes, nextra, nbetween, split_criteria=SPLIT_CRITERIA): # Gather nodes by some criteria: nodes_by_base_cfile = {} for node in nodes: c_filename = self.getbasecfilefornode(node, basecname) if c_filename in nodes_by_base_cfile: nodes_by_base_cfile[c_filename].append(node) else: nodes_by_base_cfile[c_filename] = [node] # produce a sequence of nodes, grouped into files # which have no more than SPLIT_CRITERIA lines for basecname in sorted(nodes_by_base_cfile): iternodes = iter(nodes_by_base_cfile[basecname]) done = [False] def subiter(): used = nextra for node in iternodes: impl = '\n'.join(list(node.implementation())).split('\n') if not impl: continue cost = len(impl) + nbetween yield node, impl del impl if used + cost > split_criteria: # split if criteria met, unless we would produce nothing. raise StopIteration used += cost done[0] = True while not done[0]: yield self.uniquecname(basecname), subiter() @contextlib.contextmanager def write_on_included_file(self, f, name): fi = self.makefile(name) print >> f, '#include "%s"' % name yield fi fi.close() @contextlib.contextmanager def write_on_maybe_separate_source(self, f, name): print >> f, '/* %s */' % name if self.one_source_file: yield f else: fi = self.makefile(name) yield fi fi.close() def gen_readable_parts_of_source(self, f): split_criteria_big = SPLIT_CRITERIA if py.std.sys.platform != "win32": if self.database.gcpolicy.need_no_typeptr(): pass # XXX gcc uses toooooons of memory??? else: split_criteria_big = SPLIT_CRITERIA * 4 # # All declarations # with self.write_on_included_file(f, 'structdef.h') as fi: gen_structdef(fi, self.database) with self.write_on_included_file(f, 'forwarddecl.h') as fi: gen_forwarddecl(fi, self.database) with self.write_on_included_file(f, 'preimpl.h') as fi: gen_preimpl(fi, self.database) # # Implementation of functions and global structures and arrays # print >> f print >> f, '/***********************************************************/' print >> f, '/*** Implementations ***/' print >> f print >> f, '#define PYPY_FILE_NAME "%s"' % os.path.basename(f.name) print >> f, '#include "src/g_include.h"' if self.database.reverse_debugger: print >> f, '#include "revdb_def.h"' print >> f nextralines = 11 + 1 for name, nodeiter in self.splitnodesimpl('nonfuncnodes.c', self.othernodes, nextralines, 1): with self.write_on_maybe_separate_source(f, name) as fc: if fc is not f: print >> fc, '/***********************************************************/' print >> fc, '/*** Non-function Implementations ***/' print >> fc print >> fc, '#include "common_header.h"' print >> fc, '#include "structdef.h"' print >> fc, '#include "forwarddecl.h"' print >> fc, '#include "preimpl.h"' print >> fc print >> fc, '#include "src/g_include.h"' print >> fc print >> fc, MARKER for node, impl in nodeiter: print >> fc, '\n'.join(impl) print >> fc, MARKER print >> fc, '/***********************************************************/' nextralines = 12 for name, nodeiter in self.splitnodesimpl('implement.c', self.funcnodes, nextralines, 1, split_criteria_big): with self.write_on_maybe_separate_source(f, name) as fc: if fc is not f: print >> fc, '/***********************************************************/' print >> fc, '/*** Implementations ***/' print >> fc print >> fc, '#include "common_header.h"' print >> fc, '#include "structdef.h"' print >> fc, '#include "forwarddecl.h"' print >> fc, '#include "preimpl.h"' print >> fc, '#define PYPY_FILE_NAME "%s"' % name print >> fc, '#include "src/g_include.h"' if self.database.reverse_debugger: print >> fc, '#include "revdb_def.h"' print >> fc print >> fc, MARKER for node, impl in nodeiter: print >> fc, '\n'.join(impl) print >> fc, MARKER print >> fc, '/***********************************************************/' print >> f def gen_structdef(f, database): structdeflist = database.getstructdeflist() print >> f, '/***********************************************************/' print >> f, '/*** Structure definitions ***/' print >> f print >> f, "#ifndef _PYPY_STRUCTDEF_H" print >> f, "#define _PYPY_STRUCTDEF_H" for node in structdeflist: if hasattr(node, 'forward_decl'): if node.forward_decl: print >> f, node.forward_decl elif node.name is not None: print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: for line in node.definition(): print >> f, line gen_threadlocal_structdef(f, database) print >> f, "#endif" def gen_threadlocal_structdef(f, database): from rpython.translator.c.support import cdecl print >> f bk = database.translator.annotator.bookkeeper fields = list(bk.thread_local_fields) fields.sort(key=lambda field: field.fieldname) for field in fields: print >> f, ('#define RPY_TLOFS_%s offsetof(' % field.fieldname + 'struct pypy_threadlocal_s, %s)' % field.fieldname) if fields: print >> f, '#define RPY_TLOFSFIRST RPY_TLOFS_%s' % fields[0].fieldname else: print >> f, '#define RPY_TLOFSFIRST sizeof(struct pypy_threadlocal_s)' print >> f, 'struct pypy_threadlocal_s {' print >> f, '\tint ready;' print >> f, '\tchar *stack_end;' print >> f, '\tstruct pypy_threadlocal_s *prev, *next;' # note: if the four fixed fields above are changed, you need # to adapt threadlocal.c's linkedlist_head declaration too for field in fields: typename = database.gettype(field.FIELDTYPE) print >> f, '\t%s;' % cdecl(typename, field.fieldname) print >> f, '};' print >> f def gen_forwarddecl(f, database): print >> f, '/***********************************************************/' print >> f, '/*** Forward declarations ***/' print >> f print >> f, "#ifndef _PYPY_FORWARDDECL_H" print >> f, "#define _PYPY_FORWARDDECL_H" for node in database.globalcontainers(): for line in node.forward_declaration(): print >> f, line print >> f, "#endif" def gen_preimpl(f, database): f.write('#ifndef _PY_PREIMPL_H\n#define _PY_PREIMPL_H\n') if database.translator is None or database.translator.rtyper is None: return preimplementationlines = pre_include_code_lines( database, database.translator.rtyper) for line in preimplementationlines: print >> f, line f.write('#endif /* _PY_PREIMPL_H */\n') def gen_startupcode(f, database): # generate the start-up code and put it into a function print >> f, 'void RPython_StartupCode(void) {' bk = database.translator.annotator.bookkeeper if bk.thread_local_fields: print >> f, '\tRPython_ThreadLocals_ProgramInit();' for line in database.gcpolicy.gc_startup_code(): print >> f,"\t" + line # put float infinities in global constants, we should not have so many of them for now to make # a table+loop preferable for dest, value in database.late_initializations: print >> f, "\t%s = %s;" % (dest, value) for node in database.containerlist: lines = list(node.startupcode()) if lines: for line in lines: print >> f, '\t'+line for ll_init in database.translator._call_at_startup: print >> f, '\t%s();\t/* call_at_startup */' % (database.get(ll_init),) print >> f, '}' def commondefs(defines): from rpython.rlib.rarithmetic import LONG_BIT, LONGLONG_BIT defines['PYPY_LONG_BIT'] = LONG_BIT defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(database, eci): srcdir = py.path.local(__file__).join('..', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'mem.c', srcdir / 'exception.c', srcdir / 'rtyper.c', # ifdef HAVE_RTYPER srcdir / 'support.c', srcdir / 'profiling.c', srcdir / 'debug_print.c', srcdir / 'debug_traceback.c', # ifdef HAVE_RTYPER srcdir / 'asm.c', srcdir / 'instrument.c', srcdir / 'int.c', srcdir / 'stack.c', srcdir / 'threadlocal.c', ] if _CYGWIN: files.append(srcdir / 'cygwin_wait.c') if database.reverse_debugger: from rpython.translator.revdb import gencsupp files += gencsupp.extra_files() return eci.merge(ExternalCompilationInfo(separate_module_files=files)) def gen_source(database, modulename, targetdir, eci, defines={}, split=False): if isinstance(targetdir, str): targetdir = py.path.local(targetdir) filename = targetdir.join(modulename + '.c') f = filename.open('w') incfilename = targetdir.join('common_header.h') fi = incfilename.open('w') fi.write('#ifndef _PY_COMMON_HEADER_H\n#define _PY_COMMON_HEADER_H\n') # # Header # print >> f, '#include "common_header.h"' print >> f commondefs(defines) for key, value in defines.items(): print >> fi, '#define %s %s' % (key, value) eci.write_c_header(fi) print >> fi, '#include "src/g_prerequisite.h"' fi.write('#endif /* _PY_COMMON_HEADER_H*/\n') fi.close() # # 1) All declarations # 2) Implementation of functions and global structures and arrays # sg = SourceGenerator(database) sg.set_strategy(targetdir, split) sg.gen_readable_parts_of_source(f) headers_to_precompile = sg.headers_to_precompile[:] headers_to_precompile.insert(0, incfilename) gen_startupcode(f, database) f.close() if 'PYPY_INSTRUMENT' in defines: fi = incfilename.open('a') n = database.instrument_ncounter print >>fi, "#define PYPY_INSTRUMENT_NCOUNTER %d" % n fi.close() if database.reverse_debugger: from rpython.translator.revdb import gencsupp gencsupp.write_revdb_def_file(database, targetdir.join('revdb_def.h')) eci = add_extra_files(database, eci) eci = eci.convert_sources_to_files() return eci, filename, sg.getextrafiles(), headers_to_precompile