# # FILE $Id: doc_collect.py,v 1.13 1998/05/19 19:40:48 dlarsson Exp $ # # DESCRIPTION Collect info about modules, and generate Manual page. # # AUTHOR SEISY/LKSB Daniel Larsson # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of ABB Industrial Systems # not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior permission. # # ABB INDUSTRIAL SYSTEMS DISCLAIMS ALL WARRANTIES WITH REGARD TO # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND # FITNESS, IN NO EVENT SHALL ABB INDUSTRIAL SYSTEMS BE LIABLE # FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # # Copyright (C) ABB Industrial Systems AB, 1996 # Unpublished work. All Rights Reserved. # # HISTORY: # $Log: /Gendoc/doc_collect.py $ # # 3 98-05-25 22:13 Daniel # Removed obsolete 'codehack' module use. # Revision 1.13 1998/05/19 19:40:48 dlarsson # Merged in changes from home. # # # 2 98-04-01 14:11 Daniel # Added Python 1.5 features to deduce the module from a class # (thanks to mcfletch!). # # 1 98-04-01 13:15 Daniel # Revision 1.12 1998/02/05 18:13:47 dlarsson # Added 'BuiltinFunctionCollector' and initial support for "class like" types, # such as Jim Fulton's ExtensionClass. # # Revision 1.11 1998/02/04 17:57:49 dlarsson # 'whichmodule' now returns a module object, not its name. This caused gendoc # to skip all classes. # # Revision 1.10 1998/01/19 11:37:48 dlarsson # Fixes for Python 1.5: # - pickle.whichmodule interface changed # - sys.builtin_module_names is now a tuple (not a list) # # Revision 1.9 1996/09/04 14:35:18 omfadmin # Removed setext code. # # Revision 1.8 1996/08/26 20:34:24 omfadmin # Can now generate doc for private things (name begins with single # underscore) (Robin Friedrich). # # Revision 1.7 1996/07/12 15:51:29 omfadmin # Oops, forgot to remove debug prints. # # Revision 1.6 1996/07/12 15:46:18 omfadmin # Added alias do manual pages. # # Revision 1.5 1996/07/11 16:40:15 omfadmin # Moved some docstring code to the new docutil module. This is # now shared between parser and import mode # # Revision 1.4 1996/07/10 03:29:44 omfadmin # Added links from inherited methods in synopsis to description. # # Revision 1.3 1996/07/10 03:00:34 omfadmin # Lots of improvements. Important things: # # - Better doc generation of inherited methods. # - Now skips aliases (Should probably do something more sensible). # - __author__ is special. # # Revision 1.2 1996/06/13 17:56:48 omfadmin # Removed the restrictive copyright. # # Revision 1.1 1995/03/20 21:46:25 dlarsson # Initial revision # # """Collect information about a module by traversing its dictionary, and traverse the objects found there (classes, functions).""" __author__ = "Daniel Larsson, dlarsson@sw.seisy.abb.se" __version__ = '$Revision: 3 $' import ManualPage import regex, string, docutil from types import * include_private_methods = 0 # Verbosity level VERBOSE = 0 # *** This is of course overly simplified: *** # Everything that contains an equal sign, or starts # with a comment character ('#') is a code segment. # Those that don't aren't. from regex_syntax import RE_SYNTAX_EGREP old_syntax = regex.set_syntax(RE_SYNTAX_EGREP) _code = regex.compile('[a-zA-Z0-9]\(|=|^['+string.whitespace+']*# ') regex.set_syntax(old_syntax) def _getdoc(obj): if hasattr(obj, '__doc__') and obj.__doc__: return docutil.stripleadingtabs(obj.__doc__) else: return '' def _funline(fun): "Return the line nr where 'fun' is defined." import linecache, regex co = fun.func_code filename = co.co_filename try: lineno = co.co_firstlineno except AttributeError: import codehack lineno = codehack.getlineno(co) line = linecache.getline(filename, lineno) line = line[:string.find(line, ':')] return line[regex.match('['+string.whitespace+']*', line):] def _isClass(cls): """Determines if 'cls' is a class. A class is either a standard Python class object, or an extension type mimicking the class protocol.""" try: cls.__bases__ return 1 except: return 0 _classmap = {} def whichmoduleobj(cls): """Figure out the module in which a class occurs. Search sys.modules for the module. Cache in classmap. Return a module object. If the class cannot be found, raise exception. """ ### Altered by mcfletch 98.03.31 to use 1.5 stuff if hasattr( cls, '__module__' ): try: import sys return sys.modules[ cls.__module__ ] # is this always imported??? except: pass ### End mcfletch alterations if _classmap.has_key(cls): return _classmap[cls] # # Grab a function object and look in its code object for the co_filename # attribute to determine the module name funcs = filter(lambda member: type(member) == FunctionType, cls.__dict__.values()) if funcs: filename = funcs[0].func_code.co_filename import os module = os.path.splitext(os.path.basename(filename))[0] d = {} exec 'import %s' % module in d _classmap[cls] = d[module] return d[module] # import sys clsname = cls.__name__ for name, module in sys.modules.items(): if module and module.__name__ != '__main__' and \ hasattr(module, clsname) and \ getattr(module, clsname) is cls: break else: raise AttributeError, 'Cannot find module for class %s' % clsname _classmap[cls] = module return module def all_methods(clazz, methoddict=None): """Return all methods of a class. Returns a dictionary ~class:[methods]~, containing all inherited methods. Overridden methods are removed from base classes. The class argument is not a Python class, but a ClassCollect instance. """ if methoddict is None: methoddict = {} for base in clazz._bases_: all_methods(base, methoddict) methods = filter(lambda k: k.__class__ == FunctionCollect, clazz._children_.values()) methods = map(lambda fun: fun.name(), methods) # Remove overridden methods in all base classes for mlist in methoddict.values(): for m in methods: if m in mlist: mlist.remove(m) # Add myself to dictionary methoddict[clazz] = methods return methoddict # To determine the type of a variable, we do `type(var)` # to convert the type to a string. Then we extract the # type string with this regular expression _type = regex.compile("") _ignore_value = regex.compile("<[A-Za-z]+ object at [0-9a-f]+>") class ItemCollect: """Base class for document collectors.""" def __init__(self, obj, name=None): """Initialize collector with object and object name""" self._obj_ = obj if name: self._name_ = name else: self._name_ = obj.__name__ self._module_ = None if hasattr(obj, '__doc__'): doc = obj.__doc__ else: doc = '' self._oneliner_, self._doc_ = docutil.split_doc(doc) def collect(self): """Collect doc info""" pass def class_child(self): """Inform object it is a class child. If it is a child, it should not split the oneliner doc from the rest of the doc string, so we merge them back again. This is a bit of a kludge. """ if self._oneliner_: if self._doc_: self._doc_ = self._oneliner_+'\n\n'+self._doc_ else: self._doc_ = self._oneliner_ def has_doc(self): """Does the object have a document?""" return self._doc_ != '' def name(self): "Returns the name of the object." return self._name_ def type(self): "Returns the type string for the object." type_str = `type(self._obj_)` if _type.search(type_str) != -1: return _type.group(1) else: return '' def short_head(self): "A short head is the type string + the name." return self.type()+' '+self.name() def head(self): "Returns 'short_head' + the object's value." value = `self._obj_` # If the value matches the _ignore_value regular expression, # don't bother writing it. if _ignore_value.match(value) != -1: value = '...' elif len(value) > 30: value = value[:26]+' ...' return self.short_head()+' = '+value def index_txt(self): """The index text is used to generate markers for indices.""" return self.name()+', '+self.type() def module(self): """Return this object's module (not 100% accurate).""" return self._module_ def set_module(self, module): """Set this object's module.""" self._module_ = module def write_doc(self, manpage, sect = None): """Generate documentation for this object. **TODO**: It would be nice if I could recognize definition lists somehow, such as this one: an item the definition for item another item a long definition spanning more than one line. Maybe also recognize something like: an item a definition """ docutil.docregex_parse(manpage, sect, self._doc_) def write(self, manpage, lvl=0): "Generate a document for this object under the DESCRIPTION section." # Put a section inside 'DESCRIPTION' describing this item sect = manpage.section('DESCRIPTION') # MARKER HERE sect = manpage.section(self.head(), sect, self.index_txt(), self.module().index_txt(), search=0) # Don't search for existing section self.write_doc(manpage, sect) return '' class InstanceCollect(ItemCollect): def has_doc(self): "Instances 'inherit' their class' documentation. We skip that here." return 0 def type(self): "The type string of an instance is its class' name." return self._obj_.__class__.__name__ class ClassCollect(ItemCollect): "Collect information about a class" SKIP = ['__doc__', '__builtins__'] def __init__(self, classobj, name): ItemCollect.__init__(self, classobj, name) def collect(self): dict = self._obj_.__dict__ keys = dict.keys() keys.sort() self._children_ = {} if include_private_methods: # this will always fail to inhibit since names can't start with * inhibitor = '*' else: # Inhibit names beginning with a single underscore inhibitor = '_' for member in keys: if (member[0] != inhibitor or member[1] == '_') and member not in self.SKIP: if VERBOSE > 0: print 'Class %s: Adding %s' % (self.name(), member) doc = collector(dict[member], member, self.module()) if doc: self._children_[member] = doc # Inform object it is a class child. # This has to do with how the doc string # is produced (see doc in # ItemCollect.class_child). doc.class_child() self._bases_ = [] for base in self._obj_.__bases__: module_collector = collector(whichmoduleobj(base)) self._bases_.append(collector(base, base.__name__, module_collector)) def head(self): "Return class head" head = self.short_head() if self._bases_: from string import joinfields bases = joinfields(map(lambda i: i.name(), self._bases_), ', ') head = head + '(' + bases +')' return head def write(self, manpage=None, lvl=1): "Generate a document for this class" # Create a new manual page for me. manpage = ManualPage.ManualPage(self.module().name()+'-'+self.name()) manpage.set_author(self.module().author) # Figure out which are the inherited methods inherited = all_methods(self) # MARKER HERE title = 'Class '+self.name() if self._oneliner_: title = title+' - '+self._oneliner_ manpage.title(title, self.index_txt(), self.module().index_txt()) synopsis = manpage.section('SYNOPSIS') manpage.code('import '+self.module().name()) sect = manpage.section('DESCRIPTION') self.write_doc(manpage) # self.children_write(manpage, sect) self.children_write(inherited, manpage, sect) # Generate SYNOPSIS part code = self.head()+'\n'+self.children_synopsis(manpage, lvl) # Remove myself from inherited dict del inherited[self] # Generate synopsis for inherited methods code = code + self.inherited_synopsis(self._obj_, manpage, inherited, lvl) manpage.code(code, synopsis) # Do I have any aliases? aliases = self.module()._aliases_ if aliases.has_key(self.name()): code = '' for alias in aliases[self.name()]: code = code + 'alias %s = %s\n' % (alias, self.name()) manpage.code(code, synopsis) sect = manpage.section('SEE ALSO') manpage.paragraph(manpage.reference(self.module().name()+'_overview', self.module().name()), sect) for base in self._bases_: manpage.paragraph(manpage.reference(base.module().name()+'-'+base.name(), base.name()), sect) return manpage def children_write(self, inherited, manpage, sect): "Generate docs for inherited children." all = [] for base, members in inherited.items(): all = all + map(lambda n, base=base: base._children_[n], members) all.sort(lambda x, y: cmp(x.name(), y.name())) for child in all: if child.has_doc(): # MARKER HERE subsect = manpage.section(child.head(), sect, child.index_txt(), self.index_txt(), search=0) child.write_doc(manpage, subsect) # if this is a class, do recursive generation if _isClass(child._obj_): child.children_write(all_methods(child), manpage, subsect) def children_synopsis(self, manpage, lvl): "Generate synopsis for children." code = '' keys = self._children_.keys() keys.sort() for key in keys: child = self._children_[key] if child.has_doc(): child_txt = manpage.reference(child.head(), child.head()) else: child_txt = child.head() code = code+' '*lvl+child_txt+'\n' # if this is a class, do recursive generation if _isClass(child._obj_): code = code+child.children_synopsis(manpage, lvl+1) return code def inherited_synopsis(self, clazz, manpage, inherited, lvl): code = '' for base, members in inherited.items(): members.sort() if members: code = code + '\n' + ' '*lvl + \ '# Methods inherited by %s from %s\n' % (clazz.__name__, base._obj_.__name__) for method in members: child = base._children_[method] if child.has_doc(): child_txt = manpage.reference(child.head(), child.head()) else: child_txt = child.head() code = code + ' '*lvl + child_txt + '\n' return code class FunctionCollect(ItemCollect): def __init__(self, funobj, name): ItemCollect.__init__(self, funobj, name) def head(self): "Return function head" return _funline(self._obj_) def write(self, manpage=None): # Generate new manual page manpage = ManualPage.ManualPage(self.module().name()+'-'+self.name()) manpage.set_author(self.module().author) # Second argument indicates this is an index # Third argument indicates it is a nested index # MARKER HERE title = 'Function '+self.name() if self._oneliner_: title = title+' - '+self._oneliner_ manpage.title(title, self.index_txt(), self.module().index_txt()) manpage.section('SYNOPSIS') manpage.code('import '+self.module().name()) manpage.code(self.head()) sect = manpage.section('DESCRIPTION') self.write_doc(manpage, sect) sect = manpage.section('SEE ALSO') manpage.paragraph(manpage.reference(self.module().name()+'_overview', self.module().name()), sect) return manpage class BuiltinFunctionCollect(ItemCollect): def __init__(self, funobj, name): ItemCollect.__init__(self, funobj, name) def head(self): "Return function head" if self._oneliner_: return self._oneliner_ else: return "built-in function " + self.name() def write(self, manpage=None): # Generate new manual page manpage = ManualPage.ManualPage(self.module().name()+'-'+self.name()) manpage.set_author(self.module().author) # Second argument indicates this is an index # Third argument indicates it is a nested index # MARKER HERE title = 'Built-in function '+self.name() if self._oneliner_: title = title+' - '+self._oneliner_ manpage.title(title, self.index_txt(), self.module().index_txt()) manpage.section('SYNOPSIS') manpage.code('import '+self.module().name()) manpage.code(self.head()) sect = manpage.section('DESCRIPTION') self.write_doc(manpage, sect) sect = manpage.section('SEE ALSO') manpage.paragraph(manpage.reference(self.module().name()+'_overview', self.module().name()), sect) return manpage class ModuleCollect(ItemCollect): SKIP = ['__doc__', '__name__', '__builtins__', 'test'] def __init__(self, module, name): ItemCollect.__init__(self, module) self.author = None self._module_refs_ = [] self._classes_ = [] self._functions_ = [] self.__builtinFuns = [] self._others_ = [] self._aliases_ = {} self.__isBuiltin = not module.__dict__.has_key('__file__') from os import path self.__inPython = not self.__isBuiltin and \ path.splitext(module.__file__)[1] not in ('.py', '.pyw') self._child_map_ = { ItemCollect: self._others_, InstanceCollect: self._others_, FunctionCollect: self._functions_, ClassCollect: self._classes_, BuiltinFunctionCollect: self.__builtinFuns } def collect(self): import sys # from pickle import whichmodule dict = self._obj_.__dict__ # Keep track of collected objects so we can # names referring to the same object collected_objects = [] keys = dict.keys() keys.sort() for member in keys: value = dict[member] if VERBOSE: print "Found", value, "(%s)" % type(value) if type(value) == ModuleType: if member not in tuple(sys.builtin_module_names) + ('__',): self._module_refs_.append(value) elif member[0] == '_' and member[1] != '_': if VERBOSE > 1: print 'Module %s: Skipping private %s' % (self.name(), member) elif member in self.SKIP: if VERBOSE > 1: print 'Module %s: Skipping %s (in SKIP list)' % (self.name(), member) elif _isClass(value) and \ whichmoduleobj(value).__name__ != self.name(): if VERBOSE > 1: print 'Module %s: Skipping %s (class not defined in module)' % (self.name(), member) elif member == '__author__': self.author = value elif member in collected_objects: continue else: if VERBOSE > 0: if _isClass(value): print 'Module %s: Adding %s (%s)' % (self.name(), member, whichmoduleobj(value)) else: print 'Module %s: Adding %s' % (self.name(), member) collected_objects.append(member) collect = None # Aliases have preallocated collectors try: name = value.__name__ if name != member and type(value) != type(type('')): # It must be an alias! if VERBOSE > 0: print 'Module %s: Adding alias %s for %s' % (self.name(), member, name) if not self._aliases_.has_key(name): self._aliases_[name] = [] self._aliases_[name].append(member) except AttributeError: name = member doc = collector(value, name, self, collect) if doc and name == member: self._child_map_[doc.__class__].append(doc) def write(self, manpage=None): "Generate a set of documents for this module." # Generate new manual page manpage = ManualPage.ManualPage(self.name()+'_overview') manpage.set_author(self.author) # MARKER HERE title = 'Module '+self.name() if self._oneliner_: title = title+' - '+self._oneliner_ manpage.title(title, self.index_txt()) synopsis = manpage.section('SYNOPSIS') # Write module description sect = manpage.section('DESCRIPTION') see_also = manpage.section('SEE ALSO') self.write_doc(manpage, sect) # Let children generate doc. They can either generate a new # document, or add things to the module document. fun = lambda rest, next, \ f=self.synopsis, sect=synopsis, manpage=manpage: \ rest+f(next, manpage, sect) children = [('Classes', self._classes_), ('Functions', self._functions_ + self.__builtinFuns), ('Variables', self._others_)] pages = reduce(fun, children, []) module_refs = '' for module in self._module_refs_: module_refs = module_refs + \ manpage.reference(module.__name__ + '_overview', module.__name__) + '\n' if module_refs: manpage.code(module_refs, see_also) return (manpage, pages) def synopsis(self, children, manpage, sect): title, child_list = children code = '' files = [] for child in child_list: ref=child.head() if child.has_doc() or child.__class__ in [ClassCollect]: file = child.write(manpage) if file: ref = manpage.reference(self.name()+'-'+child.name(), child.head()) files.append(file) elif VERBOSE > 1: print 'Module %s: No doc string for %s' % (self.name(), child.name()) code = code+ref+'\n' if code: code = '# '+title+'\n'+code manpage.code(code, sect) return files _doctype_map = {} _doctype_map[ClassType] = ClassCollect _doctype_map[FunctionType] = FunctionCollect _doctype_map[BuiltinFunctionType]=BuiltinFunctionCollect _doctype_map[ModuleType] = ModuleCollect _doctype_map[InstanceType] = InstanceCollect # Dictionary of all collectors available collectors = {} def collector(obj, name=None, module=None, collect=None): global collectors if collect: collect.set_module(module) collect.collect() return collect if VERBOSE > 2: print "Available collectors %s" % collectors.keys() try: if collectors.has_key(obj): return collectors[obj] else: col = make_collector(obj, name) col.set_module(module) collectors[obj] = col col.collect() except TypeError: col = make_collector(obj, name) col.set_module(module) col.collect() return col def make_collector(obj, name=None): try: if _doctype_map.has_key(type(obj)): return _doctype_map[type(obj)](obj, name) elif _isClass(obj): return ClassCollector(obj, name) else: return ItemCollect(obj, name) except AccessError: n = name if not n: n = `obj` print "Couldn't get type of", n, "(Access denied)" def doc_collect(module_list): """Return a list of collectors given a list of modules""" return map(collector, module_list) def gendoc(collect_list): """Generate documentation given a list of collectors A list of manual pages is returned.""" manpages = [] for collector in collect_list: manpages.append(collector.write()) return manpages def test(): "Uh...well...soon I'll do this properly" import OMFmisc, OMFdefs VERBOSE=2 collectors = doc_collect([OMFmisc, OMFdefs]) print gendoc(collectors) import pickle doc = make_collector(pickle) doc.write() if __name__ == '__main__': test()