""" Attempt to generate templates for module reference with Sphinx To include extension modules, first identify them as valid in the ``_uri2path`` method, then handle them in the ``_parse_module_with_import`` script. Notes ----- This parsing is based on import and introspection of modules. Previously functions and classes were found by parsing the text of .py files. Extension modules should be discovered and included as well. This is a modified version of a script originally shipped with the PyMVPA project, then adapted for use first in NIPY and then in skimage. PyMVPA is an MIT-licensed project. """ # Stdlib imports import os import re from inspect import getmodule from types import BuiltinFunctionType, FunctionType # suppress print statements (warnings for empty files) DEBUG = True class ApiDocWriter(object): ''' Class for automatic detection and parsing of API docs to Sphinx-parsable reST format''' # only separating first two levels rst_section_levels = ['*', '=', '-', '~', '^'] def __init__(self, package_name, rst_extension='.txt', package_skip_patterns=None, module_skip_patterns=None, other_defines=True ): ''' Initialize package for parsing Parameters ---------- package_name : string Name of the top-level package. *package_name* must be the name of an importable package rst_extension : string, optional Extension for reST files, default '.rst' package_skip_patterns : None or sequence of {strings, regexps} Sequence of strings giving URIs of packages to be excluded Operates on the package path, starting at (including) the first dot in the package path, after *package_name* - so, if *package_name* is ``sphinx``, then ``sphinx.util`` will result in ``.util`` being passed for searching by these regexps. If is None, gives default. Default is: ['\.tests$'] module_skip_patterns : None or sequence Sequence of strings giving URIs of modules to be excluded Operates on the module name including preceding URI path, back to the first dot after *package_name*. For example ``sphinx.util.console`` results in the string to search of ``.util.console`` If is None, gives default. Default is: ['\.setup$', '\._'] other_defines : {True, False}, optional Whether to include classes and functions that are imported in a particular module but not defined there. ''' if package_skip_patterns is None: package_skip_patterns = ['\\.tests$'] if module_skip_patterns is None: module_skip_patterns = ['\\.setup$', '\\._'] self.package_name = package_name self.rst_extension = rst_extension self.package_skip_patterns = package_skip_patterns self.module_skip_patterns = module_skip_patterns self.other_defines = other_defines def get_package_name(self): return self._package_name def set_package_name(self, package_name): ''' Set package_name >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx >>> docwriter.root_path == sphinx.__path__[0] True >>> docwriter.package_name = 'docutils' >>> import docutils >>> docwriter.root_path == docutils.__path__[0] True ''' # It's also possible to imagine caching the module parsing here self._package_name = package_name root_module = self._import(package_name) self.root_path = root_module.__path__[-1] self.written_modules = None package_name = property(get_package_name, set_package_name, None, 'get/set package_name') def _import(self, name): ''' Import namespace package ''' mod = __import__(name) components = name.split('.') for comp in components[1:]: mod = getattr(mod, comp) return mod def _get_object_name(self, line): ''' Get second token in line >>> docwriter = ApiDocWriter('sphinx') >>> docwriter._get_object_name(" def func(): ") 'func' >>> docwriter._get_object_name(" class Klass(object): ") 'Klass' >>> docwriter._get_object_name(" class Klass: ") 'Klass' ''' name = line.split()[1].split('(')[0].strip() # in case we have classes which are not derived from object # ie. old style classes return name.rstrip(':') def _uri2path(self, uri): ''' Convert uri to absolute filepath Parameters ---------- uri : string URI of python module to return path for Returns ------- path : None or string Returns None if there is no valid path for this URI Otherwise returns absolute file system path for URI Examples -------- >>> docwriter = ApiDocWriter('sphinx') >>> import sphinx >>> modpath = sphinx.__path__[0] >>> res = docwriter._uri2path('sphinx.builder') >>> res == os.path.join(modpath, 'builder.py') True >>> res = docwriter._uri2path('sphinx') >>> res == os.path.join(modpath, '__init__.py') True >>> docwriter._uri2path('sphinx.does_not_exist') ''' if uri == self.package_name: return os.path.join(self.root_path, '__init__.py') path = uri.replace(self.package_name + '.', '') path = path.replace('.', os.path.sep) path = os.path.join(self.root_path, path) # XXX maybe check for extensions as well? if os.path.exists(path + '.py'): # file path += '.py' elif os.path.exists(os.path.join(path, '__init__.py')): path = os.path.join(path, '__init__.py') else: return None return path def _path2uri(self, dirpath): ''' Convert directory path to uri ''' package_dir = self.package_name.replace('.', os.path.sep) relpath = dirpath.replace(self.root_path, package_dir) if relpath.startswith(os.path.sep): relpath = relpath[1:] return relpath.replace(os.path.sep, '.') def _parse_module(self, uri): ''' Parse module defined in *uri* ''' filename = self._uri2path(uri) if filename is None: print(filename, 'erk') # nothing that we could handle here. return ([], []) f = open(filename, 'rt') functions, classes = self._parse_lines(f) f.close() return functions, classes def _parse_module_with_import(self, uri): """Look for functions and classes in an importable module. Parameters ---------- uri : str The name of the module to be parsed. This module needs to be importable. Returns ------- functions : list of str A list of (public) function names in the module. classes : list of str A list of (public) class names in the module. """ mod = __import__(uri, fromlist=[uri.split('.')[-1]]) # find all public objects in the module. obj_strs = [obj for obj in dir(mod) if not obj.startswith('_')] functions = [] classes = [] for obj_str in obj_strs: # find the actual object from its string representation if obj_str not in mod.__dict__: continue obj = mod.__dict__[obj_str] # Check if function / class defined in module if not self.other_defines and not getmodule(obj) == mod: continue # figure out if obj is a function or class if isinstance(obj, (FunctionType, BuiltinFunctionType)): functions.append(obj_str) else: try: issubclass(obj, object) classes.append(obj_str) except TypeError: # not a function or class pass return functions, classes def _parse_lines(self, linesource): ''' Parse lines of text for functions and classes ''' functions = [] classes = [] for line in linesource: if line.startswith('def ') and line.count('('): # exclude private stuff name = self._get_object_name(line) if not name.startswith('_'): functions.append(name) elif line.startswith('class '): # exclude private stuff name = self._get_object_name(line) if not name.startswith('_'): classes.append(name) else: pass functions.sort() classes.sort() return functions, classes def generate_api_doc(self, uri): '''Make autodoc documentation template string for a module Parameters ---------- uri : string python location of module - e.g 'sphinx.builder' Returns ------- head : string Module name, table of contents. body : string Function and class docstrings. ''' # get the names of all classes and functions functions, classes = self._parse_module_with_import(uri) if not len(functions) and not len(classes) and DEBUG: print('WARNING: Empty -', uri) # dbg # Make a shorter version of the uri that omits the package name for # titles uri_short = re.sub(r'^%s\.' % self.package_name, '', uri) head = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' body = '' # Set the chapter title to read 'module' for all modules except for the # main packages if '.' in uri_short: title = 'Module: :mod:`' + uri_short + '`' head += title + '\n' + self.rst_section_levels[2] * len(title) else: title = ':mod:`' + uri_short + '`' head += title + '\n' + self.rst_section_levels[1] * len(title) head += '\n.. automodule:: ' + uri + '\n' head += '\n.. currentmodule:: ' + uri + '\n' body += '\n.. currentmodule:: ' + uri + '\n\n' for c in classes: body += '\n:class:`' + c + '`\n' \ + self.rst_section_levels[3] * \ (len(c)+9) + '\n\n' body += '\n.. autoclass:: ' + c + '\n' # must NOT exclude from index to keep cross-refs working body += ' :members:\n' \ ' :undoc-members:\n' \ ' :show-inheritance:\n' \ '\n' \ ' .. automethod:: __init__\n\n' head += '.. autosummary::\n\n' for f in classes + functions: head += ' ' + f + '\n' head += '\n' for f in functions: # must NOT exclude from index to keep cross-refs working body += f + '\n' body += self.rst_section_levels[3] * len(f) + '\n' body += '\n.. autofunction:: ' + f + '\n\n' return head, body def _survives_exclude(self, matchstr, match_type): ''' Returns True if *matchstr* does not match patterns ``self.package_name`` removed from front of string if present Examples -------- >>> dw = ApiDocWriter('sphinx') >>> dw._survives_exclude('sphinx.okpkg', 'package') True >>> dw.package_skip_patterns.append('^\\.badpkg$') >>> dw._survives_exclude('sphinx.badpkg', 'package') False >>> dw._survives_exclude('sphinx.badpkg', 'module') True >>> dw._survives_exclude('sphinx.badmod', 'module') True >>> dw.module_skip_patterns.append('^\\.badmod$') >>> dw._survives_exclude('sphinx.badmod', 'module') False ''' if match_type == 'module': patterns = self.module_skip_patterns elif match_type == 'package': patterns = self.package_skip_patterns else: raise ValueError('Cannot interpret match type "%s"' % match_type) # Match to URI without package name L = len(self.package_name) if matchstr[:L] == self.package_name: matchstr = matchstr[L:] for pat in patterns: try: pat.search except AttributeError: pat = re.compile(pat) if pat.search(matchstr): return False return True def discover_modules(self): ''' Return module sequence discovered from ``self.package_name`` Parameters ---------- None Returns ------- mods : sequence Sequence of module names within ``self.package_name`` Examples -------- >>> dw = ApiDocWriter('sphinx') >>> mods = dw.discover_modules() >>> 'sphinx.util' in mods True >>> dw.package_skip_patterns.append('\.util$') >>> 'sphinx.util' in dw.discover_modules() False >>> ''' modules = [self.package_name] # raw directory parsing for dirpath, dirnames, filenames in os.walk(self.root_path): # Check directory names for packages root_uri = self._path2uri(os.path.join(self.root_path, dirpath)) # Normally, we'd only iterate over dirnames, but since # dipy does not import a whole bunch of modules we'll # include those here as well (the *.py filenames). filenames = [f[:-3] for f in filenames if f.endswith('.py') and not f.startswith('__init__')] for filename in filenames: package_uri = '/'.join((dirpath, filename)) for subpkg_name in dirnames + filenames: package_uri = '.'.join((root_uri, subpkg_name)) package_path = self._uri2path(package_uri) if (package_path and self._survives_exclude(package_uri, 'package')): modules.append(package_uri) return sorted(modules) def write_modules_api(self, modules, outdir): # upper-level modules main_module = modules[0].split('.')[0] ulms = ['.'.join(m.split('.')[:2]) if m.count('.') >= 1 else m.split('.')[0] for m in modules] from collections import OrderedDict module_by_ulm = OrderedDict() for v, k in zip(modules, ulms): if k in module_by_ulm: module_by_ulm[k].append(v) else: module_by_ulm[k] = [v] written_modules = [] for ulm, mods in module_by_ulm.items(): print("Generating docs for %s:" % ulm) document_head = [] document_body = [] for m in mods: print(" -> " + m) head, body = self.generate_api_doc(m) document_head.append(head) document_body.append(body) out_module = ulm + self.rst_extension outfile = os.path.join(outdir, out_module) fileobj = open(outfile, 'wt') fileobj.writelines(document_head + document_body) fileobj.close() written_modules.append(out_module) self.written_modules = written_modules def write_api_docs(self, outdir): """Generate API reST files. Parameters ---------- outdir : string Directory name in which to store files We create automatic filenames for each module Returns ------- None Notes ----- Sets self.written_modules to list of written modules """ if not os.path.exists(outdir): os.mkdir(outdir) # compose list of modules modules = self.discover_modules() self.write_modules_api(modules, outdir) def write_index(self, outdir, froot='gen', relative_to=None): """Make a reST API index file from written files Parameters ---------- path : string Filename to write index to outdir : string Directory to which to write generated index file froot : string, optional root (filename without extension) of filename to write to Defaults to 'gen'. We add ``self.rst_extension``. relative_to : string path to which written filenames are relative. This component of the written file path will be removed from outdir, in the generated index. Default is None, meaning, leave path as it is. """ if self.written_modules is None: raise ValueError('No modules written') # Get full filename path path = os.path.join(outdir, froot+self.rst_extension) # Path written into index is relative to rootpath if relative_to is not None: relpath = ( outdir + os.path.sep).replace(relative_to + os.path.sep, '') else: relpath = outdir idx = open(path, 'wt') w = idx.write w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') title = "API Reference" w(title + "\n") w("=" * len(title) + "\n\n") w('.. toctree::\n\n') for f in self.written_modules: w(' %s\n' % os.path.join(relpath, f)) idx.close()