# -*- coding: utf-8 -*- #--------------------------------------------------------------------------- # Name: sphinxtools/utilities.py # Author: Andrea Gavana # # Created: 30-Nov-2010 # Copyright: (c) 2010-2018 by Total Control Software # License: wxWindows License #--------------------------------------------------------------------------- # Standard library imports import sys import os import codecs import shutil import re if sys.version_info < (3,): import cPickle as pickle from UserDict import UserDict string_base = basestring else: import pickle from collections import UserDict string_base = str # Phoenix-specific imports from .templates import TEMPLATE_CONTRIB from .constants import IGNORE, PUNCTUATION, MODULENAME_REPLACE from .constants import CPP_ITEMS, VERSION, VALUE_MAP from .constants import RE_KEEP_SPACES, EXTERN_INHERITANCE from .constants import DOXYROOT, SPHINXROOT, WIDGETS_IMAGES_ROOT # ----------------------------------------------------------------------- # class ODict(UserDict): """ An ordered dict (odict). This is a dict which maintains an order to its items; the order is rather like that of a list, in that new items are, by default, added to the end, but items can be rearranged. .. note:: Note that updating an item (setting a value where the key is already in the dict) is not considered to create a new item, and does not affect the position of that key in the order. However, if an item is deleted, then a new item with the same key is added, this is considered a new item. """ def __init__(self, dict = None): self._keys = [] UserDict.__init__(self, dict) def __delitem__(self, key): UserDict.__delitem__(self, key) self._keys.remove(key) def __setitem__(self, key, item): UserDict.__setitem__(self, key, item) if key not in self._keys: self._keys.append(key) def clear(self): UserDict.clear(self) self._keys = [] def copy(self): dict = UserDict.copy(self) dict._keys = self._keys[:] return dict def items(self): return list(zip(self._keys, list(self.values()))) def keys(self): return self._keys def popitem(self): try: key = self._keys[-1] except IndexError: raise KeyError('dictionary is empty') val = self[key] del self[key] return (key, val) def setdefault(self, key, failobj = None): UserDict.setdefault(self, key, failobj) if key not in self._keys: self._keys.append(key) def update(self, dict): UserDict.update(self, dict) for key in list(dict.keys()): if key not in self._keys: self._keys.append(key) def values(self): return list(map(self.get, self._keys)) # ----------------------------------------------------------------------- # def isNumeric(input_string): """ Checks if the string `input_string` actually represents a number. :param string `input_string`: any string. :rtype: `bool` :returns: ``True`` if the `input_string` can be converted to a number (any number), ``False`` otherwise. """ try: float(input_string) return True except ValueError: return False # ----------------------------------------------------------------------- # def countSpaces(text): """ Counts the number of spaces before and after a string. :param string `text`: any string. :rtype: `tuple` :returns: a tuple representing the number of spaces before and after the text. """ space_before = ' '*(len(text) - len(text.lstrip(' '))) space_after = ' '*(len(text) - len(text.rstrip(' '))) return space_before, space_after # ----------------------------------------------------------------------- # def underscore2Capitals(string): """ Replaces the underscore letter in a string with the following letter capitalized. :param string `string`: the string to be analyzed. :rtype: `string` """ items = string.split('_')[1:] newstr = '' for item in items: newstr += item.capitalize() return newstr # ----------------------------------------------------------------------- # def replaceCppItems(line): """ Replaces various C++ specific stuff with more Pythonized version of them. :param string `line`: any string. :rtype: `string` """ items = RE_KEEP_SPACES.split(line) # This should get rid of substitutions like "float buffered"... no_conversion = ['click', 'buffer', 'precision'] newstr = [] for n, item in enumerate(items): if item in CPP_ITEMS: continue if 'wxString' == item: item = 'string' elif 'wxCoord' == item: item = 'int' elif 'time_t' == item: item = 'int' elif item == 'char': item = 'int' elif item == 'double': if len(items) > n+2: target = items[n+2].lower() replace = True for excluded in no_conversion: if target.startswith(excluded): replace = False break if replace: item = 'float' if len(item.replace('``', '')) > 2: if '*' in item: try: eval(item) item = item.replace('*', 'x') except: # Avoid replacing standalone '&&' and similar for cpp in CPP_ITEMS[0:2]: item = item.replace(cpp, '') newstr.append(item) newstr = ''.join(newstr) newstr = newstr.replace(' *)', ' )') return newstr # ----------------------------------------------------------------------- # def pythonizeType(ptype, is_param): """ Replaces various C++ specific stuff with more Pythonized version of them, for parameter lists and return types (i.e., the `:param:` and `:rtype:` ReST roles). :param string `ptype`: any string; :param bool `is_param`: ``True`` if this is a parameter description, ``False`` if it is a return type. :rtype: `string` """ from etgtools.tweaker_tools import removeWxPrefix if 'size_t' in ptype: ptype = 'int' elif 'wx.' in ptype: ptype = ptype[3:] # *** else: ptype = wx2Sphinx(replaceCppItems(ptype))[1] ptype = ptype.replace('::', '.').replace('*&', '') ptype = ptype.replace('int const', 'int') ptype = ptype.replace('Uint32', 'int').replace('**', '').replace('Int32', 'int') ptype = ptype.replace('FILE', 'file') ptype = ptype.replace('boolean', 'bool') for item in ['unsignedchar', 'unsignedint', 'unsignedlong', 'unsigned']: ptype = ptype.replace(item, 'int') ptype = ptype.strip() ptype = removeWxPrefix(ptype) if '. wx' in ptype: #*** ptype = ptype.replace('. wx', '.') plower = ptype.lower() if plower == 'double': ptype = 'float' if plower in ['string', 'char', 'artid', 'artclient']: ptype = 'string' if plower in ['coord', 'byte', 'fileoffset', 'short', 'time_t', 'intptr', 'uintptr', 'windowid']: ptype = 'int' if plower in ['longlong']: ptype = 'long' cpp = ['ArrayString', 'ArrayInt', 'ArrayDouble'] python = ['list of strings', 'list of integers', 'list of floats'] for c, p in zip(cpp, python): ptype = ptype.replace(c, p) if 'Image.' in ptype: ptype = ptype.split('.')[-1] if 'FileName' in ptype: ptype = 'string' if ptype.endswith('&'): ptype = ptype[0:-1] if ' ' not in ptype: ptype = ':class:`%s`'%ptype else: if is_param and '.' in ptype: modules = list(MODULENAME_REPLACE.values()) modules.sort() modules = modules[1:] if ptype.split('.')[0] + '.' in modules: ptype = ':ref:`%s`'%ptype return ptype # ----------------------------------------------------------------------- # def convertToPython(text): """ Converts the input `text` into a more ReSTified version of it. This involves the following steps: 1. Any C++ specific declaration (like ``unsigned``, ``size_t`` and so on is removed. 2. Lines starting with "Include file" or "#include" are ignored. 3. Uppercase constants (i.e., like ID_ANY, HORIZONTAL and so on) are converted into inline literals (i.e., ``ID_ANY``, ``HORIZONTAL``). 4. The "wx" prefix is removed from all the words in the input `text`. #*** :param string `text`: any string. :rtype: `string` """ from etgtools.tweaker_tools import removeWxPrefix from etgtools.item_module_map import ItemModuleMap newlines = [] unwanted = ['Include file', '#include'] for line in text.splitlines(): newline = [] for remove in unwanted: if remove in line: line = line[0:line.index(remove)] break spacer = ' '*(len(line) - len(line.lstrip())) line = replaceCppItems(line) for word in RE_KEEP_SPACES.split(line): if word == VERSION: newline.append(word) continue newword = word for s in PUNCTUATION: newword = newword.replace(s, "") if newword in VALUE_MAP: word = word.replace(newword, VALUE_MAP[newword]) newline.append(word) continue if newword not in IGNORE and not newword.startswith('wx.'): word = removeWxPrefix(word) newword = removeWxPrefix(newword) if "::" in word and not word.endswith("::"): # Bloody SetCursorEvent... word = word.replace("::wx", ".") # *** word = word.replace("::", ".") word = "`%s`" % word newline.append(word) continue if (newword.upper() == newword and newword not in PUNCTUATION and newword not in IGNORE and len(newword.strip()) > 1 and not isNumeric(newword) and newword not in ['DC', 'GCDC']): if '``' not in newword and '()' not in word and '**' not in word: word = word.replace(newword, "``%s``" % ItemModuleMap().get_fullname(newword)) word = word.replace('->', '.') newline.append(word) newline = spacer + ''.join(newline) newline = newline.replace(':see:', '.. seealso::') newline = newline.replace(':note:', '.. note::') newlines.append(newline) formatted = "\n".join(newlines) formatted = formatted.replace('\\', '\\\\') return formatted # ----------------------------------------------------------------------- # def findDescendants(element, tag, descendants=None): """ Finds and returns all descendants of a specific `xml.etree.ElementTree.Element` whose tag matches the input `tag`. :param xml.etree.ElementTree.Element `element`: the XML element we want to examine. :param string `tag`: the target tag we are looking for. :param list `descendants`: a list of already-found descendants or ``None`` if this is the first call to the function. :rtype: `list` .. note:: This is a recursive function, and it is only used in the `etgtools.extractors.py` script. """ if descendants is None: descendants = [] for childElement in element: if childElement.tag == tag: descendants.append(childElement) descendants = findDescendants(childElement, tag, descendants) return descendants # ----------------------------------------------------------------------- # def findControlImages(elementOrString): """ Given the input `element` (an instance of `xml.etree.ElementTree.Element` or a plain string) representing a Phoenix class description, this function will scan the doxygen image folder ``DOXYROOT`` to look for a widget screenshot. If this class indeed represents a widget and a screenshot is found, it is then copied to the appropriate Sphinx input folder ``WIDGETS_IMAGES_ROOT`` in one of its sub-folders (``wxmsw``, ``wxgtk``, ``wxmac``) depending on which platform the screenshot was taken. :param `elementOrString`: the XML element we want to examine (an instance of xml.etree.ElementTree.Element) or a plain string (usually for wx.lib). :rtype: `list` :returns: A list of image paths, every element of it representing a screenshot on a different platform. An empty list if returned if no screenshots have been found. .. note:: If a screenshot doesn't exist for one (or more) platform but it exists for others, the missing images will be replaced by the "no_appearance.png" file (you can find it inside the ``WIDGETS_IMAGES_ROOT`` folder. """ from etgtools.tweaker_tools import removeWxPrefix if isinstance(elementOrString, string_base): class_name = py_class_name = elementOrString.lower() else: element = elementOrString class_name = element.pyName if element.pyName else removeWxPrefix(element.name) py_class_name = wx2Sphinx(class_name)[1] class_name = class_name.lower() py_class_name = py_class_name.lower() image_folder = os.path.join(DOXYROOT, 'images') appearance = ODict() for sub_folder in ['wxmsw', 'wxmac', 'wxgtk']: png_file = class_name + '.png' appearance[sub_folder] = '' possible_image = os.path.join(image_folder, sub_folder, png_file) new_path = os.path.join(WIDGETS_IMAGES_ROOT, sub_folder) py_png_file = py_class_name + '.png' new_file = os.path.join(new_path, py_png_file) if os.path.isfile(new_file): appearance[sub_folder] = py_png_file elif os.path.isfile(possible_image): if not os.path.isdir(new_path): os.makedirs(new_path) if not os.path.isfile(new_file): shutil.copyfile(possible_image, new_file) appearance[sub_folder] = py_png_file if not any(list(appearance.values())): return [] for sub_folder, image in list(appearance.items()): if not image: appearance[sub_folder] = '../no_appearance.png' return list(appearance.values()) # ----------------------------------------------------------------------- # def makeSummary(class_name, item_list, template, kind, add_tilde=True): """ This function generates a table containing a method/property name and a shortened version of its docstrings. :param string `class_name`: the class name containing the method/property lists; :param list `item_list`: a list of tuples like `(method/property name, short docstrings)`; :param string `template`: the template to use (from `sphinxtools/templates.py`, can be the ``TEMPLATE_METHOD_SUMMARY`` or the ``TEMPLATE_PROPERTY_SUMMARY``; :param string `kind`: can be ``:meth:`` or ``:attr:`` or ``:ref:`` or ``:mod:``; :param bool `add_tilde`: ``True`` to add the ``~`` character in front of the first summary table column, ``False`` otherwise. :rtype: `string` """ maxlen = 0 for method, simple_docs in item_list: substr = ':%s:`~%s`'%(kind, method) maxlen = max(maxlen, len(substr)) maxlen = max(80, maxlen) summary = '='*maxlen + ' ' + '='*80 + "\n" format = '%-' + str(maxlen) + 's %s' for method, simple_docs in item_list: if add_tilde: substr = ':%s:`~%s`'%(kind, method) else: substr = ':%s:`%s`'%(kind, method) new_docs = simple_docs if kind == 'meth': regex = re.findall(r':meth:\S+', simple_docs) for regs in regex: if '.' in regs: continue meth_name = regs[regs.index('`')+1:regs.rindex('`')] newstr = ':meth:`~%s.%s`'%(class_name, meth_name) new_docs = new_docs.replace(regs, newstr, 1) if '===' in new_docs: new_docs = '' elif new_docs.rstrip().endswith(':'): # Avoid Sphinx warnings new_docs = new_docs.rstrip(':') summary += format%(substr, new_docs) + '\n' summary += '='*maxlen + ' ' + '='*80 + "\n" return template % summary # ----------------------------------------------------------------------- # header = """\ .. wxPython Phoenix documentation This file was generated by Phoenix's sphinx generator and associated tools, do not edit by hand. Copyright: (c) 2011-2018 by Total Control Software License: wxWindows License """ def writeSphinxOutput(stream, filename, append=False): """ Writes the text contained in the `stream` to the `filename` output file. :param StringIO.StringIO `stream`: the stream where the text lives. :param string `filename`: the output file we want to write the text in; :param bool `append`: ``True`` to append to the file, ``False`` to simply write to it. """ text_file = os.path.join(SPHINXROOT, filename) text = stream.getvalue() mode = 'a' if append else 'w' with codecs.open(text_file, mode, encoding='utf-8') as fid: if mode == 'w': fid.write(header) fid.write('.. include:: headings.inc\n\n') fid.write(text) # ----------------------------------------------------------------------- # def chopDescription(text): """ Given the (possibly multiline) input text, this function will get the first non-blank line up to the next newline character. :param string `text`: any string. :rtype: `string` """ description = '' for line in text.splitlines(): line = line.strip() if not line or line.startswith('..') or line.startswith('|'): continue description = line break return description # ----------------------------------------------------------------------- # class PickleFile(object): """ A class to help simplify loading and saving data to pickle files. """ def __init__(self, fileName): self.fileName = fileName def __enter__(self): self.read() return self def __exit__(self, exc_type, exc_val, exc_tb): self.write(self.items) def read(self): if os.path.isfile(self.fileName): with open(self.fileName, 'rb') as fid: items = pickle.load(fid) else: items = {} self.items = items return items def write(self, items): with open(self.fileName, 'wb') as fid: pickle.dump(items, fid) # ----------------------------------------------------------------------- # def pickleItem(description, current_module, name, kind): """ This function pickles/unpickles a dictionary containing class names as keys and class brief description (chopped docstrings) as values to build the main class index for Sphinx **or** the Phoenix standalone function names as keys and their full description as values to build the function page. This step is necessary as the function names/description do not come out in alphabetical order from the ``etg`` process. :param string `description`: the function/class description. :param string `current_module`: the harmonized module name for this class or function (see ``MODULENAME_REPLACE`` in `sphinxtools/constants.py`). :param string `name`: the function/class name. :param string `kind`: can be `function` or `class`. """ if kind == 'function': pickle_file = os.path.join(SPHINXROOT, current_module + 'functions.pkl') else: pickle_file = os.path.join(SPHINXROOT, current_module + '1moduleindex.pkl') with PickleFile(pickle_file) as pf: pf.items[name] = description # ----------------------------------------------------------------------- # def pickleClassInfo(class_name, element, short_description): """ Saves some information about a class in a pickle-compatible file., i.e. the list of methods in that class and its super-classes. :param string `class_name`: the name of the class we want to pickle; :param xml.etree.ElementTree.Element `element`: the XML element we want to examine; :param string `short_description`: the class short description (if any). """ method_list, bases = [], [] for method, description in element.method_list: method_list.append(method) for base in element.bases: bases.append(wx2Sphinx(base)[1]) pickle_file = os.path.join(SPHINXROOT, 'class_summary.pkl') with PickleFile(pickle_file) as pf: pf.items[class_name] = (method_list, bases, short_description) # ----------------------------------------------------------------------- # def pickleFunctionInfo(fullname, short_description): """ Saves the short description for each function, used for generating the summary pages later. """ pickle_file = os.path.join(SPHINXROOT, 'function_summary.pkl') with PickleFile(pickle_file) as pf: pf.items[fullname] = short_description # ----------------------------------------------------------------------- # def wx2Sphinx(name): """ Converts a wxWidgets specific string into a Phoenix-ReST-ready string. :param string `name`: any string. """ from etgtools.tweaker_tools import removeWxPrefix if '<' in name: name = name[0:name.index('<')] name = name.strip() newname = fullname = removeWxPrefix(name) if '.' in newname and len(newname) > 3: lookup, remainder = newname.split('.') remainder = '.%s' % remainder else: lookup = newname remainder = '' from etgtools.item_module_map import ItemModuleMap imm = ItemModuleMap() if lookup in imm: fullname = imm[lookup] + lookup + remainder return newname, fullname # ----------------------------------------------------------------------- # RAW_1 = """ %s.. raw:: html %s
""" RAW_2 = """ %s.. raw:: html %s
""" def formatContributedSnippets(kind, contrib_snippets): """ This method will include and properly ReST-ify contributed snippets of wxPython code (at the moment only 2 snippets are available), by including the Python code into the ReST files and allowing the user to show/hide the snippets using a JavaScript "Accordion" script thanks to the ``.. raw::`` directive (default for snippets is to be hidden). :param string `kind`: can be "method", "function" or "class" depending on the current item being scanned by the `sphinxgenerator.py` tool; :param list `contrib_snippets`: a list of file names (with the ``*.py`` extension) containing the contributed snippets of code. Normally these snippets live in the ``SPHINXROOT/rest_substitutions/snippets/python/contrib`` folder. """ spacer = '' if kind == 'function': spacer = 3*' ' elif kind == 'method': spacer = 6*' ' if kind == 'class': text = TEMPLATE_CONTRIB else: text = '\n' + spacer + '|contributed| **Contributed Examples:**\n\n' for indx, snippet in enumerate(contrib_snippets): with open(snippet, 'rt') as fid: lines = fid.readlines() user = lines[0].replace('##', '').strip() onlyfile = os.path.split(snippet)[1] download = os.path.join(SPHINXROOT, '_downloads', onlyfile) if not os.path.isfile(download): shutil.copyfile(snippet, download) text += RAW_1%(spacer, spacer) text += '\n' + spacer + 'Example %d - %s (:download:`download <_downloads/%s>`):\n\n'%(indx+1, user, onlyfile) text += spacer + '.. literalinclude:: _downloads/%s\n'%onlyfile text += spacer + ' :lines: 2-\n\n' text += RAW_2%(spacer, spacer) text += '\n\n%s|\n\n'%spacer return text def formatExternalLink(fullname, inheritance=False): """ Analyzes the input `fullname` string to check whether a class description is actually coming from an external documentation tool (like http://docs.python.org/library/ or http://docs.scipy.org/doc/numpy/reference/generated/). If the method finds such an external link, the associated inheritance diagram (if `inheritance` is ``True``) or the ``:ref:`` directive are modified accordingly to link it to the correct external documentation. :param string `fullname`: the fully qualified name for a class, method or function, i.e. `exceptions.Exception` or `threading.Thread`; :param bool `inheritance`: ``True`` if the call is coming from :mod:`inheritance`, ``False`` otherwise. """ if fullname.count('.') == 0: if not inheritance: return ':class:`%s`'%fullname return '' parts = fullname.split('.') possible_external = parts[-2] + '.' real_name = '.'.join(parts[-2:]) if possible_external.startswith('_'): # funny ctypes... possible_external = possible_external[1:] real_name = real_name[1:] if possible_external not in EXTERN_INHERITANCE: if not inheritance: return ':class:`%s`'%fullname return '' base_address = EXTERN_INHERITANCE[possible_external] if 'numpy' in real_name: htmlpage = '%s.html#%s'%(real_name.lower(), real_name) else: htmlpage = '%shtml#%s'%(possible_external.lower(), real_name) if inheritance: full_page = '"%s"'%(base_address + htmlpage) else: full_page = '`%s <%s>`_'%(fullname, base_address + htmlpage) return full_page def isPython3(): """ Returns ``True`` if we are using Python 3.x. """ return sys.version_info >= (3, )