are all formatted in the same way, because in plain-text it's
# difficult to make the distinctions clear (and anyway, we don't use them
# much differently in the libSBML code).
placeholders = []
p = re.compile('@verbatim(.+?)@endverbatim\n', re.DOTALL)
body = p.sub(lambda match: store_text(match, 1, placeholders, "\n"), body)
p = re.compile('@code({[.\w]+?})?(.+?)@endcode\n', re.DOTALL)
body = p.sub(lambda match: store_text(match, 2, placeholders, "\n"), body)
# Remove things that have been commented out with HTML comments.
p = re.compile(r'', re.DOTALL)
body = p.sub(r'', body)
# Replace some basic formatting with the plain contents, because we have
# no good way of representing the formatting in plain text.
for tag in ['strong', 'em', 'i', 'b', 'code', 'span', 'div', 'a']:
p = re.compile(r'<(?P' + tag + r')(\s+.*?)?>(.*?)',
re.DOTALL|re.IGNORECASE)
body = p.sub(r'\3', body)
# Translate basic HTML constructs.
body = re.sub(r'<', '<', body)
body = re.sub(r'≤', '<=', body)
body = re.sub(r'>', '>', body)
body = re.sub(r'≥', '>=', body)
body = re.sub(r'≠', '!=', body)
body = re.sub(r'π', 'Pi', body)
body = re.sub(r'"', '\\"', body)
body = re.sub(r'@', '@', body)
body = re.sub(r'"', '\\\\"', body)
body = re.sub(r'&[lr]d?quo;', '\\\\"', body)
body = re.sub(r' ', ' ', body)
body = re.sub(r'–', '-', body)
body = re.sub(r'—', ' -- ', body)
body = re.sub(r'
\s*', '\n', body)
body = re.sub(r'\s*', '\n', body)
body = re.sub(r'\s*', '', body)
body = re.sub(r'\s*', '\n' + ' '*list_indent + '* ', body)
body = re.sub(r'\s*', '', body)
# We probably should do and repeatedly, in case there's
# nesting involved, but in our docs we've never nested them.
p = re.compile(r'(.+?)', re.DOTALL|re.IGNORECASE)
body = p.sub(r'_\1', body)
p = re.compile(r'\s*(.+?)\s*', re.DOTALL|re.IGNORECASE)
body = p.sub(r'^\1', body)
# We don't want our blocks to be wrapped, but we do want basic HTML
# processing done, so we wait until now to store them. We also treat our
# elements for signature descriptions specially first. This also
# uses _@_@_@_@_ to be replaced later with spaces, to maintain indentation.
p = re.compile(r"(.+?)
", re.DOTALL|re.IGNORECASE)
body = p.sub(r"\n_@_@_@_@_\1\n", body)
p = re.compile(r'(.+?) ', re.DOTALL|re.IGNORECASE)
body = p.sub(lambda match: store_text(match, 2, placeholders, "\n"), body)
# Doing after
is easier than coming up with a hairy regexp to
# avoid the wrong match.
body = re.sub(r'\s*', '\n', body)
body = re.sub(r'
', '', body)
# Now finish rewriting Doxygen tags.
body = rewrite_see(body)
# Remove excess inter-paragraph spacing prior to rewrapping paragraphs,
# or the extra spaces at the beginning of blank lines can introduce
# extra leading spaces in the first lines of the paragraphs.
body = re.sub(r'\n *\n *\n *', '\n\n', body)
body = re.sub(r'(?]*?>(.*?)', re.DOTALL|re.IGNORECASE)
body = p.sub(lambda match: rewrite_htmltable_guarded(match), body)
p = re.compile(r"%%%%{(.+?)}%%%%", re.DOTALL|re.IGNORECASE)
body = p.sub(lambda match: store_text(match, 1, placeholders), body)
# Wrap paragraphs, so that the text is more readable.
p = re.compile(r'(.+?)(\n *\n|\Z)', re.DOTALL)
body = p.sub(lambda match: rewrite_fill_paragraph(match, line_width), body)
# Handle "@image html". Do this after wrapping, or else the contents of
# what we insert will end up wrapped.
p = re.compile(r'@image\s+html\s+(\S+).*?(\n *\n)', re.DOTALL)
body = p.sub(lambda match: rewrite_image(match, graphics_dir, quietly), body)
# Handle @section and other headings, adding underlining to them. Also
# handle
tags. This all needs to be done last, after wrapping
# paragraphs.
p = re.compile(r'^ *
?', re.MULTILINE)
body = p.sub('_'*line_width + '\n', body)
p = re.compile(r'@(sub)?section\s+\S+\s+(.+?)(\n *\n)', re.DOTALL|re.IGNORECASE)
body = p.sub(lambda match: rewrite_section_heading(match, line_width), body)
p = re.compile(r'()\s*(.+?)
(\n *\n)', re.DOTALL|re.IGNORECASE)
body = p.sub(lambda match: rewrite_section_heading(match, line_width), body)
# Now replace verbatim block placeholders.
p = re.compile(r'{{{{(\d+)}}}}')
body = p.sub(lambda match: replace_stored_text(match, placeholders), body)
# Remove excess inter-paragraph spacing one more time, to normalize
# spacing above and below verbatim's.
body = re.sub(r'\n *\n *\n *\n', '\n\n', body)
body = re.sub(r'\n *\n *\n', '\n\n', body)
# Remove other hacks
body = re.sub('_@_@_@_@_', ' ', body)
# And we're done.
return body.strip() + '\n'
def rewrite_if_else(match):
# Our possible conditional elements and their meanings are:
#
# a language name: java, python, csharp, perl, cpp, conly
# special terms: clike (= C or C++)
#
# The special variants are because Doxygen doesn't have a way to indicate a
# conjunction like "if not C or C++". We have to have special smarts here
# for notclike.
ifnot = match.group(1) == '@ifnot'
cond = match.group(2)
if ((not ifnot) and (cond == 'python')) \
or (ifnot and (cond != 'python' or cond == 'clike')):
text = match.group(3)
elif match.group(5) == '@else':
text = match.group(6)
else:
text = ''
return text
def store_text(match, group_number, placeholder_list, padding=""):
text = match.group(group_number)
placeholder_list.append(padding + text + padding)
index = len(placeholder_list) - 1
return '\n\n{{{{' + str(index) + '}}}}\n\n'
def replace_stored_text(match, placeholder_list):
index = int(match.group(1))
text = placeholder_list[index]
text = re.sub(r'\n', '\n ', text)
body = ' ' + text.rstrip() + '\n'
return body
def rewrite_see(text):
def replace_sees(segment):
subtext = segment.group(0)
matches = []
words = []
for m in re.finditer(r'(@see\s+([^\n]+\n))', subtext):
matches.append(m)
words.append(m.group(2).strip())
inner_pre_text = subtext[:matches[0].start(0)]
replacement_text = 'See also ' + ', '.join(words) + '.\n'
inner_post_text = subtext[matches[-1].end(0):]
return inner_pre_text + replacement_text + inner_post_text
p = re.compile(r'(@see.*?\n(\s*\n|\Z))', re.DOTALL|re.MULTILINE)
return re.sub(p, replace_sees, text)
def rewrite_ref(match):
name = match.group(1)
for key, value in known_refs.items():
if name == key:
return value
def rewrite_section_heading(match, line_width):
is_subsection = (match.group(1) != None)
heading_text = match.group(2)
tail_whitespace = match.group(3)
if is_subsection:
underline_char = '.'
else:
underline_char = '='
heading_text = wrap_paragraph(heading_text, line_width).strip()
return heading_text + "\n" + underline_char*70 + tail_whitespace
# When expanding @htmlinclude directives, it first checks to see if a
# version of the named file, but with a .txt extension, exists in the same
# location where it finds the .html file. If the .txt eversion exists, it
# includes that instead of the .html file. (This allows hand-formatted
# text files to be used, which is useful for files containing tables,
# because the Python HTML parser doesn't handle tables.)
def rewrite_htmlinclude(match, include_dir, quietly):
file_path = os.path.join(include_dir, match.group(1))
trailing_char = match.group(2)
if not valid_file(file_path):
if not quietly:
print("Warning: unable to expand @htmlinclude '" + match.group(1) + "'")
return ''
# First, try to see if there's a .txt version. If so, use that.
txt_file = re.sub(r'html', 'txt', file_path, re.IGNORECASE)
if valid_file(txt_file):
contents = read_file_contents(txt_file)
return rewrite_included_contents(contents) + trailing_char
else: # No txt file; proceed with .html file.
# 2017-11-25 Now that we use PrettyTable, I'm not
# sure it's worth doing the stuff below. Still, keeping it here
# (commented out) for future reference.
#
# file = open(file_path, 'r')
# writer = RewritePydocStringWriter()
# parser = None
# try:
# parser = RewritePydocHTMLParser(AbstractFormatter(writer))
# except:
# parser = RewritePydocHTMLParser()
# parser.feed(file.read())
# parser.close()
# file.close()
# return rewrite_included_contents(writer.get_text()) + trailing_char
contents = read_file_contents(file_path)
return rewrite_included_contents(contents) + trailing_char
# When expanding @image directives, it looks for a file with the extension
# .txt in the same directory where it finds the .jpg file. If the .txt
# version exists, it includes that; if it doesn't exist, it does not
# include anything. (Since the docstrings are plain-text, no other action
# seems sensible in this context.)
def rewrite_image(match, graphics_dir, quietly):
file_name = match.group(1)
txt_file = re.sub(r'.(png|jpg)', '.txt', file_name, re.IGNORECASE)
file_path = os.path.join(graphics_dir, txt_file)
trailing_space = match.group(2)
if not valid_file(file_path):
if not quietly:
print("Warning: unable to open '" + txt_file
+ "' to replace '" + file_name + "'")
return ''
contents = read_file_contents(file_path)
return rewrite_included_contents(contents) + trailing_space
def rewrite_htmltable_guarded(match):
headings = []
body = []
num_columns = 0
row_pattern = re.compile(r']*?>(.*?)', re.DOTALL|re.IGNORECASE)
th_pattern = re.compile(r']*?>(.*?)', re.DOTALL|re.IGNORECASE)
td_pattern = re.compile(r']*?>(.*?)', re.DOTALL|re.IGNORECASE)
for row in re.finditer(row_pattern, match.group(1)):
heads = [c.group(1).replace('\n', ' ').strip() for c in re.finditer(th_pattern, row.group(1))]
if not empty_list(heads):
headings.append(heads)
data = [c.group(1).replace('\n', ' ').strip() for c in re.finditer(td_pattern, row.group(1))]
if not empty_list(data):
body.append(data)
num_columns = max(num_columns, len(heads), len(data))
# Pad rows that don't have the same number of columns, or else PrettyTable
# chokes on the input.
for row in headings:
for x in range(0, num_columns - len(row)):
row.append("")
for row in body:
for x in range(0, num_columns - len(row)):
row.append("")
# Now generate the table structure and return it.
if not empty_list(headings):
table = PrettyTable(headings[0])
else:
table = PrettyTable()
table.header = False
for row in body:
table.add_row(row)
# For the table borders, we only insert borders if the table has a heading.
# This more often matches the way we use tables in libSBML.
table.border = not empty_list(headings)
table.align = "l"
table.right_padding_width = 1
table.left_padding_width = 1
return "%%%%{" + "\n" + table.get_string() + "\n" + "}%%%%"
def rewrite_included_contents(contents):
contents = re.sub(r'"', '\\"', contents) # Quote all double quotes.
return contents
def rewrite_fill_paragraph(match, line_width):
text = match.group(1)
return wrap_paragraph(text, line_width)
def wrap_paragraph(text, line_width):
return textwrap.fill(text, width = line_width).strip() + '\n\n'
def expanded_path(path):
if path: return os.path.expanduser(os.path.expandvars(path))
else: return ''
def read_file_contents(file):
file_stream = open(file, 'r')
contents = file_stream.read()
file_stream.close()
return contents
def write_output_file(file, contents):
output = open(file, 'w')
output.write(contents)
output.close()
def valid_file(file, quiet=False):
if not os.path.exists(file):
return False
elif not os.path.isfile(file):
return False
else:
return True
def valid_directory(dir, quiet=False):
if not os.path.exists(dir):
return False
elif not os.path.isdir(dir):
return False
else:
return True
# The following is based on http://stackoverflow.com/a/1605679/743730
# specifically the one-line version by user "Stephan202".
def empty_list(data):
return all(map(empty_list, data)) if isinstance(data, list) else False
# The following came from a StackOverflow posting by "Soldier.moth" in 2011:
# http://stackoverflow.com/questions/930303/python-string-cleanup-manipulation-accented-characters
# I added 0xa0, which is unbreakable space, because the Python HTML parser was
# producing that character.
def latin1_to_ascii (unicrap):
"""This replaces UNICODE Latin-1 characters with
something equivalent in 7-bit ASCII. All characters in the standard
7-bit ASCII range are preserved. In the 8th bit range all the Latin-1
accented letters are stripped of their accents. Most symbol characters
are converted to something meaningful. Anything not converted is deleted.
"""
xlate = {
0xc0:'A', 0xc1:'A', 0xc2:'A', 0xc3:'A', 0xc4:'A', 0xc5:'A',
0xc6:'Ae', 0xc7:'C', 0xc8:'E', 0xc9:'E', 0xca:'E', 0xcb:'E',
0xcc:'I', 0xcd:'I', 0xce:'I', 0xcf:'I', 0xd0:'Th', 0xd1:'N',
0xd2:'O', 0xd3:'O', 0xd4:'O', 0xd5:'O', 0xd6:'O', 0xd8:'O',
0xd9:'U', 0xda:'U', 0xdb:'U', 0xdc:'U', 0xdd:'Y', 0xde:'th',
0xdf:'ss', 0xe0:'a', 0xe1:'a', 0xe2:'a', 0xe3:'a', 0xe4:'a',
0xe5:'a', 0xe6:'ae', 0xe7:'c', 0xe8:'e', 0xe9:'e', 0xea:'e',
0xeb:'e', 0xec:'i', 0xed:'i', 0xee:'i', 0xef:'i', 0xf0:'th',
0xf1:'n', 0xf2:'o', 0xf3:'o', 0xf4:'o', 0xf5:'o', 0xf6:'o',
0xf8:'o', 0xf9:'u', 0xfa:'u', 0xfb:'u', 0xfc:'u', 0xfd:'y',
0xfe:'th', 0xff:'y', 0xa0: ' ', 0xa1:'!', 0xa2:'{cent}',
0xa3:'{pound}', 0xa4:'{currency}', 0xa5:'{yen}', 0xa6:'|',
0xa7:'{section}', 0xa8:'{umlaut}', 0xa9:'{C}', 0xaa:'{^a}',
0xab:'<<', 0xac:'{not}', 0xad:'-', 0xae:'{R}', 0xaf:'_',
0xb0:'{degrees}', 0xb1:'{+/-}', 0xb2:'{^2}', 0xb3:'{^3}', 0xb4:"'",
0xb5:'{micro}', 0xb6:'{paragraph}', 0xb7:'*', 0xb8:'{cedilla}',
0xb9:'{^1}', 0xba:'{^o}', 0xbb:'>>', 0xbc:'{1/4}', 0xbd:'{1/2}',
0xbe:'{3/4}', 0xbf:'?', 0xd7:'*', 0xf7:'/'
}
r = ''
for i in unicrap:
if xlate.has_key(ord(i)):
r += xlate[ord(i)]
elif ord(i) >= 0x80:
pass
else:
r += i
return r
def main():
args = parse_cmdline()
input_file = get_input_file_name(args)
output_file = get_output_file_name(args)
include_dir = get_include_dir(args)
graphics_dir = get_graphics_dir(args)
quietly = get_quiet_flag(args)
# Sanity-check the arguments.
if not valid_file(input_file, quietly):
if not quietly: print("Error: cannot read file '" + input_file + "'")
sys.exit(1)
elif include_dir and not valid_directory(include_dir, quietly):
if not quietly: print("Error: cannot access directory '" + include_dir + "'")
sys.exit(1)
elif graphics_dir and not valid_directory(graphics_dir, quietly):
if not quietly: print("Error: cannot access directory '" + graphics_dir + "'")
sys.exit(1)
# Let's do this thing.
results = rewrite(read_file_contents(input_file), include_dir,
graphics_dir, quietly)
write_output_file(output_file, results)
if __name__ == '__main__':
main()