and ),
these are kept together as a single chunk.
Lists are formatted using parse_paragraph_list().
Empty lines are stripped from the output.
Indentation (i.e. lines starting with ":") is ignored.
Called from parse_paragraphs() method.
"""
s = self.plain(markup)
# Add an extra linebreak between the last list item
# and the normal line following after it, so they don't stick together, e.g.
# **[[Alin Magic]], magic used in the videogame ''[[Rise of Nations: Rise of Legends]]''
# In '''popular culture''':
# * [[Magic (film)|''Magic'' (film)]], a 1978 film starring Anthony Hopkins and Ann-Margret
s = re.sub(re.compile("\n([*#;].*?)\n([^*#?])", re.DOTALL), "\n\\1\n\n\\2", s)
# This keeps list items with linebreaks
# between them nicely together.
s = re.sub("\n{2,3}([*#;])", "\n\\1", s)
chunks = []
ch = ""
i = 1
for chunk in s.split("\n"):
if chunk.startswith(":"):
chunk = chunk.lstrip(":")
if len(chunk.strip()) > 1:
# Leave out taxoboxes and infoboxes.
if not chunk.startswith("|"):
ch += chunk + "\n"
if ch.strip() != "":
if not re.search("^[ *#;]", chunk):
ch = self.parse_paragraph_list(ch)
chunks.append(ch.rstrip())
ch = ""
if ch.strip() != "":
ch = self.parse_paragraph_list(ch)
chunks.append(ch.strip())
return chunks
def parse_paragraph_list(self, markup, indent="\t"):
""" Formats bullets and numbering of Wikipedia lists.
List items are marked by "*", "#" or ";" at the start of a line.
We treat ";" the same as "*",
and replace "#" with real numbering (e.g. "2.").
Sublists (e.g. *** and ###) get indented by tabs.
Called from parse_paragraphs() method.
"""
def lastleft(ch, str):
n = 0
while n < len(str) and str[n] == ch: n += 1
return n
tally = [1 for i in range(10)]
chunks = markup.split("\n")
for i in range(len(chunks)):
if chunks[i].startswith("#"):
j = min(lastleft("#", chunks[i]), len(tally)-1)
chunks[i] = indent*(j-1) + str(tally[j])+". " + chunks[i][j:]
chunks[i] = chunks[i].replace(". ", ". ")
tally[j] += 1
# Reset the numbering of sublists.
for k in range(j+1, len(tally)):
tally[k] = 1
if chunks[i].startswith(";"):
chunks[i] = "*" + chunks[i][1:]
if chunks[i].startswith("*"):
j = lastleft("*", chunks[i])
chunks[i] = indent*(j-1) + "* " + chunks[i][j:]
chunks[i] = chunks[i].replace("* ", "* ")
return "\n".join(chunks)
def parse_paragraph_heading_depth(self, markup):
""" Returns the depth of a heading.
The depth determines parent and child relations,
which headings (and hence which paragraphs) are a child to a heading higher up.
Returns 0 for =, 1 for ==, etc.
Called from parse_paragraphs() method.
"""
return markup.count("=")/2 - 1
def connect_paragraph(self, paragraph, paragraphs):
""" Create parent/child links to other paragraphs.
The paragraphs parameters is a list of all the paragraphs
parsed up till now.
The parent is the previous paragraph whose depth is less.
The parent's children include this paragraph.
Called from parse_paragraphs() method.
"""
if paragraph.depth > 0:
n = range(len(paragraphs))
n.reverse()
for i in n:
if paragraphs[i].depth == paragraph.depth-1:
paragraph.parent = paragraphs[i]
paragraphs[i].children.append(paragraph)
break
return paragraph
def parse_paragraph_references(self, markup):
""" Updates references with content from specific paragraphs.
The "references", "notes", "external links" paragraphs
are double-checked for references. Not all items in the list
might have been referenced inside the article, or the item
might contain more info than we initially parsed from it.
Called from parse_paragraphs() method.
"""
for chunk in markup.split("\n"):
# We already parsed this, it contains the self.ref mark.
# See if we can strip more notes from it.
m = re.search(self.ref+"\(([0-9]*?)\)", chunk)
if m:
chunk = chunk.strip("* ")
chunk = chunk.replace(m.group(0), "")
chunk = self.plain(chunk)
i = int(m.group(1))
if chunk != "":
self.references[i-1].note = chunk
# If it's not a citation we don't have this reference yet.
elif chunk.strip().startswith("*") \
and chunk.find("{{cite") < 0:
chunk = chunk.strip("* ")
chunk = self.plain(chunk)
if chunk != "":
r = WikipediaReference()
r.note = chunk
self.references.append(r)
def parse_paragraphs(self, markup):
""" Returns a list of paragraphs in the markup.
A paragraph has a title and multiple lines of plain text.
A paragraph might have parent and child paragraphs,
denoting subtitles or bigger chapters.
A paragraph might have links to additional articles.
Formats numbered lists by replacing # by 1.
Formats bulleted sublists like ** or *** with indentation.
"""
# Paragraphs to exclude.
refs = ["references", "notes", "notes and references", "external links", "further reading"]
exclude = ["see also", "media", "gallery", "related topics", "lists", "gallery", "images"]
exclude.extend(refs)
paragraphs = []
paragraph = WikipediaParagraph(self.title)
paragraph_data = ""
for chunk in markup.split("\n"):
# Strip each line of whitespace,
# unless it's a preformatted line (starts with a space).
if not chunk.startswith(" "):
chunk = chunk.strip()
# A title wrapped in "=", "==", "==="...
# denotes a new paragraphs section.
if chunk.startswith("="):
if paragraph.title.lower() in refs \
or (paragraph.parent and paragraph.parent.title.lower() in refs):
self.parse_paragraph_references(paragraph_data)
paragraph.extend(self.parse_paragraph(paragraph_data))
paragraphs.append(paragraph)
# Initialise a new paragraph.
# Create parent/child links to other paragraphs.
title = chunk.strip().strip("=")
title = self.plain(title)
paragraph = WikipediaParagraph(title)
paragraph.depth = self.parse_paragraph_heading_depth(chunk)
if paragraph.title.lower() not in exclude:
paragraph = self.connect_paragraph(paragraph, paragraphs)
paragraph_data = ""
# Underneath a title might be links to in-depth articles,
# e.g. Main articles: Computer program and Computer programming
# which in wiki markup would be {{main|Computer program|Computer programming}}
# The second line corrects" {{Main|Credit (finance)}} or {{Main|Usury}}".
elif re.search(re.compile("^{{main", re.I), chunk):
paragraph.main = [link.strip("} ") for link in chunk.split("|")[1:]]
paragraph.main = [re.sub(re.compile("}}.*?{{main", re.I), "", link)
for link in paragraph.main]
# At the bottom might be links to related articles,
# e.g. See also: Abundance of the chemical elements
# which in wiki markup would be {{see also|Abundance of the chemical elements}}
elif re.search(re.compile("^{{see {0,1}also", re.I), chunk):
paragraph.related = [link.strip("} ") for link in chunk.split("|")[1:]]
# Accumulate the data in this paragraph,
# we'll process it once a new paragraph starts.
else:
paragraph_data += chunk +"\n"
# Append the last paragraph.
if paragraph.title.lower() in refs \
or (paragraph.parent and paragraph.parent.title.lower() in refs):
self.parse_paragraph_references(paragraph_data)
paragraph.extend(self.parse_paragraph(paragraph_data))
paragraphs.append(paragraph)
# The "See also" paragraph is an enumeration of links
# which we already parsed so don't show them.
# We also did references, and other paragraphs are not that relevant.
paragraphs_exclude = []
for paragraph in paragraphs:
if paragraph.title.lower() not in exclude \
and not (paragraph.parent and paragraph.parent.title.lower() in exclude):
paragraphs_exclude.append(paragraph)
if len(paragraphs_exclude) == 1 and \
len(paragraphs_exclude[0]) == 0:
return []
return paragraphs_exclude
def parse_table_row(self, markup, row):
""" Parses a row of cells in a Wikipedia table.
Cells in the row are separated by "||".
A "!" indicates a row of heading columns.
Each cell can contain properties before a "|",
# e.g. align="right" | Cell 2 (right aligned).
"""
if row == None:
row = WikipediaTableRow()
markup = markup.replace("!!", "||")
for cell in markup.lstrip("|!").split("||"):
# The "|" after the properties can't be part of a link.
i = cell.find("|")
j = cell.find("[[")
if i>0 and (j<0 or i 0:
tables.append(table)
return tables
def parse_references(self, markup):
""" Returns a list of references found in the markup.
References appear inline as [ footnotes,
http:// external links, or {{cite}} citations.
We replace it with (1)-style footnotes.
Additional references data is gathered in
parse_paragraph_references() when we parse paragraphs.
References can also appear in image descriptions,
tables and taxoboxes, so they might not always pop up in a paragraph.
The plain() method finally replaces (1) by [1].
"""
references = []
# A Wikipedia reference note looks like:
# ][In 1946, [[ENIAC]] consumed an estimated 174 kW.
# By comparison, a typical personal computer may use around 400 W;
# over four hundred times less. {{Ref harvard|kempf1961|Kempf 1961|a}}]
m = re.findall(self.re["reference"], markup)
for reference in m:
reference = re.sub("[ {0,1}cite", "][{{cite", reference)
if not reference.strip().startswith("[http://") and \
not re.search("\{\{cite", reference):
r = WikipediaReference()
r.note = self.plain(re.sub("", "", reference))
if r.note != "":
references.append(r)
p = " "+self.ref+"("+str(len(references))+")"
markup = markup.replace(reference, p, 1)
else:
# References containing a citation or url
# are better handled by the next patterns.
pass
# A Wikipedia citation looks like:
# {{cite journal
# | last = Einstein
# | first = Albert
# | authorlink = Albert Einstein
# | title = Sidelights on Relativity (Geometry and Experience)
# | publisher = P. Dutton., Co
# | date = 1923}}
m = re.findall(self.re["citation"], markup)
for citation in m:
c = citation.replace("\n", "")
r = WikipediaReference()
for key in r.__dict__.keys():
value = re.search("\| {0,1}"+key+"(.*?)[\|}]", c)
if value:
value = value.group(1)
value = value.replace("link", "")
value = value.strip().strip(" =[]")
value = self.plain(value)
setattr(r, key, value)
if r.first != "" and r.last != "":
r.author = r.first + " " + r.last
references.append(r)
p = " "+self.ref+"("+str(len(references))+")"
markup = markup.replace(citation, p, 1)
# A Wikipedia embedded url looks like:
# [http://www.pbs.org/wnet/hawking/html/home.html ''Stephen Hawking's Universe'']
m = re.findall(self.re["url"], markup)
for url in m:
r = WikipediaReference()
i = url.find(" ")
if i > 0:
r.url = url[:i].strip()
r.note = self.plain(url[i:])
else:
r.url = url.strip()
references.append(r)
p = r.note+" "+self.ref+"("+str(len(references))+")"
markup = markup.replace("["+url+"]", p, 1)
# Since we parsed all citations first and then all notes and urls,
# the ordering will not be correct in the markup,
# e.g. (1) (11) (12) (2) (3).
sorted = []
m = re.findall(self.ref+"\(([0-9]*)\)", markup)
for i in m:
sorted.append(references[int(i)-1])
markup = markup.replace(
self.ref+"("+i+")",
self.ref+"**("+str(len(sorted))+")"
)
markup = markup.replace(self.ref+"**", self.ref)
for r in references:
if r not in sorted:
sorted.append(r)
references = sorted
return references, markup.strip()
def parse_categories(self, markup):
""" Returns a list of categories the page belongs to.
# A Wikipedia category link looks like:
# [[Category:Computing]]
# This indicates the page is included in the given category.
# If "Category" is preceded by ":" this indicates a link to a category.
"""
categories = []
m = re.findall(self.re["category"], markup)
for category in m:
category = category.split("|")
page = category[0].strip()
display = u""
if len(category) > 1:
display = category[1].strip()
#if not categories.has_key(page):
# categories[page] = WikipediaLink(page, u"", display)
if not page in categories:
categories.append(page)
return categories
def parse_translations(self, markup):
""" Returns a dictionary of translations for the page title.
A Wikipedia language link looks like: [[af:Rekenaar]].
The parser will also fetch links like "user:" and "media:"
but these are stripped against the dictionary of
Wikipedia languages.
You can get a translated page by searching Wikipedia
with the appropriate language code and supplying
the translated title as query.
"""
global languages
translations = {}
m = re.findall(self.re["translation"], markup)
for language, translation in m:
if language in languages:
translations[language] = translation
return translations
def parse_disambiguation(self, markup):
""" Gets the Wikipedia disambiguation page for this article.
A Wikipedia disambiguation link refers to other pages
with the same title but of smaller significance,
e.g. {{dablink|For the IEEE magazine see [[Computer (magazine)]].}}
"""
m = re.search(self.re["disambiguation"], markup)
if m:
return self.parse_links(m.group(1))
else:
return []
def parse_important(self, markup):
""" Returns a list of words that appear in bold in the article.
Things like table titles are not added to the list,
these are probably bold because it makes the layout nice,
not necessarily because they are important.
"""
important = []
table_titles = [table.title for table in self.tables]
m = re.findall(self.re["bold"], markup)
for bold in m:
bold = self.plain(bold)
if not bold in table_titles:
important.append(bold.lower())
return important
### DRAWING UTILITIES ################################################################################
def is_preformatted(str):
""" Determines if an item in a paragraph is preformatted.
If all of the lines in the markup start with a " "
this indicates preformatted text.
Preformatted is usually used for programming code.
"""
for chunk in str.split("\n"):
if not chunk.startswith(" "):
return False
return True
def is_list(str):
""" Determines if an item in a paragraph is a list.
If all of the lines in the markup start with a "*" or "1."
this indicates a list as parsed by parse_paragraphs().
It can be drawn with draw_list().
"""
for chunk in str.split("\n"):
chunk = chunk.replace("\t", "")
if not chunk.lstrip().startswith("*") \
and not re.search(r"^([0-9]{1,3}\. )", chunk.lstrip()):
return False
return True
def is_math(str):
""" Determines if an item in a paragraph is a LaTeX math equation.
Math equations are wrapped in tags.
They can be drawn as an image using draw_math().
"""
str = str.strip()
if str.startswith(""):
return True
else:
return False
def draw_math(str, x, y, alpha=1.0):
""" Uses mimetex to generate a GIF-image from the LaTeX equation.
"""
try: from web import _ctx
except: pass
str = re.sub("", "", str.strip())
img = mimetex.gif(str)
w, h = _ctx.imagesize(img)
_ctx.image(img, x, y, alpha=alpha)
return w, h
def textwidth(str):
"""textwidth() reports incorrectly when lineheight() is smaller than 1.0
"""
try: from web import _ctx
except: pass
l = _ctx.lineheight()
_ctx.lineheight(1)
w = _ctx.textwidth(str)
_ctx.lineheight(l)
return w
def draw_list(markup, x, y, w, padding=5, callback=None):
""" Draws list markup with indentation in NodeBox.
Draw list markup at x, y coordinates
using indented bullets or numbers.
The callback is a command that takes a str and an int.
"""
try: from web import _ctx
except: pass
i = 1
for chunk in markup.split("\n"):
if callback != None:
callback(chunk, i)
m = re.search("^([0-9]{1,3}\. )", chunk.lstrip())
if m:
indent = re.search("[0-9]", chunk).start()*padding*2
bullet = m.group(1)
dx = textwidth("000.")
chunk = chunk.lstrip(m.group(1)+"\t")
if chunk.lstrip().startswith("*"):
indent = chunk.find("*")*padding*2
bullet = u"•"
dx = textwidth("*")
chunk = chunk.lstrip("* \t")
_ctx.text(bullet, x+indent, y)
dx += padding + indent
_ctx.text(chunk, x+dx, y, width=w-dx)
y += _ctx.textheight(chunk, width=w-dx)
y += _ctx.textheight(" ") * 0.25
i += 1
def draw_table(table, x, y, w, padding=5):
""" This is a very poor algorithm to draw Wikipedia tables in NodeBox.
"""
try: from web import _ctx
except: pass
f = _ctx.fill()
_ctx.stroke(f)
h = _ctx.textheight(" ") + padding*2
row_y = y
if table.title != "":
_ctx.fill(f)
_ctx.rect(x, row_y, w, h)
_ctx.fill(1)
_ctx.text(table.title, x+padding, row_y+_ctx.fontsize()+ padding)
row_y += h
# A table of flags marking how long a cell
# from a previous row is still spanning in a column.
rowspans = [1 for i in range(10)]
previous_cell_w = 0
for row in table:
cell_x = x
# The width of a cell is the total table width
# evenly divided by the number of cells.
# Previous rows' cells still spanning will push cells
# to the right and decrease their width.
cell_w = 1.0 * w
cell_w -= previous_cell_w * len([n for n in rowspans if n > 1])
cell_w /= len(row)
# The height of each cell is the highest cell in the row.
# The height depends on the amount of text in the cell.
cell_h = 0
for cell in row:
this_h = _ctx.textheight(cell, width=cell_w-padding*2) + padding*2
cell_h = max(cell_h, this_h)
# Traverse each cell in this row.
i = 0
for cell in row:
# If a previous row's cell is still spanning,
# push this cell to the right.
if rowspans[i] > 1:
rowspans[i] -= 1
cell_x += previous_cell_w
i += 1
# Get the rowspan attribute for this cell.
m = re.search("rowspan=\"(.*?)\"", cell.properties)
if m:
rowspan = int(m.group(1))
rowspans[i] = rowspan
else:
rowspan = 1
# Padded cell text.
# Horizontal line above each cell.
# Vertical line before each cell.
_ctx.fill(f)
_ctx.text(cell, cell_x+padding, row_y+_ctx.fontsize()+padding, cell_w-padding*2)
_ctx.line(cell_x, row_y, cell_x+cell_w, row_y)
if cell_x > x:
_ctx.nofill()
_ctx.line(cell_x, row_y, cell_x, row_y+cell_h)
cell_x += cell_w
i += 1
# Move to next row.
row_y += cell_h
previous_cell_w = cell_w
# Table's bounding rectangle.
_ctx.nofill()
_ctx.rect(x, y, w, row_y-y)
### WIKIPEDIASEARCH ##################################################################################
class WikipediaSearch(WikipediaPage, URLAccumulator):
def _api_request(self, q, language="en"):
url = "http://"+language+".wikipedia.org/w/api.php"
url += "?action=query&redirects&format=xml&prop=revisions&rvprop=content&titles="
url += quote(q)
return url
def __init__(self, q, language="en", light=False, wait=10, asynchronous=False, cached=True,
case_sensitive=False, full_strip=True):
""" A download manager for Wikipedia pages.
WikipediaSearch is a combination of
URLAccumulator that handles asynchronous and cached web downloads and
WikipediaPage that parses XML retrieved from the Wikipedia API.
Retrieves the latest revision.
Redirects are handled by the Wikipedia server.
"""
self._light = light
self._full_strip = full_strip
if cached:
cache = "wikipedia"
else:
cache = None
if not case_sensitive:
q = str(q.lower())
q = q.replace(" ", "_")
url = self._api_request(q, language)
URLAccumulator.__init__(self, url, wait, asynchronous, cache, type=".xml", throttle=2)
def load(self, data):
dom = minidom.parseString(self.data)
page = dom.getElementsByTagName("page")[0]
title = page.getAttribute("title")
try:
rev = dom.getElementsByTagName("rev")[0]
data = rev.childNodes[0].nodeValue.strip()
except:
if not self.error:
self.error = WikipediaPageMissing()
data = ""
WikipediaPage.__init__(self, title, data, light=self._light, full_strip=self._full_strip)
def search(q, language="en", light=False, wait=10, asynchronous=False, cached=True,
case_sensitive=False, full_strip=True):
return WikipediaSearch(q, language, light, wait, asynchronous, cached, case_sensitive, full_strip)
######################################################################################################
# Some interesting things...
# Redirects are now handled by the Wikipedia server but for some reason I'm keeping this code around.
# The superscript could be used to format references and footnotes.
def is_redirect(page):
m = re.search(r"#REDIRECT \[\[.*?\]\]", page)
if m and len(m.group(0)) == len(page):
return True
else:
return False
def redirect(page):
m = re.search(r"#REDIRECT \[\[(.*?)\]\]", page)
if m:
return m.group(1)
else:
return None
def superscript(number):
digits = [
u"\u2070",
u"\u2071",
u"\u2072",
u"\u2073",
u"\u2074",
u"\u2075",
u"\u2076",
u"\u2077",
u"\u2078",
u"\u2079",
]
s = u""
for digit in str(number):
s += digits[int(digit)]
return s
######################################################################################################
]