############################################################################### # # This file is part of the Utopia Documents application. # Copyright (c) 2008-2014 Lost Island Labs # # # Utopia Documents is free software: you can redistribute it and/or modify # it under the terms of the GNU GENERAL PUBLIC LICENSE VERSION 3 as # published by the Free Software Foundation. # # Utopia Documents is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # In addition, as a special exception, the copyright holders give # permission to link the code of portions of this program with the OpenSSL # library under certain conditions as described in each individual source # file, and distribute linked combinations including the two. # # You must obey the GNU General Public License in all respects for all of # the code used other than OpenSSL. If you modify file(s) with this # exception, you may extend this exception to your version of the file(s), # but you are not obligated to do so. If you do not wish to do so, delete # this exception statement from your version. # # You should have received a copy of the GNU General Public License # along with Utopia Documents. If not, see # ############################################################################### import base64 import common.eutils import common.utils import copy import re import spineapi import utopia.document from lxml import etree namespaces = {'xlink':'http://www.w3.org/1999/xlink'} entities = { 'AElig': [0xc6], 'AMP': [0x26], 'Aacute': [0xc1], 'Abreve': [0x102], 'Acirc': [0xc2], 'Acy': [0x410], 'Afr': [0x1d504], 'Agrave': [0xc0], 'Alpha': [0x391], 'Amacr': [0x100], 'And': [0x2a53], 'Aogon': [0x104], 'Aopf': [0x1d538], 'ApplyFunction': [0x2061], 'Aring': [0xc5], 'Ascr': [0x1d49c], 'Assign': [0x2254], 'Atilde': [0xc3], 'Auml': [0xc4], 'Backslash': [0x2216], 'Barv': [0x2ae7], 'Barwed': [0x2306], 'Bcy': [0x411], 'Because': [0x2235], 'Bernoullis': [0x212c], 'Beta': [0x392], 'Bfr': [0x1d505], 'Bopf': [0x1d539], 'Breve': [0x2d8], 'Bscr': [0x212c], 'Bumpeq': [0x224e], 'CHcy': [0x427], 'COPY': [0xa9], 'Cacute': [0x106], 'Cap': [0x22d2], 'CapitalDifferentialD': [0x2145], 'Cayleys': [0x212d], 'Ccaron': [0x10c], 'Ccedil': [0xc7], 'Ccirc': [0x108], 'Cconint': [0x2230], 'Cdot': [0x10a], 'Cedilla': [0xb8], 'CenterDot': [0xb7], 'Cfr': [0x212d], 'Chi': [0x3a7], 'CircleDot': [0x2299], 'CircleMinus': [0x2296], 'CirclePlus': [0x2295], 'CircleTimes': [0x2297], 'ClockwiseContourIntegral': [0x2232], 'CloseCurlyDoubleQuote': [0x201d], 'CloseCurlyQuote': [0x2019], 'Colon': [0x2237], 'Colone': [0x2a74], 'Congruent': [0x2261], 'Conint': [0x222f], 'ContourIntegral': [0x222e], 'Copf': [0x2102], 'Coproduct': [0x2210], 'CounterClockwiseContourIntegral': [0x2233], 'Cross': [0x2a2f], 'Cscr': [0x1d49e], 'Cup': [0x22d3], 'CupCap': [0x224d], 'DD': [0x2145], 'DDotrahd': [0x2911], 'DJcy': [0x402], 'DScy': [0x405], 'DZcy': [0x40f], 'Dagger': [0x2021], 'Darr': [0x21a1], 'Dashv': [0x2ae4], 'Dcaron': [0x10e], 'Dcy': [0x414], 'Del': [0x2207], 'Delta': [0x394], 'Dfr': [0x1d507], 'DiacriticalAcute': [0xb4], 'DiacriticalDot': [0x2d9], 'DiacriticalDoubleAcute': [0x2dd], 'DiacriticalGrave': [0x60], 'DiacriticalTilde': [0x2dc], 'Diamond': [0x22c4], 'DifferentialD': [0x2146], 'Dopf': [0x1d53b], 'Dot': [0xa8], 'DotDot': [0x20dc], 'DotEqual': [0x2250], 'DoubleContourIntegral': [0x222f], 'DoubleDot': [0xa8], 'DoubleDownArrow': [0x21d3], 'DoubleLeftArrow': [0x21d0], 'DoubleLeftRightArrow': [0x21d4], 'DoubleLeftTee': [0x2ae4], 'DoubleLongLeftArrow': [0x27f8], 'DoubleLongLeftRightArrow': [0x27fa], 'DoubleLongRightArrow': [0x27f9], 'DoubleRightArrow': [0x21d2], 'DoubleRightTee': [0x22a8], 'DoubleUpArrow': [0x21d1], 'DoubleUpDownArrow': [0x21d5], 'DoubleVerticalBar': [0x2225], 'DownArrow': [0x2193], 'DownArrowBar': [0x2913], 'DownArrowUpArrow': [0x21f5], 'DownBreve': [0x311], 'DownLeftRightVector': [0x2950], 'DownLeftTeeVector': [0x295e], 'DownLeftVector': [0x21bd], 'DownLeftVectorBar': [0x2956], 'DownRightTeeVector': [0x295f], 'DownRightVector': [0x21c1], 'DownRightVectorBar': [0x2957], 'DownTee': [0x22a4], 'DownTeeArrow': [0x21a7], 'Downarrow': [0x21d3], 'Dscr': [0x1d49f], 'Dstrok': [0x110], 'ENG': [0x14a], 'ETH': [0xd0], 'Eacute': [0xc9], 'Ecaron': [0x11a], 'Ecirc': [0xca], 'Ecy': [0x42d], 'Edot': [0x116], 'Efr': [0x1d508], 'Egrave': [0xc8], 'Element': [0x2208], 'Emacr': [0x112], 'EmptySmallSquare': [0x25fb], 'EmptyVerySmallSquare': [0x25ab], 'Eogon': [0x118], 'Eopf': [0x1d53c], 'Epsilon': [0x395], 'Equal': [0x2a75], 'EqualTilde': [0x2242], 'Equilibrium': [0x21cc], 'Escr': [0x2130], 'Esim': [0x2a73], 'Eta': [0x397], 'Euml': [0xcb], 'Exists': [0x2203], 'ExponentialE': [0x2147], 'Fcy': [0x424], 'Ffr': [0x1d509], 'FilledSmallSquare': [0x25fc], 'FilledVerySmallSquare': [0x25aa], 'Fopf': [0x1d53d], 'ForAll': [0x2200], 'Fouriertrf': [0x2131], 'Fscr': [0x2131], 'GJcy': [0x403], 'GT': [0x3e], 'Gamma': [0x393], 'Gammad': [0x3dc], 'Gbreve': [0x11e], 'Gcedil': [0x122], 'Gcirc': [0x11c], 'Gcy': [0x413], 'Gdot': [0x120], 'Gfr': [0x1d50a], 'Gg': [0x22d9], 'Gopf': [0x1d53e], 'GreaterEqual': [0x2265], 'GreaterEqualLess': [0x22db], 'GreaterFullEqual': [0x2267], 'GreaterGreater': [0x2aa2], 'GreaterLess': [0x2277], 'GreaterSlantEqual': [0x2a7e], 'GreaterTilde': [0x2273], 'Gscr': [0x1d4a2], 'Gt': [0x226b], 'HARDcy': [0x42a], 'Hacek': [0x2c7], 'Hat': [0x5e], 'Hcirc': [0x124], 'Hfr': [0x210c], 'HilbertSpace': [0x210b], 'Hopf': [0x210d], 'HorizontalLine': [0x2500], 'Hscr': [0x210b], 'Hstrok': [0x126], 'HumpDownHump': [0x224e], 'HumpEqual': [0x224f], 'IEcy': [0x415], 'IJlig': [0x132], 'IOcy': [0x401], 'Iacute': [0xcd], 'Icirc': [0xce], 'Icy': [0x418], 'Idot': [0x130], 'Ifr': [0x2111], 'Igrave': [0xcc], 'Im': [0x2111], 'Imacr': [0x12a], 'ImaginaryI': [0x2148], 'Implies': [0x21d2], 'Int': [0x222c], 'Integral': [0x222b], 'Intersection': [0x22c2], 'InvisibleComma': [0x2063], 'InvisibleTimes': [0x2062], 'Iogon': [0x12e], 'Iopf': [0x1d540], 'Iota': [0x399], 'Iscr': [0x2110], 'Itilde': [0x128], 'Iukcy': [0x406], 'Iuml': [0xcf], 'Jcirc': [0x134], 'Jcy': [0x419], 'Jfr': [0x1d50d], 'Jopf': [0x1d541], 'Jscr': [0x1d4a5], 'Jsercy': [0x408], 'Jukcy': [0x404], 'KHcy': [0x425], 'KJcy': [0x40c], 'Kappa': [0x39a], 'Kcedil': [0x136], 'Kcy': [0x41a], 'Kfr': [0x1d50e], 'Kopf': [0x1d542], 'Kscr': [0x1d4a6], 'LJcy': [0x409], 'LT': [0x3c], 'Lacute': [0x139], 'Lambda': [0x39b], 'Lang': [0x27ea], 'Laplacetrf': [0x2112], 'Larr': [0x219e], 'Lcaron': [0x13d], 'Lcedil': [0x13b], 'Lcy': [0x41b], 'LeftAngleBracket': [0x27e8], 'LeftArrow': [0x2190], 'LeftArrowBar': [0x21e4], 'LeftArrowRightArrow': [0x21c6], 'LeftCeiling': [0x2308], 'LeftDoubleBracket': [0x27e6], 'LeftDownTeeVector': [0x2961], 'LeftDownVector': [0x21c3], 'LeftDownVectorBar': [0x2959], 'LeftFloor': [0x230a], 'LeftRightArrow': [0x2194], 'LeftRightVector': [0x294e], 'LeftTee': [0x22a3], 'LeftTeeArrow': [0x21a4], 'LeftTeeVector': [0x295a], 'LeftTriangle': [0x22b2], 'LeftTriangleBar': [0x29cf], 'LeftTriangleEqual': [0x22b4], 'LeftUpDownVector': [0x2951], 'LeftUpTeeVector': [0x2960], 'LeftUpVector': [0x21bf], 'LeftUpVectorBar': [0x2958], 'LeftVector': [0x21bc], 'LeftVectorBar': [0x2952], 'Leftarrow': [0x21d0], 'Leftrightarrow': [0x21d4], 'LessEqualGreater': [0x22da], 'LessFullEqual': [0x2266], 'LessGreater': [0x2276], 'LessLess': [0x2aa1], 'LessSlantEqual': [0x2a7d], 'LessTilde': [0x2272], 'Lfr': [0x1d50f], 'Ll': [0x22d8], 'Lleftarrow': [0x21da], 'Lmidot': [0x13f], 'LongLeftArrow': [0x27f5], 'LongLeftRightArrow': [0x27f7], 'LongRightArrow': [0x27f6], 'Longleftarrow': [0x27f8], 'Longleftrightarrow': [0x27fa], 'Longrightarrow': [0x27f9], 'Lopf': [0x1d543], 'LowerLeftArrow': [0x2199], 'LowerRightArrow': [0x2198], 'Lscr': [0x2112], 'Lsh': [0x21b0], 'Lstrok': [0x141], 'Lt': [0x226a], 'Map': [0x2905], 'Mcy': [0x41c], 'MediumSpace': [0x205f], 'Mellintrf': [0x2133], 'Mfr': [0x1d510], 'MinusPlus': [0x2213], 'Mopf': [0x1d544], 'Mscr': [0x2133], 'Mu': [0x39c], 'NJcy': [0x40a], 'Nacute': [0x143], 'Ncaron': [0x147], 'Ncedil': [0x145], 'Ncy': [0x41d], 'NegativeMediumSpace': [0x200b], 'NegativeThickSpace': [0x200b], 'NegativeThinSpace': [0x200b], 'NegativeVeryThinSpace': [0x200b], 'NestedGreaterGreater': [0x226b], 'NestedLessLess': [0x226a], 'NewLine': [0xa], 'Nfr': [0x1d511], 'NoBreak': [0x2060], 'NonBreakingSpace': [0xa0], 'Nopf': [0x2115], 'Not': [0x2aec], 'NotCongruent': [0x2262], 'NotCupCap': [0x226d], 'NotDoubleVerticalBar': [0x2226], 'NotElement': [0x2209], 'NotEqual': [0x2260], 'NotEqualTilde': [0x2242, 0x338], 'NotExists': [0x2204], 'NotGreater': [0x226f], 'NotGreaterEqual': [0x2271], 'NotGreaterFullEqual': [0x2267, 0x338], 'NotGreaterGreater': [0x226b, 0x338], 'NotGreaterLess': [0x2279], 'NotGreaterSlantEqual': [0x2a7e, 0x338], 'NotGreaterTilde': [0x2275], 'NotHumpDownHump': [0x224e, 0x338], 'NotHumpEqual': [0x224f, 0x338], 'NotLeftTriangle': [0x22ea], 'NotLeftTriangleBar': [0x29cf, 0x338], 'NotLeftTriangleEqual': [0x22ec], 'NotLess': [0x226e], 'NotLessEqual': [0x2270], 'NotLessGreater': [0x2278], 'NotLessLess': [0x226a, 0x338], 'NotLessSlantEqual': [0x2a7d, 0x338], 'NotLessTilde': [0x2274], 'NotNestedGreaterGreater': [0x2aa2, 0x338], 'NotNestedLessLess': [0x2aa1, 0x338], 'NotPrecedes': [0x2280], 'NotPrecedesEqual': [0x2aaf, 0x338], 'NotPrecedesSlantEqual': [0x22e0], 'NotReverseElement': [0x220c], 'NotRightTriangle': [0x22eb], 'NotRightTriangleBar': [0x29d0, 0x338], 'NotRightTriangleEqual': [0x22ed], 'NotSquareSubset': [0x228f, 0x338], 'NotSquareSubsetEqual': [0x22e2], 'NotSquareSuperset': [0x2290, 0x338], 'NotSquareSupersetEqual': [0x22e3], 'NotSubset': [0x2282, 0x20d2], 'NotSubsetEqual': [0x2288], 'NotSucceeds': [0x2281], 'NotSucceedsEqual': [0x2ab0, 0x338], 'NotSucceedsSlantEqual': [0x22e1], 'NotSucceedsTilde': [0x227f, 0x338], 'NotSuperset': [0x2283, 0x20d2], 'NotSupersetEqual': [0x2289], 'NotTilde': [0x2241], 'NotTildeEqual': [0x2244], 'NotTildeFullEqual': [0x2247], 'NotTildeTilde': [0x2249], 'NotVerticalBar': [0x2224], 'Nscr': [0x1d4a9], 'Ntilde': [0xd1], 'Nu': [0x39d], 'OElig': [0x152], 'Oacute': [0xd3], 'Ocirc': [0xd4], 'Ocy': [0x41e], 'Odblac': [0x150], 'Ofr': [0x1d512], 'Ograve': [0xd2], 'Omacr': [0x14c], 'Omega': [0x3a9], 'Omicron': [0x39f], 'Oopf': [0x1d546], 'OpenCurlyDoubleQuote': [0x201c], 'OpenCurlyQuote': [0x2018], 'Or': [0x2a54], 'Oscr': [0x1d4aa], 'Oslash': [0xd8], 'Otilde': [0xd5], 'Otimes': [0x2a37], 'Ouml': [0xd6], 'OverBar': [0x203e], 'OverBrace': [0x23de], 'OverBracket': [0x23b4], 'OverParenthesis': [0x23dc], 'PartialD': [0x2202], 'Pcy': [0x41f], 'Pfr': [0x1d513], 'Phi': [0x3a6], 'Pi': [0x3a0], 'PlusMinus': [0xb1], 'Poincareplane': [0x210c], 'Popf': [0x2119], 'Pr': [0x2abb], 'Precedes': [0x227a], 'PrecedesEqual': [0x2aaf], 'PrecedesSlantEqual': [0x227c], 'PrecedesTilde': [0x227e], 'Prime': [0x2033], 'Product': [0x220f], 'Proportion': [0x2237], 'Proportional': [0x221d], 'Pscr': [0x1d4ab], 'Psi': [0x3a8], 'QUOT': [0x22], 'Qfr': [0x1d514], 'Qopf': [0x211a], 'Qscr': [0x1d4ac], 'RBarr': [0x2910], 'REG': [0xae], 'Racute': [0x154], 'Rang': [0x27eb], 'Rarr': [0x21a0], 'Rarrtl': [0x2916], 'Rcaron': [0x158], 'Rcedil': [0x156], 'Rcy': [0x420], 'Re': [0x211c], 'ReverseElement': [0x220b], 'ReverseEquilibrium': [0x21cb], 'ReverseUpEquilibrium': [0x296f], 'Rfr': [0x211c], 'Rho': [0x3a1], 'RightAngleBracket': [0x27e9], 'RightArrow': [0x2192], 'RightArrowBar': [0x21e5], 'RightArrowLeftArrow': [0x21c4], 'RightCeiling': [0x2309], 'RightDoubleBracket': [0x27e7], 'RightDownTeeVector': [0x295d], 'RightDownVector': [0x21c2], 'RightDownVectorBar': [0x2955], 'RightFloor': [0x230b], 'RightTee': [0x22a2], 'RightTeeArrow': [0x21a6], 'RightTeeVector': [0x295b], 'RightTriangle': [0x22b3], 'RightTriangleBar': [0x29d0], 'RightTriangleEqual': [0x22b5], 'RightUpDownVector': [0x294f], 'RightUpTeeVector': [0x295c], 'RightUpVector': [0x21be], 'RightUpVectorBar': [0x2954], 'RightVector': [0x21c0], 'RightVectorBar': [0x2953], 'Rightarrow': [0x21d2], 'Ropf': [0x211d], 'RoundImplies': [0x2970], 'Rrightarrow': [0x21db], 'Rscr': [0x211b], 'Rsh': [0x21b1], 'RuleDelayed': [0x29f4], 'SHCHcy': [0x429], 'SHcy': [0x428], 'SOFTcy': [0x42c], 'Sacute': [0x15a], 'Sc': [0x2abc], 'Scaron': [0x160], 'Scedil': [0x15e], 'Scirc': [0x15c], 'Scy': [0x421], 'Sfr': [0x1d516], 'ShortDownArrow': [0x2193], 'ShortLeftArrow': [0x2190], 'ShortRightArrow': [0x2192], 'ShortUpArrow': [0x2191], 'Sigma': [0x3a3], 'SmallCircle': [0x2218], 'Sopf': [0x1d54a], 'Sqrt': [0x221a], 'Square': [0x25a1], 'SquareIntersection': [0x2293], 'SquareSubset': [0x228f], 'SquareSubsetEqual': [0x2291], 'SquareSuperset': [0x2290], 'SquareSupersetEqual': [0x2292], 'SquareUnion': [0x2294], 'Sscr': [0x1d4ae], 'Star': [0x22c6], 'Sub': [0x22d0], 'Subset': [0x22d0], 'SubsetEqual': [0x2286], 'Succeeds': [0x227b], 'SucceedsEqual': [0x2ab0], 'SucceedsSlantEqual': [0x227d], 'SucceedsTilde': [0x227f], 'SuchThat': [0x220b], 'Sum': [0x2211], 'Sup': [0x22d1], 'Superset': [0x2283], 'SupersetEqual': [0x2287], 'Supset': [0x22d1], 'THORN': [0xde], 'TRADE': [0x2122], 'TSHcy': [0x40b], 'TScy': [0x426], 'Tab': [0x9], 'Tau': [0x3a4], 'Tcaron': [0x164], 'Tcedil': [0x162], 'Tcy': [0x422], 'Tfr': [0x1d517], 'Therefore': [0x2234], 'Theta': [0x398], 'ThickSpace': [0x205f, 0x200a], 'ThinSpace': [0x2009], 'Tilde': [0x223c], 'TildeEqual': [0x2243], 'TildeFullEqual': [0x2245], 'TildeTilde': [0x2248], 'Topf': [0x1d54b], 'TripleDot': [0x20db], 'Tscr': [0x1d4af], 'Tstrok': [0x166], 'Uacute': [0xda], 'Uarr': [0x219f], 'Uarrocir': [0x2949], 'Ubrcy': [0x40e], 'Ubreve': [0x16c], 'Ucirc': [0xdb], 'Ucy': [0x423], 'Udblac': [0x170], 'Ufr': [0x1d518], 'Ugrave': [0xd9], 'Umacr': [0x16a], 'UnderBar': [0x5f], 'UnderBrace': [0x23df], 'UnderBracket': [0x23b5], 'UnderParenthesis': [0x23dd], 'Union': [0x22c3], 'UnionPlus': [0x228e], 'Uogon': [0x172], 'Uopf': [0x1d54c], 'UpArrow': [0x2191], 'UpArrowBar': [0x2912], 'UpArrowDownArrow': [0x21c5], 'UpDownArrow': [0x2195], 'UpEquilibrium': [0x296e], 'UpTee': [0x22a5], 'UpTeeArrow': [0x21a5], 'Uparrow': [0x21d1], 'Updownarrow': [0x21d5], 'UpperLeftArrow': [0x2196], 'UpperRightArrow': [0x2197], 'Upsi': [0x3d2], 'Upsilon': [0x3a5], 'Uring': [0x16e], 'Uscr': [0x1d4b0], 'Utilde': [0x168], 'Uuml': [0xdc], 'VDash': [0x22ab], 'Vbar': [0x2aeb], 'Vcy': [0x412], 'Vdash': [0x22a9], 'Vdashl': [0x2ae6], 'Vee': [0x22c1], 'Verbar': [0x2016], 'Vert': [0x2016], 'VerticalBar': [0x2223], 'VerticalLine': [0x7c], 'VerticalSeparator': [0x2758], 'VerticalTilde': [0x2240], 'VeryThinSpace': [0x200a], 'Vfr': [0x1d519], 'Vopf': [0x1d54d], 'Vscr': [0x1d4b1], 'Vvdash': [0x22aa], 'Wcirc': [0x174], 'Wedge': [0x22c0], 'Wfr': [0x1d51a], 'Wopf': [0x1d54e], 'Wscr': [0x1d4b2], 'Xfr': [0x1d51b], 'Xi': [0x39e], 'Xopf': [0x1d54f], 'Xscr': [0x1d4b3], 'YAcy': [0x42f], 'YIcy': [0x407], 'YUcy': [0x42e], 'Yacute': [0xdd], 'Ycirc': [0x176], 'Ycy': [0x42b], 'Yfr': [0x1d51c], 'Yopf': [0x1d550], 'Yscr': [0x1d4b4], 'Yuml': [0x178], 'ZHcy': [0x416], 'Zacute': [0x179], 'Zcaron': [0x17d], 'Zcy': [0x417], 'Zdot': [0x17b], 'ZeroWidthSpace': [0x200b], 'Zeta': [0x396], 'Zfr': [0x2128], 'Zopf': [0x2124], 'Zscr': [0x1d4b5], 'aacute': [0xe1], 'abreve': [0x103], 'ac': [0x223e], 'acE': [0x223e, 0x333], 'acd': [0x223f], 'acirc': [0xe2], 'acute': [0xb4], 'acy': [0x430], 'aelig': [0xe6], 'af': [0x2061], 'afr': [0x1d51e], 'agrave': [0xe0], 'alefsym': [0x2135], 'aleph': [0x2135], 'alpha': [0x3b1], 'amacr': [0x101], 'amalg': [0x2a3f], 'amp': [0x26], 'and': [0x2227], 'andand': [0x2a55], 'andd': [0x2a5c], 'andslope': [0x2a58], 'andv': [0x2a5a], 'ang': [0x2220], 'ange': [0x29a4], 'angle': [0x2220], 'angmsd': [0x2221], 'angmsdaa': [0x29a8], 'angmsdab': [0x29a9], 'angmsdac': [0x29aa], 'angmsdad': [0x29ab], 'angmsdae': [0x29ac], 'angmsdaf': [0x29ad], 'angmsdag': [0x29ae], 'angmsdah': [0x29af], 'angrt': [0x221f], 'angrtvb': [0x22be], 'angrtvbd': [0x299d], 'angsph': [0x2222], 'angst': [0xc5], 'angzarr': [0x237c], 'aogon': [0x105], 'aopf': [0x1d552], 'ap': [0x2248], 'apE': [0x2a70], 'apacir': [0x2a6f], 'ape': [0x224a], 'apid': [0x224b], 'apos': [0x27], 'approx': [0x2248], 'approxeq': [0x224a], 'aring': [0xe5], 'ascr': [0x1d4b6], 'ast': [0x2a], 'asymp': [0x2248], 'asympeq': [0x224d], 'atilde': [0xe3], 'auml': [0xe4], 'awconint': [0x2233], 'awint': [0x2a11], 'bNot': [0x2aed], 'backcong': [0x224c], 'backepsilon': [0x3f6], 'backprime': [0x2035], 'backsim': [0x223d], 'backsimeq': [0x22cd], 'barvee': [0x22bd], 'barwed': [0x2305], 'barwedge': [0x2305], 'bbrk': [0x23b5], 'bbrktbrk': [0x23b6], 'bcong': [0x224c], 'bcy': [0x431], 'bdquo': [0x201e], 'becaus': [0x2235], 'because': [0x2235], 'bemptyv': [0x29b0], 'bepsi': [0x3f6], 'bernou': [0x212c], 'beta': [0x3b2], 'beth': [0x2136], 'between': [0x226c], 'bfr': [0x1d51f], 'bigcap': [0x22c2], 'bigcirc': [0x25ef], 'bigcup': [0x22c3], 'bigodot': [0x2a00], 'bigoplus': [0x2a01], 'bigotimes': [0x2a02], 'bigsqcup': [0x2a06], 'bigstar': [0x2605], 'bigtriangledown': [0x25bd], 'bigtriangleup': [0x25b3], 'biguplus': [0x2a04], 'bigvee': [0x22c1], 'bigwedge': [0x22c0], 'bkarow': [0x290d], 'blacklozenge': [0x29eb], 'blacksquare': [0x25aa], 'blacktriangle': [0x25b4], 'blacktriangledown': [0x25be], 'blacktriangleleft': [0x25c2], 'blacktriangleright': [0x25b8], 'blank': [0x2423], 'blk12': [0x2592], 'blk14': [0x2591], 'blk34': [0x2593], 'block': [0x2588], 'bne': [0x3d, 0x20e5], 'bnequiv': [0x2261, 0x20e5], 'bnot': [0x2310], 'bopf': [0x1d553], 'bot': [0x22a5], 'bottom': [0x22a5], 'bowtie': [0x22c8], 'boxDL': [0x2557], 'boxDR': [0x2554], 'boxDl': [0x2556], 'boxDr': [0x2553], 'boxH': [0x2550], 'boxHD': [0x2566], 'boxHU': [0x2569], 'boxHd': [0x2564], 'boxHu': [0x2567], 'boxUL': [0x255d], 'boxUR': [0x255a], 'boxUl': [0x255c], 'boxUr': [0x2559], 'boxV': [0x2551], 'boxVH': [0x256c], 'boxVL': [0x2563], 'boxVR': [0x2560], 'boxVh': [0x256b], 'boxVl': [0x2562], 'boxVr': [0x255f], 'boxbox': [0x29c9], 'boxdL': [0x2555], 'boxdR': [0x2552], 'boxdl': [0x2510], 'boxdr': [0x250c], 'boxh': [0x2500], 'boxhD': [0x2565], 'boxhU': [0x2568], 'boxhd': [0x252c], 'boxhu': [0x2534], 'boxminus': [0x229f], 'boxplus': [0x229e], 'boxtimes': [0x22a0], 'boxuL': [0x255b], 'boxuR': [0x2558], 'boxul': [0x2518], 'boxur': [0x2514], 'boxv': [0x2502], 'boxvH': [0x256a], 'boxvL': [0x2561], 'boxvR': [0x255e], 'boxvh': [0x253c], 'boxvl': [0x2524], 'boxvr': [0x251c], 'bprime': [0x2035], 'breve': [0x2d8], 'brvbar': [0xa6], 'bscr': [0x1d4b7], 'bsemi': [0x204f], 'bsim': [0x223d], 'bsime': [0x22cd], 'bsol': [0x5c], 'bsolb': [0x29c5], 'bsolhsub': [0x27c8], 'bull': [0x2022], 'bullet': [0x2022], 'bump': [0x224e], 'bumpE': [0x2aae], 'bumpe': [0x224f], 'bumpeq': [0x224f], 'cacute': [0x107], 'cap': [0x2229], 'capand': [0x2a44], 'capbrcup': [0x2a49], 'capcap': [0x2a4b], 'capcup': [0x2a47], 'capdot': [0x2a40], 'caps': [0x2229, 0xfe00], 'caret': [0x2041], 'caron': [0x2c7], 'ccaps': [0x2a4d], 'ccaron': [0x10d], 'ccedil': [0xe7], 'ccirc': [0x109], 'ccups': [0x2a4c], 'ccupssm': [0x2a50], 'cdot': [0x10b], 'cedil': [0xb8], 'cemptyv': [0x29b2], 'cent': [0xa2], 'centerdot': [0xb7], 'cfr': [0x1d520], 'chcy': [0x447], 'check': [0x2713], 'checkmark': [0x2713], 'chi': [0x3c7], 'cir': [0x25cb], 'cirE': [0x29c3], 'circ': [0x2c6], 'circeq': [0x2257], 'circlearrowleft': [0x21ba], 'circlearrowright': [0x21bb], 'circledR': [0xae], 'circledS': [0x24c8], 'circledast': [0x229b], 'circledcirc': [0x229a], 'circleddash': [0x229d], 'cire': [0x2257], 'cirfnint': [0x2a10], 'cirmid': [0x2aef], 'cirscir': [0x29c2], 'clubs': [0x2663], 'clubsuit': [0x2663], 'colon': [0x3a], 'colone': [0x2254], 'coloneq': [0x2254], 'comma': [0x2c], 'commat': [0x40], 'comp': [0x2201], 'compfn': [0x2218], 'complement': [0x2201], 'complexes': [0x2102], 'cong': [0x2245], 'congdot': [0x2a6d], 'conint': [0x222e], 'copf': [0x1d554], 'coprod': [0x2210], 'copy': [0xa9], 'copysr': [0x2117], 'crarr': [0x21b5], 'cross': [0x2717], 'cscr': [0x1d4b8], 'csub': [0x2acf], 'csube': [0x2ad1], 'csup': [0x2ad0], 'csupe': [0x2ad2], 'ctdot': [0x22ef], 'cudarrl': [0x2938], 'cudarrr': [0x2935], 'cuepr': [0x22de], 'cuesc': [0x22df], 'cularr': [0x21b6], 'cularrp': [0x293d], 'cup': [0x222a], 'cupbrcap': [0x2a48], 'cupcap': [0x2a46], 'cupcup': [0x2a4a], 'cupdot': [0x228d], 'cupor': [0x2a45], 'cups': [0x222a, 0xfe00], 'curarr': [0x21b7], 'curarrm': [0x293c], 'curlyeqprec': [0x22de], 'curlyeqsucc': [0x22df], 'curlyvee': [0x22ce], 'curlywedge': [0x22cf], 'curren': [0xa4], 'curvearrowleft': [0x21b6], 'curvearrowright': [0x21b7], 'cuvee': [0x22ce], 'cuwed': [0x22cf], 'cwconint': [0x2232], 'cwint': [0x2231], 'cylcty': [0x232d], 'dArr': [0x21d3], 'dHar': [0x2965], 'dagger': [0x2020], 'daleth': [0x2138], 'darr': [0x2193], 'dash': [0x2010], 'dashv': [0x22a3], 'dbkarow': [0x290f], 'dblac': [0x2dd], 'dcaron': [0x10f], 'dcy': [0x434], 'dd': [0x2146], 'ddagger': [0x2021], 'ddarr': [0x21ca], 'ddotseq': [0x2a77], 'deg': [0xb0], 'delta': [0x3b4], 'demptyv': [0x29b1], 'dfisht': [0x297f], 'dfr': [0x1d521], 'dharl': [0x21c3], 'dharr': [0x21c2], 'diam': [0x22c4], 'diamond': [0x22c4], 'diamondsuit': [0x2666], 'diams': [0x2666], 'die': [0xa8], 'digamma': [0x3dd], 'disin': [0x22f2], 'div': [0xf7], 'divide': [0xf7], 'divideontimes': [0x22c7], 'divonx': [0x22c7], 'djcy': [0x452], 'dlcorn': [0x231e], 'dlcrop': [0x230d], 'dollar': [0x24], 'dopf': [0x1d555], 'dot': [0x2d9], 'doteq': [0x2250], 'doteqdot': [0x2251], 'dotminus': [0x2238], 'dotplus': [0x2214], 'dotsquare': [0x22a1], 'doublebarwedge': [0x2306], 'downarrow': [0x2193], 'downdownarrows': [0x21ca], 'downharpoonleft': [0x21c3], 'downharpoonright': [0x21c2], 'drbkarow': [0x2910], 'drcorn': [0x231f], 'drcrop': [0x230c], 'dscr': [0x1d4b9], 'dscy': [0x455], 'dsol': [0x29f6], 'dstrok': [0x111], 'dtdot': [0x22f1], 'dtri': [0x25bf], 'dtrif': [0x25be], 'duarr': [0x21f5], 'duhar': [0x296f], 'dwangle': [0x29a6], 'dzcy': [0x45f], 'dzigrarr': [0x27ff], 'eDDot': [0x2a77], 'eDot': [0x2251], 'eacute': [0xe9], 'easter': [0x2a6e], 'ecaron': [0x11b], 'ecir': [0x2256], 'ecirc': [0xea], 'ecolon': [0x2255], 'ecy': [0x44d], 'edot': [0x117], 'ee': [0x2147], 'efDot': [0x2252], 'efr': [0x1d522], 'eg': [0x2a9a], 'egrave': [0xe8], 'egs': [0x2a96], 'egsdot': [0x2a98], 'el': [0x2a99], 'elinters': [0x23e7], 'ell': [0x2113], 'els': [0x2a95], 'elsdot': [0x2a97], 'emacr': [0x113], 'empty': [0x2205], 'emptyset': [0x2205], 'emptyv': [0x2205], 'emsp': [0x2003], 'emsp13': [0x2004], 'emsp14': [0x2005], 'eng': [0x14b], 'ensp': [0x2002], 'eogon': [0x119], 'eopf': [0x1d556], 'epar': [0x22d5], 'eparsl': [0x29e3], 'eplus': [0x2a71], 'epsi': [0x3b5], 'epsilon': [0x3b5], 'epsiv': [0x3f5], 'eqcirc': [0x2256], 'eqcolon': [0x2255], 'eqsim': [0x2242], 'eqslantgtr': [0x2a96], 'eqslantless': [0x2a95], 'equals': [0x3d], 'equest': [0x225f], 'equiv': [0x2261], 'equivDD': [0x2a78], 'eqvparsl': [0x29e5], 'erDot': [0x2253], 'erarr': [0x2971], 'escr': [0x212f], 'esdot': [0x2250], 'esim': [0x2242], 'eta': [0x3b7], 'eth': [0xf0], 'euml': [0xeb], 'euro': [0x20ac], 'excl': [0x21], 'exist': [0x2203], 'expectation': [0x2130], 'exponentiale': [0x2147], 'fallingdotseq': [0x2252], 'fcy': [0x444], 'female': [0x2640], 'ffilig': [0xfb03], 'fflig': [0xfb00], 'ffllig': [0xfb04], 'ffr': [0x1d523], 'filig': [0xfb01], 'fjlig': [0x66, 0x6a], 'flat': [0x266d], 'fllig': [0xfb02], 'fltns': [0x25b1], 'fnof': [0x192], 'fopf': [0x1d557], 'forall': [0x2200], 'fork': [0x22d4], 'forkv': [0x2ad9], 'fpartint': [0x2a0d], 'frac12': [0xbd], 'frac13': [0x2153], 'frac14': [0xbc], 'frac15': [0x2155], 'frac16': [0x2159], 'frac18': [0x215b], 'frac23': [0x2154], 'frac25': [0x2156], 'frac34': [0xbe], 'frac35': [0x2157], 'frac38': [0x215c], 'frac45': [0x2158], 'frac56': [0x215a], 'frac58': [0x215d], 'frac78': [0x215e], 'frasl': [0x2044], 'frown': [0x2322], 'fscr': [0x1d4bb], 'gE': [0x2267], 'gEl': [0x2a8c], 'gacute': [0x1f5], 'gamma': [0x3b3], 'gammad': [0x3dd], 'gap': [0x2a86], 'gbreve': [0x11f], 'gcirc': [0x11d], 'gcy': [0x433], 'gdot': [0x121], 'ge': [0x2265], 'gel': [0x22db], 'geq': [0x2265], 'geqq': [0x2267], 'geqslant': [0x2a7e], 'ges': [0x2a7e], 'gescc': [0x2aa9], 'gesdot': [0x2a80], 'gesdoto': [0x2a82], 'gesdotol': [0x2a84], 'gesl': [0x22db, 0xfe00], 'gesles': [0x2a94], 'gfr': [0x1d524], 'gg': [0x226b], 'ggg': [0x22d9], 'gimel': [0x2137], 'gjcy': [0x453], 'gl': [0x2277], 'glE': [0x2a92], 'gla': [0x2aa5], 'glj': [0x2aa4], 'gnE': [0x2269], 'gnap': [0x2a8a], 'gnapprox': [0x2a8a], 'gne': [0x2a88], 'gneq': [0x2a88], 'gneqq': [0x2269], 'gnsim': [0x22e7], 'gopf': [0x1d558], 'grave': [0x60], 'gscr': [0x210a], 'gsim': [0x2273], 'gsime': [0x2a8e], 'gsiml': [0x2a90], 'gt': [0x3e], 'gtcc': [0x2aa7], 'gtcir': [0x2a7a], 'gtdot': [0x22d7], 'gtlPar': [0x2995], 'gtquest': [0x2a7c], 'gtrapprox': [0x2a86], 'gtrarr': [0x2978], 'gtrdot': [0x22d7], 'gtreqless': [0x22db], 'gtreqqless': [0x2a8c], 'gtrless': [0x2277], 'gtrsim': [0x2273], 'gvertneqq': [0x2269, 0xfe00], 'gvnE': [0x2269, 0xfe00], 'hArr': [0x21d4], 'hairsp': [0x200a], 'half': [0xbd], 'hamilt': [0x210b], 'hardcy': [0x44a], 'harr': [0x2194], 'harrcir': [0x2948], 'harrw': [0x21ad], 'hbar': [0x210f], 'hcirc': [0x125], 'hearts': [0x2665], 'heartsuit': [0x2665], 'hellip': [0x2026], 'hercon': [0x22b9], 'hfr': [0x1d525], 'hksearow': [0x2925], 'hkswarow': [0x2926], 'hoarr': [0x21ff], 'homtht': [0x223b], 'hookleftarrow': [0x21a9], 'hookrightarrow': [0x21aa], 'hopf': [0x1d559], 'horbar': [0x2015], 'hscr': [0x1d4bd], 'hslash': [0x210f], 'hstrok': [0x127], 'hybull': [0x2043], 'hyphen': [0x2010], 'iacute': [0xed], 'ic': [0x2063], 'icirc': [0xee], 'icy': [0x438], 'iecy': [0x435], 'iexcl': [0xa1], 'iff': [0x21d4], 'ifr': [0x1d526], 'igrave': [0xec], 'ii': [0x2148], 'iiiint': [0x2a0c], 'iiint': [0x222d], 'iinfin': [0x29dc], 'iiota': [0x2129], 'ijlig': [0x133], 'imacr': [0x12b], 'image': [0x2111], 'imagline': [0x2110], 'imagpart': [0x2111], 'imath': [0x131], 'imof': [0x22b7], 'imped': [0x1b5], 'in': [0x2208], 'incare': [0x2105], 'infin': [0x221e], 'infintie': [0x29dd], 'inodot': [0x131], 'int': [0x222b], 'intcal': [0x22ba], 'integers': [0x2124], 'intercal': [0x22ba], 'intlarhk': [0x2a17], 'intprod': [0x2a3c], 'iocy': [0x451], 'iogon': [0x12f], 'iopf': [0x1d55a], 'iota': [0x3b9], 'iprod': [0x2a3c], 'iquest': [0xbf], 'iscr': [0x1d4be], 'isin': [0x2208], 'isinE': [0x22f9], 'isindot': [0x22f5], 'isins': [0x22f4], 'isinsv': [0x22f3], 'isinv': [0x2208], 'it': [0x2062], 'itilde': [0x129], 'iukcy': [0x456], 'iuml': [0xef], 'jcirc': [0x135], 'jcy': [0x439], 'jfr': [0x1d527], 'jmath': [0x237], 'jopf': [0x1d55b], 'jscr': [0x1d4bf], 'jsercy': [0x458], 'jukcy': [0x454], 'kappa': [0x3ba], 'kappav': [0x3f0], 'kcedil': [0x137], 'kcy': [0x43a], 'kfr': [0x1d528], 'kgreen': [0x138], 'khcy': [0x445], 'kjcy': [0x45c], 'kopf': [0x1d55c], 'kscr': [0x1d4c0], 'lAarr': [0x21da], 'lArr': [0x21d0], 'lAtail': [0x291b], 'lBarr': [0x290e], 'lE': [0x2266], 'lEg': [0x2a8b], 'lHar': [0x2962], 'lacute': [0x13a], 'laemptyv': [0x29b4], 'lagran': [0x2112], 'lambda': [0x3bb], 'lang': [0x27e8], 'langd': [0x2991], 'langle': [0x27e8], 'lap': [0x2a85], 'laquo': [0xab], 'larr': [0x2190], 'larrb': [0x21e4], 'larrbfs': [0x291f], 'larrfs': [0x291d], 'larrhk': [0x21a9], 'larrlp': [0x21ab], 'larrpl': [0x2939], 'larrsim': [0x2973], 'larrtl': [0x21a2], 'lat': [0x2aab], 'latail': [0x2919], 'late': [0x2aad], 'lates': [0x2aad, 0xfe00], 'lbarr': [0x290c], 'lbbrk': [0x2772], 'lbrace': [0x7b], 'lbrack': [0x5b], 'lbrke': [0x298b], 'lbrksld': [0x298f], 'lbrkslu': [0x298d], 'lcaron': [0x13e], 'lcedil': [0x13c], 'lceil': [0x2308], 'lcub': [0x7b], 'lcy': [0x43b], 'ldca': [0x2936], 'ldquo': [0x201c], 'ldquor': [0x201e], 'ldrdhar': [0x2967], 'ldrushar': [0x294b], 'ldsh': [0x21b2], 'le': [0x2264], 'leftarrow': [0x2190], 'leftarrowtail': [0x21a2], 'leftharpoondown': [0x21bd], 'leftharpoonup': [0x21bc], 'leftleftarrows': [0x21c7], 'leftrightarrow': [0x2194], 'leftrightarrows': [0x21c6], 'leftrightharpoons': [0x21cb], 'leftrightsquigarrow': [0x21ad], 'leftthreetimes': [0x22cb], 'leg': [0x22da], 'leq': [0x2264], 'leqq': [0x2266], 'leqslant': [0x2a7d], 'les': [0x2a7d], 'lescc': [0x2aa8], 'lesdot': [0x2a7f], 'lesdoto': [0x2a81], 'lesdotor': [0x2a83], 'lesg': [0x22da, 0xfe00], 'lesges': [0x2a93], 'lessapprox': [0x2a85], 'lessdot': [0x22d6], 'lesseqgtr': [0x22da], 'lesseqqgtr': [0x2a8b], 'lessgtr': [0x2276], 'lesssim': [0x2272], 'lfisht': [0x297c], 'lfloor': [0x230a], 'lfr': [0x1d529], 'lg': [0x2276], 'lgE': [0x2a91], 'lhard': [0x21bd], 'lharu': [0x21bc], 'lharul': [0x296a], 'lhblk': [0x2584], 'ljcy': [0x459], 'll': [0x226a], 'llarr': [0x21c7], 'llcorner': [0x231e], 'llhard': [0x296b], 'lltri': [0x25fa], 'lmidot': [0x140], 'lmoust': [0x23b0], 'lmoustache': [0x23b0], 'lnE': [0x2268], 'lnap': [0x2a89], 'lnapprox': [0x2a89], 'lne': [0x2a87], 'lneq': [0x2a87], 'lneqq': [0x2268], 'lnsim': [0x22e6], 'loang': [0x27ec], 'loarr': [0x21fd], 'lobrk': [0x27e6], 'longleftarrow': [0x27f5], 'longleftrightarrow': [0x27f7], 'longmapsto': [0x27fc], 'longrightarrow': [0x27f6], 'looparrowleft': [0x21ab], 'looparrowright': [0x21ac], 'lopar': [0x2985], 'lopf': [0x1d55d], 'loplus': [0x2a2d], 'lotimes': [0x2a34], 'lowast': [0x2217], 'lowbar': [0x5f], 'loz': [0x25ca], 'lozenge': [0x25ca], 'lozf': [0x29eb], 'lpar': [0x28], 'lparlt': [0x2993], 'lrarr': [0x21c6], 'lrcorner': [0x231f], 'lrhar': [0x21cb], 'lrhard': [0x296d], 'lrm': [0x200e], 'lrtri': [0x22bf], 'lsaquo': [0x2039], 'lscr': [0x1d4c1], 'lsh': [0x21b0], 'lsim': [0x2272], 'lsime': [0x2a8d], 'lsimg': [0x2a8f], 'lsqb': [0x5b], 'lsquo': [0x2018], 'lsquor': [0x201a], 'lstrok': [0x142], 'lt': [0x3c], 'ltcc': [0x2aa6], 'ltcir': [0x2a79], 'ltdot': [0x22d6], 'lthree': [0x22cb], 'ltimes': [0x22c9], 'ltlarr': [0x2976], 'ltquest': [0x2a7b], 'ltrPar': [0x2996], 'ltri': [0x25c3], 'ltrie': [0x22b4], 'ltrif': [0x25c2], 'lurdshar': [0x294a], 'luruhar': [0x2966], 'lvertneqq': [0x2268, 0xfe00], 'lvnE': [0x2268, 0xfe00], 'mDDot': [0x223a], 'macr': [0xaf], 'male': [0x2642], 'malt': [0x2720], 'maltese': [0x2720], 'map': [0x21a6], 'mapsto': [0x21a6], 'mapstodown': [0x21a7], 'mapstoleft': [0x21a4], 'mapstoup': [0x21a5], 'marker': [0x25ae], 'mcomma': [0x2a29], 'mcy': [0x43c], 'mdash': [0x2014], 'measuredangle': [0x2221], 'mfr': [0x1d52a], 'mho': [0x2127], 'micro': [0xb5], 'mid': [0x2223], 'midast': [0x2a], 'midcir': [0x2af0], 'middot': [0xb7], 'minus': [0x2212], 'minusb': [0x229f], 'minusd': [0x2238], 'minusdu': [0x2a2a], 'mlcp': [0x2adb], 'mldr': [0x2026], 'mnplus': [0x2213], 'models': [0x22a7], 'mopf': [0x1d55e], 'mp': [0x2213], 'mscr': [0x1d4c2], 'mstpos': [0x223e], 'mu': [0x3bc], 'multimap': [0x22b8], 'mumap': [0x22b8], 'nGg': [0x22d9, 0x338], 'nGt': [0x226b, 0x20d2], 'nGtv': [0x226b, 0x338], 'nLeftarrow': [0x21cd], 'nLeftrightarrow': [0x21ce], 'nLl': [0x22d8, 0x338], 'nLt': [0x226a, 0x20d2], 'nLtv': [0x226a, 0x338], 'nRightarrow': [0x21cf], 'nVDash': [0x22af], 'nVdash': [0x22ae], 'nabla': [0x2207], 'nacute': [0x144], 'nang': [0x2220, 0x20d2], 'nap': [0x2249], 'napE': [0x2a70, 0x338], 'napid': [0x224b, 0x338], 'napos': [0x149], 'napprox': [0x2249], 'natur': [0x266e], 'natural': [0x266e], 'naturals': [0x2115], 'nbsp': [0xa0], 'nbump': [0x224e, 0x338], 'nbumpe': [0x224f, 0x338], 'ncap': [0x2a43], 'ncaron': [0x148], 'ncedil': [0x146], 'ncong': [0x2247], 'ncongdot': [0x2a6d, 0x338], 'ncup': [0x2a42], 'ncy': [0x43d], 'ndash': [0x2013], 'ne': [0x2260], 'neArr': [0x21d7], 'nearhk': [0x2924], 'nearr': [0x2197], 'nearrow': [0x2197], 'nedot': [0x2250, 0x338], 'nequiv': [0x2262], 'nesear': [0x2928], 'nesim': [0x2242, 0x338], 'nexist': [0x2204], 'nexists': [0x2204], 'nfr': [0x1d52b], 'ngE': [0x2267, 0x338], 'nge': [0x2271], 'ngeq': [0x2271], 'ngeqq': [0x2267, 0x338], 'ngeqslant': [0x2a7e, 0x338], 'nges': [0x2a7e, 0x338], 'ngsim': [0x2275], 'ngt': [0x226f], 'ngtr': [0x226f], 'nhArr': [0x21ce], 'nharr': [0x21ae], 'nhpar': [0x2af2], 'ni': [0x220b], 'nis': [0x22fc], 'nisd': [0x22fa], 'niv': [0x220b], 'njcy': [0x45a], 'nlArr': [0x21cd], 'nlE': [0x2266, 0x338], 'nlarr': [0x219a], 'nldr': [0x2025], 'nle': [0x2270], 'nleftarrow': [0x219a], 'nleftrightarrow': [0x21ae], 'nleq': [0x2270], 'nleqq': [0x2266, 0x338], 'nleqslant': [0x2a7d, 0x338], 'nles': [0x2a7d, 0x338], 'nless': [0x226e], 'nlsim': [0x2274], 'nlt': [0x226e], 'nltri': [0x22ea], 'nltrie': [0x22ec], 'nmid': [0x2224], 'nopf': [0x1d55f], 'not': [0xac], 'notin': [0x2209], 'notinE': [0x22f9, 0x338], 'notindot': [0x22f5, 0x338], 'notinva': [0x2209], 'notinvb': [0x22f7], 'notinvc': [0x22f6], 'notni': [0x220c], 'notniva': [0x220c], 'notnivb': [0x22fe], 'notnivc': [0x22fd], 'npar': [0x2226], 'nparallel': [0x2226], 'nparsl': [0x2afd, 0x20e5], 'npart': [0x2202, 0x338], 'npolint': [0x2a14], 'npr': [0x2280], 'nprcue': [0x22e0], 'npre': [0x2aaf, 0x338], 'nprec': [0x2280], 'npreceq': [0x2aaf, 0x338], 'nrArr': [0x21cf], 'nrarr': [0x219b], 'nrarrc': [0x2933, 0x338], 'nrarrw': [0x219d, 0x338], 'nrightarrow': [0x219b], 'nrtri': [0x22eb], 'nrtrie': [0x22ed], 'nsc': [0x2281], 'nsccue': [0x22e1], 'nsce': [0x2ab0, 0x338], 'nscr': [0x1d4c3], 'nshortmid': [0x2224], 'nshortparallel': [0x2226], 'nsim': [0x2241], 'nsime': [0x2244], 'nsimeq': [0x2244], 'nsmid': [0x2224], 'nspar': [0x2226], 'nsqsube': [0x22e2], 'nsqsupe': [0x22e3], 'nsub': [0x2284], 'nsubE': [0x2ac5, 0x338], 'nsube': [0x2288], 'nsubset': [0x2282, 0x20d2], 'nsubseteq': [0x2288], 'nsubseteqq': [0x2ac5, 0x338], 'nsucc': [0x2281], 'nsucceq': [0x2ab0, 0x338], 'nsup': [0x2285], 'nsupE': [0x2ac6, 0x338], 'nsupe': [0x2289], 'nsupset': [0x2283, 0x20d2], 'nsupseteq': [0x2289], 'nsupseteqq': [0x2ac6, 0x338], 'ntgl': [0x2279], 'ntilde': [0xf1], 'ntlg': [0x2278], 'ntriangleleft': [0x22ea], 'ntrianglelefteq': [0x22ec], 'ntriangleright': [0x22eb], 'ntrianglerighteq': [0x22ed], 'nu': [0x3bd], 'num': [0x23], 'numero': [0x2116], 'numsp': [0x2007], 'nvDash': [0x22ad], 'nvHarr': [0x2904], 'nvap': [0x224d, 0x20d2], 'nvdash': [0x22ac], 'nvge': [0x2265, 0x20d2], 'nvgt': [0x3e, 0x20d2], 'nvinfin': [0x29de], 'nvlArr': [0x2902], 'nvle': [0x2264, 0x20d2], 'nvlt': [0x3c, 0x20d2], 'nvltrie': [0x22b4, 0x20d2], 'nvrArr': [0x2903], 'nvrtrie': [0x22b5, 0x20d2], 'nvsim': [0x223c, 0x20d2], 'nwArr': [0x21d6], 'nwarhk': [0x2923], 'nwarr': [0x2196], 'nwarrow': [0x2196], 'nwnear': [0x2927], 'oS': [0x24c8], 'oacute': [0xf3], 'oast': [0x229b], 'ocir': [0x229a], 'ocirc': [0xf4], 'ocy': [0x43e], 'odash': [0x229d], 'odblac': [0x151], 'odiv': [0x2a38], 'odot': [0x2299], 'odsold': [0x29bc], 'oelig': [0x153], 'ofcir': [0x29bf], 'ofr': [0x1d52c], 'ogon': [0x2db], 'ograve': [0xf2], 'ogt': [0x29c1], 'ohbar': [0x29b5], 'ohm': [0x3a9], 'oint': [0x222e], 'olarr': [0x21ba], 'olcir': [0x29be], 'olcross': [0x29bb], 'oline': [0x203e], 'olt': [0x29c0], 'omacr': [0x14d], 'omega': [0x3c9], 'omicron': [0x3bf], 'omid': [0x29b6], 'ominus': [0x2296], 'oopf': [0x1d560], 'opar': [0x29b7], 'operp': [0x29b9], 'oplus': [0x2295], 'or': [0x2228], 'orarr': [0x21bb], 'ord': [0x2a5d], 'order': [0x2134], 'orderof': [0x2134], 'ordf': [0xaa], 'ordm': [0xba], 'origof': [0x22b6], 'oror': [0x2a56], 'orslope': [0x2a57], 'orv': [0x2a5b], 'oscr': [0x2134], 'oslash': [0xf8], 'osol': [0x2298], 'otilde': [0xf5], 'otimes': [0x2297], 'otimesas': [0x2a36], 'ouml': [0xf6], 'ovbar': [0x233d], 'par': [0x2225], 'para': [0xb6], 'parallel': [0x2225], 'parsim': [0x2af3], 'parsl': [0x2afd], 'part': [0x2202], 'pcy': [0x43f], 'percnt': [0x25], 'period': [0x2e], 'permil': [0x2030], 'perp': [0x22a5], 'pertenk': [0x2031], 'pfr': [0x1d52d], 'phi': [0x3c6], 'phiv': [0x3d5], 'phmmat': [0x2133], 'phone': [0x260e], 'pi': [0x3c0], 'pitchfork': [0x22d4], 'piv': [0x3d6], 'planck': [0x210f], 'planckh': [0x210e], 'plankv': [0x210f], 'plus': [0x2b], 'plusacir': [0x2a23], 'plusb': [0x229e], 'pluscir': [0x2a22], 'plusdo': [0x2214], 'plusdu': [0x2a25], 'pluse': [0x2a72], 'plusmn': [0xb1], 'plussim': [0x2a26], 'plustwo': [0x2a27], 'pm': [0xb1], 'pointint': [0x2a15], 'popf': [0x1d561], 'pound': [0xa3], 'pr': [0x227a], 'prE': [0x2ab3], 'prap': [0x2ab7], 'prcue': [0x227c], 'pre': [0x2aaf], 'prec': [0x227a], 'precapprox': [0x2ab7], 'preccurlyeq': [0x227c], 'preceq': [0x2aaf], 'precnapprox': [0x2ab9], 'precneqq': [0x2ab5], 'precnsim': [0x22e8], 'precsim': [0x227e], 'prime': [0x2032], 'primes': [0x2119], 'prnE': [0x2ab5], 'prnap': [0x2ab9], 'prnsim': [0x22e8], 'prod': [0x220f], 'profalar': [0x232e], 'profline': [0x2312], 'profsurf': [0x2313], 'prop': [0x221d], 'propto': [0x221d], 'prsim': [0x227e], 'prurel': [0x22b0], 'pscr': [0x1d4c5], 'psi': [0x3c8], 'puncsp': [0x2008], 'qfr': [0x1d52e], 'qint': [0x2a0c], 'qopf': [0x1d562], 'qprime': [0x2057], 'qscr': [0x1d4c6], 'quaternions': [0x210d], 'quatint': [0x2a16], 'quest': [0x3f], 'questeq': [0x225f], 'quot': [0x22], 'rAarr': [0x21db], 'rArr': [0x21d2], 'rAtail': [0x291c], 'rBarr': [0x290f], 'rHar': [0x2964], 'race': [0x223d, 0x331], 'racute': [0x155], 'radic': [0x221a], 'raemptyv': [0x29b3], 'rang': [0x27e9], 'rangd': [0x2992], 'range': [0x29a5], 'rangle': [0x27e9], 'raquo': [0xbb], 'rarr': [0x2192], 'rarrap': [0x2975], 'rarrb': [0x21e5], 'rarrbfs': [0x2920], 'rarrc': [0x2933], 'rarrfs': [0x291e], 'rarrhk': [0x21aa], 'rarrlp': [0x21ac], 'rarrpl': [0x2945], 'rarrsim': [0x2974], 'rarrtl': [0x21a3], 'rarrw': [0x219d], 'ratail': [0x291a], 'ratio': [0x2236], 'rationals': [0x211a], 'rbarr': [0x290d], 'rbbrk': [0x2773], 'rbrace': [0x7d], 'rbrack': [0x5d], 'rbrke': [0x298c], 'rbrksld': [0x298e], 'rbrkslu': [0x2990], 'rcaron': [0x159], 'rcedil': [0x157], 'rceil': [0x2309], 'rcub': [0x7d], 'rcy': [0x440], 'rdca': [0x2937], 'rdldhar': [0x2969], 'rdquo': [0x201d], 'rdquor': [0x201d], 'rdsh': [0x21b3], 'real': [0x211c], 'realine': [0x211b], 'realpart': [0x211c], 'reals': [0x211d], 'rect': [0x25ad], 'reg': [0xae], 'rfisht': [0x297d], 'rfloor': [0x230b], 'rfr': [0x1d52f], 'rhard': [0x21c1], 'rharu': [0x21c0], 'rharul': [0x296c], 'rho': [0x3c1], 'rhov': [0x3f1], 'rightarrow': [0x2192], 'rightarrowtail': [0x21a3], 'rightharpoondown': [0x21c1], 'rightharpoonup': [0x21c0], 'rightleftarrows': [0x21c4], 'rightleftharpoons': [0x21cc], 'rightrightarrows': [0x21c9], 'rightsquigarrow': [0x219d], 'rightthreetimes': [0x22cc], 'ring': [0x2da], 'risingdotseq': [0x2253], 'rlarr': [0x21c4], 'rlhar': [0x21cc], 'rlm': [0x200f], 'rmoust': [0x23b1], 'rmoustache': [0x23b1], 'rnmid': [0x2aee], 'roang': [0x27ed], 'roarr': [0x21fe], 'robrk': [0x27e7], 'ropar': [0x2986], 'ropf': [0x1d563], 'roplus': [0x2a2e], 'rotimes': [0x2a35], 'rpar': [0x29], 'rpargt': [0x2994], 'rppolint': [0x2a12], 'rrarr': [0x21c9], 'rsaquo': [0x203a], 'rscr': [0x1d4c7], 'rsh': [0x21b1], 'rsqb': [0x5d], 'rsquo': [0x2019], 'rsquor': [0x2019], 'rthree': [0x22cc], 'rtimes': [0x22ca], 'rtri': [0x25b9], 'rtrie': [0x22b5], 'rtrif': [0x25b8], 'rtriltri': [0x29ce], 'ruluhar': [0x2968], 'rx': [0x211e], 'sacute': [0x15b], 'sbquo': [0x201a], 'sc': [0x227b], 'scE': [0x2ab4], 'scap': [0x2ab8], 'scaron': [0x161], 'sccue': [0x227d], 'sce': [0x2ab0], 'scedil': [0x15f], 'scirc': [0x15d], 'scnE': [0x2ab6], 'scnap': [0x2aba], 'scnsim': [0x22e9], 'scpolint': [0x2a13], 'scsim': [0x227f], 'scy': [0x441], 'sdot': [0x22c5], 'sdotb': [0x22a1], 'sdote': [0x2a66], 'seArr': [0x21d8], 'searhk': [0x2925], 'searr': [0x2198], 'searrow': [0x2198], 'sect': [0xa7], 'semi': [0x3b], 'seswar': [0x2929], 'setminus': [0x2216], 'setmn': [0x2216], 'sext': [0x2736], 'sfr': [0x1d530], 'sfrown': [0x2322], 'sharp': [0x266f], 'shchcy': [0x449], 'shcy': [0x448], 'shortmid': [0x2223], 'shortparallel': [0x2225], 'shy': [0xad], 'sigma': [0x3c3], 'sigmaf': [0x3c2], 'sigmav': [0x3c2], 'sim': [0x223c], 'simdot': [0x2a6a], 'sime': [0x2243], 'simeq': [0x2243], 'simg': [0x2a9e], 'simgE': [0x2aa0], 'siml': [0x2a9d], 'simlE': [0x2a9f], 'simne': [0x2246], 'simplus': [0x2a24], 'simrarr': [0x2972], 'slarr': [0x2190], 'smallsetminus': [0x2216], 'smashp': [0x2a33], 'smeparsl': [0x29e4], 'smid': [0x2223], 'smile': [0x2323], 'smt': [0x2aaa], 'smte': [0x2aac], 'smtes': [0x2aac, 0xfe00], 'softcy': [0x44c], 'sol': [0x2f], 'solb': [0x29c4], 'solbar': [0x233f], 'sopf': [0x1d564], 'spades': [0x2660], 'spadesuit': [0x2660], 'spar': [0x2225], 'sqcap': [0x2293], 'sqcaps': [0x2293, 0xfe00], 'sqcup': [0x2294], 'sqcups': [0x2294, 0xfe00], 'sqsub': [0x228f], 'sqsube': [0x2291], 'sqsubset': [0x228f], 'sqsubseteq': [0x2291], 'sqsup': [0x2290], 'sqsupe': [0x2292], 'sqsupset': [0x2290], 'sqsupseteq': [0x2292], 'squ': [0x25a1], 'square': [0x25a1], 'squarf': [0x25aa], 'squf': [0x25aa], 'srarr': [0x2192], 'sscr': [0x1d4c8], 'ssetmn': [0x2216], 'ssmile': [0x2323], 'sstarf': [0x22c6], 'star': [0x2606], 'starf': [0x2605], 'straightepsilon': [0x3f5], 'straightphi': [0x3d5], 'strns': [0xaf], 'sub': [0x2282], 'subE': [0x2ac5], 'subdot': [0x2abd], 'sube': [0x2286], 'subedot': [0x2ac3], 'submult': [0x2ac1], 'subnE': [0x2acb], 'subne': [0x228a], 'subplus': [0x2abf], 'subrarr': [0x2979], 'subset': [0x2282], 'subseteq': [0x2286], 'subseteqq': [0x2ac5], 'subsetneq': [0x228a], 'subsetneqq': [0x2acb], 'subsim': [0x2ac7], 'subsub': [0x2ad5], 'subsup': [0x2ad3], 'succ': [0x227b], 'succapprox': [0x2ab8], 'succcurlyeq': [0x227d], 'succeq': [0x2ab0], 'succnapprox': [0x2aba], 'succneqq': [0x2ab6], 'succnsim': [0x22e9], 'succsim': [0x227f], 'sum': [0x2211], 'sung': [0x266a], 'sup': [0x2283], 'sup1': [0xb9], 'sup2': [0xb2], 'sup3': [0xb3], 'supE': [0x2ac6], 'supdot': [0x2abe], 'supdsub': [0x2ad8], 'supe': [0x2287], 'supedot': [0x2ac4], 'suphsol': [0x27c9], 'suphsub': [0x2ad7], 'suplarr': [0x297b], 'supmult': [0x2ac2], 'supnE': [0x2acc], 'supne': [0x228b], 'supplus': [0x2ac0], 'supset': [0x2283], 'supseteq': [0x2287], 'supseteqq': [0x2ac6], 'supsetneq': [0x228b], 'supsetneqq': [0x2acc], 'supsim': [0x2ac8], 'supsub': [0x2ad4], 'supsup': [0x2ad6], 'swArr': [0x21d9], 'swarhk': [0x2926], 'swarr': [0x2199], 'swarrow': [0x2199], 'swnwar': [0x292a], 'szlig': [0xdf], 'target': [0x2316], 'tau': [0x3c4], 'tbrk': [0x23b4], 'tcaron': [0x165], 'tcedil': [0x163], 'tcy': [0x442], 'tdot': [0x20db], 'telrec': [0x2315], 'tfr': [0x1d531], 'there4': [0x2234], 'therefore': [0x2234], 'theta': [0x3b8], 'thetasym': [0x3d1], 'thetav': [0x3d1], 'thickapprox': [0x2248], 'thicksim': [0x223c], 'thinsp': [0x2009], 'thkap': [0x2248], 'thksim': [0x223c], 'thorn': [0xfe], 'tilde': [0x2dc], 'times': [0xd7], 'timesb': [0x22a0], 'timesbar': [0x2a31], 'timesd': [0x2a30], 'tint': [0x222d], 'toea': [0x2928], 'top': [0x22a4], 'topbot': [0x2336], 'topcir': [0x2af1], 'topf': [0x1d565], 'topfork': [0x2ada], 'tosa': [0x2929], 'tprime': [0x2034], 'trade': [0x2122], 'triangle': [0x25b5], 'triangledown': [0x25bf], 'triangleleft': [0x25c3], 'trianglelefteq': [0x22b4], 'triangleq': [0x225c], 'triangleright': [0x25b9], 'trianglerighteq': [0x22b5], 'tridot': [0x25ec], 'trie': [0x225c], 'triminus': [0x2a3a], 'triplus': [0x2a39], 'trisb': [0x29cd], 'tritime': [0x2a3b], 'trpezium': [0x23e2], 'tscr': [0x1d4c9], 'tscy': [0x446], 'tshcy': [0x45b], 'tstrok': [0x167], 'twixt': [0x226c], 'twoheadleftarrow': [0x219e], 'twoheadrightarrow': [0x21a0], 'uArr': [0x21d1], 'uHar': [0x2963], 'uacute': [0xfa], 'uarr': [0x2191], 'ubrcy': [0x45e], 'ubreve': [0x16d], 'ucirc': [0xfb], 'ucy': [0x443], 'udarr': [0x21c5], 'udblac': [0x171], 'udhar': [0x296e], 'ufisht': [0x297e], 'ufr': [0x1d532], 'ugrave': [0xf9], 'uharl': [0x21bf], 'uharr': [0x21be], 'uhblk': [0x2580], 'ulcorn': [0x231c], 'ulcorner': [0x231c], 'ulcrop': [0x230f], 'ultri': [0x25f8], 'umacr': [0x16b], 'uml': [0xa8], 'uogon': [0x173], 'uopf': [0x1d566], 'uparrow': [0x2191], 'updownarrow': [0x2195], 'upharpoonleft': [0x21bf], 'upharpoonright': [0x21be], 'uplus': [0x228e], 'upsi': [0x3c5], 'upsih': [0x3d2], 'upsilon': [0x3c5], 'upuparrows': [0x21c8], 'urcorn': [0x231d], 'urcorner': [0x231d], 'urcrop': [0x230e], 'uring': [0x16f], 'urtri': [0x25f9], 'uscr': [0x1d4ca], 'utdot': [0x22f0], 'utilde': [0x169], 'utri': [0x25b5], 'utrif': [0x25b4], 'uuarr': [0x21c8], 'uuml': [0xfc], 'uwangle': [0x29a7], 'vArr': [0x21d5], 'vBar': [0x2ae8], 'vBarv': [0x2ae9], 'vDash': [0x22a8], 'vangrt': [0x299c], 'varepsilon': [0x3f5], 'varkappa': [0x3f0], 'varnothing': [0x2205], 'varphi': [0x3d5], 'varpi': [0x3d6], 'varpropto': [0x221d], 'varr': [0x2195], 'varrho': [0x3f1], 'varsigma': [0x3c2], 'varsubsetneq': [0x228a, 0xfe00], 'varsubsetneqq': [0x2acb, 0xfe00], 'varsupsetneq': [0x228b, 0xfe00], 'varsupsetneqq': [0x2acc, 0xfe00], 'vartheta': [0x3d1], 'vartriangleleft': [0x22b2], 'vartriangleright': [0x22b3], 'vcy': [0x432], 'vdash': [0x22a2], 'vee': [0x2228], 'veebar': [0x22bb], 'veeeq': [0x225a], 'vellip': [0x22ee], 'verbar': [0x7c], 'vert': [0x7c], 'vfr': [0x1d533], 'vltri': [0x22b2], 'vnsub': [0x2282, 0x20d2], 'vnsup': [0x2283, 0x20d2], 'vopf': [0x1d567], 'vprop': [0x221d], 'vrtri': [0x22b3], 'vscr': [0x1d4cb], 'vsubnE': [0x2acb, 0xfe00], 'vsubne': [0x228a, 0xfe00], 'vsupnE': [0x2acc, 0xfe00], 'vsupne': [0x228b, 0xfe00], 'vzigzag': [0x299a], 'wcirc': [0x175], 'wedbar': [0x2a5f], 'wedge': [0x2227], 'wedgeq': [0x2259], 'weierp': [0x2118], 'wfr': [0x1d534], 'wopf': [0x1d568], 'wp': [0x2118], 'wr': [0x2240], 'wreath': [0x2240], 'wscr': [0x1d4cc], 'xcap': [0x22c2], 'xcirc': [0x25ef], 'xcup': [0x22c3], 'xdtri': [0x25bd], 'xfr': [0x1d535], 'xhArr': [0x27fa], 'xharr': [0x27f7], 'xi': [0x3be], 'xlArr': [0x27f8], 'xlarr': [0x27f5], 'xmap': [0x27fc], 'xnis': [0x22fb], 'xodot': [0x2a00], 'xopf': [0x1d569], 'xoplus': [0x2a01], 'xotime': [0x2a02], 'xrArr': [0x27f9], 'xrarr': [0x27f6], 'xscr': [0x1d4cd], 'xsqcup': [0x2a06], 'xuplus': [0x2a04], 'xutri': [0x25b3], 'xvee': [0x22c1], 'xwedge': [0x22c0], 'yacute': [0xfd], 'yacy': [0x44f], 'ycirc': [0x177], 'ycy': [0x44b], 'yen': [0xa5], 'yfr': [0x1d536], 'yicy': [0x457], 'yopf': [0x1d56a], 'yscr': [0x1d4ce], 'yucy': [0x44e], 'yuml': [0xff], 'zacute': [0x17a], 'zcaron': [0x17e], 'zcy': [0x437], 'zdot': [0x17c], 'zeetrf': [0x2128], 'zeta': [0x3b6], 'zfr': [0x1d537], 'zhcy': [0x436], 'zigrarr': [0x21dd], 'zopf': [0x1d56b], 'zscr': [0x1d4cf], 'zwj': [0x200d], 'zwnj': [0x200c], } def query(element, queries): matches = {} for key, query in queries.iteritems(): is_text = query.endswith('/text()') if is_text: query = query[:len(query)-7] results = element.xpath(query, namespaces=namespaces) if len(results) > 0: matches[key] = [] if is_text: results = [etree.tostring(results[0], method="text", encoding=unicode)] for result in results: if isinstance(result, str): result = result.decode('utf8') if isinstance(result, unicode): result = re.sub(r'\s+', ' ', result.strip()) if len(results) > 1: matches[key].append(result) else: matches[key] = result return matches def parse(nlm): '''Parse NLM for useful information''' if nlm is not None: # Parse the NLM parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding='utf8', resolve_entities=True) try: nlm = etree.fromstring(nlm, parser) #print etree.tostring(nlm, pretty_print=True, encoding='utf8') except: # Error parsing the NLM raise return # Bail early # If not article, unwrap to article if nlm.tag != 'article': nlm = nlm.find('article') # If now None, no article found if nlm is None: return # Bail invalid XML ####################################################################################### # Parse various useful bits of metadata queries = { 'publication-title': 'front/journal-meta//journal-title[1]/text()', 'issn': 'front/journal-meta/issn[@pub-type="epub"]/text()', 'publisher': 'front/journal-meta/publisher/publisher-name/text()', 'pmcid': 'front/article-meta/article-id[@pub-id-type="pmc"]/text()', 'pmid': 'front/article-meta/article-id[@pub-id-type="pmid"]/text()', 'doi': 'front/article-meta/article-id[@pub-id-type="doi"]/text()', 'pii': 'front/article-meta/article-id[@pub-id-type="pii"]/text()', 'title': 'front/article-meta/title-group/article-title/text()', 'altTitle': 'front/article-meta/title-group/alt-title[1]/text()', } metadata = query(nlm, queries) ####################################################################################### # Parse citation information queries = { 'id': '@id', 'label': 'label/text()', 'type': '(citation|element-citation|mixed-citation|nlm-citation)/@publication-type | (citation|element-citation|mixed-citation|nlm-citation)/@citation-type', 'title': '(citation|element-citation|mixed-citation|nlm-citation)/article-title/text()', 'year': '(citation|element-citation|mixed-citation|nlm-citation)/year/text()', 'publication-title': '(citation|element-citation|mixed-citation|nlm-citation)/source/text()', 'volume': '(citation|element-citation|mixed-citation|nlm-citation)/volume/text()', 'pagefrom': '(citation|element-citation|mixed-citation|nlm-citation)/fpage/text()', 'pageto': '(citation|element-citation|mixed-citation|nlm-citation)/lpage/text()', 'pmid': '(citation|element-citation|mixed-citation|nlm-citation)/pub-id[@pub-id-type="pmid"]/text()', 'doi': '(citation|element-citation|mixed-citation|nlm-citation)/pub-id[@pub-id-type="doi"]/text()', 'pmcid': '(citation|element-citation|mixed-citation|nlm-citation)/pub-id[@pub-id-type="pmc"]/text()', 'pii': '(citation|element-citation|mixed-citation|nlm-citation)/pub-id[@pub-id-type="pii"]/text()', 'url': '(citation|element-citation|mixed-citation|nlm-citation)/ext-link[@ext-link-type="uri"]/@xlink:href', 'etree': 'citation|element-citation|mixed-citation|nlm-citation', } metadata['citations'] = [] metadata['citations_by_id'] = {} for ref in nlm.xpath('back/ref-list/ref'): citation = query(ref, queries) authorelems = ref.xpath('(citation|element-citation|mixed-citation|nlm-citation)/person-group[@person-group-type="author"]/name') authorelems += ref.xpath('(citation|element-citation|mixed-citation|nlm-citation)/person-group[@person-group-type="author"]/string-name') authorelems += ref.xpath('(citation|element-citation|mixed-citation|nlm-citation)/name') authors = [u'{0}, {1}'.format(a.findtext('surname', ''), a.findtext('given-names', '')).strip(', ') for a in authorelems] citation['authors'] = authors citation['contexts'] = [] metadata['citations'].append(citation) if 'id' in citation: metadata['citations_by_id'][citation['id']] = citation # Does this look like a structured citation? structured = len(authors) > 0 or 'title' in citation or 'year' in citation if not structured and 'etree' in citation: html = etree.Element('span') xml = citation['etree'] html.text = xml.text for node in xml: if node.tag == 'ext-link': a = etree.Element('a') url = node.get('{{{0}}}href'.format(namespaces['xlink'])) a.set('href', url) a.text = url a.tail = node.tail html.append(a) else: html.append(copy.deepcopy(node)) def tostring(node): return etree.tostring(node, encoding=unicode) citation['unstructured'] = (html.text or '') + ''.join(map(tostring, html)) if 'doi' not in citation and 'url' in citation: url = citation['url'] if url is not None and url.startswith('http://dx.doi.org/'): citation['doi'] = url[18:] del citation['url'] citation['displayText'] = common.utils.format_citation(citation) ####################################################################################### # Parse in-text citations if present min_length = 10 max_length = 20 for paragraph in nlm.xpath('front/article-meta/abstract//p | body//p'): text_stack = [paragraph.text or ''] xref_stack = [None] for elem in paragraph: if len(elem.xpath('self::*[@ref-type="bibr"][@rid]')) > 0: text_stack.append(etree.tostring(elem, method='text', encoding=unicode, with_tail=False)) text_stack.append(elem.tail or '') xref = metadata['citations_by_id'].get(elem.get('rid')) if xref is not None: xref_stack += [[xref], None] else: xref_stack += [[], None] elif isinstance(elem, etree._Entity): points = entities.get(elem.text[1:-1]) if points is not None: text_stack[-1] += ''.join((unichr(p) for p in points)) else: text_stack[-1] += etree.tostring(elem, encoding=unicode) else: if elem.get('position') == 'float': text_stack[-1] += elem.tail or '' else: text_stack[-1] += etree.tostring(elem, method='text', encoding=unicode) # Find and collapse ranges in the text for i in xrange(len(xref_stack) - 3, 1, -2): text = text_stack[i].strip() #print len(text), (text in u'-\u2010\u2011\u2012\u2013\u2014\u2015'), u''.join(text_stack[i-1:i+2]).encode('utf8') # if this text is a dash, we need to coalesce the text fragments if len(text) == 1 and text in u'-\u2010\u2011\u2012\u2013\u2014\u2015': text_stack[i-1:i+2] = [u''.join(text_stack[i-1:i+2])] xref_stack[i-1:i+2] = [xref_stack[i-1] + xref_stack[i+1]] #for text in text_stack: # print text.encode('utf8') # Then make sure we resolve the implied citations for i in xrange(1, len(xref_stack), 2): # Get actual cross references xrefs = xref_stack[i] # Expand cross references try: if len(xrefs) == 2: labelfrom = int(xrefs[0].get('label')) labelto = int(xrefs[1].get('label')) candidates = {} midlabels = [unicode(midlabel) for midlabel in xrange(labelfrom+1, labelto)] for candidate in metadata['citations']: if candidate.get('label') in midlabels: candidates[int(candidate.get('label'))] = candidate xrefs[1:-1] = candidates.values() except (ValueError, TypeError): pass # Find and collapse lists in the text for i in xrange(len(xref_stack) - 3, 1, -2): text = text_stack[i].strip() # if this text is a comma, we need to coalesce the text fragments if len(text) == 1 and text == ',': text_stack[i-1:i+2] = [u''.join(text_stack[i-1:i+2])] xref_stack[i-1:i+2] = [xref_stack[i-1] + xref_stack[i+1]] # Expand citations to include brackets (on both sides) for i in xrange(len(xref_stack) - 2, 0, -2): before = text_stack[i-1].strip()[-1:] text = text_stack[i].strip() after = text_stack[i+1].strip()[:1] # if this text is a comma, we need to coalesce the text fragments #print before.encode('utf'), after.encode('utf') if len(before) > 0 and before in '({[' and len(after) > 0 and after in ')}]': text_stack[i-1] = re.sub(r'[({[](\s*)$', r'\1', text_stack[i-1]) text_stack[i+1] = re.sub(r'^(\s*)[)}\]]', r'\1', text_stack[i+1]) text_stack[i] = before + text_stack[i] + after #print repr(text_stack) for i in xrange(1, len(xref_stack), 2): # Get context before = u' '.join(text_stack[:i]).strip() label = text_stack[i].strip() after = u' '.join(text_stack[i+1:]).strip() # Strip out extraneous brackets if len(xref_stack[i]) > 1: # Hack to differentiate single / multiple citations # as multiple numbers tend not to have spaces between them label = re.sub(ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,])\s?[({[]?', r'\1', label) else: label = re.sub(ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,]\s?)[({[]?', r'\1', label) # Normalise context before = re.sub(r'\s+', ' ', before)[-max_length:].strip() label = re.sub(r'\s+', ' ', label) after = re.sub(r'\s+', ' ', after)[:max_length].strip() #print (before.encode('utf8'), label.encode('utf8'), after.encode('utf8')) if len(before + after) > min_length: for xref in xref_stack[i]: xref['contexts'].append((before, label, after)) #print xref_stack[i] ####################################################################################### # Parse tables if present metadata['tables'] = {} for table_wrap in nlm.xpath('//table-wrap'): id = table_wrap.get('id') if id is not None: queries = { 'label': 'label/text()', 'caption': '(caption/title|caption/p)[1]', 'caption_raw': 'caption[1]', 'doi': 'object-id[@pub-id-type="doi"]/text()', 'xml': ".//*[local-name() = 'table'][1]", } table = query(table_wrap, queries) if 'caption' not in table and 'caption_raw' in table: table['caption'] = table['caption_raw'] if 'caption' in table: table['caption'] = re.sub(r'\s+', ' ', etree.tostring(table['caption'], method='text', encoding=unicode).strip()) #print table if 'xml' in table: table['xml'] = etree.tostring(table['xml'], encoding='utf8') metadata['tables'][id] = table # Return the metadata return metadata